test(e2e): hermetic Tester Army suite for proxy use cases (MASTRA-768)

.github/workflows/ci.yml

@@ -19,3 +19,35 @@ jobs:
           cache: true
       - run: go build ./...
       - run: go test ./...
+
+  e2e-hermetic:
+    runs-on: ubuntu-latest
+    env:
+      E2E_TELEMETRY_DISABLED: "1"
+      MASTRA_TELEMETRY_DISABLED: "1"
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+      - run: go build -o e2e/.bin/cli-proxy-api ./cmd/server
+      - uses: pnpm/action-setup@v4
+        with:
+          package_json_file: e2e/package.json
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+          cache-dependency-path: e2e/pnpm-lock.yaml
+      - run: pnpm --dir e2e install --frozen-lockfile
+      - run: pnpm --dir e2e run fmt:check
+      - run: pnpm --dir e2e run lint
+      - run: pnpm --dir e2e exec e2e run --target hermetic --output .e2e/ci
+      - if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-hermetic-report
+          path: |
+            e2e/.e2e/ci/report.json
+            e2e/.e2e/logs/hermetic.log

e2e/.gitignore

@@ -0,0 +1,3 @@
+.e2e/
+.bin/
+node_modules/

e2e/.oxfmtrc.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "./node_modules/oxfmt/configuration_schema.json",
+  "printWidth": 100,
+  "singleQuote": true
+}

e2e/e2e.config.ts

@@ -0,0 +1,34 @@
+import type { E2EConfig } from 'e2e';
+import { defineEngine } from 'e2e/engine';
+
+// No UI to drive: the runner starts app.command, probes readyUrl and exposes app.baseUrl.
+const http = (platform: string) =>
+  defineEngine({ name: 'http', version: '1', spiVersion: 1, platform });
+
+export default {
+  tests: 'tests/**/*.e2e.ts',
+  retries: 0,
+  trace: 'off',
+  targets: [
+    {
+      engine: http('hermetic'),
+      app: {
+        url: 'http://127.0.0.1:0',
+        readyUrl: 'http://127.0.0.1:{port}/healthz',
+        command: {
+          executable: 'node',
+          args: ['hermetic/start.mjs'],
+          // Loopback is never proxied, so this blocks only the binary's own internet fetches.
+          env: {
+            PORT: '{port}',
+            HTTPS_PROXY: 'http://127.0.0.1:9',
+            MASTRA_TELEMETRY_DISABLED: '1',
+          },
+          log: '.e2e/logs/hermetic.log',
+        },
+      },
+    },
+    // Live tests read CLIPROXY_LIVE_URL themselves: the runner refuses plain HTTP off loopback.
+    { engine: http('live') },
+  ],
+} satisfies E2EConfig;

e2e/hermetic/mock-upstream.mjs

@@ -0,0 +1,112 @@
+// Mock Codex and Claude upstream. A credential whose key ends in `-limited` answers Codex
+// 429 usage_limit_reached; every other request gets a complete SSE stream (or a Claude JSON
+// message when the request does not stream). GET /_mock/requests returns what it received.
+import { createServer } from 'node:http';
+
+const requests = [];
+const resetAt = () => Math.floor(Date.now() / 1000) + 3600;
+
+const sse = (res, headers, events) => {
+  res.writeHead(200, { 'content-type': 'text/event-stream', ...headers });
+  for (const data of events) res.write(`event: ${data.type}\ndata: ${JSON.stringify(data)}\n\n`);
+  res.end();
+};
+
+const codex = (res, key, body) => {
+  if (key.endsWith('-limited')) {
+    res.writeHead(429, { 'content-type': 'application/json' });
+    const error = { type: 'usage_limit_reached', message: 'You have hit your usage limit.' };
+    res.end(JSON.stringify({ error: { ...error, resets_at: resetAt(), resets_in_seconds: 3600 } }));
+    return;
+  }
+  const item = {
+    type: 'message',
+    id: 'msg_mock',
+    role: 'assistant',
+    status: 'completed',
+    content: [{ type: 'output_text', text: '{"ok":true}', annotations: [] }],
+  };
+  const response = { id: 'resp_mock', object: 'response', model: body.model, output: [] };
+  const usage = { input_tokens: 5, output_tokens: 3, total_tokens: 8 };
+  sse(
+    res,
+    {
+      'x-codex-primary-used-percent': '10',
+      'x-codex-primary-window-minutes': '300',
+      'x-codex-primary-reset-at': String(resetAt()),
+      'x-codex-secondary-used-percent': '20',
+      'x-codex-secondary-window-minutes': '10080',
+      'x-codex-secondary-reset-at': String(resetAt()),
+    },
+    [
+      { type: 'response.created', response: { ...response, status: 'in_progress' } },
+      { type: 'response.output_item.added', output_index: 0, item: { ...item, content: [] } },
+      {
+        type: 'response.output_text.delta',
+        output_index: 0,
+        content_index: 0,
+        delta: item.content[0].text,
+      },
+      { type: 'response.output_item.done', output_index: 0, item },
+      { type: 'response.completed', response: { ...response, status: 'completed', usage } },
+    ],
+  );
+};
+
+const claude = (res, body) => {
+  const headers = {
+    'anthropic-ratelimit-unified-status': 'allowed',
+    'anthropic-ratelimit-unified-5h-status': 'allowed',
+    'anthropic-ratelimit-unified-5h-utilization': '0.1',
+    'anthropic-ratelimit-unified-5h-reset': String(resetAt()),
+    'anthropic-ratelimit-unified-7d-status': 'allowed',
+    'anthropic-ratelimit-unified-7d-utilization': '0.2',
+    'anthropic-ratelimit-unified-7d-reset': String(resetAt()),
+  };
+  const message = {
+    id: 'msg_mock',
+    type: 'message',
+    role: 'assistant',
+    model: body.model,
+    content: [],
+    stop_reason: null,
+    stop_sequence: null,
+    usage: { input_tokens: 5, output_tokens: 0 },
+  };
+  if (!body.stream) {
+    res.writeHead(200, { 'content-type': 'application/json', ...headers });
+    const content = [{ type: 'text', text: 'ok' }];
+    res.end(JSON.stringify({ ...message, content, stop_reason: 'end_turn' }));
+    return;
+  }
+  sse(res, headers, [
+    { type: 'message_start', message },
+    { type: 'content_block_start', index: 0, content_block: { type: 'text', text: '' } },
+    { type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text: 'ok' } },
+    { type: 'content_block_stop', index: 0 },
+    { type: 'message_delta', delta: { stop_reason: 'end_turn' }, usage: { output_tokens: 1 } },
+    { type: 'message_stop' },
+  ]);
+};
+
+export const startMock = () =>
+  new Promise((resolve) => {
+    const server = createServer(async (req, res) => {
+      if (req.method === 'GET' && req.url === '/_mock/requests') {
+        res.writeHead(200, { 'content-type': 'application/json' });
+        res.end(JSON.stringify(requests));
+        return;
+      }
+      let raw = '';
+      for await (const chunk of req) raw += chunk;
+      const body = JSON.parse(raw);
+      const key = (req.headers['x-api-key'] ?? req.headers.authorization ?? '').replace(
+        'Bearer ',
+        '',
+      );
+      requests.push({ path: req.url, headers: req.headers, body, key });
+      if (req.url === '/v1/responses') codex(res, key, body);
+      else claude(res, body);
+    });
+    server.listen(0, '127.0.0.1', () => resolve(`http://127.0.0.1:${server.address().port}`));
+  });

e2e/hermetic/start.mjs

@@ -0,0 +1,45 @@
+// Starts the mock upstream, writes a proxy config whose API-key credentials point at it, and
+// runs the candidate binary on $PORT. Tests read the mock URL from .e2e/mock.json.
+import { spawn } from 'node:child_process';
+import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { startMock } from './mock-upstream.mjs';
+
+const url = await startMock();
+mkdirSync('.e2e', { recursive: true });
+writeFileSync('.e2e/mock.json', JSON.stringify({ url }));
+
+const dir = mkdtempSync(join(tmpdir(), 'cliproxy-e2e-'));
+mkdirSync(join(dir, 'auths'));
+const codex = (key, model, priority = 0) => ({
+  'api-key': key,
+  'base-url': `${url}/v1`,
+  priority,
+  models: [{ name: model }],
+});
+// YAML accepts JSON. The failover and cooldown tests own their models and credentials;
+// failover-a has the higher priority, so the proxy tries it first.
+const config = {
+  host: '127.0.0.1',
+  port: Number(process.env.PORT),
+  'auth-dir': join(dir, 'auths'),
+  'api-keys': ['e2e-client-key'],
+  'codex-api-key': [
+    codex('codex-main', 'gpt-5.6-sol'),
+    codex('failover-a-limited', 'e2e-failover', 1),
+    codex('failover-b', 'e2e-failover'),
+    codex('cooled-a-limited', 'e2e-cooled'),
+    codex('cooled-b-limited', 'e2e-cooled'),
+  ],
+  'claude-api-key': [
+    { 'api-key': 'claude-main', 'base-url': url, models: [{ name: 'claude-sonnet-4-6' }] },
+  ],
+};
+writeFileSync(join(dir, 'config.yaml'), JSON.stringify(config, null, 2));
+
+const proxy = spawn('.bin/cli-proxy-api', ['-config', join(dir, 'config.yaml'), '-local-model'], {
+  stdio: 'inherit',
+});
+for (const signal of ['SIGTERM', 'SIGINT']) process.on(signal, () => proxy.kill(signal));
+proxy.on('exit', (code) => process.exit(code ?? 1));

e2e/lib/proxy.ts

@@ -0,0 +1,32 @@
+import { readFileSync } from 'node:fs';
+
+export const clientKey = 'e2e-client-key';
+
+/** POSTs JSON to the proxy with the client key. */
+export const post = (baseUrl: string | undefined, path: string, body: unknown, headers = {}) =>
+  fetch(new URL(path, baseUrl), {
+    method: 'POST',
+    headers: {
+      authorization: `Bearer ${clientKey}`,
+      'content-type': 'application/json',
+      ...headers,
+    },
+    body: JSON.stringify(body),
+  });
+
+/** A Responses `input` holding one user message. */
+export const input = (text: string) => [
+  { type: 'message', role: 'user', content: [{ type: 'input_text', text }] },
+];
+
+/** The mock upstream's URL, written by hermetic/start.mjs. */
+export const mockUrl = (): string =>
+  JSON.parse(readFileSync(new URL('../.e2e/mock.json', import.meta.url), 'utf8')).url;
+
+export type UpstreamRequest = { path: string; body: Record<string, unknown>; key: string };
+
+/** Requests the mock received whose body contains `marker`, oldest first. */
+export async function upstreamRequests(marker: string): Promise<UpstreamRequest[]> {
+  const all: UpstreamRequest[] = await (await fetch(`${mockUrl()}/_mock/requests`)).json();
+  return all.filter((request) => JSON.stringify(request.body).includes(marker));
+}

e2e/lib/sse.ts

@@ -0,0 +1,32 @@
+export type SseEvent = { event: string | undefined; data: string };
+
+/** Reads a server-sent event stream to its end, or until `signal` aborts. */
+export async function readSse(response: Response, signal: AbortSignal): Promise<SseEvent[]> {
+  const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader();
+  const cancel = () => void reader.cancel(signal.reason);
+  signal.addEventListener('abort', cancel, { once: true });
+  const events: SseEvent[] = [];
+  let buffer = '';
+  try {
+    for (;;) {
+      const { value, done } = await reader.read();
+      signal.throwIfAborted();
+      if (done) break;
+      buffer += value.replaceAll('\r\n', '\n');
+      let end;
+      while ((end = buffer.indexOf('\n\n')) >= 0) {
+        const lines = buffer.slice(0, end).split('\n');
+        buffer = buffer.slice(end + 2);
+        const field = (name: string) =>
+          lines
+            .filter((line) => line.startsWith(`${name}:`))
+            .map((line) => line.slice(name.length + 1).trimStart());
+        const data = field('data');
+        if (data.length > 0) events.push({ event: field('event')[0], data: data.join('\n') });
+      }
+    }
+  } finally {
+    signal.removeEventListener('abort', cancel);
+  }
+  return events;
+}

e2e/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "cliproxyapi-e2e",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "fmt": "oxfmt .",
+    "fmt:check": "oxfmt --check .",
+    "lint": "oxlint --deny-warnings ."
+  },
+  "devDependencies": {
+    "e2e": "0.16.0",
+    "oxfmt": "0.71.0",
+    "oxlint": "1.86.0",
+    "zod": "4.6.1"
+  },
+  "engines": {
+    "node": ">=22.12.0"
+  },
+  "packageManager": "pnpm@11.21.0"
+}