Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 6 additions & 15 deletions e2e/lib/live.ts
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
import { AgentError, expect } from 'e2e';

// Tiny input at the lowest effort answers in seconds; OSWE-278's 429 takes about 10.5 s (one
// proxy retry). 60 s leaves headroom and stays under the runner's 120 s test timeout, so a
// stall surfaces as this request's error, not as TEST_TIMEOUT.
// Tiny input at the lowest effort answers in seconds. 60 s leaves headroom and stays under the
// runner's 120 s test timeout, so a stall surfaces as this request's error, not as TEST_TIMEOUT.
const BOUND_MS = 60_000;

type Request = { body?: unknown; headers?: Record<string, string>; keyless?: boolean };

/**
* Sends one request to CLIPROXY_LIVE_URL and classifies failures that are not the product's:
* missing inputs and a rejected key are configuration (exit 2); an unreachable proxy and a
* provider 429 or 5xx are infrastructure (exit 3). `known` accepts a failure the test asserts
* itself, matched on status and body.
* provider 429 or 5xx are infrastructure (exit 3).
*/
export async function live(
path: string,
request: Request = {},
known?: (status: number, body: string) => boolean,
) {
export async function live(path: string, request: Request = {}) {
const base = process.env.CLIPROXY_LIVE_URL;
if (!base) throw new AgentError('TEST_SETUP_FAILED', 'CLIPROXY_LIVE_URL is not set');
const key = process.env.CLIPROXY_CLIENT_KEY;
Expand Down Expand Up @@ -48,11 +42,8 @@ export async function live(
throw new AgentError('AUTH_CREDENTIAL_INVALID', `${path} rejected CLIPROXY_CLIENT_KEY (401)`);
}
if (status === 429 || status >= 500) {
const text = await response.clone().text();
if (!known?.(status, text)) {
const excerpt = text.slice(0, 300);
throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${path} answered ${status}: ${excerpt}`);
}
const excerpt = (await response.text()).slice(0, 300);
throw new AgentError('ENVIRONMENT_UNAVAILABLE', `${path} answered ${status}: ${excerpt}`);
}
return { response, signal };
}
Expand Down
48 changes: 15 additions & 33 deletions e2e/tests/live.e2e.ts
Original file line number Diff line number Diff line change
Expand Up @@ -108,36 +108,18 @@ test('live: Responses on claude-sonnet-4-6 completes', onLive, async () => {
expect((await response.json()).status).toBe('completed');
});

// OSWE-278's response as captured from studio2. A genuine rate limit carries a descriptive
// message, so it stays an infrastructure failure (exit 3).
const isOswe278 = (status: number, body: string) => {
try {
const { error } = JSON.parse(body);
return status === 429 && error?.type === 'rate_limit_error' && error?.message === 'Error';
} catch {
return false;
}
};

// Known failure: this asserts the bug. When OSWE-278 is fixed the status changes and the test
// fails visibly; then assert 200 here instead.
test(
'live: OSWE-278 known failure, native Messages on claude-sonnet-4-6 returns 429',
onLive,
async () => {
const { response } = await live(
'/v1/messages',
{
body: {
model: 'claude-sonnet-4-6',
max_tokens: 16,
messages: [{ role: 'user', content: say }],
},
headers: anthropic,
},
isOswe278,
);
expect(response.status).toBe(429);
expect((await response.json()).error).toEqual({ type: 'rate_limit_error', message: 'Error' });
},
);
test('live: native Messages on claude-sonnet-4-6 completes', onLive, async () => {
const { response } = await live('/v1/messages', {
body: {
model: 'claude-sonnet-4-6',
max_tokens: 16,
messages: [{ role: 'user', content: say }],
},
headers: anthropic,
});
expect(response.status).toBe(200);
expectTrace(response);
const message = await response.json();
expect(message.type).toBe('message');
expect(message.stop_reason).toBe('end_turn');
});
Loading