diff --git a/CHANGELOG.md b/CHANGELOG.md index df91973..79450bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ +## 0.7.2 + +### Fixed + +- HTTP advice is backend-aware: `safeError(error, backend?)` names the backend's key variable on a 401 (`Check OPENROUTER_API_KEY.` on OpenRouter, `Check TYPESAFE_API_KEY.` by default), a 402 now says `Insufficient credits. Add credits at https://openrouter.ai/credits.` on OpenRouter and `Check your account balance.` elsewhere without marking the key unusable, and a 429 appends `Retry after seconds.` when the response carries a numeric `Retry-After` header. One-argument `safeError` calls are unchanged. + ## 0.7.1 ### Fixed diff --git a/docs/api.md b/docs/api.md index 690f1e1..814f55d 100644 --- a/docs/api.md +++ b/docs/api.md @@ -36,7 +36,7 @@ result.answers.severity.score; // 0..2, may be fractional `DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. -`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified. +`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, keys, or your submitted state, and no header value except a numeric `Retry-After` count in seconds (quoted by the 429 advice as `Retry after seconds.`). The advice is backend-aware: a 401 says `Check TYPESAFE_API_KEY.` or `Check OPENROUTER_API_KEY.`, and a 402 says `Check your account balance.` except on OpenRouter, which says `Insufficient credits. Add credits at https://openrouter.ai/credits.` `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified. ## Admission diff --git a/package-lock.json b/package-lock.json index c59c9ed..0a44ed7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.7.1", + "version": "0.7.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.7.1", + "version": "0.7.2", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index 8268a97..7e008f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.7.1", + "version": "0.7.2", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT", diff --git a/src/client.ts b/src/client.ts index fbde528..eb9118b 100644 --- a/src/client.ts +++ b/src/client.ts @@ -251,7 +251,7 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { } return models.map(card => card?.name).filter((name): name is string => typeof name === "string" && name.length > 0 && name.length <= 100); } catch (error) { - throw safeError(error); + throw safeError(error, backend); } }, async evaluate(input: SystemOneRequest, callOptions: EvaluationOptions = {}): Promise> { @@ -285,7 +285,7 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { } return { ...result, elapsedMs: Math.round(performance.now() - start) }; } catch (error) { - const safe = safeError(error); + const safe = safeError(error, backend); // The request was submitted, so it counts even when it fails; the reason stays visible in `authState()`. usage.requestsFailed += 1; ledger.recordFailure(); diff --git a/src/errors.ts b/src/errors.ts index 997882a..9d480be 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,8 +1,13 @@ import { APIError, APIConnectionError, APITimeoutError, APIUserAbortError } from "@typesafe-ai/sdk"; +import { DECISIONS_BACKENDS, TYPESAFE_KEY_ENV, backendConfig } from "./backends.js"; +import type { BackendConfig, TypeSafeBackend } from "./backends.js"; export type IntegrationErrorCode = "configuration" | "validation" | "budget" | "aborted" | "timeout" | "http" | "connection" | "response"; -/** Safe to display: never contains upstream bodies, headers, keys, or submitted state. */ +/** + * Safe to display: never contains upstream bodies, keys, or submitted state. The only header value a message can + * quote is a numeric Retry-After count in seconds, which cannot carry a secret. + */ export class TypeSafeIntegrationError extends Error { override readonly name = "TypeSafeIntegrationError"; constructor( @@ -14,14 +19,31 @@ export class TypeSafeIntegrationError extends Error { } } -export function safeError(error: unknown): TypeSafeIntegrationError { +/** A numeric Retry-After delay in seconds; dates, blanks, and anything else stay out of the message. */ +function retryAfterSeconds(error: APIError): number | undefined { + const raw = error.headers?.get("retry-after")?.trim(); + if (!raw) return undefined; + const seconds = Number(raw); + return Number.isSafeInteger(seconds) && seconds >= 0 ? seconds : undefined; +} + +/** + * Classify an error into a message safe to display. `backend` names the key variable the 401 advice tells the user + * to check and selects the 402 wording; omitting it keeps the one-argument call and assumes the default TypeSafe key. + */ +export function safeError(error: unknown, backend?: TypeSafeBackend | BackendConfig): TypeSafeIntegrationError { if (error instanceof TypeSafeIntegrationError) return error; if (error instanceof APIUserAbortError) return new TypeSafeIntegrationError("aborted", "TypeSafe request cancelled; an already submitted request may still be billed."); if (error instanceof APITimeoutError) return new TypeSafeIntegrationError("timeout", "TypeSafe request timed out; it was not retried and may still be billed."); if (error instanceof APIError) { - const advice = error.status === 401 ? "Check TYPESAFE_API_KEY." + const config = backend === undefined ? undefined : typeof backend === "string" ? backendConfig(backend) : backend; + const keyEnv = config?.keyEnv ?? TYPESAFE_KEY_ENV; + const openrouter = config !== undefined && config.host === DECISIONS_BACKENDS.openrouter.host; + const retry = error.status === 429 ? retryAfterSeconds(error) : undefined; + const advice = error.status === 401 ? `Check ${keyEnv}.` + : error.status === 402 ? (openrouter ? "Insufficient credits. Add credits at https://openrouter.ai/credits." : "Check your account balance.") : error.status === 403 ? "Check your account access and model permissions." - : error.status === 429 ? "Check your account quota and try again later." + : error.status === 429 ? `Check your account quota and try again later.${retry === undefined ? "" : ` Retry after ${retry} seconds.`}` : error.status === 400 || error.status === 422 ? "Check the question format and model limits." : "Try again later or check the service status."; return new TypeSafeIntegrationError("http", `TypeSafe returned HTTP ${error.status}. ${advice} No automatic retry was made.`, error.status); diff --git a/tests/client.test.ts b/tests/client.test.ts index ac58364..3801fd9 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -4,6 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { after, before, test } from "node:test"; import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js"; +import { safeError } from "../src/errors.js"; +import { APIError } from "@typesafe-ai/sdk"; import type { Questions, SystemOneRequest } from "../src/index.js"; export function responseFor(questions: Questions): Response { @@ -186,6 +188,53 @@ test("HTTP errors are classified, never retried, and do not expose response secr } }); +test("HTTP advice names the backend's key, covers 402, and quotes a numeric Retry-After", async () => { + const savedTypesafeKey = process.env.TYPESAFE_API_KEY; + const savedOpenRouterKey = process.env.OPENROUTER_API_KEY; + // authState({ backend }).usable needs a key present for each backend; the client itself takes its key below. + process.env.TYPESAFE_API_KEY = "offline-env-key-0123456789abcdef"; + process.env.OPENROUTER_API_KEY = "offline-or-key-0123456789abcdef"; + const cases = [ + { backend: "typesafe", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made." }, + { backend: "openrouter", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check OPENROUTER_API_KEY. No automatic retry was made." }, + { backend: "typesafe", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Check your account balance. No automatic retry was made." }, + { backend: "openrouter", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Insufficient credits. Add credits at https://openrouter.ai/credits. No automatic retry was made." }, + { backend: "typesafe", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." }, + { backend: "openrouter", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." }, + { backend: "typesafe", status: 429, retryAfter: false, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. No automatic retry was made." }, + ] as const; + try { + for (const c of cases) { + clearAuthState(); + const client = createTypeSafe({ + apiKey: "test-key", + backend: c.backend, + fetch: async () => Response.json({ error: { code: "x", message: "never-print-me" } }, { status: c.status, headers: c.retryAfter ? { "Retry-After": "7" } : {} }), + }); + await assert.rejects(client.evaluate(sample()), error => { + assert.ok(error instanceof TypeSafeIntegrationError); + assert.equal(error.code, "http"); + assert.equal(error.status, c.status); + assert.equal(error.message, c.message); + assert.equal(error.message.includes("never-print-me"), false); + return true; + }); + // A 401 rejects the key; a 402 is billing, so it must leave it usable (REJECTED_STATUSES stays {401, 403}). + assert.equal(authState({ backend: c.backend }).usable, c.usable, `${c.backend} ${c.status}`); + } + } finally { + clearAuthState(); + if (savedTypesafeKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedTypesafeKey; + if (savedOpenRouterKey === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = savedOpenRouterKey; + } +}); + +test("safeError keeps its one-argument form and defaults the key advice", () => { + const own = new TypeSafeIntegrationError("http", "kept", 401); + assert.equal(safeError(own), own); + assert.equal(safeError(new APIError(401, undefined, new Headers())).message, "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made."); +}); + test("cancellation before submission does not consume an attempt", async () => { const client = createTypeSafe({ apiKey: "test-key", fetch: async () => { throw new Error("must not run"); } }); await assert.rejects(client.evaluate(sample(), { signal: AbortSignal.abort() }), hasCode("aborted"));