Backend-aware advice in HTTP errors: key variable, 402, Retry-After

CHANGELOG.md

@@ -4,6 +4,12 @@
 
 <!-- Empty. Next release starts here. -->
 
+## 0.7.2
+
+### Fixed
+
+- HTTP advice is backend-aware: `safeError(error, backend?)` names the backend's key variable on a 401 (`Check OPENROUTER_API_KEY.` on OpenRouter, `Check TYPESAFE_API_KEY.` by default), a 402 now says `Insufficient credits. Add credits at https://openrouter.ai/credits.` on OpenRouter and `Check your account balance.` elsewhere without marking the key unusable, and a 429 appends `Retry after <n> seconds.` when the response carries a numeric `Retry-After` header. One-argument `safeError` calls are unchanged.
+
 ## 0.7.1
 
 ### Fixed

docs/api.md

@@ -36,7 +36,7 @@ result.answers.severity.score;   // 0..2, may be fractional
 
 `DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send.
 
-`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified.
+`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, keys, or your submitted state, and no header value except a numeric `Retry-After` count in seconds (quoted by the 429 advice as `Retry after <n> seconds.`). The advice is backend-aware: a 401 says `Check TYPESAFE_API_KEY.` or `Check OPENROUTER_API_KEY.`, and a 402 says `Check your account balance.` except on OpenRouter, which says `Insufficient credits. Add credits at https://openrouter.ai/credits.` `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified.
 
 ## Admission
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-typesafe",
-  "version": "0.7.1",
+  "version": "0.7.2",
   "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.",
   "type": "module",
   "license": "MIT",

src/client.ts

@@ -251,7 +251,7 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
         }
         return models.map(card => card?.name).filter((name): name is string => typeof name === "string" && name.length > 0 && name.length <= 100);
       } catch (error) {
-        throw safeError(error);
+        throw safeError(error, backend);
       }
     },
     async evaluate<Q extends Questions>(input: SystemOneRequest<Q>, callOptions: EvaluationOptions = {}): Promise<Evaluation<Q>> {
@@ -285,7 +285,7 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
         }
         return { ...result, elapsedMs: Math.round(performance.now() - start) };
       } catch (error) {
-        const safe = safeError(error);
+        const safe = safeError(error, backend);
         // The request was submitted, so it counts even when it fails; the reason stays visible in `authState()`.
         usage.requestsFailed += 1;
         ledger.recordFailure();

src/errors.ts

@@ -1,8 +1,13 @@
 import { APIError, APIConnectionError, APITimeoutError, APIUserAbortError } from "@typesafe-ai/sdk";
+import { DECISIONS_BACKENDS, TYPESAFE_KEY_ENV, backendConfig } from "./backends.js";
+import type { BackendConfig, TypeSafeBackend } from "./backends.js";
 
 export type IntegrationErrorCode = "configuration" | "validation" | "budget" | "aborted" | "timeout" | "http" | "connection" | "response";
 
-/** Safe to display: never contains upstream bodies, headers, keys, or submitted state. */
+/**
+ * Safe to display: never contains upstream bodies, keys, or submitted state. The only header value a message can
+ * quote is a numeric Retry-After count in seconds, which cannot carry a secret.
+ */
 export class TypeSafeIntegrationError extends Error {
   override readonly name = "TypeSafeIntegrationError";
   constructor(
@@ -14,14 +19,31 @@ export class TypeSafeIntegrationError extends Error {
   }
 }
 
-export function safeError(error: unknown): TypeSafeIntegrationError {
+/** A numeric Retry-After delay in seconds; dates, blanks, and anything else stay out of the message. */
+function retryAfterSeconds(error: APIError): number | undefined {
+  const raw = error.headers?.get("retry-after")?.trim();
+  if (!raw) return undefined;
+  const seconds = Number(raw);
+  return Number.isSafeInteger(seconds) && seconds >= 0 ? seconds : undefined;
+}
+
+/**
+ * Classify an error into a message safe to display. `backend` names the key variable the 401 advice tells the user
+ * to check and selects the 402 wording; omitting it keeps the one-argument call and assumes the default TypeSafe key.
+ */
+export function safeError(error: unknown, backend?: TypeSafeBackend | BackendConfig): TypeSafeIntegrationError {
   if (error instanceof TypeSafeIntegrationError) return error;
   if (error instanceof APIUserAbortError) return new TypeSafeIntegrationError("aborted", "TypeSafe request cancelled; an already submitted request may still be billed.");
   if (error instanceof APITimeoutError) return new TypeSafeIntegrationError("timeout", "TypeSafe request timed out; it was not retried and may still be billed.");
   if (error instanceof APIError) {
-    const advice = error.status === 401 ? "Check TYPESAFE_API_KEY."
+    const config = backend === undefined ? undefined : typeof backend === "string" ? backendConfig(backend) : backend;
+    const keyEnv = config?.keyEnv ?? TYPESAFE_KEY_ENV;
+    const openrouter = config !== undefined && config.host === DECISIONS_BACKENDS.openrouter.host;
+    const retry = error.status === 429 ? retryAfterSeconds(error) : undefined;
+    const advice = error.status === 401 ? `Check ${keyEnv}.`
+      : error.status === 402 ? (openrouter ? "Insufficient credits. Add credits at https://openrouter.ai/credits." : "Check your account balance.")
       : error.status === 403 ? "Check your account access and model permissions."
-      : error.status === 429 ? "Check your account quota and try again later."
+      : error.status === 429 ? `Check your account quota and try again later.${retry === undefined ? "" : ` Retry after ${retry} seconds.`}`
       : error.status === 400 || error.status === 422 ? "Check the question format and model limits."
       : "Try again later or check the service status.";
     return new TypeSafeIntegrationError("http", `TypeSafe returned HTTP ${error.status}. ${advice} No automatic retry was made.`, error.status);

tests/client.test.ts

@@ -4,6 +4,8 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { after, before, test } from "node:test";
 import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js";
+import { safeError } from "../src/errors.js";
+import { APIError } from "@typesafe-ai/sdk";
 import type { Questions, SystemOneRequest } from "../src/index.js";
 
 export function responseFor(questions: Questions): Response {
@@ -186,6 +188,53 @@ test("HTTP errors are classified, never retried, and do not expose response secr
   }
 });
 
+test("HTTP advice names the backend's key, covers 402, and quotes a numeric Retry-After", async () => {
+  const savedTypesafeKey = process.env.TYPESAFE_API_KEY;
+  const savedOpenRouterKey = process.env.OPENROUTER_API_KEY;
+  // authState({ backend }).usable needs a key present for each backend; the client itself takes its key below.
+  process.env.TYPESAFE_API_KEY = "offline-env-key-0123456789abcdef";
+  process.env.OPENROUTER_API_KEY = "offline-or-key-0123456789abcdef";
+  const cases = [
+    { backend: "typesafe", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made." },
+    { backend: "openrouter", status: 401, retryAfter: true, usable: false, message: "TypeSafe returned HTTP 401. Check OPENROUTER_API_KEY. No automatic retry was made." },
+    { backend: "typesafe", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Check your account balance. No automatic retry was made." },
+    { backend: "openrouter", status: 402, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 402. Insufficient credits. Add credits at https://openrouter.ai/credits. No automatic retry was made." },
+    { backend: "typesafe", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." },
+    { backend: "openrouter", status: 429, retryAfter: true, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. Retry after 7 seconds. No automatic retry was made." },
+    { backend: "typesafe", status: 429, retryAfter: false, usable: true, message: "TypeSafe returned HTTP 429. Check your account quota and try again later. No automatic retry was made." },
+  ] as const;
+  try {
+    for (const c of cases) {
+      clearAuthState();
+      const client = createTypeSafe({
+        apiKey: "test-key",
+        backend: c.backend,
+        fetch: async () => Response.json({ error: { code: "x", message: "never-print-me" } }, { status: c.status, headers: c.retryAfter ? { "Retry-After": "7" } : {} }),
+      });
+      await assert.rejects(client.evaluate(sample()), error => {
+        assert.ok(error instanceof TypeSafeIntegrationError);
+        assert.equal(error.code, "http");
+        assert.equal(error.status, c.status);
+        assert.equal(error.message, c.message);
+        assert.equal(error.message.includes("never-print-me"), false);
+        return true;
+      });
+      // A 401 rejects the key; a 402 is billing, so it must leave it usable (REJECTED_STATUSES stays {401, 403}).
+      assert.equal(authState({ backend: c.backend }).usable, c.usable, `${c.backend} ${c.status}`);
+    }
+  } finally {
+    clearAuthState();
+    if (savedTypesafeKey === undefined) delete process.env.TYPESAFE_API_KEY; else process.env.TYPESAFE_API_KEY = savedTypesafeKey;
+    if (savedOpenRouterKey === undefined) delete process.env.OPENROUTER_API_KEY; else process.env.OPENROUTER_API_KEY = savedOpenRouterKey;
+  }
+});
+
+test("safeError keeps its one-argument form and defaults the key advice", () => {
+  const own = new TypeSafeIntegrationError("http", "kept", 401);
+  assert.equal(safeError(own), own);
+  assert.equal(safeError(new APIError(401, undefined, new Headers())).message, "TypeSafe returned HTTP 401. Check TYPESAFE_API_KEY. No automatic retry was made.");
+});
+
 test("cancellation before submission does not consume an attempt", async () => {
   const client = createTypeSafe({ apiKey: "test-key", fetch: async () => { throw new Error("must not run"); } });
   await assert.rejects(client.evaluate(sample(), { signal: AbortSignal.abort() }), hasCode("aborted"));