fix: ask each backend for its own model list

CHANGELOG.md

@@ -4,6 +4,18 @@
 
 <!-- Empty. Next release starts here. -->
 
+## 0.7.1
+
+### Fixed
+
+- `listModels()` asks each backend for its own model list. It requested the SDK's `/v1/models` on every backend, so on OpenRouter it fetched an HTML page and always failed; it now uses `/api/v1/models` and reads the list from OpenRouter's `data` field (#11).
+- `listModels()` on OpenRouter returns model ids (`vendor/model`), the values `model:` accepts, instead of display names.
+- A public model list no longer proves a key. OpenRouter serves its list without checking the key, so a successful `listModels()` there leaves `authState({ backend: "openrouter" })` unverified rather than recording a garbage key as verified.
+
+### Added
+
+- `BackendConfig` gains optional `modelsPath`, `modelsField`, `modelsIdField`, and `modelsVerifyKey`, documented in the API reference.
+
 ## 0.7.0
 
 ### Fixed

docs/api.md

@@ -34,9 +34,9 @@ result.answers.severity.score;   // 0..2, may be fractional
 | `ledger` | the store next to the key | Inject a ledger in tests |
 | `fetch` | global fetch | Inject a transport for offline tests |
 
-`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own path, `path`. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send.
+`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send.
 
-`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`.
+`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified.
 
 ## Admission
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-typesafe",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.",
   "type": "module",
   "license": "MIT",

src/backends.ts

@@ -10,6 +10,14 @@ export interface BackendConfig {
   keyEnv?: string;
   /** Request path, when the backend does not serve the SDK's own `/v1/systemone`. */
   path?: string;
+  /** Request path for the model list, when the backend does not serve the SDK's own `/v1/models`. */
+  modelsPath?: string;
+  /** Field the model list arrives in, when the backend does not use the SDK's own `models`. */
+  modelsField?: string;
+  /** Entry field carrying the id callers pass as `model:`, when the SDK's own `name` is only a label. */
+  modelsIdField?: string;
+  /** Whether the model list checks the key. A public list accepts any key, so it proves nothing. Absent means it does. */
+  modelsVerifyKey?: boolean;
 }
 
 /** The backend every key and auth function assumes when none is named. */
@@ -21,7 +29,16 @@ export const TYPESAFE_KEY_ENV = "TYPESAFE_API_KEY";
 /** Registry of known judgment backends. Extendable by callers. */
 export const DECISIONS_BACKENDS: Record<TypeSafeBackend, BackendConfig> = {
   typesafe: { label: "TypeSafe", host: "https://api.typesafe.ai", keyEnv: TYPESAFE_KEY_ENV },
-  openrouter: { label: "OpenRouter", host: "https://openrouter.ai", keyEnv: "OPENROUTER_API_KEY", path: "/api/alpha/decisions" },
+  openrouter: {
+    label: "OpenRouter",
+    host: "https://openrouter.ai",
+    keyEnv: "OPENROUTER_API_KEY",
+    path: "/api/alpha/decisions",
+    modelsPath: "/api/v1/models",
+    modelsField: "data",
+    modelsIdField: "id",
+    modelsVerifyKey: false,
+  },
 };
 
 /** The registry entry for a backend name; a `configuration` error for a name the registry does not know. */

src/client.ts

@@ -2,7 +2,7 @@ import { TypeSafeClient } from "@typesafe-ai/sdk";
 import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk";
 import { recordAuthFailure, recordAuthVerified } from "./auth.js";
 import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, usesTypesafeKey } from "./backends.js";
-import type { TypeSafeBackend } from "./backends.js";
+import type { BackendConfig, TypeSafeBackend } from "./backends.js";
 import type { BatchEvaluation, BatchOptions } from "./batch.js";
 import { evaluateAll, evaluateMany } from "./batch.js";
 import { keySituation } from "./credentials.js";
@@ -14,12 +14,48 @@ import type { BlockedCap, SpendCaps, UsageLedger, UsageReport } from "./usage.js
 export { DECISIONS_BACKENDS, DEFAULT_BACKEND } from "./backends.js";
 export type { BackendConfig, TypeSafeBackend } from "./backends.js";
 
-/** The path the SDK appends to whatever base URL it is given. */
+/** The paths the SDK appends to whatever base URL it is given. */
 const SDK_PATH = "/v1/systemone";
+const SDK_MODELS_PATH = "/v1/models";
 
-/** Send the SDK's fixed path to the backend's own, preserving any caller-supplied transport. */
-function backendFetch(path: string, inner: Fetch = fetch): Fetch {
-  return (input, init) => inner(String(input).replace(SDK_PATH, path), init);
+/**
+ * Send the SDK's fixed paths to the backend's own, preserving any caller-supplied transport. A backend that serves
+ * its model list under another path also gets that list renamed to the field the SDK reads.
+ */
+function backendFetch(backend: BackendConfig, inner: Fetch = fetch): Fetch {
+  const { path, modelsPath, modelsField } = backend;
+  return async (input, init) => {
+    const url = String(input);
+    const models = modelsPath !== undefined && url.includes(SDK_MODELS_PATH);
+    const rewrite = models ? modelsPath : path;
+    const response = await inner(rewrite === undefined ? url : url.replace(models ? SDK_MODELS_PATH : SDK_PATH, rewrite), init);
+    return models && modelsField !== undefined ? translateModels(response, backend) : response;
+  };
+}
+
+/**
+ * Hand the SDK the list it expects: the field it reads, and the entry value callers pass as `model:` when the backend
+ * labels models differently. Status and headers survive; a body without the declared field is passed through
+ * unchanged, so the SDK still reports its own shape error.
+ */
+async function translateModels(response: Response, backend: BackendConfig): Promise<Response> {
+  const { modelsField, modelsIdField } = backend;
+  const text = await response.text();
+  let wire: unknown;
+  try { wire = JSON.parse(text); } catch { wire = undefined; }
+  const list = modelsField !== undefined && wire !== null && typeof wire === "object" ? (wire as Record<string, unknown>)[modelsField] : undefined;
+  const headers = new Headers(response.headers);
+  // The body is replaced, so a copied length would describe the old one.
+  headers.delete("content-length");
+  headers.delete("content-encoding");
+  const send = (body: string) => new Response(body, { status: response.status, statusText: response.statusText, headers });
+  if (!Array.isArray(list)) return send(text);
+  const models = list.map(entry => {
+    if (modelsIdField === undefined || entry === null || typeof entry !== "object") return entry;
+    const id = (entry as Record<string, unknown>)[modelsIdField];
+    return typeof id === "string" && id.length > 0 ? { ...entry, name: id } : entry;
+  });
+  return send(JSON.stringify({ models }));
 }
 
 export interface TypeSafeOptions {
@@ -148,7 +184,6 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
   const backendName: TypeSafeBackend = options.backend ?? DEFAULT_BACKEND;
   const backend = backendConfig(backendName);
   const baseURL = backend.host;
-  const backendPath = backend.path;
 
   if (!apiKey) {
     // The same resolution that authState() and ensureApiKey() report, so the status line and the request agree.
@@ -170,7 +205,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
     ...(options.maxInputTokensPerDay === undefined ? {} : { maxInputTokensPerDay: positiveInteger(options.maxInputTokensPerDay, "maxInputTokensPerDay") }),
     ...(options.maxUsdPerDay === undefined ? {} : { maxUsdPerDay: positiveNumber(options.maxUsdPerDay, "maxUsdPerDay") }),
   }, capsFromEnvironment());
-  const transport = backendPath ? backendFetch(backendPath, options.fetch) : options.fetch;
+  // A backend that serves its own paths gets a transport that rewrites them; the default backend keeps the caller's.
+  const transport = backend.path !== undefined || backend.modelsPath !== undefined ? backendFetch(backend, options.fetch) : options.fetch;
   const model = options.model ?? (backendName === "openrouter" ? "typesafe/jev-1.13" : "jev-latest");
   if (typeof model !== "string" || !model.trim() || model.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters.");
   // Do not inherit SDK debug logging or alternate destinations from the environment.
@@ -208,7 +244,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe {
       try {
         const models = await client.models.list(callOptions);
         if (!Array.isArray(models)) throw new TypeSafeIntegrationError("response", "TypeSafe returned an unexpected model list.");
-        if (!verificationRecorded) {
+        // A backend that serves its list publicly accepts any key, so a success there proves nothing about one.
+        if (backend.modelsVerifyKey !== false && !verificationRecorded) {
           verificationRecorded = true;
           recordAuthVerified();
         }

tests/client.test.ts

@@ -3,7 +3,7 @@ import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { after, before, test } from "node:test";
-import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, TypeSafeIntegrationError } from "../src/index.js";
+import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js";
 import type { Questions, SystemOneRequest } from "../src/index.js";
 
 export function responseFor(questions: Questions): Response {
@@ -296,6 +296,61 @@ test("known backend is called at its full request URL", async () => {
   }
 });
 
+test("listModels asks each backend for its own model list", async () => {
+  const backends = [
+    ["typesafe", "https://api.typesafe.ai/v1/models", { models: [{ name: "jev-latest" }] }, ["jev-latest"]],
+    ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["vendor/model"]],
+  ] as const;
+  for (const [backend, expectedUrl, wire, expected] of backends) {
+    let capturedUrl = "";
+    const client = createTypeSafe({
+      apiKey: "test-key",
+      backend,
+      fetch: async (url) => { capturedUrl = String(url); return Response.json(wire); },
+    });
+    assert.deepEqual(await client.listModels(), expected);
+    assert.equal(capturedUrl, expectedUrl);
+  }
+});
+
+test("a model list without the backend's declared field keeps the SDK's own shape error", async () => {
+  const client = createTypeSafe({
+    apiKey: "test-key",
+    backend: "openrouter",
+    fetch: async () => Response.json({ items: [{ name: "not the declared field" }] }),
+  });
+  await assert.rejects(client.listModels(), (error: unknown) => {
+    assert.ok(error instanceof TypeSafeIntegrationError);
+    assert.equal(error.code, "response");
+    return true;
+  });
+});
+
+test("the model list is renamed only for the backend that declares another field", async () => {
+  const client = createTypeSafe({
+    apiKey: "test-key",
+    backend: "typesafe",
+    fetch: async () => Response.json({ data: [{ name: "not the SDK's field" }] }),
+  });
+  await assert.rejects(client.listModels(), (error: unknown) => {
+    assert.ok(error instanceof TypeSafeIntegrationError);
+    assert.equal(error.code, "response");
+    return true;
+  });
+});
+
+test("a public model list leaves the auth state unverified", async () => {
+  clearAuthState();
+  const client = createTypeSafe({
+    apiKey: "garbage-key",
+    backend: "openrouter",
+    fetch: async () => Response.json({ data: [{ id: "vendor/model", name: "Vendor: Model" }] }),
+  });
+  assert.deepEqual(await client.listModels(), ["vendor/model"]);
+  // openrouter.ai serves this list to anyone, so a success says nothing about the key.
+  assert.equal(authState({ backend: "openrouter" }).verified, false);
+});
+
 test("openrouter backend uses default model typesafe/jev-1.13", async () => {
   let sentModel: string | undefined;
   const client = createTypeSafe({