From 8609c6e8df42dd930c1294955f09217cfc40cf74 Mon Sep 17 00:00:00 2001 From: SamuelMauricioL Date: Tue, 22 Sep 2026 18:21:46 -0500 Subject: [PATCH 1/3] fix: ask each backend for its own model list listModels() requested the SDK's own /v1/models against the backend's host. Only the judgment path was rewritten, so an OpenRouter client asked for a path that host does not serve: openrouter.ai answers it with its app page and the SDK rejects the body. listModels() could never verify a key on that backend, which is the budget-free way to prove one. The transport wrapper now rewrites the model-list path too, and renames the list to the field the SDK reads when the registry declares another. Both are registry data, so the SDK stays untouched and the TypeSafe backend keeps the caller's fetch exactly as before. Closes #11 --- docs/api.md | 2 +- src/backends.ts | 13 ++++++++++++- src/client.ts | 41 ++++++++++++++++++++++++++++++++++------- tests/client.test.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 9 deletions(-) diff --git a/docs/api.md b/docs/api.md index ec24250..8bfc2de 100644 --- a/docs/api.md +++ b/docs/api.md @@ -34,7 +34,7 @@ result.answers.severity.score; // 0..2, may be fractional | `ledger` | the store next to the key | Inject a ledger in tests | | `fetch` | global fetch | Inject a transport for offline tests | -`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own path, `path`. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. +`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath` and `modelsField` for the model list — the list arrives under OpenRouter's `data` field and is renamed to the `models` the SDK reads. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. `evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`. diff --git a/src/backends.ts b/src/backends.ts index 84dcee8..2a9133c 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -10,6 +10,10 @@ export interface BackendConfig { keyEnv?: string; /** Request path, when the backend does not serve the SDK's own `/v1/systemone`. */ path?: string; + /** Request path for the model list, when the backend does not serve the SDK's own `/v1/models`. */ + modelsPath?: string; + /** Field the model list arrives in, when the backend does not use the SDK's own `models`. */ + modelsField?: string; } /** The backend every key and auth function assumes when none is named. */ @@ -21,7 +25,14 @@ export const TYPESAFE_KEY_ENV = "TYPESAFE_API_KEY"; /** Registry of known judgment backends. Extendable by callers. */ export const DECISIONS_BACKENDS: Record = { typesafe: { label: "TypeSafe", host: "https://api.typesafe.ai", keyEnv: TYPESAFE_KEY_ENV }, - openrouter: { label: "OpenRouter", host: "https://openrouter.ai", keyEnv: "OPENROUTER_API_KEY", path: "/api/alpha/decisions" }, + openrouter: { + label: "OpenRouter", + host: "https://openrouter.ai", + keyEnv: "OPENROUTER_API_KEY", + path: "/api/alpha/decisions", + modelsPath: "/api/v1/models", + modelsField: "data", + }, }; /** The registry entry for a backend name; a `configuration` error for a name the registry does not know. */ diff --git a/src/client.ts b/src/client.ts index 985df03..c711c3a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -2,7 +2,7 @@ import { TypeSafeClient } from "@typesafe-ai/sdk"; import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk"; import { recordAuthFailure, recordAuthVerified } from "./auth.js"; import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, usesTypesafeKey } from "./backends.js"; -import type { TypeSafeBackend } from "./backends.js"; +import type { BackendConfig, TypeSafeBackend } from "./backends.js"; import type { BatchEvaluation, BatchOptions } from "./batch.js"; import { evaluateAll, evaluateMany } from "./batch.js"; import { keySituation } from "./credentials.js"; @@ -14,12 +14,39 @@ import type { BlockedCap, SpendCaps, UsageLedger, UsageReport } from "./usage.js export { DECISIONS_BACKENDS, DEFAULT_BACKEND } from "./backends.js"; export type { BackendConfig, TypeSafeBackend } from "./backends.js"; -/** The path the SDK appends to whatever base URL it is given. */ +/** The paths the SDK appends to whatever base URL it is given. */ const SDK_PATH = "/v1/systemone"; +const SDK_MODELS_PATH = "/v1/models"; -/** Send the SDK's fixed path to the backend's own, preserving any caller-supplied transport. */ -function backendFetch(path: string, inner: Fetch = fetch): Fetch { - return (input, init) => inner(String(input).replace(SDK_PATH, path), init); +/** + * Send the SDK's fixed paths to the backend's own, preserving any caller-supplied transport. A backend that serves + * its model list under another path also gets that list renamed to the field the SDK reads. + */ +function backendFetch(backend: BackendConfig, inner: Fetch = fetch): Fetch { + const { path, modelsPath, modelsField } = backend; + return async (input, init) => { + const url = String(input); + const models = modelsPath !== undefined && url.includes(SDK_MODELS_PATH); + const rewrite = models ? modelsPath : path; + const response = await inner(rewrite === undefined ? url : url.replace(models ? SDK_MODELS_PATH : SDK_PATH, rewrite), init); + return models && modelsField !== undefined ? renameModelsField(response, modelsField) : response; + }; +} + +/** + * Hand the SDK the field it reads when a backend names the model list differently. Status and headers survive; a body + * without the declared field is passed through unchanged, so the SDK still reports its own shape error. + */ +async function renameModelsField(response: Response, field: string): Promise { + const text = await response.text(); + let wire: unknown; + try { wire = JSON.parse(text); } catch { wire = undefined; } + const list = wire !== null && typeof wire === "object" ? (wire as Record)[field] : undefined; + const headers = new Headers(response.headers); + // The body is replaced, so a copied length would describe the old one. + headers.delete("content-length"); + headers.delete("content-encoding"); + return new Response(Array.isArray(list) ? JSON.stringify({ models: list }) : text, { status: response.status, statusText: response.statusText, headers }); } export interface TypeSafeOptions { @@ -148,7 +175,6 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { const backendName: TypeSafeBackend = options.backend ?? DEFAULT_BACKEND; const backend = backendConfig(backendName); const baseURL = backend.host; - const backendPath = backend.path; if (!apiKey) { // The same resolution that authState() and ensureApiKey() report, so the status line and the request agree. @@ -170,7 +196,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { ...(options.maxInputTokensPerDay === undefined ? {} : { maxInputTokensPerDay: positiveInteger(options.maxInputTokensPerDay, "maxInputTokensPerDay") }), ...(options.maxUsdPerDay === undefined ? {} : { maxUsdPerDay: positiveNumber(options.maxUsdPerDay, "maxUsdPerDay") }), }, capsFromEnvironment()); - const transport = backendPath ? backendFetch(backendPath, options.fetch) : options.fetch; + // A backend that serves its own paths gets a transport that rewrites them; the default backend keeps the caller's. + const transport = backend.path !== undefined || backend.modelsPath !== undefined ? backendFetch(backend, options.fetch) : options.fetch; const model = options.model ?? (backendName === "openrouter" ? "typesafe/jev-1.13" : "jev-latest"); if (typeof model !== "string" || !model.trim() || model.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters."); // Do not inherit SDK debug logging or alternate destinations from the environment. diff --git a/tests/client.test.ts b/tests/client.test.ts index 1edcf97..12fbe16 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -296,6 +296,49 @@ test("known backend is called at its full request URL", async () => { } }); +test("listModels asks each backend for its own model list", async () => { + const backends = [ + ["typesafe", "https://api.typesafe.ai/v1/models", { models: [{ name: "jev-latest" }] }, ["jev-latest"]], + ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["Vendor: Model"]], + ] as const; + for (const [backend, expectedUrl, wire, expected] of backends) { + let capturedUrl = ""; + const client = createTypeSafe({ + apiKey: "test-key", + backend, + fetch: async (url) => { capturedUrl = String(url); return Response.json(wire); }, + }); + assert.deepEqual(await client.listModels(), expected); + assert.equal(capturedUrl, expectedUrl); + } +}); + +test("a model list without the backend's declared field keeps the SDK's own shape error", async () => { + const client = createTypeSafe({ + apiKey: "test-key", + backend: "openrouter", + fetch: async () => Response.json({ items: [{ name: "not the declared field" }] }), + }); + await assert.rejects(client.listModels(), (error: unknown) => { + assert.ok(error instanceof TypeSafeIntegrationError); + assert.equal(error.code, "response"); + return true; + }); +}); + +test("the model list is renamed only for the backend that declares another field", async () => { + const client = createTypeSafe({ + apiKey: "test-key", + backend: "typesafe", + fetch: async () => Response.json({ data: [{ name: "not the SDK's field" }] }), + }); + await assert.rejects(client.listModels(), (error: unknown) => { + assert.ok(error instanceof TypeSafeIntegrationError); + assert.equal(error.code, "response"); + return true; + }); +}); + test("openrouter backend uses default model typesafe/jev-1.13", async () => { let sentModel: string | undefined; const client = createTypeSafe({ From 94273b53d377831503cdf6f9a2cec389adf5f2ac Mon Sep 17 00:00:00 2001 From: SamuelMauricioL Date: Tue, 22 Sep 2026 18:58:08 -0500 Subject: [PATCH 2/3] fix: keep a public model list from proving a key, and return ids Review feedback on the model-list change. openrouter.ai serves /api/v1/models to anyone, so a success there says nothing about the key. Recording it as verified would report a missing or garbage OpenRouter key as good, which the unreachable path never did. The registry now says whether a list checks the key, and only a list that does writes the verification record. OpenRouter entries carry the id that `model:` accepts and a display name, so the entry's id is promoted to the name listModels() returns. The TypeSafe backend declares neither and is unchanged. --- docs/api.md | 4 ++-- src/backends.ts | 6 ++++++ src/client.ts | 24 +++++++++++++++++------- tests/client.test.ts | 16 ++++++++++++++-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/docs/api.md b/docs/api.md index 8bfc2de..690f1e1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -34,9 +34,9 @@ result.answers.severity.score; // 0..2, may be fractional | `ledger` | the store next to the key | Inject a ledger in tests | | `fetch` | global fetch | Inject a transport for offline tests | -`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath` and `modelsField` for the model list — the list arrives under OpenRouter's `data` field and is renamed to the `models` the SDK reads. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. +`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. -`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`. +`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified. ## Admission diff --git a/src/backends.ts b/src/backends.ts index 2a9133c..9b9f9e2 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -14,6 +14,10 @@ export interface BackendConfig { modelsPath?: string; /** Field the model list arrives in, when the backend does not use the SDK's own `models`. */ modelsField?: string; + /** Entry field carrying the id callers pass as `model:`, when the SDK's own `name` is only a label. */ + modelsIdField?: string; + /** Whether the model list checks the key. A public list accepts any key, so it proves nothing. Absent means it does. */ + modelsVerifyKey?: boolean; } /** The backend every key and auth function assumes when none is named. */ @@ -32,6 +36,8 @@ export const DECISIONS_BACKENDS: Record = { path: "/api/alpha/decisions", modelsPath: "/api/v1/models", modelsField: "data", + modelsIdField: "id", + modelsVerifyKey: false, }, }; diff --git a/src/client.ts b/src/client.ts index c711c3a..fbde528 100644 --- a/src/client.ts +++ b/src/client.ts @@ -29,24 +29,33 @@ function backendFetch(backend: BackendConfig, inner: Fetch = fetch): Fetch { const models = modelsPath !== undefined && url.includes(SDK_MODELS_PATH); const rewrite = models ? modelsPath : path; const response = await inner(rewrite === undefined ? url : url.replace(models ? SDK_MODELS_PATH : SDK_PATH, rewrite), init); - return models && modelsField !== undefined ? renameModelsField(response, modelsField) : response; + return models && modelsField !== undefined ? translateModels(response, backend) : response; }; } /** - * Hand the SDK the field it reads when a backend names the model list differently. Status and headers survive; a body - * without the declared field is passed through unchanged, so the SDK still reports its own shape error. + * Hand the SDK the list it expects: the field it reads, and the entry value callers pass as `model:` when the backend + * labels models differently. Status and headers survive; a body without the declared field is passed through + * unchanged, so the SDK still reports its own shape error. */ -async function renameModelsField(response: Response, field: string): Promise { +async function translateModels(response: Response, backend: BackendConfig): Promise { + const { modelsField, modelsIdField } = backend; const text = await response.text(); let wire: unknown; try { wire = JSON.parse(text); } catch { wire = undefined; } - const list = wire !== null && typeof wire === "object" ? (wire as Record)[field] : undefined; + const list = modelsField !== undefined && wire !== null && typeof wire === "object" ? (wire as Record)[modelsField] : undefined; const headers = new Headers(response.headers); // The body is replaced, so a copied length would describe the old one. headers.delete("content-length"); headers.delete("content-encoding"); - return new Response(Array.isArray(list) ? JSON.stringify({ models: list }) : text, { status: response.status, statusText: response.statusText, headers }); + const send = (body: string) => new Response(body, { status: response.status, statusText: response.statusText, headers }); + if (!Array.isArray(list)) return send(text); + const models = list.map(entry => { + if (modelsIdField === undefined || entry === null || typeof entry !== "object") return entry; + const id = (entry as Record)[modelsIdField]; + return typeof id === "string" && id.length > 0 ? { ...entry, name: id } : entry; + }); + return send(JSON.stringify({ models })); } export interface TypeSafeOptions { @@ -235,7 +244,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { try { const models = await client.models.list(callOptions); if (!Array.isArray(models)) throw new TypeSafeIntegrationError("response", "TypeSafe returned an unexpected model list."); - if (!verificationRecorded) { + // A backend that serves its list publicly accepts any key, so a success there proves nothing about one. + if (backend.modelsVerifyKey !== false && !verificationRecorded) { verificationRecorded = true; recordAuthVerified(); } diff --git a/tests/client.test.ts b/tests/client.test.ts index 12fbe16..ac58364 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { after, before, test } from "node:test"; -import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, TypeSafeIntegrationError } from "../src/index.js"; +import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js"; import type { Questions, SystemOneRequest } from "../src/index.js"; export function responseFor(questions: Questions): Response { @@ -299,7 +299,7 @@ test("known backend is called at its full request URL", async () => { test("listModels asks each backend for its own model list", async () => { const backends = [ ["typesafe", "https://api.typesafe.ai/v1/models", { models: [{ name: "jev-latest" }] }, ["jev-latest"]], - ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["Vendor: Model"]], + ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["vendor/model"]], ] as const; for (const [backend, expectedUrl, wire, expected] of backends) { let capturedUrl = ""; @@ -339,6 +339,18 @@ test("the model list is renamed only for the backend that declares another field }); }); +test("a public model list leaves the auth state unverified", async () => { + clearAuthState(); + const client = createTypeSafe({ + apiKey: "garbage-key", + backend: "openrouter", + fetch: async () => Response.json({ data: [{ id: "vendor/model", name: "Vendor: Model" }] }), + }); + assert.deepEqual(await client.listModels(), ["vendor/model"]); + // openrouter.ai serves this list to anyone, so a success says nothing about the key. + assert.equal(authState({ backend: "openrouter" }).verified, false); +}); + test("openrouter backend uses default model typesafe/jev-1.13", async () => { let sentModel: string | undefined; const client = createTypeSafe({ From 91acd571f8018c738790ebf82b7ecde85b1faf6b Mon Sep 17 00:00:00 2001 From: Ryan Gapac Date: Wed, 23 Sep 2026 08:31:24 +0800 Subject: [PATCH 3/3] chore: release 0.7.1 --- CHANGELOG.md | 12 ++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1056af8..df91973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ +## 0.7.1 + +### Fixed + +- `listModels()` asks each backend for its own model list. It requested the SDK's `/v1/models` on every backend, so on OpenRouter it fetched an HTML page and always failed; it now uses `/api/v1/models` and reads the list from OpenRouter's `data` field (#11). +- `listModels()` on OpenRouter returns model ids (`vendor/model`), the values `model:` accepts, instead of display names. +- A public model list no longer proves a key. OpenRouter serves its list without checking the key, so a successful `listModels()` there leaves `authState({ backend: "openrouter" })` unverified rather than recording a garbage key as verified. + +### Added + +- `BackendConfig` gains optional `modelsPath`, `modelsField`, `modelsIdField`, and `modelsVerifyKey`, documented in the API reference. + ## 0.7.0 ### Fixed diff --git a/package-lock.json b/package-lock.json index a9089fe..c59c9ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index f89eba9..8268a97 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT",