diff --git a/CHANGELOG.md b/CHANGELOG.md index 1056af8..df91973 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ +## 0.7.1 + +### Fixed + +- `listModels()` asks each backend for its own model list. It requested the SDK's `/v1/models` on every backend, so on OpenRouter it fetched an HTML page and always failed; it now uses `/api/v1/models` and reads the list from OpenRouter's `data` field (#11). +- `listModels()` on OpenRouter returns model ids (`vendor/model`), the values `model:` accepts, instead of display names. +- A public model list no longer proves a key. OpenRouter serves its list without checking the key, so a successful `listModels()` there leaves `authState({ backend: "openrouter" })` unverified rather than recording a garbage key as verified. + +### Added + +- `BackendConfig` gains optional `modelsPath`, `modelsField`, `modelsIdField`, and `modelsVerifyKey`, documented in the API reference. + ## 0.7.0 ### Fixed diff --git a/docs/api.md b/docs/api.md index ec24250..690f1e1 100644 --- a/docs/api.md +++ b/docs/api.md @@ -34,9 +34,9 @@ result.answers.severity.score; // 0..2, may be fractional | `ledger` | the store next to the key | Inject a ledger in tests | | `fetch` | global fetch | Inject a transport for offline tests | -`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own path, `path`. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. +`DECISIONS_BACKENDS` is the registry behind `backend`: each entry carries `label`, `host`, `keyEnv`, and, when the service does not serve the SDK's own paths, `path` for the judgment request plus `modelsPath`, `modelsField`, and `modelsIdField` for the model list — OpenRouter's list arrives under `data` and is renamed to the `models` the SDK reads, with each entry's `id` promoted to the `name` that `listModels()` returns. `modelsVerifyKey: false` marks a backend whose model list is public, and therefore proves nothing about the key. `DEFAULT_BACKEND` is `"typesafe"`. The TypeSafe backend takes its key from `TYPESAFE_API_KEY`, then the `/typesafe login` store. Every other backend reads only its own environment variable (`OPENROUTER_API_KEY` for OpenRouter): the store holds a TypeSafe key, and a login verifies against api.typesafe.ai, so neither applies elsewhere. Pass the same `backend` to `authState`, `keySituation`, and `ensureApiKey` so what you report matches what you send. -`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`. +`evaluate(request, { signal })` validates before sending and rejects with `TypeSafeIntegrationError`. `code` is one of `configuration`, `validation`, `budget`, `aborted`, `timeout`, `http`, `connection`, `response`; messages never contain upstream bodies, headers, keys, or your submitted state. `listModels()` verifies the key without counting toward `maxRequests`, except on a backend whose model list is public (`modelsVerifyKey: false`), which accepts any key and leaves the auth state unverified. ## Admission diff --git a/package-lock.json b/package-lock.json index a9089fe..c59c9ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "license": "MIT", "dependencies": { "@typesafe-ai/sdk": "^0.6.0", diff --git a/package.json b/package.json index f89eba9..8268a97 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pi-typesafe", - "version": "0.7.0", + "version": "0.7.1", "description": "TypeSafe AI (Jev) decisions for Pi: batched Choice/Score/Noul evaluation tool, terminal playground, and a typed API other extensions build on.", "type": "module", "license": "MIT", diff --git a/src/backends.ts b/src/backends.ts index 84dcee8..9b9f9e2 100644 --- a/src/backends.ts +++ b/src/backends.ts @@ -10,6 +10,14 @@ export interface BackendConfig { keyEnv?: string; /** Request path, when the backend does not serve the SDK's own `/v1/systemone`. */ path?: string; + /** Request path for the model list, when the backend does not serve the SDK's own `/v1/models`. */ + modelsPath?: string; + /** Field the model list arrives in, when the backend does not use the SDK's own `models`. */ + modelsField?: string; + /** Entry field carrying the id callers pass as `model:`, when the SDK's own `name` is only a label. */ + modelsIdField?: string; + /** Whether the model list checks the key. A public list accepts any key, so it proves nothing. Absent means it does. */ + modelsVerifyKey?: boolean; } /** The backend every key and auth function assumes when none is named. */ @@ -21,7 +29,16 @@ export const TYPESAFE_KEY_ENV = "TYPESAFE_API_KEY"; /** Registry of known judgment backends. Extendable by callers. */ export const DECISIONS_BACKENDS: Record = { typesafe: { label: "TypeSafe", host: "https://api.typesafe.ai", keyEnv: TYPESAFE_KEY_ENV }, - openrouter: { label: "OpenRouter", host: "https://openrouter.ai", keyEnv: "OPENROUTER_API_KEY", path: "/api/alpha/decisions" }, + openrouter: { + label: "OpenRouter", + host: "https://openrouter.ai", + keyEnv: "OPENROUTER_API_KEY", + path: "/api/alpha/decisions", + modelsPath: "/api/v1/models", + modelsField: "data", + modelsIdField: "id", + modelsVerifyKey: false, + }, }; /** The registry entry for a backend name; a `configuration` error for a name the registry does not know. */ diff --git a/src/client.ts b/src/client.ts index 985df03..fbde528 100644 --- a/src/client.ts +++ b/src/client.ts @@ -2,7 +2,7 @@ import { TypeSafeClient } from "@typesafe-ai/sdk"; import type { Fetch, Questions, SystemOneRequest, SystemOneResult } from "@typesafe-ai/sdk"; import { recordAuthFailure, recordAuthVerified } from "./auth.js"; import { DEFAULT_BACKEND, TYPESAFE_KEY_ENV, backendConfig, usesTypesafeKey } from "./backends.js"; -import type { TypeSafeBackend } from "./backends.js"; +import type { BackendConfig, TypeSafeBackend } from "./backends.js"; import type { BatchEvaluation, BatchOptions } from "./batch.js"; import { evaluateAll, evaluateMany } from "./batch.js"; import { keySituation } from "./credentials.js"; @@ -14,12 +14,48 @@ import type { BlockedCap, SpendCaps, UsageLedger, UsageReport } from "./usage.js export { DECISIONS_BACKENDS, DEFAULT_BACKEND } from "./backends.js"; export type { BackendConfig, TypeSafeBackend } from "./backends.js"; -/** The path the SDK appends to whatever base URL it is given. */ +/** The paths the SDK appends to whatever base URL it is given. */ const SDK_PATH = "/v1/systemone"; +const SDK_MODELS_PATH = "/v1/models"; -/** Send the SDK's fixed path to the backend's own, preserving any caller-supplied transport. */ -function backendFetch(path: string, inner: Fetch = fetch): Fetch { - return (input, init) => inner(String(input).replace(SDK_PATH, path), init); +/** + * Send the SDK's fixed paths to the backend's own, preserving any caller-supplied transport. A backend that serves + * its model list under another path also gets that list renamed to the field the SDK reads. + */ +function backendFetch(backend: BackendConfig, inner: Fetch = fetch): Fetch { + const { path, modelsPath, modelsField } = backend; + return async (input, init) => { + const url = String(input); + const models = modelsPath !== undefined && url.includes(SDK_MODELS_PATH); + const rewrite = models ? modelsPath : path; + const response = await inner(rewrite === undefined ? url : url.replace(models ? SDK_MODELS_PATH : SDK_PATH, rewrite), init); + return models && modelsField !== undefined ? translateModels(response, backend) : response; + }; +} + +/** + * Hand the SDK the list it expects: the field it reads, and the entry value callers pass as `model:` when the backend + * labels models differently. Status and headers survive; a body without the declared field is passed through + * unchanged, so the SDK still reports its own shape error. + */ +async function translateModels(response: Response, backend: BackendConfig): Promise { + const { modelsField, modelsIdField } = backend; + const text = await response.text(); + let wire: unknown; + try { wire = JSON.parse(text); } catch { wire = undefined; } + const list = modelsField !== undefined && wire !== null && typeof wire === "object" ? (wire as Record)[modelsField] : undefined; + const headers = new Headers(response.headers); + // The body is replaced, so a copied length would describe the old one. + headers.delete("content-length"); + headers.delete("content-encoding"); + const send = (body: string) => new Response(body, { status: response.status, statusText: response.statusText, headers }); + if (!Array.isArray(list)) return send(text); + const models = list.map(entry => { + if (modelsIdField === undefined || entry === null || typeof entry !== "object") return entry; + const id = (entry as Record)[modelsIdField]; + return typeof id === "string" && id.length > 0 ? { ...entry, name: id } : entry; + }); + return send(JSON.stringify({ models })); } export interface TypeSafeOptions { @@ -148,7 +184,6 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { const backendName: TypeSafeBackend = options.backend ?? DEFAULT_BACKEND; const backend = backendConfig(backendName); const baseURL = backend.host; - const backendPath = backend.path; if (!apiKey) { // The same resolution that authState() and ensureApiKey() report, so the status line and the request agree. @@ -170,7 +205,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { ...(options.maxInputTokensPerDay === undefined ? {} : { maxInputTokensPerDay: positiveInteger(options.maxInputTokensPerDay, "maxInputTokensPerDay") }), ...(options.maxUsdPerDay === undefined ? {} : { maxUsdPerDay: positiveNumber(options.maxUsdPerDay, "maxUsdPerDay") }), }, capsFromEnvironment()); - const transport = backendPath ? backendFetch(backendPath, options.fetch) : options.fetch; + // A backend that serves its own paths gets a transport that rewrites them; the default backend keeps the caller's. + const transport = backend.path !== undefined || backend.modelsPath !== undefined ? backendFetch(backend, options.fetch) : options.fetch; const model = options.model ?? (backendName === "openrouter" ? "typesafe/jev-1.13" : "jev-latest"); if (typeof model !== "string" || !model.trim() || model.length > 100) throw new TypeSafeIntegrationError("configuration", "model must be a nonempty string of at most 100 characters."); // Do not inherit SDK debug logging or alternate destinations from the environment. @@ -208,7 +244,8 @@ export function createTypeSafe(options: TypeSafeOptions = {}): TypeSafe { try { const models = await client.models.list(callOptions); if (!Array.isArray(models)) throw new TypeSafeIntegrationError("response", "TypeSafe returned an unexpected model list."); - if (!verificationRecorded) { + // A backend that serves its list publicly accepts any key, so a success there proves nothing about one. + if (backend.modelsVerifyKey !== false && !verificationRecorded) { verificationRecorded = true; recordAuthVerified(); } diff --git a/tests/client.test.ts b/tests/client.test.ts index 1edcf97..ac58364 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -3,7 +3,7 @@ import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { after, before, test } from "node:test"; -import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, TypeSafeIntegrationError } from "../src/index.js"; +import { createTypeSafe, choice, noul, score, normalizeEvaluationRequest, parseEvaluationRequest, authState, clearAuthState, TypeSafeIntegrationError } from "../src/index.js"; import type { Questions, SystemOneRequest } from "../src/index.js"; export function responseFor(questions: Questions): Response { @@ -296,6 +296,61 @@ test("known backend is called at its full request URL", async () => { } }); +test("listModels asks each backend for its own model list", async () => { + const backends = [ + ["typesafe", "https://api.typesafe.ai/v1/models", { models: [{ name: "jev-latest" }] }, ["jev-latest"]], + ["openrouter", "https://openrouter.ai/api/v1/models", { data: [{ id: "vendor/model", name: "Vendor: Model" }] }, ["vendor/model"]], + ] as const; + for (const [backend, expectedUrl, wire, expected] of backends) { + let capturedUrl = ""; + const client = createTypeSafe({ + apiKey: "test-key", + backend, + fetch: async (url) => { capturedUrl = String(url); return Response.json(wire); }, + }); + assert.deepEqual(await client.listModels(), expected); + assert.equal(capturedUrl, expectedUrl); + } +}); + +test("a model list without the backend's declared field keeps the SDK's own shape error", async () => { + const client = createTypeSafe({ + apiKey: "test-key", + backend: "openrouter", + fetch: async () => Response.json({ items: [{ name: "not the declared field" }] }), + }); + await assert.rejects(client.listModels(), (error: unknown) => { + assert.ok(error instanceof TypeSafeIntegrationError); + assert.equal(error.code, "response"); + return true; + }); +}); + +test("the model list is renamed only for the backend that declares another field", async () => { + const client = createTypeSafe({ + apiKey: "test-key", + backend: "typesafe", + fetch: async () => Response.json({ data: [{ name: "not the SDK's field" }] }), + }); + await assert.rejects(client.listModels(), (error: unknown) => { + assert.ok(error instanceof TypeSafeIntegrationError); + assert.equal(error.code, "response"); + return true; + }); +}); + +test("a public model list leaves the auth state unverified", async () => { + clearAuthState(); + const client = createTypeSafe({ + apiKey: "garbage-key", + backend: "openrouter", + fetch: async () => Response.json({ data: [{ id: "vendor/model", name: "Vendor: Model" }] }), + }); + assert.deepEqual(await client.listModels(), ["vendor/model"]); + // openrouter.ai serves this list to anyone, so a success says nothing about the key. + assert.equal(authState({ backend: "openrouter" }).verified, false); +}); + test("openrouter backend uses default model typesafe/jev-1.13", async () => { let sentModel: string | undefined; const client = createTypeSafe({