diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a512c46 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +# CI: typecheck, unit tests, and the prompt/contract drift guard on every +# push and pull request. patch-package runs via the postinstall hook, so a +# plain `npm ci` reproduces the local toolchain exactly. +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run typecheck + - run: npm test + - run: npm run check:drift diff --git a/__tests__/scorecard.test.tsx b/__tests__/scorecard.test.tsx new file mode 100644 index 0000000..060dd03 --- /dev/null +++ b/__tests__/scorecard.test.tsx @@ -0,0 +1,87 @@ +/** + * GenUI scorecard rig: scores every exemplar screen in scorecard/corpus on + * contract validity (parser + zod props) and headless render success through + * BOTH design-system libraries, then writes the score table to + * scorecard/SCORECARD.md. Run with `npm run scorecard`. + */ +import { readFileSync, readdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { Renderer } from "@openuidev/react-lang"; +import type { Library } from "@openuidev/react-lang"; +import React from "react"; +import { act, create, type ReactTestRenderer } from "react-test-renderer"; + +// WebView has no native module under jest - MapView renders as null here. +jest.mock("react-native-webview", () => ({ WebView: () => null })); + +// eslint-disable-next-line import/first +import { genosLibrary } from "../src/genos/library"; +// eslint-disable-next-line import/first +import { buildGenosLibrary } from "../src/genos/ui/contract"; +// eslint-disable-next-line import/first +import { materialRenderers } from "../src/genos/ui/material"; +// eslint-disable-next-line import/first +import { formatScorecard, scoreContract, type ScreenScore } from "../scorecard/harness"; + +const materialLibrary = buildGenosLibrary(materialRenderers); + +const corpusDir = join(__dirname, "..", "scorecard", "corpus"); +const files = readdirSync(corpusDir) + .filter((f) => f.endsWith(".txt")) + .sort(); + +function renderCheck(library: Library, program: string): { ok: boolean; note?: string } { + let tree: ReactTestRenderer | undefined; + try { + act(() => { + tree = create(); + }); + const json = tree?.toJSON(); + act(() => tree?.unmount()); + return json ? { ok: true } : { ok: false, note: "rendered an empty tree" }; + } catch (err) { + const message = err instanceof Error ? err.message.split("\n")[0] : String(err); + return { ok: false, note: message.slice(0, 120) }; + } +} + +const scores: ScreenScore[] = files.map((file) => { + const source = readFileSync(join(corpusDir, file), "utf-8").trim(); + const score = scoreContract(file.replace(/\.txt$/, ""), source, genosLibrary); + if (score.contractOk) { + const c = renderCheck(genosLibrary, source); + score.cupertinoOk = c.ok; + if (c.note) score.renderNotes.push(`cupertino: ${c.note}`); + const m = renderCheck(materialLibrary, source); + score.materialOk = m.ok; + if (m.note) score.renderNotes.push(`material: ${m.note}`); + } + return score; +}); + +writeFileSync( + join(corpusDir, "..", "SCORECARD.md"), + formatScorecard(scores, Object.keys(genosLibrary.components).sort()), +); + +describe("GenUI scorecard", () => { + it("has at least 10 corpus screens", () => { + expect(files.length).toBeGreaterThanOrEqual(10); + }); + + it("covers every contract component across the corpus", () => { + const used = new Set(scores.flatMap((s) => s.components)); + const missing = Object.keys(genosLibrary.components).filter((c) => !used.has(c)); + expect(missing).toEqual([]); + }); + + for (const score of scores) { + it(`${score.name}: contract-valid and renders in both design systems`, () => { + expect(score.propErrors).toEqual([]); + expect(score.unknownComponents).toEqual([]); + expect(score.contractOk).toBe(true); + expect(score.cupertinoOk).toBe(true); + expect(score.materialOk).toBe(true); + }); + } +}); diff --git a/__tests__/stream.test.ts b/__tests__/stream.test.ts new file mode 100644 index 0000000..94943dd --- /dev/null +++ b/__tests__/stream.test.ts @@ -0,0 +1,404 @@ +/** + * Unit tests for the SSE streaming layer: chunk-boundary parsing, the manual + * UTF-8 fallback decoder, tool-call accumulation, the round budget, the + * residual-buffer flush, the stall watchdog, and the friendly error mapping. + * expo/fetch is mocked with scripted ReadableStream-like readers; the tools + * module is mocked so tool rounds never touch the network. + */ + +jest.mock("expo-secure-store", () => ({ + getItemAsync: async () => null, + setItemAsync: async () => {}, + deleteItemAsync: async () => {}, +})); +// expo/fetch pulls the winter runtime, which jest-expo's env can't load. +jest.mock("expo/fetch", () => ({ fetch: jest.fn() })); +jest.mock("../src/genos/tools/search", () => ({ + TOOLS_PROMPT_SECTION: "TOOLS_SECTION", + TOOL_DEFS: [{ type: "function", function: { name: "web_search" } }], + executeTool: jest.fn(async () => "search results"), + toolsAvailable: jest.fn(() => true), +})); + +import { fetch as expoFetch } from "expo/fetch"; +import { cerebrasKey } from "../src/config"; +import { NEEDS_LIVE_DATA, streamScreen } from "../src/genos/stream"; +import type { StreamEndInfo } from "../src/genos/stream"; +import { executeTool, toolsAvailable } from "../src/genos/tools/search"; + +const fetchMock = expoFetch as unknown as jest.Mock; +const executeToolMock = executeTool as unknown as jest.Mock; +const toolsAvailableMock = toolsAvailable as unknown as jest.Mock; + +const enc = new TextEncoder(); + +function makeReader(chunks: Uint8Array[]) { + let i = 0; + return { + read: jest.fn(async () => + i < chunks.length + ? { done: false as const, value: chunks[i++] } + : { done: true as const, value: undefined }, + ), + cancel: jest.fn(async () => {}), + }; +} + +function sseResponse(chunks: string[], status = 200) { + return { + status, + ok: status >= 200 && status < 300, + body: { getReader: () => makeReader(chunks.map((c) => enc.encode(c))) }, + text: async () => chunks.join(""), + }; +} + +function errorResponse(status: number, body = "") { + return { status, ok: false, body: null, text: async () => body }; +} + +function dataLine(chunk: unknown): string { + return `data: ${JSON.stringify(chunk)}\n`; +} + +function contentChunk(text: string, finishReason: string | null = null) { + return { choices: [{ delta: { content: text }, finish_reason: finishReason }] }; +} + +interface RunResult { + deltas: string[]; + info?: StreamEndInfo; + error?: Error; +} + +function run(handlers: { + onToolRound?: (calls: Array<{ name: string; args: Record }>) => "continue" | "abort"; + signal?: AbortSignal; +}): Promise { + const deltas: string[] = []; + return new Promise((resolve) => { + streamScreen([{ role: "user", content: "make a screen" }], { + onDelta: (t) => deltas.push(t), + onDone: (info) => resolve({ deltas, info }), + onError: (error) => resolve({ deltas, error }), + ...handlers, + }); + }); +} + +beforeEach(async () => { + fetchMock.mockReset(); + executeToolMock.mockClear(); + toolsAvailableMock.mockReturnValue(true); + await cerebrasKey.set("test-key"); +}); + +describe("SSE parsing", () => { + it("assembles data lines split across chunk boundaries", async () => { + const full = + dataLine(contentChunk("Hello, ")) + dataLine(contentChunk("world", "stop")) + "data: [DONE]\n"; + // Split mid-line and mid-token. + fetchMock.mockResolvedValue(sseResponse([full.slice(0, 23), full.slice(23, 60), full.slice(60)])); + const r = await run({}); + expect(r.error).toBeUndefined(); + expect(r.deltas.join("")).toBe("Hello, world"); + expect(r.info).toEqual({ truncated: false, dropped: false }); + }); + + it("decodes multi-byte UTF-8 split across chunks via the manual decoder", async () => { + const saved = globalThis.TextDecoder; + // Force the manual fallback decoder path. + (globalThis as { TextDecoder?: typeof TextDecoder }).TextDecoder = undefined; + try { + const bytes = enc.encode(dataLine(contentChunk("tacos 🌮 é", "stop")) + "data: [DONE]\n"); + // Cut inside the 4-byte emoji sequence. + const cut = bytes.findIndex((b, i) => i > 7 && b === 0xf0) + 2; + fetchMock.mockResolvedValue({ + status: 200, + ok: true, + body: { getReader: () => makeReader([bytes.slice(0, cut), bytes.slice(cut)]) }, + text: async () => "", + }); + const r = await run({}); + expect(r.error).toBeUndefined(); + expect(r.deltas.join("")).toBe("tacos 🌮 é"); + } finally { + globalThis.TextDecoder = saved; + } + }); + + it("treats finish_reason length as truncated", async () => { + fetchMock.mockResolvedValue( + sseResponse([dataLine(contentChunk("partial", "length")) + "data: [DONE]\n"]), + ); + const r = await run({}); + expect(r.info).toEqual({ truncated: true, dropped: false }); + }); + + it("flags a stream closed without [DONE] or finish_reason as dropped", async () => { + fetchMock.mockResolvedValue(sseResponse([dataLine(contentChunk("half a screen"))])); + const r = await run({}); + expect(r.error).toBeUndefined(); + expect(r.info).toEqual({ truncated: false, dropped: true }); + }); + + it("errors when the stream drops before any content", async () => { + fetchMock.mockResolvedValue(sseResponse([])); + const r = await run({}); + expect(r.error?.message).toMatch(/dropped before any content/); + }); + + it("flushes a trailing unterminated data event after reader done", async () => { + // No trailing newline on the finish_reason chunk. + fetchMock.mockResolvedValue( + sseResponse([dataLine(contentChunk("done deal")) + dataLine(contentChunk("", "stop")).trimEnd()]), + ); + const r = await run({}); + expect(r.error).toBeUndefined(); + expect(r.deltas.join("")).toBe("done deal"); + expect(r.info).toEqual({ truncated: false, dropped: false }); + }); + + it("surfaces in-stream error payloads without raw JSON", async () => { + fetchMock.mockResolvedValue( + sseResponse([dataLine({ error: { message: "model queue exploded" } })]), + ); + const r = await run({}); + expect(r.error?.message).toBe("model queue exploded"); + }); +}); + +describe("tool-call rounds", () => { + it("accumulates tool calls by index across split argument fragments", async () => { + const round1 = + dataLine({ + choices: [ + { + delta: { + tool_calls: [ + { index: 0, id: "call_1", function: { name: "web_search", arguments: '{"que' } }, + { index: 1, id: "call_2", function: { name: "web_search", arguments: '{"query":"sf' } }, + ], + }, + }, + ], + }) + + dataLine({ + choices: [ + { + delta: { + tool_calls: [ + { index: 1, function: { arguments: ' weather"}' } }, + { index: 0, function: { arguments: 'ry":"nyc weather"}' } }, + ], + }, + }, + ], + }) + + dataLine({ choices: [{ delta: {}, finish_reason: "tool_calls" }] }) + + "data: [DONE]\n"; + const round2 = dataLine(contentChunk("here is the weather", "stop")) + "data: [DONE]\n"; + fetchMock + .mockResolvedValueOnce(sseResponse([round1])) + .mockResolvedValueOnce(sseResponse([round2])); + const onToolRound = jest.fn(() => "continue" as const); + const r = await run({ onToolRound }); + expect(r.error).toBeUndefined(); + expect(onToolRound).toHaveBeenCalledWith([ + { name: "web_search", args: { query: "nyc weather" } }, + { name: "web_search", args: { query: "sf weather" } }, + ]); + expect(executeToolMock).toHaveBeenCalledTimes(2); + expect(r.deltas.join("")).toBe("here is the weather"); + // Round 2 must include the tool outputs in the conversation. + const secondBody = JSON.parse(fetchMock.mock.calls[1][1].body); + const roles = secondBody.messages.map((m: { role: string }) => m.role); + expect(roles).toEqual(["system", "user", "assistant", "tool", "tool"]); + expect(secondBody.messages[3].content).toBe("search results"); + }); + + it("turns an onToolRound abort into NEEDS_LIVE_DATA", async () => { + fetchMock.mockResolvedValue( + sseResponse([ + dataLine({ + choices: [ + { + delta: { + tool_calls: [ + { index: 0, id: "c1", function: { name: "web_search", arguments: "{}" } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + }) + "data: [DONE]\n", + ]), + ); + const r = await run({ onToolRound: () => "abort" }); + expect(r.error?.message).toBe(NEEDS_LIVE_DATA); + expect(executeToolMock).not.toHaveBeenCalled(); + }); + + it("stops offering tools past MAX_TOOL_ROUNDS", async () => { + const toolRound = + dataLine({ + choices: [ + { + delta: { + tool_calls: [ + { index: 0, id: "c", function: { name: "web_search", arguments: "{}" } }, + ], + }, + finish_reason: "tool_calls", + }, + ], + }) + "data: [DONE]\n"; + fetchMock + .mockResolvedValueOnce(sseResponse([toolRound])) + .mockResolvedValueOnce(sseResponse([toolRound])) + .mockResolvedValueOnce(sseResponse([toolRound])) + .mockResolvedValueOnce( + sseResponse([dataLine(contentChunk("final screen", "stop")) + "data: [DONE]\n"]), + ); + const r = await run({}); + expect(r.error).toBeUndefined(); + expect(executeToolMock).toHaveBeenCalledTimes(3); + expect(fetchMock).toHaveBeenCalledTimes(4); + const lastBody = JSON.parse(fetchMock.mock.calls[3][1].body); + expect(lastBody.tools).toBeUndefined(); + expect(r.deltas.join("")).toBe("final screen"); + }); +}); + +describe("stall watchdog and abort", () => { + it("aborts with a friendly stall error after 20s without bytes", async () => { + jest.useFakeTimers(); + try { + fetchMock.mockResolvedValue({ + status: 200, + ok: true, + body: { + getReader: () => ({ + read: jest.fn(() => new Promise(() => {})), // never settles + cancel: jest.fn(async () => {}), + }), + }, + text: async () => "", + }); + const pending = run({}); + await jest.advanceTimersByTimeAsync(20_000); + const r = await pending; + expect(r.error?.message).toMatch(/stream stalled/); + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }); + + it("resets the watchdog on every received chunk", async () => { + jest.useFakeTimers(); + try { + let reads = 0; + const chunks = [dataLine(contentChunk("a")), dataLine(contentChunk("b", "stop")) + "data: [DONE]\n"]; + fetchMock.mockResolvedValue({ + status: 200, + ok: true, + body: { + getReader: () => ({ + // Each read takes 15s of fake time - under the 20s limit. + read: jest.fn( + () => + new Promise((resolve) => { + const i = reads++; + setTimeout( + () => + resolve( + i < chunks.length + ? { done: false, value: enc.encode(chunks[i]) } + : { done: true, value: undefined }, + ), + 15_000, + ); + }), + ), + cancel: jest.fn(async () => {}), + }), + }, + text: async () => "", + }); + const pending = run({}); + await jest.advanceTimersByTimeAsync(50_000); + const r = await pending; + expect(r.error).toBeUndefined(); + expect(r.deltas.join("")).toBe("ab"); + expect(jest.getTimerCount()).toBe(0); + } finally { + jest.useRealTimers(); + } + }); + + it("settles silently when the caller aborts mid-stream", async () => { + const controller = new AbortController(); + fetchMock.mockResolvedValue({ + status: 200, + ok: true, + body: { + getReader: () => ({ + read: jest.fn(() => new Promise(() => {})), + cancel: jest.fn(async () => {}), + }), + }, + text: async () => "", + }); + const onError = jest.fn(); + let settled = false; + streamScreen([{ role: "user", content: "hi" }], { + onDelta: () => {}, + onDone: () => {}, + onError, + signal: controller.signal, + }); + await Promise.resolve(); + controller.abort(); + await new Promise((r) => setTimeout(r, 10)); + settled = true; + expect(settled).toBe(true); + expect(onError).not.toHaveBeenCalled(); + }); +}); + +describe("friendly error mapping", () => { + it("rejects the key on 401/403", async () => { + fetchMock.mockResolvedValue(errorResponse(401, '{"error":"bad key"}')); + const r = await run({}); + expect(r.error?.message).toMatch(/rejected the API key/); + expect(cerebrasKey.get()).toBeNull(); + }); + + it("maps 429 to a rate-limit message", async () => { + fetchMock.mockResolvedValue(errorResponse(429)); + const r = await run({}); + expect(r.error?.message).toMatch(/rate-limiting/); + }); + + it("maps 5xx to a provider-unavailable message", async () => { + fetchMock.mockResolvedValue(errorResponse(503)); + const r = await run({}); + expect(r.error?.message).toMatch(/unavailable/); + }); + + it("sanitizes other HTTP error bodies (no raw JSON, capped length)", async () => { + const body = JSON.stringify({ error: { message: "x".repeat(400) } }); + fetchMock.mockResolvedValue(errorResponse(400, body)); + const r = await run({}); + expect(r.error?.message).toMatch(/request failed \(HTTP 400\)/); + expect(r.error?.message).not.toMatch(/[{}"]/); + expect(r.error?.message.length).toBeLessThanOrEqual(200); + }); + + it("maps fetch rejections to a network failure message", async () => { + fetchMock.mockRejectedValue(new TypeError("Network request failed")); + const r = await run({}); + expect(r.error?.message).toMatch(/network request failed/); + }); +}); diff --git a/docs/scorecard.md b/docs/scorecard.md new file mode 100644 index 0000000..d22b672 --- /dev/null +++ b/docs/scorecard.md @@ -0,0 +1,64 @@ +# GenUI Scorecard + +A small, repo-local quality gate for generated UI: a corpus of exemplar +openui-lang screens scored against the native component contract and both +design-system renderer sets. Run it with: + +```bash +npm run scorecard +``` + +This executes `__tests__/scorecard.test.tsx` under the jest-expo rig and +rewrites `scorecard/SCORECARD.md` with the score table. + +## What each screen is scored on + +1. **Contract validity** - the program parses to a complete (non-truncated) + tree with a `root`, every component used exists in + `src/genos/ui/contract.tsx`, and every component's props pass its zod + schema. The parser wraps sub-components as `ElementNode`s while the zod + `.ref` schemas expect plain props objects, so the harness + (`scorecard/harness.ts`) recursively unwraps elements before calling + `safeParse`. Parser-reported errors (`unknown-component`, + `missing-required`, `null-required`) count as failures too. +2. **Headless render success** - each screen is rendered with + `react-test-renderer` through the Cupertino library + (`src/genos/library.ts`) and the Material library + (`buildGenosLibrary(materialRenderers)`). A screen passes when rendering + neither throws nor produces an empty tree. `react-native-webview` is + mocked, so `MapView` renders as null under jest - map behavior is not + covered by this score. +3. **Unknown components** - any component name the program uses that is not + in the contract. Unknown components render as NOTHING (react-lang returns + null), so a non-empty list always means a real user-facing gap. + +The table also reports component coverage: every contract component (all 33, +including the `TabItem` / `SelectItem` / `Series` sub-components) must appear +in at least one corpus screen, and the test suite fails if any are missing. + +## Adding a corpus entry + +1. Add `.txt` to `scorecard/corpus/` containing one complete + openui-lang program (`root = Card(...)` required). +2. Follow the prompt's syntax rules: positional arguments, one statement per + line, and every non-`root` variable referenced by a parent - unreferenced + statements are dropped before rendering. +3. Optional positional arguments cannot be skipped or passed as explicit + `null` - omit trailing arguments instead. +4. Run `npm run scorecard`. New screens are picked up automatically and must + pass all three checks; if the screen exercises a component no other screen + uses, the coverage check credits it. + +Exemplars should read like real model output - the four original entries +(`settings`, `wallet`, `chat`, `trip`) are the prompt exemplars also used by +`__tests__/render.test.tsx` and `__tests__/render-material.test.tsx`. + +## Scope + +The scorecard measures whether screens the model is *expected* to emit +actually survive this app's contract and renderers. It complements the +upstream discussion in thesysdev/openui#786 on evaluating generative-UI +output quality; it is a CI-friendly smoke gate for this fork, not a general +benchmark of model quality. Pair it with `npm run check:drift` +(`scripts/check-contract-drift.mjs`), which guards the other direction: the +embedded system prompt must not document components the contract lacks. diff --git a/package.json b/package.json index 42ce9aa..752e3f3 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,9 @@ "postinstall": "patch-package", "generate:prompt": "node scripts/embed-prompt.mjs", "typecheck": "tsc --noEmit", - "test": "jest" + "test": "jest", + "check:drift": "node scripts/check-contract-drift.mjs", + "scorecard": "jest __tests__/scorecard.test.tsx" }, "private": true, "license": "MIT", diff --git a/scorecard/SCORECARD.md b/scorecard/SCORECARD.md new file mode 100644 index 0000000..798cc84 --- /dev/null +++ b/scorecard/SCORECARD.md @@ -0,0 +1,24 @@ +# GenUI Scorecard + +Generated by `npm run scorecard` - do not edit by hand. See docs/scorecard.md. + +| screen | contract | cupertino | material | unknown components | +| --- | --- | --- | --- | --- | +| analytics | pass | pass | pass | - | +| article | pass | pass | pass | - | +| booking | pass | pass | pass | - | +| chat | pass | pass | pass | - | +| fitness | pass | pass | pass | - | +| gallery | pass | pass | pass | - | +| player | pass | pass | pass | - | +| profile | pass | pass | pass | - | +| settings | pass | pass | pass | - | +| transit | pass | pass | pass | - | +| trip | pass | pass | pass | - | +| wallet | pass | pass | pass | - | + +**12/12 screens pass all three checks.** + +## Component coverage: 33/33 + +Every contract component is covered. diff --git a/scorecard/corpus/analytics.txt b/scorecard/corpus/analytics.txt new file mode 100644 index 0000000..9bc6f3c --- /dev/null +++ b/scorecard/corpus/analytics.txt @@ -0,0 +1,11 @@ +root = Card([header, headline, bars, lines, pie, hbars]) +header = CardHeader("Q1 Revenue", "Product analytics · FY2026") +headline = HeroStat("$1.84M", "NET REVENUE", "+14.2% vs Q4") +bars = BarChart(["Jan", "Feb", "Mar"], [direct, partner], "grouped") +direct = Series("Direct", [410, 480, 520]) +partner = Series("Partner", [180, 210, 260]) +lines = LineChart(["W1", "W2", "W3", "W4"], [signups], "linear") +signups = Series("Signups", [820, 940, 910, 1120]) +pie = PieChart(["Subscriptions", "Services", "Licensing"], [62, 24, 14], "donut", "circular") +hbars = HorizontalBarChart(["Enterprise", "Growth", "Starter"], [deals], "stacked") +deals = Series("Deals closed", [38, 64, 121]) diff --git a/scorecard/corpus/article.txt b/scorecard/corpus/article.txt new file mode 100644 index 0000000..1dea92e --- /dev/null +++ b/scorecard/corpus/article.txt @@ -0,0 +1,5 @@ +root = Card([header, callout, body, facts]) +header = CardHeader("The Sunday Essay", "Reading time · 6 min") +callout = TextCallout("info", "Editors' pick", "Selected by the features desk for weekend reading.") +body = TextContent("Cities reorganize themselves around the schedules of their night workers. Long after the last train, bakeries fire their ovens and the wholesale markets begin to hum.", "default") +facts = KVList([{label: "Author", value: "Lena Ortiz"}, {label: "Published", value: "Mar 2, 2026"}, {label: "Section", value: "Features"}], "ABOUT THIS PIECE") diff --git a/scorecard/corpus/booking.txt b/scorecard/corpus/booking.txt new file mode 100644 index 0000000..91d256c --- /dev/null +++ b/scorecard/corpus/booking.txt @@ -0,0 +1,19 @@ +root = Card([header, book]) +header = CardHeader("Book a Table", "Sakura Sushi · Indiranagar") +book = Form("reservation", actions, [nameField, partyField, dateField, notesField, budgetField]) +nameField = FormControl("Name", nameInput) +nameInput = Input("guest_name", "Full name…", "text") +partyField = FormControl("Party size", partySelect, "Including children") +partySelect = Select("party_size", [two, three, four], "Select…") +two = SelectItem("2", "2 guests") +three = SelectItem("3", "3 guests") +four = SelectItem("4", "4 guests") +dateField = FormControl("Date", dateInput) +dateInput = DatePicker("reservation_date", "single") +notesField = FormControl("Requests", notesInput) +notesInput = TextArea("notes", "Allergies, seating preferences…", 3) +budgetField = FormControl("Budget per person", budgetSlider) +budgetSlider = Slider("budget", "continuous", 20, 200, 5, [60], "USD") +actions = Buttons([confirm, cancel]) +confirm = Button("Request table", Action([@ToAssistant("Submit the reservation")]), "primary") +cancel = Button("Cancel", Action([@ToAssistant("Cancel the booking")]), "tertiary") diff --git a/scorecard/corpus/chat.txt b/scorecard/corpus/chat.txt new file mode 100644 index 0000000..bcc3dc4 --- /dev/null +++ b/scorecard/corpus/chat.txt @@ -0,0 +1,8 @@ +root = Card([header, thread, compose]) +header = CardHeader("Maya Chen", "mobile · active now") +thread = Bubbles([{text: "Are we still on for dinner tonight?", me: false, time: "2:02 PM"}, {text: "Yes! Sakura Sushi at 7?", me: true}]) +compose = Form("reply", btns, [msg]) +msg = FormControl("Message", input) +input = Input("text", "Message…") +btns = Buttons([send]) +send = Button("Send", Action([@ToAssistant("Send the reply")]), "primary") diff --git a/scorecard/corpus/fitness.txt b/scorecard/corpus/fitness.txt new file mode 100644 index 0000000..b8af257 --- /dev/null +++ b/scorecard/corpus/fitness.txt @@ -0,0 +1,8 @@ +root = Card([header, steps, tiles, trend, splits, note]) +header = CardHeader("Training Week", "Mar 2 - 8, 2026") +steps = HeroStat("12,480", "STEPS TODAY", "Goal 10,000 · 125%") +tiles = StatTiles([{label: "Runs", value: "4", delta: "+1", icon: "running"}, {label: "Distance", value: "31.2 km", delta: "+8%", icon: "map-pin"}]) +trend = LineChart(["Mon", "Tue", "Wed", "Thu", "Fri"], [pace], "natural") +pace = Series("Pace", [6.1, 5.8, 5.9, 5.6, 5.4]) +splits = KVList([{label: "Longest run", value: "12.4 km"}, {label: "Avg heart rate", value: "142 bpm"}, {label: "Elevation", value: "310 m"}], "WEEK TOTALS") +note = TextCallout("warning", "Recovery day tomorrow", "Two hard sessions in a row - keep Friday easy.") diff --git a/scorecard/corpus/gallery.txt b/scorecard/corpus/gallery.txt new file mode 100644 index 0000000..a32c3ef --- /dev/null +++ b/scorecard/corpus/gallery.txt @@ -0,0 +1,4 @@ +root = Card([header, hero, grid]) +header = CardHeader("Kyoto in Autumn", "42 photos · November 2025") +hero = ImageBlock("https://images.unsplash.com/photo-1493976040374-85c8e12f0c0e", "Kiyomizu-dera at dusk") +grid = PhotoGrid([{src: "https://images.unsplash.com/photo-1545569341-9eb8b30979d9", alt: "Torii gates"}, {src: "https://images.unsplash.com/photo-1493780474015-ba834fd0ce2f", alt: "Maple leaves"}, {src: "https://images.unsplash.com/photo-1528360983277-13d401cdc186", alt: "Bamboo grove"}]) diff --git a/scorecard/corpus/player.txt b/scorecard/corpus/player.txt new file mode 100644 index 0000000..b94b08b --- /dev/null +++ b/scorecard/corpus/player.txt @@ -0,0 +1,10 @@ +root = Card([header, hero, tabs]) +header = CardHeader("Now Playing", "Midnight Runners · Synthwave") +hero = ImageBlock("https://images.unsplash.com/photo-1470225620780-dba8ba36b745", "Album art: Neon Skyline") +tabs = Tabs([upnext, lyrics]) +upnext = TabItem("Up Next", [queue]) +queue = ListBlock([s1, s2]) +s1 = ListItem("Neon Skyline", "Midnight Runners", "music", "3:42") +s2 = ListItem("Chrome Rivers", "Midnight Runners", "music", "4:05") +lyrics = TabItem("Lyrics", [verse]) +verse = TextContent("Streetlights bend across the boulevard, engines humming low.", "default") diff --git a/scorecard/corpus/profile.txt b/scorecard/corpus/profile.txt new file mode 100644 index 0000000..e9a0f73 --- /dev/null +++ b/scorecard/corpus/profile.txt @@ -0,0 +1,9 @@ +root = Card([header, hero, chips, bio, details, prefs]) +header = CardHeader("Priya Nair", "Product designer · Bengaluru") +hero = ImageBlock("https://images.unsplash.com/photo-1494790108377-be9c29b29330", "Profile photo") +chips = Chips(["About", "Work", "Reviews"]) +bio = TextContent("Designing calm interfaces for busy people. Previously at two fintech startups; now independent.", "default") +details = KVList([{label: "Location", value: "Indiranagar"}, {label: "Experience", value: "8 years"}, {label: "Rate", value: "$95/hr"}], "DETAILS") +prefs = ListBlock([avail, notif], "PREFERENCES") +avail = Toggle("Available for projects", true, "briefcase") +notif = Toggle("Weekly digest", false, "bell") diff --git a/scorecard/corpus/settings.txt b/scorecard/corpus/settings.txt new file mode 100644 index 0000000..5d10ffe --- /dev/null +++ b/scorecard/corpus/settings.txt @@ -0,0 +1,8 @@ +root = Card([header, connectivity, display]) +header = CardHeader("Settings") +connectivity = ListBlock([wifi, bt], "CONNECTIVITY") +wifi = ListItem("Wi-Fi", null, "wifi", "HomeNet", Action([@ToAssistant("Open the Wi-Fi settings screen")])) +bt = ListItem("Bluetooth", null, "bluetooth", "On", Action([@ToAssistant("Open the Bluetooth settings screen")])) +display = ListBlock([dark, bright], "DISPLAY") +dark = Toggle("Dark Mode", true, "moon") +bright = ListItem("Brightness", "Auto-adjusting", "sun", "68%", Action([@ToAssistant("Open the brightness screen")])) diff --git a/scorecard/corpus/transit.txt b/scorecard/corpus/transit.txt new file mode 100644 index 0000000..0c21448 --- /dev/null +++ b/scorecard/corpus/transit.txt @@ -0,0 +1,7 @@ +root = Card([header, alert, map, stops]) +header = CardHeader("Dinner Tonight", "Sakura Sushi · 1.2 km away") +alert = TextCallout("success", "Table confirmed", "Reservation for two at 7:00 PM, held for 15 minutes.") +map = MapView("Sakura Sushi, Indiranagar, Bengaluru", 15) +stops = ListBlock([walk, metro]) +walk = ListItem("Walk to metro", "6 min via 100 Feet Rd", "map-pin", "0.4 km") +metro = ListItem("Green Line", "Towards Silk Institute", "tram", "4 stops", Action([@ToAssistant("Show the full route")])) diff --git a/scorecard/corpus/trip.txt b/scorecard/corpus/trip.txt new file mode 100644 index 0000000..b8a2752 --- /dev/null +++ b/scorecard/corpus/trip.txt @@ -0,0 +1,9 @@ +root = Card([header, chips, tiles, chart, itin]) +header = CardHeader("Goa Getaway", "JULY 11 - 13, 2026") +chips = Chips(["Full Plan", "Beaches", "Nightlife"]) +tiles = StatTiles([{label: "Budget", value: "$640", delta: "-12%", icon: "wallet"}, {label: "Weather", value: "28°C", icon: "cloud-rain"}]) +chart = AreaChart(["Fri", "Sat", "Sun"], [spend], "natural") +spend = Series("Spend", [120, 260, 180]) +itin = ListBlock([d1, tg], "Weekend itinerary") +d1 = ListItem("Friday: North Goa", "Baga Beach & Sunset Dinner", "map-pin", "Day 1", Action([@ToAssistant("Open Friday detail")])) +tg = Toggle("Notifications", true, "bell") diff --git a/scorecard/corpus/wallet.txt b/scorecard/corpus/wallet.txt new file mode 100644 index 0000000..721c0eb --- /dev/null +++ b/scorecard/corpus/wallet.txt @@ -0,0 +1,8 @@ +root = Card([header, balance, tiles, chart, txs]) +header = CardHeader("Wallet", "Personal · ··4821") +balance = HeroStat("$8,427.50", "AVAILABLE BALANCE", "+$1,204 vs last month") +tiles = StatTiles([{label: "Spent", value: "$2,318", delta: "-12%", icon: "arrow-down-right"}, {label: "Saved", value: "$940", delta: "+8%", icon: "piggy-bank"}]) +chart = AreaChart(["Feb", "Mar", "Apr"], [spend], "natural") +spend = Series("Spending", [2100, 1890, 2480]) +txs = ListBlock([t1]) +t1 = ListItem("Blue Tokai Coffee", "Today, 9:12 AM", "coffee", "-$6.40", Action([@ToAssistant("Open the transaction detail screen")])) diff --git a/scorecard/harness.ts b/scorecard/harness.ts new file mode 100644 index 0000000..edf5eab --- /dev/null +++ b/scorecard/harness.ts @@ -0,0 +1,167 @@ +/** + * GenUI scorecard harness. Scores a corpus of openui-lang exemplar screens on + * three axes: + * (a) contract validity - every component exists in the library and its + * (recursively unwrapped) props pass the component's zod schema, plus + * the openui-lang parser reports no validation errors; + * (b) headless render success through both design-system libraries + * (filled in by the jest-expo rig in __tests__/scorecard.test.tsx); + * (c) the unknown-component list - names the model used that render as + * NOTHING in this app (react-lang returns null for unknown components). + * See docs/scorecard.md for the methodology. + */ +import { createParser } from "@openuidev/react-lang"; +import type { ElementNode, Library } from "@openuidev/react-lang"; + +export interface ScreenScore { + name: string; + contractOk: boolean; + /** zod/parser prop validation failures, "Component.path: message". */ + propErrors: string[]; + unknownComponents: string[]; + /** Sorted names of every component the screen uses. */ + components: string[]; + /** null until the render rig fills them in. */ + cupertinoOk: boolean | null; + materialOk: boolean | null; + renderNotes: string[]; +} + +function isElement(value: unknown): value is ElementNode { + return ( + !!value && typeof value === "object" && (value as { type?: unknown }).type === "element" + ); +} + +/** + * The parser wraps sub-components as ElementNodes ({type, typeName, props, + * partial}), while a component's zod schema (via `.ref`) expects the plain + * props object. Unwrap recursively so safeParse checks the real shape. + */ +function unwrap(value: unknown): unknown { + if (isElement(value)) return unwrapProps(value.props); + if (Array.isArray(value)) return value.map(unwrap); + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value as Record).map(([k, v]) => [k, unwrap(v)]), + ); + } + return value; +} + +function unwrapProps(props: Record): Record { + return Object.fromEntries(Object.entries(props).map(([k, v]) => [k, unwrap(v)])); +} + +/** Depth-first collection of every element in a parsed tree. */ +export function collectElements(root: ElementNode | null): ElementNode[] { + const out: ElementNode[] = []; + const walk = (value: unknown) => { + if (isElement(value)) { + out.push(value); + Object.values(value.props).forEach(walk); + } else if (Array.isArray(value)) { + value.forEach(walk); + } else if (value && typeof value === "object") { + Object.values(value).forEach(walk); + } + }; + walk(root); + return out; +} + +/** Score (a) contract validity and (c) unknown components for one screen. */ +export function scoreContract(name: string, source: string, library: Library): ScreenScore { + const score: ScreenScore = { + name, + contractOk: false, + propErrors: [], + unknownComponents: [], + components: [], + cupertinoOk: null, + materialOk: null, + renderNotes: [], + }; + + let result; + try { + result = createParser(library.toJSONSchema()).parse(source); + } catch (err) { + score.propErrors.push(`parse failed: ${err instanceof Error ? err.message : String(err)}`); + return score; + } + + const unknown = new Set(); + for (const err of result.meta.errors) { + if (err.code === "unknown-component") unknown.add(err.component); + else score.propErrors.push(`${err.component}${err.path}: ${err.message}`); + } + + const elements = collectElements(result.root); + score.components = [...new Set(elements.map((el) => el.typeName))].sort(); + for (const el of elements) { + const def = library.components[el.typeName]; + if (!def) { + unknown.add(el.typeName); + continue; + } + const check = def.props.safeParse(unwrapProps(el.props)); + if (!check.success) { + for (const issue of check.error.issues.slice(0, 3)) { + score.propErrors.push(`${el.typeName}.${issue.path.join(".")}: ${issue.message}`); + } + } + } + + if (result.meta.incomplete) score.propErrors.push("program parses as incomplete/truncated"); + if (!result.root) score.propErrors.push("no root element parsed"); + + score.unknownComponents = [...unknown].sort(); + score.contractOk = + result.root !== null && + !result.meta.incomplete && + score.unknownComponents.length === 0 && + score.propErrors.length === 0; + return score; +} + +function mark(value: boolean | null): string { + if (value === null) return "-"; + return value ? "pass" : "FAIL"; +} + +/** Render the full scorecard as a markdown table (written to SCORECARD.md). */ +export function formatScorecard(scores: ScreenScore[], contractComponents: string[]): string { + const lines: string[] = [ + "# GenUI Scorecard", + "", + "Generated by `npm run scorecard` - do not edit by hand. See docs/scorecard.md.", + "", + "| screen | contract | cupertino | material | unknown components |", + "| --- | --- | --- | --- | --- |", + ]; + for (const s of scores) { + lines.push( + `| ${s.name} | ${mark(s.contractOk)} | ${mark(s.cupertinoOk)} | ${mark(s.materialOk)} | ${s.unknownComponents.join(", ") || "-"} |`, + ); + } + + const failing = scores.filter((s) => !s.contractOk || s.cupertinoOk === false || s.materialOk === false); + lines.push("", `**${scores.length - failing.length}/${scores.length} screens pass all three checks.**`, ""); + for (const s of failing) { + lines.push(`### ${s.name}`); + for (const e of s.propErrors) lines.push(`- ${e}`); + for (const n of s.renderNotes) lines.push(`- ${n}`); + lines.push(""); + } + + const used = new Set(scores.flatMap((s) => s.components)); + const missing = contractComponents.filter((c) => !used.has(c)).sort(); + lines.push( + `## Component coverage: ${used.size}/${contractComponents.length}`, + "", + missing.length > 0 ? `Not covered: ${missing.join(", ")}` : "Every contract component is covered.", + "", + ); + return lines.join("\n"); +} diff --git a/scripts/check-contract-drift.mjs b/scripts/check-contract-drift.mjs new file mode 100644 index 0000000..7b79109 --- /dev/null +++ b/scripts/check-contract-drift.mjs @@ -0,0 +1,121 @@ +/** + * Contract-drift guard: the embedded system prompt is the model's component + * vocabulary, and `src/genos/ui/contract.tsx` is what the app can actually + * render. If the two drift apart the model emits components that render as + * NOTHING (react-lang returns null for unknown components), so this script + * fails CI on any mismatch. Run with `npm run check:drift`; regenerate the + * prompt with `npm run generate:prompt` after contract changes. + * + * Checks, both directions: + * 1. every component defined in the contract is mentioned in the prompt; + * 2. every component documented in the prompt (signature lines) exists in + * the contract; + * 3. every component-looking call token anywhere in the prompt resolves to + * a contract component, a known non-component callable, or the allowlist. + * + * Documented allowlist (prompt-side names that are intentionally NOT in the + * native contract): + * - Stack: appears only in the positional-arguments syntax example, not a + * real component. + * - CheckBoxGroup, RadioGroup: web-contract components the native renderer + * sets do not implement; scripts/embed-prompt.mjs strips them from the + * prompt at embed time, so they must never reappear - but if they ever + * do (hand-edited prompt), the failure message should say "stripped", + * not "unknown". + * + * Non-component callables in the prompt (action steps and syntax prose, not + * components): Action, OS, ToAssistant, OpenUrl, TypeName. + */ +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const contractSrc = readFileSync(join(root, "src/genos/ui/contract.tsx"), "utf-8"); +const promptSrc = readFileSync(join(root, "src/genos/generated/system-prompt.ts"), "utf-8"); + +// The generated module embeds the prompt as one JSON string literal. +let prompt; +try { + prompt = JSON.parse(promptSrc.slice(promptSrc.indexOf("= ") + 2).replace(/;\s*$/, "")); +} catch (err) { + console.error( + `check-contract-drift: could not parse src/genos/generated/system-prompt.ts: ${err.message}`, + ); + process.exit(1); +} + +/** Prompt-side names intentionally absent from the native contract. */ +const ALLOWLIST = new Map([ + ["Stack", "syntax example only (positional arguments)"], + ["CheckBoxGroup", "web-only component, stripped by scripts/embed-prompt.mjs"], + ["RadioGroup", "web-only component, stripped by scripts/embed-prompt.mjs"], +]); + +/** Call tokens in the prompt that are action steps or prose, not components. */ +const NON_COMPONENTS = new Set(["Action", "OS", "ToAssistant", "OpenUrl", "TypeName"]); + +const contractNames = new Set( + [...contractSrc.matchAll(/defineComponent\(\{\s*name: "([^"]+)"/g)].map((m) => m[1]), +); +if (contractNames.size === 0) { + console.error("check-contract-drift: no defineComponent names found in contract.tsx"); + process.exit(1); +} + +// Components documented by a signature line at the start of a prompt line. +const documented = new Set( + [...prompt.matchAll(/^([A-Z][A-Za-z]+)\(/gm)].map((m) => m[1]), +); +// Every component-looking call token in the whole prompt. +const referenced = new Set( + [...prompt.matchAll(/\b([A-Z][a-zA-Z]+)\(/g)].map((m) => m[1]), +); + +const problems = []; + +// Direction 1: contract -> prompt (word-boundary mention anywhere). +for (const name of [...contractNames].sort()) { + if (!new RegExp(`\\b${name}\\b`).test(prompt)) { + problems.push(`contract component "${name}" is never mentioned in the prompt`); + } +} + +// Direction 2: documented prompt components -> contract. +for (const name of [...documented].sort()) { + if (NON_COMPONENTS.has(name)) continue; + if (ALLOWLIST.has(name)) { + problems.push( + `"${name}" is documented in the prompt but is allowlisted (${ALLOWLIST.get(name)}) - it must be stripped at embed time`, + ); + continue; + } + if (!contractNames.has(name)) { + problems.push(`prompt documents "${name}" but it is not in the contract (renders as nothing)`); + } +} + +// Direction 3: any call token elsewhere in the prompt must resolve. +for (const name of [...referenced].sort()) { + if (NON_COMPONENTS.has(name) || contractNames.has(name) || ALLOWLIST.has(name)) continue; + problems.push( + `prompt references "${name}(...)" which is neither a contract component nor a known callable`, + ); +} + +if (problems.length > 0) { + console.error( + `check-contract-drift: ${problems.length} drift problem(s) between\n` + + "src/genos/ui/contract.tsx and src/genos/generated/system-prompt.ts:\n\n" + + problems.map((p) => ` - ${p}`).join("\n") + + "\n\nFix the contract or regenerate the prompt (npm run generate:prompt).\n" + + "If a prompt-only name is intentional, document it in the ALLOWLIST in\n" + + "scripts/check-contract-drift.mjs.", + ); + process.exit(1); +} + +console.log( + `check-contract-drift: OK - ${contractNames.size} contract components, ` + + `${documented.size} documented in prompt, ${ALLOWLIST.size} allowlisted exceptions`, +); diff --git a/src/genos/stream.ts b/src/genos/stream.ts index 33e24d9..4cb4cdf 100644 --- a/src/genos/stream.ts +++ b/src/genos/stream.ts @@ -39,6 +39,18 @@ export const NEEDS_LIVE_DATA = "needs live data"; /** Rounds that may end in tool calls before we force a plain generation. */ const MAX_TOOL_ROUNDS = 3; +/** A read gap longer than this mid-stream is treated as a stalled socket. */ +const STALL_TIMEOUT_MS = 20_000; + +/** + * Provider error bodies are raw JSON; flatten to one short line so provider + * internals never land verbatim in the error UI. + */ +function sanitizeDetail(raw: string): string { + const flat = raw.replace(/[{}\[\]"]/g, " ").replace(/\s+/g, " ").trim(); + return flat.slice(0, 160); +} + interface StreamHandlers { onDelta: (text: string) => void; onDone: (info: StreamEndInfo) => void; @@ -105,6 +117,56 @@ function createUtf8Decoder(): (chunk: Uint8Array) => string { }; } +/** + * One read with an inactivity watchdog. reader.read() never settles if the + * socket stalls without closing, so a byte gap longer than STALL_TIMEOUT_MS + * fails the round with a friendly stall error; a caller abort also settles + * the read even when the underlying socket ignores the signal. The timer is + * cleared on every settle path - no dangling handles. + */ +function readWithWatchdog( + reader: ReadableStreamDefaultReader, + signal: AbortSignal | undefined, +): Promise> { + return new Promise((resolve, reject) => { + let timer: ReturnType | undefined; + const cleanup = () => { + if (timer !== undefined) clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + }; + const fail = (err: Error) => { + cleanup(); + reader.cancel().catch(() => {}); + reject(err); + }; + const onAbort = () => fail(new Error("aborted")); + timer = setTimeout( + () => + fail( + new Error( + `stream stalled - no data for ${Math.round(STALL_TIMEOUT_MS / 1000)}s, try again`, + ), + ), + STALL_TIMEOUT_MS, + ); + if (signal?.aborted) { + onAbort(); + return; + } + signal?.addEventListener("abort", onAbort, { once: true }); + reader.read().then( + (r) => { + cleanup(); + resolve(r); + }, + (err) => { + cleanup(); + reject(err); + }, + ); + }); +} + /** System prompt + optional tools section + today's date line. */ function systemPrompt(): string { const today = new Date().toLocaleDateString("en-US", { @@ -139,29 +201,41 @@ async function streamRound( const apiKey = cerebrasKey.get(); if (!apiKey) throw new Error("No Cerebras API key set"); - const res = await expoFetch(`${CEREBRAS_BASE_URL}/chat/completions`, { - method: "POST", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model: GENOS_MODEL, - messages: [{ role: "system", content: systemPrompt() }, ...convo], - ...(includeTools ? { tools: TOOL_DEFS } : {}), - stream: true, - temperature: 0.8, - max_completion_tokens: 3072, - }), - signal, - }); + let res: Awaited>; + try { + res = await expoFetch(`${CEREBRAS_BASE_URL}/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: GENOS_MODEL, + messages: [{ role: "system", content: systemPrompt() }, ...convo], + ...(includeTools ? { tools: TOOL_DEFS } : {}), + stream: true, + temperature: 0.8, + max_completion_tokens: 3072, + }), + signal, + }); + } catch (err) { + if (signal?.aborted || (err instanceof Error && err.name === "AbortError")) throw err; + throw new Error("network request failed - check the connection and try again"); + } if (res.status === 401 || res.status === 403) { cerebrasKey.markRejected(apiKey); throw new Error("Cerebras rejected the API key - enter a valid key"); } + if (res.status === 429) { + throw new Error("the AI provider is rate-limiting requests - wait a moment and try again"); + } + if (res.status >= 500) { + throw new Error("the AI provider is unavailable right now - try again shortly"); + } if (!res.ok || !res.body) { - const detail = await res.text().catch(() => ""); - throw new Error(detail.slice(0, 500) || `HTTP ${res.status}`); + const detail = sanitizeDetail(await res.text().catch(() => "")); + throw new Error(detail ? `request failed (HTTP ${res.status}): ${detail}` : `HTTP ${res.status}`); } const reader = res.body.getReader(); @@ -172,67 +246,72 @@ async function streamRound( let content = ""; const toolCalls = new Map(); + const handleLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed.startsWith("data:")) return; + const payload = trimmed.slice(5).trim(); + if (!payload) return; + if (payload === "[DONE]") { + sawDone = true; + return; + } + + let chunk: { + error?: { message?: string } | string; + choices?: Array<{ + delta?: { + content?: string; + tool_calls?: Array<{ + index?: number; + id?: string; + function?: { name?: string; arguments?: string }; + }>; + }; + finish_reason?: string | null; + }>; + }; + try { + chunk = JSON.parse(payload); + } catch { + return; + } + if (chunk.error) { + const msg = typeof chunk.error === "string" ? chunk.error : chunk.error.message; + throw new Error(sanitizeDetail(msg ?? "") || "stream error"); + } + const choice = chunk.choices?.[0]; + if (choice?.finish_reason) finishReason = choice.finish_reason; + const delta = choice?.delta; + if (delta?.content) { + content += delta.content; + onDelta(delta.content); + } + for (const tc of delta?.tool_calls ?? []) { + const idx = tc.index ?? 0; + const cur = toolCalls.get(idx) ?? { + id: "", + type: "function" as const, + function: { name: "", arguments: "" }, + }; + if (tc.id) cur.id = tc.id; + if (tc.function?.name) cur.function.name = tc.function.name; + if (tc.function?.arguments) cur.function.arguments += tc.function.arguments; + toolCalls.set(idx, cur); + } + }; + for (;;) { - const { done, value } = await reader.read(); + const { done, value } = await readWithWatchdog(reader, signal); if (done) break; buffer += decode(value); const lines = buffer.split("\n"); buffer = lines.pop() ?? ""; - for (const line of lines) { - const trimmed = line.trim(); - if (!trimmed.startsWith("data:")) continue; - const payload = trimmed.slice(5).trim(); - if (!payload) continue; - if (payload === "[DONE]") { - sawDone = true; - continue; - } - - let chunk: { - error?: { message?: string } | string; - choices?: Array<{ - delta?: { - content?: string; - tool_calls?: Array<{ - index?: number; - id?: string; - function?: { name?: string; arguments?: string }; - }>; - }; - finish_reason?: string | null; - }>; - }; - try { - chunk = JSON.parse(payload); - } catch { - continue; - } - if (chunk.error) { - const msg = typeof chunk.error === "string" ? chunk.error : chunk.error.message; - throw new Error(msg || "stream error"); - } - const choice = chunk.choices?.[0]; - if (choice?.finish_reason) finishReason = choice.finish_reason; - const delta = choice?.delta; - if (delta?.content) { - content += delta.content; - onDelta(delta.content); - } - for (const tc of delta?.tool_calls ?? []) { - const idx = tc.index ?? 0; - const cur = toolCalls.get(idx) ?? { - id: "", - type: "function" as const, - function: { name: "", arguments: "" }, - }; - if (tc.id) cur.id = tc.id; - if (tc.function?.name) cur.function.name = tc.function.name; - if (tc.function?.arguments) cur.function.arguments += tc.function.arguments; - toolCalls.set(idx, cur); - } - } + for (const line of lines) handleLine(line); } + // Providers should end lines, but a proxy that closes without a trailing + // newline must not lose the final event (often the finish_reason chunk). + if (buffer.trim()) handleLine(buffer); const dropped = !sawDone && !finishReason; if (dropped && !content && toolCalls.size === 0) {