From 939deec4ce0963373738db8cfc1dbc966d6141ce Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 29 Aug 2026 09:41:57 -0400 Subject: [PATCH] fix(tui): reduce cached transcript remount work --- packages/tui/perf/tabs.md | 106 ++++++++ packages/tui/src/routes/session/index.tsx | 15 +- packages/tui/src/routes/session/rows.ts | 29 +- packages/tui/test/app-lifecycle.test.tsx | 247 ++++++++++++++++++ .../tui/test/cli/tui/session-rows.test.ts | 95 +++++++ script/bench-tui-tabs.ts | 231 ++++++++++++++++ 6 files changed, 708 insertions(+), 15 deletions(-) create mode 100644 packages/tui/perf/tabs.md create mode 100644 script/bench-tui-tabs.ts diff --git a/packages/tui/perf/tabs.md b/packages/tui/perf/tabs.md new file mode 100644 index 000000000000..8bc56c5dfa48 --- /dev/null +++ b/packages/tui/perf/tabs.md @@ -0,0 +1,106 @@ +# Session Tab Switching + +## Run + +Install OpenCode Drive, then run from the repository root: + +```sh +PERF_RUN=before \ +OPENCODE_DRIVE_MEDIA_DIR="$PWD/.cache/tui-switch/media" \ +opencode-drive run script/bench-tui-tabs.ts +``` + +Use a new `PERF_RUN` for each run; existing result directories are not overwritten. +`PERF_TARGET` selects another source worktree and defaults to the working directory. +`PERF_OUTPUT` overrides the result root, which defaults to `/.cache/tui-switch`. +`PERF_CONTENT=prose` replaces the Markdown with equal-byte-size plain text. + +Drive checks the script, creates an isolated server/home/project, imports synthetic +sessions through the real API, and launches the real TUI components. It never +connects to the elected background service. Only the final streaming correctness +check prompts a model, and that model is simulated. + +Run benchmarks serially, without simultaneous tests or builds. The script records +the target revision and its production TUI/Client diff, every completed action in +`samples.jsonl`, summary statistics, terminal frames, and Drive artifact metadata. +It retains failed runs' completed samples. Result and media directories are local +artifacts, not files to commit. + +## Workload + +- SHORT: 20 messages, 256 text bytes per assistant. +- LONG: 2,000 messages with the same text sizes and a comparable latest page. +- LARGE: 20 messages, 32 KiB per assistant. +- Every fifth assistant includes a completed synthetic read-tool result. +- Historical fixtures carry creation/completion times but omit stream-end/token + accounting. Correctness tests exercise complete timing and token metadata. +- Markdown deliberately repeats small fenced blocks, about 170 per LARGE + assistant. It is a stress fixture, not a typical-response latency claim. + +The TUI starts after import. It opens sessions through the picker, switches with +the real keybindings, loads all LONG history, and returns to both its tail and a +saved head anchor. Each warm category has one explicitly retained warm-up sample +(`sample: 0`) and eight measured observations. Initial opens and first/last-message +navigation are single observations and are not included in warm medians. + +Location caveat: this runner supplies `info.location` but omits the import API's +top-level `location`. Imported sessions therefore use the isolated server's +working directory, not the fixture `files` directory. The results describe warm +cross-Location sessions, not same-Location restoration or cold project loading. +Both directories are private synthetic fixtures; no live user Location is used. + +`actionMs` is Drive's action RPC duration, including its forced render and UI-tree +inspection. `visibleMs` additionally waits for a destination marker, with 20 ms +polling. Neither is physical-terminal input-to-paint latency or a styled-content +completion guarantee. Fixed inter-action pacing occurs outside the measured +interval and is not used instead of readiness checks. + +The final check streams an incomplete ordinary fence, completes it, verifies its +final displayed text, and checks the server's completed assistant projection. +Normal tests separately cover custom Markdown and footer correctness. + +## Initial Experiments + +Base: `849824efd2`, Bun 1.3.14, OpenTUI 0.5.9, Apple M2 Max, 120x40, source builds, +DevTools disabled. All results below are local action-RPC medians in milliseconds, +eight measured observations per cell. They are not release-binary guarantees. + +| Scenario | Base repeat | Markdown ordering | Ordering + footer index | Footer confirmation | +| ------------------------------ | ----------: | ----------------: | ----------------------: | ------------------: | +| Latest 20 of LONG retained | 28.6 | 28.8 | 26.6 | 26.2 | +| All 2,000 retained, tail | 107.7 | 107.9 | 66.9 | 60.5 | +| All 2,000 retained, saved head | 115.3 | 118.7 | 71.9 | 69.3 | +| Dense Markdown | 1,615.5 | 832.1 | 830.4 | 820.6 | + +These are the `before-02`, `markdown-02`, `footer-01`, and `footer-02` runs. The earlier +base/Markdown pair independently measured 1,717.3 -> 842.6 ms for dense Markdown; +that earlier runner did not yet include the saved-head category, so its results +are not pooled into the table. Ordinary short/latest-page differences are near +the noise floor and are not claimed as improvements. + +### Kept + +- Configure Markdown's custom renderer before content. OpenTUI's `renderNode` + setter otherwise clears populated parse/block state and repeats preparation. + Keep content before `streaming` so the completion update retains final tokens. +- Share a reactive message-position index across a Session view's footers. Scan + only from each position to its preceding user/synthetic input instead of + searching and slicing entire history prefixes. This improves both tail and + historical positions rather than shifting work to the newer suffix. + +Captured content, geometry, and styling matched across the base, Markdown-only, +and both footer trials after excluding the isolated-project path footer. + +The helper dependency test deliberately supplies a known position. It verifies +that the bounded calculation does not read an unrelated prefix, not that the +whole Session ignores structural history changes. A real-App regression checks +the reactive index through prepend/reconcile and a same-length truncate/append. + +### Deferred + +- The second row reduction after a cache-hit sync still exists. Removing it + cleanly needs an explicit cache-hit/synchronization contract; this pass does + not change the public Client data API or infer freshness from array identity. +- Parsed Markdown caches, mounted-view retention, history eviction, and initial + window changes were not mixed into these experiments. +- No memory-leak or retained-heap improvement is claimed by these latency runs. diff --git a/packages/tui/src/routes/session/index.tsx b/packages/tui/src/routes/session/index.tsx index ff05284da176..7a3e57b728ce 100644 --- a/packages/tui/src/routes/session/index.tsx +++ b/packages/tui/src/routes/session/index.tsx @@ -141,6 +141,7 @@ const context = createContext<{ groupExploration: () => boolean diffWrapMode: () => "word" | "none" models: () => ModelInfo[] + messageIndex: (messageID: string) => number | undefined config: ReturnType["data"] mutatePending: (action: PendingAction, inboxID: string) => Promise pendingDelivery: (inboxID: string) => SessionInbox.Delivery | undefined @@ -180,6 +181,7 @@ export function Session(props: { const promptRef = usePromptRef() const session = createMemo(() => data.session.get(route.sessionID)) const messages = () => data.session.message.list(route.sessionID) + const messageIndexes = createMemo(() => new Map(messages().map((message, index) => [message.id, index]))) const messagesBeforeRevert = () => { const messageID = session()?.revert?.messageID if (!messageID) return messages() @@ -1349,6 +1351,7 @@ export function Session(props: { groupExploration, diffWrapMode, models, + messageIndex: (messageID) => messageIndexes().get(messageID), config, mutatePending, pendingDelivery: (inboxID) => pendingDeliveries().get(inboxID), @@ -2031,8 +2034,10 @@ function AssistantFooter(props: { message: SessionMessageAssistant }) { ?.name ?? `${props.message.model.providerID}/${props.message.model.id}`, ) const messages = createMemo(() => data.session.message.list(ctx.sessionID)) - const duration = createMemo(() => turnDuration(props.message, messages())) - const tokensPerSecond = createMemo(() => turnTokensPerSecond(props.message, messages())) + const duration = createMemo(() => turnDuration(props.message, messages(), ctx.messageIndex(props.message.id))) + const tokensPerSecond = createMemo(() => + turnTokensPerSecond(props.message, messages(), ctx.messageIndex(props.message.id)), + ) const interrupted = createMemo(() => props.message.error?.message === "Step interrupted") return ( <> @@ -2211,6 +2216,7 @@ function CompactionMessage(props: { message: Extract @@ -2658,9 +2663,10 @@ function TextPart(props: { last: boolean; part: SessionMessageAssistantText; mes return ( - {/* Apply content before streaming so completion does not freeze the previous Markdown tokens. */} + {/* Configure custom nodes before parsing; apply content before streaming so completion keeps the final tokens. */} diff --git a/packages/tui/src/routes/session/rows.ts b/packages/tui/src/routes/session/rows.ts index a3632d01f2f7..2f4f9723fc7d 100644 --- a/packages/tui/src/routes/session/rows.ts +++ b/packages/tui/src/routes/session/rows.ts @@ -346,21 +346,21 @@ export function cacheReuseDrop(previous: CacheUsage | undefined, current: CacheU return drop > 0 ? drop : undefined } -export function turnDuration(message: SessionMessageAssistant, messages: SessionMessageInfo[]) { +export function turnDuration(message: SessionMessageAssistant, messages: SessionMessageInfo[], position?: number) { if (message.time.completed === undefined) return 0 - const index = messages.findIndex((item) => item.id === message.id) - const input = messages - .slice(0, index === -1 ? messages.length : index) - .findLast((item) => item.type === "user" || item.type === "synthetic") + const index = position ?? messages.findIndex((item) => item.id === message.id) + const input = messages[inputIndex(messages, index === -1 ? messages.length : index)] return Math.max(0, message.time.completed - (input?.time.created ?? message.time.created)) } -export function turnTokensPerSecond(message: SessionMessageAssistant, messages: SessionMessageInfo[]) { - const index = messages.findIndex((item) => item.id === message.id) +export function turnTokensPerSecond( + message: SessionMessageAssistant, + messages: SessionMessageInfo[], + position?: number, +) { + const index = position ?? messages.findIndex((item) => item.id === message.id) const end = index === -1 ? messages.length : index + 1 - const start = messages - .slice(0, end) - .findLastIndex((item) => item.type === "user" || item.type === "synthetic") + const start = inputIndex(messages, end) const steps = messages .slice(start + 1, end) .filter((item): item is SessionMessageAssistant => item.type === "assistant") @@ -374,6 +374,15 @@ export function turnTokensPerSecond(message: SessionMessageAssistant, messages: return output / (duration / 1_000) } +function inputIndex(messages: SessionMessageInfo[], end: number) { + // Reading a sliced prefix subscribes every footer to unrelated historical messages. + for (let index = end - 1; index >= 0; index--) { + const message = messages[index] + if (message.type === "user" || message.type === "synthetic") return index + } + return -1 +} + function hasTokenUsage( message: SessionMessageAssistant, ): message is SessionMessageAssistant & { tokens: NonNullable } { diff --git a/packages/tui/test/app-lifecycle.test.tsx b/packages/tui/test/app-lifecycle.test.tsx index 331375434822..96cb04791cda 100644 --- a/packages/tui/test/app-lifecycle.test.tsx +++ b/packages/tui/test/app-lifecycle.test.tsx @@ -490,6 +490,253 @@ test("automatic rename refreshes the displayed title before settling, even witho } }) +test.each([80, 120])("completes custom Markdown and ordinary fences in a session at width %s", async (width) => { + await using state = await tmpdir() + const session = { + id: "ses_markdown", + title: "Markdown fixture", + projectID: "project", + location: { directory }, + agent: "build", + model: { providerID: "fixture", id: "model" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 1, updated: 2 }, + } + const initial = + "```mermaid\ngraph LR\n A[DiagramStart] --> B[DiagramEnd]\n```\n\n```latex\nx^2\n```\n\n```text\ninitial" + await using setup = await createAppFixture({ + width, + height: 55, + state: state.path, + config: { animations: false, tabs: { enabled: false }, session: { sidebar: "hide" } }, + args: { sessionID: session.id }, + fetch: (url) => { + if (url.pathname === `/api/session/${session.id}`) return json({ data: session }) + if (url.pathname === `/api/session/${session.id}/message`) + return json({ + data: [ + { + id: "msg_markdown", + type: "assistant", + agent: session.agent, + model: session.model, + time: { created: 2 }, + content: [{ type: "text", text: initial }], + }, + { + id: "msg_compaction", + type: "compaction", + time: { created: 1 }, + status: "completed", + reason: "manual", + summary: "```latex\ny^2\n```", + recent: "msg_markdown", + }, + ], + cursor: {}, + }) + if ( + url.pathname === `/api/session/${session.id}/inbox` || + url.pathname === `/api/session/${session.id}/permission` + ) + return json({ data: [] }) + return undefined + }, + }) + const streaming = await setup.waitForFrame( + (frame) => + frame.includes("initial") && + frame.includes("DiagramStart") && + frame.includes("DiagramEnd") && + frame.includes("x\u00b2") && + frame.includes("y\u00b2"), + ) + expect(streaming).toContain("Compaction") + expect(streaming).not.toContain("initial final") + expect(streaming).not.toContain("MARKDOWN_END") + + // Queue final text and completion together to exercise TextPart's reactive property order. + setup.events.emit({ + id: "evt_markdown_text_ended", + created: 3, + type: "session.text.ended", + durable: { aggregateID: session.id, seq: 1, version: 1 }, + data: { + sessionID: session.id, + assistantMessageID: "msg_markdown", + ordinal: 0, + text: `${initial} final\n\`\`\`\n\nMARKDOWN_END`, + }, + }) + setup.events.emit({ + id: "evt_markdown_step_ended", + created: 4, + type: "session.step.ended", + durable: { aggregateID: session.id, seq: 2, version: 1 }, + data: { + sessionID: session.id, + assistantMessageID: "msg_markdown", + finish: "stop", + cost: 0, + tokens: session.tokens, + }, + }) + const frame = await setup.waitForFrame( + (frame) => frame.includes("MARKDOWN_END") && frame.includes("initial final") && frame.includes("2ms"), + ) + expect(frame).toContain("DiagramStart") + expect(frame).toContain("DiagramEnd") + expect(frame).toContain("x\u00b2") + expect(frame).toContain("y\u00b2") + expect(frame).toContain("initial final") + expect(frame).not.toContain("graph LR") + expect(frame).not.toContain("x^2") + expect(frame).not.toContain("y^2") + expect(frame).not.toContain("```") +}) + +test("keeps assistant footer metrics current after history prepend and same-length replacement", async () => { + await using state = await tmpdir() + const session = { + id: "ses_footer", + title: "Footer fixture", + projectID: "project", + location: { directory }, + agent: "build", + model: { providerID: "fixture", id: "model" }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created: 100, updated: 5000 }, + } + await using setup = await createAppFixture({ + width: 100, + height: 40, + state: state.path, + config: { animations: false, tabs: { enabled: false }, session: { sidebar: "hide", tps: true } }, + args: { sessionID: session.id }, + fetch: (url) => { + if (url.pathname === `/api/session/${session.id}`) return json({ data: session }) + if (url.pathname === `/api/session/${session.id}/message`) { + if (url.searchParams.get("cursor") === "older") + return json({ + data: [ + { + id: "msg_0001", + type: "system", + text: "Earlier instructions", + description: "Prepended instructions", + time: { created: 200 }, + }, + { id: "msg_0000", type: "user", text: "Prepended input", time: { created: 100 } }, + ], + cursor: {}, + }) + return json({ + data: [ + { + id: "msg_0003", + type: "assistant", + agent: session.agent, + model: session.model, + time: { created: 2000, streamed: 3000, completed: 5000 }, + finish: "stop", + cost: 0, + tokens: { ...session.tokens, output: 20 }, + content: [{ type: "text", text: "Original answer" }], + }, + { id: "msg_0002", type: "user", text: "Current input", time: { created: 1000 } }, + ], + cursor: { next: "older" }, + }) + } + if ( + url.pathname === `/api/session/${session.id}/inbox` || + url.pathname === `/api/session/${session.id}/permission` + ) + return json({ data: [] }) + return undefined + }, + }) + + const initial = await setup.waitForFrame((frame) => frame.includes("Original answer") && frame.includes("20.0 tok/s")) + expect(initial).toContain("Current input") + expect(initial).toContain("4.0s \u00b7 20.0 tok/s") + expect(initial).not.toContain("Prepended input") + + setup.mockInput.pressKey("g", { ctrl: true }) + const prepended = await setup.waitForFrame( + (frame) => + frame.includes("Prepended input") && + frame.includes("Prepended instructions") && + frame.includes("Original answer") && + frame.includes("20.0 tok/s") && + !frame.includes("Loading session history..."), + ) + expect(prepended).toContain("Current input") + expect(prepended).toContain("Original answer") + expect(prepended).toContain("4.0s \u00b7 20.0 tok/s") + + // Remove and append in one queue batch: array length is unchanged, but the message ID is new. + setup.events.emit({ + id: "evt_footer_reverted", + created: 5500, + type: "session.revert.committed", + durable: { aggregateID: session.id, seq: 1, version: 1 }, + data: { sessionID: session.id, to: "msg_0003" }, + }) + setup.events.emit({ + id: "evt_footer_step_started", + created: 6000, + type: "session.step.started", + durable: { aggregateID: session.id, seq: 2, version: 1 }, + data: { sessionID: session.id, assistantMessageID: "msg_0004", agent: session.agent, model: session.model }, + }) + setup.events.emit({ + id: "evt_footer_text_started", + created: 6500, + type: "session.text.started", + durable: { aggregateID: session.id, seq: 3, version: 1 }, + data: { sessionID: session.id, assistantMessageID: "msg_0004", ordinal: 0 }, + }) + setup.events.emit({ + id: "evt_footer_text_ended", + created: 7500, + type: "session.text.ended", + durable: { aggregateID: session.id, seq: 4, version: 1 }, + data: { sessionID: session.id, assistantMessageID: "msg_0004", ordinal: 0, text: "Replacement answer" }, + }) + setup.events.emit({ + id: "evt_footer_step_streamed", + created: 8000, + type: "session.step.streamed", + durable: { aggregateID: session.id, seq: 5, version: 1 }, + data: { sessionID: session.id, assistantMessageID: "msg_0004" }, + }) + setup.events.emit({ + id: "evt_footer_step_ended", + created: 9000, + type: "session.step.ended", + durable: { aggregateID: session.id, seq: 6, version: 1 }, + data: { + sessionID: session.id, + assistantMessageID: "msg_0004", + finish: "stop", + cost: 0, + tokens: { ...session.tokens, output: 50 }, + }, + }) + const replaced = await setup.waitForFrame( + (frame) => frame.includes("Replacement answer") && frame.includes("25.0 tok/s"), + ) + expect(replaced).toContain("Prepended input") + expect(replaced).toContain("Prepended instructions") + expect(replaced).toContain("Current input") + expect(replaced).toContain("8.0s \u00b7 25.0 tok/s") + expect(replaced).not.toContain("Original answer") + expect(replaced).not.toContain("20.0 tok/s") +}) + test("session startup prompt is submitted exactly once", async () => { const setup = await createTestRenderer({ width: 80, height: 24, useThread: false }) const events = createEventStream() diff --git a/packages/tui/test/cli/tui/session-rows.test.ts b/packages/tui/test/cli/tui/session-rows.test.ts index bbab0257ec0d..0fd7678b7808 100644 --- a/packages/tui/test/cli/tui/session-rows.test.ts +++ b/packages/tui/test/cli/tui/session-rows.test.ts @@ -1,5 +1,7 @@ import { expect, test } from "bun:test" import type { SessionMessageAssistant, SessionMessageAssistantTool, SessionMessageInfo } from "@opencode-ai/client" +import { createMemo, createRoot } from "solid-js" +import { createStore } from "solid-js/store" import { backgroundToolRowIndex, cacheReuseDrop, @@ -46,6 +48,99 @@ test("omits turn throughput when a stream boundary is unavailable", () => { expect(turnTokensPerSecond(final, [final])).toBeUndefined() }) +test.each([false, true])( + "measures historical footers without later inputs or incomplete steps (indexed: %s)", + (indexed) => { + const step = (id: string, created: number, streamed: number, completed: number, output: number) => ({ + ...assistant(id, []), + time: { created, streamed, completed }, + tokens: { input: 1, output, reasoning: 2, cache: { read: 0, write: 0 } }, + }) + const messages: SessionMessageInfo[] = [ + step("before-input", 0, 1_000, 2_000, 5), + { type: "user", id: "input", text: "Question", time: { created: 3_000 } }, + step("first-step", 4_000, 5_000, 6_000, 10), + { type: "system", id: "system", text: "Instructions", time: { created: 6_500 } }, + step("second-step", 7_000, 8_000, 9_000, 20), + { type: "synthetic", id: "synthetic", text: "Update", time: { created: 10_000 } }, + step("after-synthetic", 11_000, 13_000, 14_000, 12), + { type: "user", id: "later-input", text: "Next question", time: { created: 15_000 } }, + assistant("incomplete", []), + ] + + expect( + messages.flatMap((message, index) => + message.type === "assistant" + ? [ + [ + turnDuration(message, messages, indexed ? index : undefined), + turnTokensPerSecond(message, messages, indexed ? index : undefined), + ], + ] + : [], + ), + ).toEqual([ + [2_000, 5], + [3_000, 10], + [6_000, 15], + [4_000, 6], + [0, undefined], + ]) + }, +) + +test("preserves missing-anchor footer fallbacks without including the absent assistant's tokens", () => { + const absent = assistant("absent", []) + absent.time = { created: 8_000, streamed: 9_000, completed: 10_000 } + absent.tokens = { input: 1, output: 900, reasoning: 0, cache: { read: 0, write: 0 } } + const stored = assistant("stored", []) + stored.time = { created: 6_000, streamed: 8_000, completed: 9_000 } + stored.tokens = { input: 1, output: 20, reasoning: 0, cache: { read: 0, write: 0 } } + const input: SessionMessageInfo = { type: "user", id: "input", text: "Question", time: { created: 5_000 } } + + expect(turnDuration(absent, [input, stored])).toBe(5_000) + expect(turnTokensPerSecond(absent, [input, stored])).toBe(10) + expect(turnDuration(absent, [stored])).toBe(2_000) + expect(turnTokensPerSecond(absent, [stored])).toBe(10) + expect(turnDuration(absent, [])).toBe(2_000) + expect(turnTokensPerSecond(absent, [])).toBeUndefined() +}) + +test("indexed tail footer calculations do not subscribe to an unrelated history prefix", () => { + createRoot((dispose) => { + try { + const final = assistant("final", []) + final.time = { created: 2_000, streamed: 3_000, completed: 5_000 } + final.tokens = { input: 1, output: 20, reasoning: 0, cache: { read: 0, write: 0 } } + const [messages, setMessages] = createStore([ + { type: "user", id: "old-input", text: "Old question", time: { created: 0 } }, + assistant("old-step", []), + { type: "user", id: "input", text: "Current question", time: { created: 1_000 } }, + final, + ]) + let runs = 0 + const footer = createMemo(() => { + runs++ + const current = messages[3] + if (current.type !== "assistant") throw new Error("Expected an assistant") + return [turnDuration(current, messages, 3), turnTokensPerSecond(current, messages, 3)] + }) + expect(footer()).toEqual([4_000, 20]) + setMessages(0, { type: "user", id: "replaced-prefix", text: "Older question", time: { created: 50 } }) + expect(footer()).toEqual([4_000, 20]) + expect(runs).toBe(1) + + setMessages(2, "time", "created", 1_500) + expect(footer()).toEqual([3_500, 20]) + setMessages(3, { ...final, time: { ...final.time, streamed: 4_000 }, tokens: { ...final.tokens, output: 60 } }) + expect(footer()).toEqual([3_500, 30]) + expect(runs).toBe(3) + } finally { + dispose() + } + }) +}) + test("filters OpenAI cache quantization from cache reuse drops", () => { const openai = { id: "gpt", providerID: "openai" } expect(cacheReuseDrop(undefined, { read: 10_000, model: openai })).toBeUndefined() diff --git a/script/bench-tui-tabs.ts b/script/bench-tui-tabs.ts new file mode 100644 index 000000000000..362fb47b7338 --- /dev/null +++ b/script/bench-tui-tabs.ts @@ -0,0 +1,231 @@ +import { $ } from "bun" +import { appendFileSync } from "node:fs" +import { mkdir } from "node:fs/promises" +import path from "node:path" +import { Effect, Schema } from "effect" +import { Llm, OpenCodeDriver, type Ui } from "opencode-drive" +import { Session } from "../packages/schema/src/session" +import { SessionMessage } from "../packages/schema/src/session-message" + +const run = process.env.PERF_RUN +if (!run) throw new Error("PERF_RUN must identify a new experiment") +const target = process.env.PERF_TARGET ?? process.cwd() +const output = path.join(process.env.PERF_OUTPUT ?? path.join(target, ".cache", "tui-switch"), run) +await mkdir(path.dirname(output), { recursive: true }) +await mkdir(output, { recursive: false }) +const revision = (await $`git -C ${target} rev-parse HEAD`.quiet().text()).trim() +await Bun.write( + path.join(output, "changes.patch"), + await $`git -C ${target} diff HEAD -- packages/tui/src packages/client/src/solid`.quiet().text(), +) +const samples: { name: string; sample: number; actionMs: number; visibleMs: number }[] = [] +const fixtures = [ + { name: "SHORT", count: 20, bytes: 256 }, + { name: "LONG", count: 2000, bytes: 256 }, + { name: "LARGE", count: 20, bytes: 32768 }, +] + +const measure = (ui: Ui, name: string, sample: number, action: Effect.Effect, marker: string) => + Effect.gen(function* () { + const start = performance.now() + yield* action + const actionMs = performance.now() - start + yield* ui.waitFor(marker, { timeout: 30_000, interval: 20 }) + const result = { name, sample, actionMs, visibleMs: performance.now() - start } + samples.push(result) + appendFileSync(path.join(output, "samples.jsonl"), JSON.stringify(result) + "\n") + console.error(JSON.stringify(result)) + // Fixed pacing is outside the timing window; the marker above determines readiness. + yield* Effect.sleep(150) + }) + +export default OpenCodeDriver.useReport( + { + keepArtifacts: true, + project: { + git: true, + files: { + "README.md": "# Synthetic tab-switch benchmark\n", + ".opencode/cli.json": JSON.stringify({ debug: { devtools: false } }), + }, + }, + config: { autoupdate: false }, + tui: { viewport: { cols: 120, rows: 40 } }, + opencode: { dev: target }, + }, + (driver) => + Effect.gen(function* () { + yield* driver.tui.close() + const template = yield* driver.opencode.session.create({ title: "Template" }) + const model = (yield* driver.opencode.model.default({ location: template.location })).data + if (!model) return yield* Effect.fail(new Error("Simulated model unavailable")) + const agent = (yield* driver.opencode.agent.list({ location: template.location })).data.find( + (item) => item.id === "build", + ) + if (!agent) return yield* Effect.fail(new Error("Build agent unavailable")) + const seeded = yield* Effect.forEach(fixtures, (fixture) => + Effect.gen(function* () { + const messages = Array.from({ length: fixture.count }, (_, index) => { + const created = 1_780_000_000_000 + index * 10_000 + const marker = + index === 0 + ? `FIRST_${fixture.name}` + : index === fixture.count - 1 + ? `END_${fixture.name}` + : `ROW_${index}` + const id = SessionMessage.ID.create() + if (index % 2 === 0) + return Schema.decodeUnknownSync(SessionMessage.Info)({ + id, + type: "user", + time: { created }, + text: `${marker} Please inspect the parser and explain the next small implementation step with a test.`, + }) + const block = + process.env.PERF_CONTENT === "prose" + ? "The parser validates input before constructing the result. Keep this boundary explicit and add a focused regression test. " + : "The parser validates input before constructing the result. Keep this boundary explicit and add a focused regression test.\n\n```ts\nconst value = parse(source)\nexpect(value.ok).toBe(true)\n```\n\n" + const tail = `\n\n${marker}` + return Schema.decodeUnknownSync(SessionMessage.Info)({ + id, + type: "assistant", + agent: agent.id, + model: { providerID: model.providerID, id: model.id }, + finish: "stop", + time: { created, completed: created + 400 }, + content: [ + ...(index % 10 === 5 + ? [ + { + type: "tool", + id: `call_${index}`, + name: "read", + state: { + status: "completed", + input: { path: "src/parser.ts" }, + content: [ + { + type: "text", + text: "1: export const parse = (source: string) => ({ ok: true, source })", + }, + ], + }, + time: { created, completed: created + 100 }, + }, + ] + : []), + { + type: "text", + text: + block.repeat(Math.ceil(fixture.bytes / block.length)).slice(0, fixture.bytes - tail.length) + tail, + }, + ], + }) + }) + return yield* driver.opencode.session.import({ + info: { + ...template, + id: Session.ID.create(), + title: `Perf ${fixture.name}`, + agent: agent.id, + model: { providerID: model.providerID, id: model.id }, + }, + messages, + }) + }), + ) + yield* driver.opencode.session.remove({ sessionID: template.id }) + const tui = yield* driver.tuis.launch("measured", { viewport: { cols: 120, rows: 40 } }) + const ui = tui.ui + yield* Effect.forEach(seeded, (session, index) => + Effect.gen(function* () { + yield* ui.press("o", { ctrl: true }) + yield* ui.waitFor("Search sessions and") + yield* ui.type(session.title ?? "") + yield* ui.waitFor(session.title ?? "") + yield* measure(ui, `cold.${fixtures[index].name}`, 0, ui.enter(), `END_${fixtures[index].name}`) + }), + ) + const select = (index: number) => ui.press(String(index + 1), { ctrl: true }) + yield* Effect.forEach(["latest20", "retained2000", "head2000", "large"], (phase) => + Effect.gen(function* () { + if (phase === "retained2000") { + yield* select(1) + yield* measure(ui, "history.first", 0, ui.press("g", { ctrl: true }), "FIRST_LONG") + yield* measure(ui, "history.latest", 0, ui.press("g", { ctrl: true, meta: true }), "END_LONG") + } + if (phase === "head2000") { + yield* select(1) + yield* ui.press("g", { ctrl: true }) + yield* ui.waitFor("FIRST_LONG") + } + const index = phase === "large" ? 2 : 1 + yield* Effect.forEach( + Array.from({ length: 9 }, (_, index) => index), + (sample) => + Effect.gen(function* () { + yield* measure(ui, `${phase}.SHORT`, sample, select(0), "END_SHORT") + yield* measure( + ui, + `${phase}.${fixtures[index].name}`, + sample, + select(index), + phase === "head2000" ? "FIRST_LONG" : `END_${fixtures[index].name}`, + ) + }), + ) + const frame = yield* ui.capture() + yield* Effect.promise(() => Bun.write(path.join(output, `${phase}.frame.json`), JSON.stringify(frame))) + if (phase === "head2000") { + yield* ui.press("g", { ctrl: true, meta: true }) + yield* ui.waitFor("END_LONG") + } + }), + ) + yield* ui.screenshot("measured-large") + yield* select(0) + yield* driver.llm.queue( + Llm.text("```text\ninitial", { delay: 10, chunkSize: 5 }), + Llm.text(" final\n```\n\nSTREAM_DONE", { delay: 10, chunkSize: 5 }), + ) + yield* ui.submit("Synthetic streaming completion check") + yield* ui.waitFor("STREAM_DONE", { timeout: 30_000 }) + yield* ui.waitFor("initial final") + yield* driver.opencode.session.wait({ sessionID: seeded[0].id }) + const final = yield* driver.opencode.message.list({ sessionID: seeded[0].id, limit: 1, order: "desc" }) + if (final.data[0]?.type !== "assistant" || !final.data[0].time.completed) + return yield* Effect.fail(new Error("Streaming completion was not projected")) + yield* ui.screenshot("streaming-completed") + const summary = [...new Set(samples.filter((sample) => sample.sample > 0).map((sample) => sample.name))].map( + (name) => { + const values = samples + .filter((sample) => sample.name === name && sample.sample > 0) + .map((sample) => sample.actionMs) + .toSorted((a, b) => a - b) + return { + name, + n: values.length, + medianMs: (values[3] + values[4]) / 2, + minMs: values[0], + maxMs: values.at(-1), + } + }, + ) + yield* Effect.promise(() => + Bun.write( + path.join(output, "results.json"), + JSON.stringify( + { run, target, revision, fixtures, content: process.env.PERF_CONTENT ?? "markdown", samples, summary }, + null, + 2, + ), + ), + ) + console.table(summary) + return summary + }), +).pipe( + Effect.tap((report) => + Effect.promise(() => Bun.write(path.join(output, "report.json"), JSON.stringify(report, null, 2))), + ), +)