row.includes("unpriced-model")) ?? "";
+
+ expect(unpricedRow).toContain("Unpriced");
+ expect(unpricedRow).not.toContain("$0.00");
+ });
+
it("sorts models by token usage when the token metric is selected", () => {
testState.metric = "tokens";
testState.breakdown = "model";
@@ -202,6 +225,7 @@ describe("UsagePage model breakdown", () => {
"expensive-model",
"token-heavy-model",
"token-heavy-cheaper-model",
+ "unpriced-model",
]);
});
});
diff --git a/apps/web/src/components/usage/UsagePage.tsx b/apps/web/src/components/usage/UsagePage.tsx
index deb05f266b98..21970c675596 100644
--- a/apps/web/src/components/usage/UsagePage.tsx
+++ b/apps/web/src/components/usage/UsagePage.tsx
@@ -15,6 +15,7 @@ import { useMemo, useRef, useState } from "react";
import {
isCompatibleUsageContractVersion,
+ isModelCostUnknown,
type DailyTotals,
type HourlyTotals,
} from "@t3tools/shared/usageMerge";
@@ -363,9 +364,13 @@ export function UsagePage() {
: formatTokens(merged.totalTokens)}
- {metric === "cost"
- ? `${formatCount(merged.sessions)} sessions · API estimate`
- : `${formatCount(merged.sessions)} sessions`}
+ {metric !== "cost"
+ ? `${formatCount(merged.sessions)} sessions`
+ : merged.costQuality.unpricedShare > 0
+ ? `${formatCount(merged.sessions)} sessions · API estimate excludes ${formatPercent(
+ merged.costQuality.unpricedShare,
+ )} unpriced records`
+ : `${formatCount(merged.sessions)} sessions · API estimate`}
@@ -511,10 +516,14 @@ export function UsagePage() {
|
- {formatUsd(model.costUsd)}
+ {isModelCostUnknown(model) ? (
+ Unpriced
+ ) : (
+ formatUsd(model.costUsd)
+ )}
|
- {formatPercent(model.costShare)}
+ {isModelCostUnknown(model) ? "—" : formatPercent(model.costShare)}
|
{formatTokens(model.totalTokens)}
diff --git a/apps/web/src/lib/incrementalHighlighting.test.ts b/apps/web/src/lib/incrementalHighlighting.test.ts
new file mode 100644
index 000000000000..36c27cea14a2
--- /dev/null
+++ b/apps/web/src/lib/incrementalHighlighting.test.ts
@@ -0,0 +1,95 @@
+import { toHtml } from "hast-util-to-html";
+import { getSharedHighlighter } from "@pierre/diffs";
+import { describe, expect, it } from "vite-plus/test";
+
+import { createIncrementalHighlightedDocument } from "./incrementalHighlighting";
+
+const samples = {
+ typescript: "/* multi\nline comment */\nconst x = `template\n${1 + 2}`;\nconst re = /abc/;\n",
+ python: '#!/usr/bin/python\nx = """multi\nline"""\nprint(x)\n',
+ bash: "#!/bin/bash\ncat <\nconst x = 1;\n\n\n",
+ markdown: "# heading\n\n```ts\nconst a = 1;\n```\n\ntext\n",
+ rust: 'fn main() {\n let x = r#"multi\nline"#;\n}\n',
+ tsx: 'const element = \n{value}\n;\n',
+ json: '{\n "value": [1,\n 2, 3]\n}\n',
+ yaml: "key: |\n multiline\n value\nnext: true\n",
+ css: '/* comment\n continued */\np::before {\n content: "text";\n}\n',
+ sql: "SELECT 'multi\nline'\nFROM table_name;\n",
+} as const;
+
+const highlighterPromise = getSharedHighlighter({
+ langs: Object.keys(samples) as Array,
+ themes: ["pierre-dark", "pierre-light"],
+ preferredHighlighter: "shiki-wasm",
+});
+
+describe("incremental code highlighting", () => {
+ it.each(Object.entries(samples))(
+ "matches full HTML at every streaming prefix in %s",
+ async (language, code) => {
+ const highlighter = await highlighterPromise;
+ for (const theme of ["pierre-dark", "pierre-light"] as const) {
+ const highlight = createIncrementalHighlightedDocument(highlighter, language, theme);
+ for (let end = 0; end <= code.length; end++) {
+ const text = code.slice(0, end);
+ expect(toHtml(highlight(text)), `${theme}, prefix ${end}`).toBe(
+ highlighter.codeToHtml(text, { lang: language, theme }),
+ );
+ }
+ }
+ },
+ );
+
+ it("resets after edits and truncation, including edits to a completed line", async () => {
+ const highlighter = await highlighterPromise;
+ const highlight = createIncrementalHighlightedDocument(
+ highlighter,
+ "typescript",
+ "pierre-dark",
+ );
+ const inputs = [
+ "/* open\ncomment\n",
+ "/* open\ncomment\n*/\nconst x = 1;",
+ "const edited = 2;\nconst x = 1;",
+ "const edited = 2;\nconst x = 10;",
+ "const edited = 2;\n",
+ "",
+ "\n\n\nconst fresh = true;\n",
+ ];
+ for (const text of inputs) {
+ expect(toHtml(highlight(text))).toBe(
+ highlighter.codeToHtml(text, { lang: "typescript", theme: "pierre-dark" }),
+ );
+ }
+ });
+
+ it.each(["text", "plaintext", "plain", "txt", "ansi"])(
+ "preserves %s without requesting grammar state",
+ async (language) => {
+ const highlighter = await highlighterPromise;
+ const highlight = createIncrementalHighlightedDocument(highlighter, language, "pierre-dark");
+ for (const text of ["plain\ntext", "\u001b[31mred\ncontinued", "\n"]) {
+ expect(toHtml(highlight(text))).toBe(
+ highlighter.codeToHtml(text, { lang: language, theme: "pierre-dark" }),
+ );
+ }
+ },
+ );
+
+ it("preserves partial CRLF and CR line endings", async () => {
+ const highlighter = await highlighterPromise;
+ const highlight = createIncrementalHighlightedDocument(
+ highlighter,
+ "typescript",
+ "pierre-dark",
+ );
+ const code = "/* multi\r\nline */\r\nconst x = 1;\r\n";
+ for (let end = 0; end <= code.length; end++) {
+ const text = code.slice(0, end);
+ expect(toHtml(highlight(text))).toBe(
+ highlighter.codeToHtml(text, { lang: "typescript", theme: "pierre-dark" }),
+ );
+ }
+ });
+});
diff --git a/apps/web/src/lib/incrementalHighlighting.ts b/apps/web/src/lib/incrementalHighlighting.ts
new file mode 100644
index 000000000000..fef3598a2152
--- /dev/null
+++ b/apps/web/src/lib/incrementalHighlighting.ts
@@ -0,0 +1,74 @@
+import type { DiffsHighlighter } from "@pierre/diffs";
+
+import type { DiffThemeName } from "./diffRendering";
+
+function codeChildren(root: ReturnType) {
+ const pre = root.children.find((node) => node.type === "element" && node.tagName === "pre");
+ if (pre?.type !== "element") throw new Error("Missing highlighted pre element");
+ const code = pre.children.find((node) => node.type === "element" && node.tagName === "code");
+ if (code?.type !== "element") throw new Error("Missing highlighted code element");
+ return code.children;
+}
+
+/** Resume tokenization after the last completed line. Keep its grammar state so
+ * multiline strings, comments, and embedded languages continue to highlight as
+ * they do in a full pass. The current line is always highlighted again.
+ */
+export function createIncrementalHighlightedDocument(
+ highlighter: DiffsHighlighter,
+ language: string,
+ theme: DiffThemeName,
+) {
+ const options = { lang: language, theme };
+ const newline = { type: "text" as const, value: "\n" };
+ let cached:
+ | {
+ prefix: string;
+ state: ReturnType;
+ children: ReturnType;
+ }
+ | undefined;
+
+ return (code: string) => {
+ // Plain text and ANSI do not have a TextMate grammar state. A CR at the end
+ // of a chunk can still become a CRLF, so keep that input on the full path.
+ if (
+ !language ||
+ ["text", "plaintext", "plain", "txt", "ansi"].includes(language) ||
+ code.includes("\r")
+ ) {
+ return highlighter.codeToHast(code, options);
+ }
+ if (cached && !code.startsWith(cached.prefix)) cached = undefined;
+ const end = code.lastIndexOf("\n") + 1;
+ if (end > (cached?.prefix.length ?? 0)) {
+ // Omit the final newline: Shiki would tokenize an extra empty line and
+ // advance the grammar state twice before we process the following line.
+ const root = highlighter.codeToHast(code.slice(cached?.prefix.length ?? 0, end - 1), {
+ ...options,
+ ...(cached ? { grammarState: cached.state } : {}),
+ });
+ const state = highlighter.getLastGrammarState(root);
+ if (!state) {
+ cached = undefined;
+ return highlighter.codeToHast(code, options);
+ }
+ cached = {
+ prefix: code.slice(0, end),
+ state,
+ children: [...(cached ? [...cached.children, newline] : []), ...codeChildren(root)],
+ };
+ }
+ const prefix = cached;
+ if (!prefix) return highlighter.codeToHast(code, options);
+ return highlighter.codeToHast(code.slice(prefix.prefix.length), {
+ ...options,
+ grammarState: prefix.state,
+ transformers: [
+ {
+ code: (node) => ({ ...node, children: [...prefix.children, newline, ...node.children] }),
+ },
+ ],
+ });
+ };
+}
diff --git a/apps/web/src/markdown-incremental.test.tsx b/apps/web/src/markdown-incremental.test.tsx
new file mode 100644
index 000000000000..cf06355f3b49
--- /dev/null
+++ b/apps/web/src/markdown-incremental.test.tsx
@@ -0,0 +1,136 @@
+import type { Root } from "mdast";
+import { renderToStaticMarkup } from "react-dom/server";
+import ReactMarkdown from "react-markdown";
+import rehypeRaw from "rehype-raw";
+import rehypeSanitize from "rehype-sanitize";
+import remarkGfm from "remark-gfm";
+import type { Plugin } from "unified";
+import { describe, expect, it } from "vite-plus/test";
+
+import { remarkCodexDirectives } from "@t3tools/client-runtime/codex-markdown-directives";
+import { remarkGithubAlerts } from "./markdown-github-alerts";
+import { createIncrementalMarkdownPlugin } from "./markdown-incremental";
+import { remarkNormalizeListItemIndentation } from "./markdown-list-indentation";
+
+function render(source: string, incremental?: Plugin<[], Root>, parsedSources?: string[]) {
+ let tree: Root | undefined;
+ const observeParsing: Plugin<[], Root> = function () {
+ const original = this.parser;
+ if (original) {
+ this.parser = (text, file) => {
+ parsedSources?.push(text);
+ return original(text, file);
+ };
+ }
+ };
+ const capture: Plugin<[], Root> = () => (root) => {
+ tree = structuredClone(root);
+ };
+ const html = renderToStaticMarkup(
+
+ {source}
+ ,
+ );
+ return { html, tree };
+}
+
+const prefix = "# Before\n\n```ts\nconst values = [1, 2];\n```\n\n";
+
+describe("incremental Markdown parsing", () => {
+ it("keeps the document prefix cached when list recovery parses contain fences", () => {
+ const source =
+ prefix +
+ "- first block\n\n ```ts\n const nested = 1;\n ```\n\n tail";
+ const incremental = createIncrementalMarkdownPlugin();
+ const parsedSources: string[] = [];
+ expect(render(source, incremental, parsedSources)).toEqual(render(source));
+ parsedSources.length = 0;
+ const next = source + " more";
+ expect(render(next, incremental, parsedSources)).toEqual(render(next));
+ expect(parsedSources).not.toContain(next);
+ expect(parsedSources.some((text) => text.startsWith("t3-markdown-inline-prefix:"))).toBe(true);
+ });
+
+ it.each([
+ "a\n===\n\nb\n---\n",
+ "- first\n\n continued\n\n- next\n",
+ "> quoted\n>\n> ```js\n> abc\n> ```\n\nend",
+ " \nhello\n\n \n\nend",
+ "[ref]\n\n[ref]: /later",
+ "a[^x]\n\n[^x]: note",
+ "a | b\n--|--\na | b\n",
+ "```\na\n```\n\nnext\n\n~~~\nb\n~~~\n\nmore",
+ "\n\n\tcode\n\nmore",
+ "text *bold*",
+ "> [!NOTE]\n> alert\n\n- [ ] task",
+ "\uFEFFtext after a byte-order mark",
+ ])("preserves the parse tree, positions, and HTML while streaming %j", (tail) => {
+ const source = prefix + tail;
+ const incremental = createIncrementalMarkdownPlugin();
+ for (let end = 0; end <= source.length; end++) {
+ const text = source.slice(0, end);
+ expect(render(text, incremental), `prefix ${end}`).toEqual(render(text));
+ }
+ });
+
+ it.each(["\r\n", "\r"])("preserves partial %j line endings", (newline) => {
+ const source = (prefix + "next\n\n```\nlast\n```\n\nend").replaceAll("\n", newline);
+ const incremental = createIncrementalMarkdownPlugin();
+ for (let end = 0; end <= source.length; end++) {
+ const text = source.slice(0, end);
+ expect(render(text, incremental)).toEqual(render(text));
+ }
+ });
+
+ it("updates earlier references when definitions arrive after the cached prefix", () => {
+ const before = "[later] and footnote[^note]\n\n" + prefix;
+ const incremental = createIncrementalMarkdownPlugin();
+ for (const tail of ["text", "[later]: /target", "[later]: /target\n\n[^note]: a note"]) {
+ expect(render(before + tail, incremental)).toEqual(render(before + tail));
+ }
+ });
+
+ it("handles edits, replacements, and repeated renders without leaking transformed nodes", () => {
+ const incremental = createIncrementalMarkdownPlugin();
+ const documents = [
+ prefix + "- first\n - second",
+ prefix + "> [!NOTE]\n> transformed alert",
+ prefix + "plain text",
+ "replacement without fences",
+ prefix.replace("Before", "Edited") + "edited prefix",
+ prefix + "plain text",
+ prefix + "plain text",
+ ];
+ for (const document of documents) {
+ expect(render(document, incremental)).toEqual(render(document));
+ }
+ });
+
+ it("does not freeze unclosed, nested, indented, or mismatched fences", () => {
+ const prefixes = [
+ "```\nopen\n\n",
+ "````\n```\n\n",
+ "> ```\n> code\n> ```\n\n",
+ "- ```\n code\n ```\n\n",
+ " ```\n code\n ```\n\n",
+ " |