From fb3d165d33b155038de717851207d25fb52fe65e Mon Sep 17 00:00:00 2001 From: Bilal Bakr <62337003+Bil0000@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:17:59 +0300 Subject: [PATCH 1/7] fix(web): show platform file manager icons in Open menu (#11228) --- apps/web/src/components/Icons.tsx | 28 +++++++++++++++++++ .../src/components/chat/OpenInPicker.test.ts | 26 +++++++++++++++++ apps/web/src/components/chat/OpenInPicker.tsx | 19 +++++++++---- apps/web/src/editorLabels.test.ts | 2 +- apps/web/src/lib/utils.test.ts | 2 +- apps/web/src/lib/utils.ts | 2 +- 6 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 apps/web/src/components/chat/OpenInPicker.test.ts diff --git a/apps/web/src/components/Icons.tsx b/apps/web/src/components/Icons.tsx index b13040152e18..99880a2512ad 100644 --- a/apps/web/src/components/Icons.tsx +++ b/apps/web/src/components/Icons.tsx @@ -2,6 +2,34 @@ import React, { type SVGProps, useId } from "react"; import { cn } from "~/lib/utils"; export type Icon = React.FC>; +export const FinderIcon: Icon = (props) => ( + + + + + +); + +export const FileExplorerIcon: Icon = (props) => ( + + + + + + +); + // Apple brand mark from Simple Icons (CC0). export const AppleIcon: Icon = (props) => (
Date: Fri, 11 Sep 2026 12:04:44 -0700 Subject: [PATCH 7/7] perf(mobile): reuse completed code lines while streaming (#11211) --- .../src/NativeMarkdownBlock.tsx | 232 +++++------------- .../src/SelectableMarkdownText.types.ts | 15 +- .../src/nativeMarkdownText.ts | 23 +- .../src/pendingCodeHighlight.test.ts | 28 +++ .../src/pendingCodeHighlight.ts | 19 ++ .../src/useHighlightedCode.test.tsx | 189 ++++++++++++++ .../src/useHighlightedCode.ts | 149 +++++++++++ .../review/incrementalSnippet.test.ts | 85 +++++++ .../src/features/review/incrementalSnippet.ts | 68 +++++ .../features/review/shikiReviewHighlighter.ts | 66 ++++- .../mobile/src/lib/nativeMarkdownText.test.ts | 93 ++++++- 11 files changed, 783 insertions(+), 184 deletions(-) create mode 100644 apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.test.ts create mode 100644 apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.ts create mode 100644 apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.test.tsx create mode 100644 apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.ts create mode 100644 apps/mobile/src/features/review/incrementalSnippet.test.ts create mode 100644 apps/mobile/src/features/review/incrementalSnippet.ts diff --git a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.tsx b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.tsx index ab75eaf32a2c..a4485ede4705 100644 --- a/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.tsx +++ b/apps/mobile/modules/t3-markdown-text/src/NativeMarkdownBlock.tsx @@ -1,10 +1,14 @@ -import { createContext, useContext, useEffect, useState } from "react"; +import { createContext, memo, useContext, useMemo } from "react"; import { Image, Platform, ScrollView, Text, useColorScheme, View } from "react-native"; import type { MarkdownNode } from "react-native-nitro-markdown/headless"; import { CopyTextButton } from "./CopyTextButton"; import { MarkdownTextPrimitive } from "./MarkdownTextPrimitive"; -import { nativeMarkdownDocumentRuns, nativeMarkdownListItemBlocks } from "./nativeMarkdownText"; +import { + nativeMarkdownDocumentRuns, + nativeMarkdownListItemBlocks, + nativeMarkdownNodePosition, +} from "./nativeMarkdownText"; import { NativeMarkdownSelectableText } from "./NativeMarkdownSelectableText"; import type { MarkdownCodeHighlighter, @@ -13,15 +17,11 @@ import type { NativeMarkdownTextStyle, SelectableMarkdownSkill, } from "./SelectableMarkdownText.types"; +import { useHighlightedCode, type HighlightedCode } from "./useHighlightedCode"; /** Set by SelectableMarkdownText so images anywhere in the block tree can use it. */ export const MarkdownImageRendererContext = createContext(null); -type HighlightedCode = ReadonlyArray>; - -const highlightedCodeCache = new Map(); -const highlightedCodePromiseCache = new Map>(); -const HIGHLIGHTED_CODE_CACHE_LIMIT = 64; const MONO_FONT_FAMILY = Platform.select({ ios: "ui-monospace", android: "monospace", @@ -29,7 +29,7 @@ const MONO_FONT_FAMILY = Platform.select({ }); function nodeKey(node: MarkdownNode, index: number): string { - return `${node.type}:${node.beg ?? index}:${node.end ?? index}`; + return `${node.type}:${nativeMarkdownNodePosition(node, index)}`; } /** Code inside markdown scales with the base text size (12pt at the default 15pt body). */ @@ -67,174 +67,74 @@ function SelectableNode(props: { ); } -function codeHighlightCacheKey( - code: string, - language: string | undefined, - theme: "light" | "dark", -): string { - return `${theme}:${language ?? "text"}:${code}`; -} - -function cacheHighlightedCode(key: string, tokens: HighlightedCode): void { - highlightedCodeCache.delete(key); - highlightedCodeCache.set(key, tokens); - - while (highlightedCodeCache.size > HIGHLIGHTED_CODE_CACHE_LIMIT) { - const oldestKey = highlightedCodeCache.keys().next().value; - if (oldestKey === undefined) { - break; - } - highlightedCodeCache.delete(oldestKey); - } -} - -function loadHighlightedCode( - code: string, - language: string | undefined, - theme: "light" | "dark", - highlightCode: MarkdownCodeHighlighter, -): Promise { - const key = codeHighlightCacheKey(code, language, theme); - const cached = highlightedCodeCache.get(key); - if (cached) { - return Promise.resolve(cached); - } - - const pending = highlightedCodePromiseCache.get(key); - if (pending) { - return pending; - } - - const promise = highlightCode({ code, language, theme }) - .then((tokens) => { - cacheHighlightedCode(key, tokens); - highlightedCodePromiseCache.delete(key); - return tokens; - }) - .catch((error) => { - highlightedCodePromiseCache.delete(key); - throw error; - }); - highlightedCodePromiseCache.set(key, promise); - return promise; -} - -function useHighlightedCode( - code: string, - language: string | undefined, - theme: "light" | "dark", - highlightCode: MarkdownCodeHighlighter, -): HighlightedCode | null { - const key = codeHighlightCacheKey(code, language, theme); - const [highlighted, setHighlighted] = useState<{ - readonly key: string; - readonly tokens: HighlightedCode | null; - }>(() => ({ - key, - tokens: highlightedCodeCache.get(key) ?? null, - })); - - useEffect(() => { - let active = true; - const cached = highlightedCodeCache.get(key); - if (cached) { - cacheHighlightedCode(key, cached); - setHighlighted({ key, tokens: cached }); - return () => { - active = false; - }; - } - - void loadHighlightedCode(code, language, theme, highlightCode) - .then((tokens) => { - if (active) { - setHighlighted({ key, tokens }); - } - }) - .catch(() => { - if (active) { - setHighlighted({ key, tokens: null }); - } - }); - return () => { - active = false; - }; - }, [code, highlightCode, key, language, theme]); - - return highlighted.key === key ? highlighted.tokens : null; -} - -function HighlightedCodeText(props: { - readonly content: string; - readonly highlighted: HighlightedCode | null; - readonly textStyle: NativeMarkdownTextStyle; +const HighlightedCodeLine = memo(function HighlightedCodeLine(props: { + readonly tokens: ReadonlyArray; + readonly color: string; + readonly newline: boolean; }) { - if (!props.highlighted) { - return ( + let offset = 0; + const children = []; + for (const token of props.tokens) { + if (!token.content) continue; + children.push( - {props.content} - + {token.content} + , ); + offset += token.content.length; } - const highlighted = props.highlighted; - let sourceOffset = 0; - const keyOccurrences = new Map(); - const keyedLines = highlighted.map((line) => { - const lineStart = sourceOffset; - const tokens = line.map((token) => { - const start = sourceOffset; - sourceOffset += token.content.length; - const signature = `${start}:${token.content}:${token.color ?? ""}:${token.fontStyle ?? ""}`; - const occurrence = keyOccurrences.get(signature) ?? 0; - keyOccurrences.set(signature, occurrence + 1); - return { key: `${signature}:${occurrence}`, token }; - }); - sourceOffset += 1; - return { - key: `line:${lineStart}:${line.map((token) => token.content).join("")}`, - tokens, - }; - }); + return ( + + {children} + {props.newline ? "\n" : ""} + + ); +}); +function HighlightedCodeText(props: { + readonly content: string; + readonly highlighted: HighlightedCode | null; + readonly textStyle: NativeMarkdownTextStyle; +}) { + // The text root provides inherited styles through context. A new style object + // would rerender every token even when its completed line is unchanged. + const fontSize = codeBlockFontSize(props.textStyle); + const lineHeight = codeBlockLineHeight(props.textStyle); + const style = useMemo( + () => ({ + color: props.textStyle.codeColor, + fontFamily: MONO_FONT_FAMILY, + fontSize, + lineHeight, + }), + [props.textStyle.codeColor, fontSize, lineHeight], + ); + let offset = 0; + const lines = []; + if (props.highlighted) { + for (const tokens of props.highlighted) { + lines.push( + , + ); + offset += tokens.reduce((length, token) => length + token.content.length, 0) + 1; + } + } return ( - - {keyedLines.map((line, lineIndex) => ( - - {line.tokens.map(({ key, token }) => ( - - {token.content} - - ))} - {lineIndex + 1 < keyedLines.length ? "\n" : ""} - - ))} + + {props.highlighted ? lines : props.content} ); } diff --git a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.types.ts b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.types.ts index 50b1cccb6602..13acb58b1a7d 100644 --- a/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.types.ts +++ b/apps/mobile/modules/t3-markdown-text/src/SelectableMarkdownText.types.ts @@ -25,11 +25,22 @@ export interface MarkdownHighlightedToken { readonly fontStyle: number | null; } -export type MarkdownCodeHighlighter = (input: { +export interface MarkdownCodeHighlightInput { + /** Identity of the mounted code block, for incremental highlighting. */ + readonly session?: object; readonly code: string; readonly language?: string | null; readonly theme: "light" | "dark"; -}) => Promise>>; +} +export interface MarkdownCodeHighlighter { + ( + input: MarkdownCodeHighlightInput, + ): Promise>>; + /** Optional synchronous result for a small append to an already warm block. */ + read?: ( + input: MarkdownCodeHighlightInput, + ) => ReadonlyArray> | undefined; +} export interface SelectableMarkdownSkill { readonly name: string; diff --git a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts index 4c8a6c4d7cd5..51dc85a2e9bf 100644 --- a/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts +++ b/apps/mobile/modules/t3-markdown-text/src/nativeMarkdownText.ts @@ -694,21 +694,31 @@ function containsRichBlock(node: MarkdownNode): boolean { return (node.children ?? []).some(containsRichBlock); } +/** + * Sibling identity for React keys. A source offset survives appends while the + * document streams; the child index is the fallback for offset-free nodes. The + * two never share a namespace, so a positioned node cannot collide with an + * offset-free sibling whose index happens to equal its offset. + */ +export function nativeMarkdownNodePosition(node: MarkdownNode, index: number): string { + return node.beg === undefined ? `index:${index}` : `offset:${node.beg}`; +} + export function nativeMarkdownDocumentChunks( document: MarkdownNode, ): ReadonlyArray { const chunks: NativeMarkdownDocumentChunk[] = []; let selectableNodes: MarkdownNode[] = []; + let selectableStart = 0; const flushSelectable = () => { - if (selectableNodes.length === 0) { + const first = selectableNodes[0]; + if (!first) { return; } - const first = selectableNodes[0]; - const last = selectableNodes.at(-1); chunks.push({ kind: "selectable", - key: `selectable:${first?.beg ?? "start"}:${last?.end ?? "end"}`, + key: `selectable:${nativeMarkdownNodePosition(first, selectableStart)}`, node: { type: "document", children: selectableNodes, @@ -719,6 +729,9 @@ export function nativeMarkdownDocumentChunks( for (const [index, child] of (document.children ?? []).entries()) { if (!containsRichBlock(child)) { + if (selectableNodes.length === 0) { + selectableStart = index; + } selectableNodes.push(child); continue; } @@ -726,7 +739,7 @@ export function nativeMarkdownDocumentChunks( flushSelectable(); chunks.push({ kind: "rich", - key: `rich:${child.type}:${child.beg ?? index}:${child.end ?? index}`, + key: `rich:${child.type}:${nativeMarkdownNodePosition(child, index)}`, node: child, }); } diff --git a/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.test.ts b/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.test.ts new file mode 100644 index 000000000000..19516e782524 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.test.ts @@ -0,0 +1,28 @@ +import { expect, it } from "vite-plus/test"; +import { pendingCodeHighlight } from "./pendingCodeHighlight"; + +it("keeps completed colors and exact current text without retaining an edited tail", () => { + const colored = [ + [{ content: "const n = 1;", color: "red", fontStyle: 0 }], + [{ content: "partial", color: "blue", fontStyle: 0 }], + ]; + const result = pendingCodeHighlight( + "const n = 1;\npartial", + "const n = 1;\nchanged\nnext", + colored, + )!; + expect(result[0]).toBe(colored[0]); + expect(result.map((line) => line.map((token) => token.content).join("")).join("\n")).toBe( + "const n = 1;\nchanged\nnext", + ); + expect( + result + .slice(1) + .flat() + .every((token) => token.color === null), + ).toBe(true); + expect( + pendingCodeHighlight("const n = 1;\npartial", "const n = 2;\npartial", colored), + ).toBeNull(); + expect(pendingCodeHighlight("partial", "other", colored)).toBeNull(); +}); diff --git a/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.ts b/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.ts new file mode 100644 index 000000000000..308997fc83a8 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/pendingCodeHighlight.ts @@ -0,0 +1,19 @@ +import type { MarkdownHighlightedToken } from "./SelectableMarkdownText.types"; + +/** Keep finished lines colored while the current line awaits highlighting. */ +export function pendingCodeHighlight( + previousCode: string, + code: string, + tokens: ReadonlyArray>, +): ReadonlyArray> | null { + const end = previousCode.lastIndexOf("\n") + 1; + if (!end || !code.startsWith(previousCode.slice(0, end))) return null; + const completedLines = previousCode.slice(0, end).split("\n").length - 1; + return [ + ...tokens.slice(0, completedLines), + ...code + .slice(end) + .split("\n") + .map((content) => [{ content, color: null, fontStyle: null }]), + ]; +} diff --git a/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.test.tsx b/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.test.tsx new file mode 100644 index 000000000000..c4fcaa597db5 --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.test.tsx @@ -0,0 +1,189 @@ +import { act, useLayoutEffect } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test"; + +import type { MarkdownCodeHighlighter } from "./SelectableMarkdownText.types"; +import { useHighlightedCode, type HighlightedCode } from "./useHighlightedCode"; + +type Deferred = { + readonly code: string; + readonly resolve: (tokens: HighlightedCode) => void; +}; + +/** One colored line per newline-separated line, so a line's color proves where it came from. */ +function tokensFor(code: string, color: string): HighlightedCode { + return code.split("\n").map((content) => [{ content, color, fontStyle: 0 }]); +} + +function lineText(tokens: HighlightedCode | null): string | null { + return tokens?.map((line) => line.map((token) => token.content).join("")).join("\n") ?? null; +} + +function lineColors(tokens: HighlightedCode | null): ReadonlyArray { + return tokens?.map((line) => line[0]?.color ?? null) ?? []; +} + +const deferred: Deferred[] = []; +const readResults = new Map(); +let renders = 0; +let result: HighlightedCode | null = null; +let root: Root; + +const highlightCode: MarkdownCodeHighlighter = Object.assign( + vi.fn( + (input: { code: string }) => + new Promise((resolve) => { + deferred.push({ code: input.code, resolve }); + }), + ), + { + read: vi.fn((input: { code: string }) => readResults.get(input.code)), + }, +); + +function Probe(props: { + readonly code: string; + readonly language?: string; + readonly theme?: "light" | "dark"; +}) { + const tokens = useHighlightedCode( + props.code, + props.language ?? "ts", + props.theme ?? "dark", + highlightCode, + ); + useLayoutEffect(() => { + renders += 1; + result = tokens; + }); + return null; +} + +async function render(props: Parameters[0]) { + await act(() => root.render()); +} + +async function resolveAsync(index: number, color: string) { + const pending = deferred[index]; + if (!pending) throw new Error(`no pending highlight at ${index}`); + await act(async () => { + pending.resolve(tokensFor(pending.code, color)); + }); +} + +beforeEach(async () => { + // The probe has no DOM output, but ReactDOM needs an event target. + const document = { + nodeType: 9, + addEventListener() {}, + removeEventListener() {}, + }; + const container = { + nodeType: 1, + tagName: "DIV", + namespaceURI: "http://www.w3.org/1999/xhtml", + ownerDocument: document, + addEventListener() {}, + removeEventListener() {}, + }; + vi.stubGlobal("document", document); + vi.stubGlobal("window", { document, HTMLIFrameElement: EventTarget }); + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + deferred.length = 0; + readResults.clear(); + renders = 0; + result = null; + vi.mocked(highlightCode).mockClear(); + vi.mocked(highlightCode.read!).mockClear(); + root = createRoot(container as unknown as HTMLElement); +}); + +afterEach(async () => { + await act(() => root.unmount()); + vi.unstubAllGlobals(); +}); + +// Every test uses a distinct language so the module-level token cache from an +// earlier test can never satisfy a later one. +describe("useHighlightedCode", () => { + it("keeps colors from the latest synchronous read when a later read misses", async () => { + const language = "sync-baseline"; + await render({ code: "a", language }); + expect(result).toBeNull(); + await resolveAsync(0, "async"); + expect(lineColors(result)).toEqual(["async"]); + + readResults.set("a\nb", tokensFor("a\nb", "sync-1")); + readResults.set("a\nb\nc", tokensFor("a\nb\nc", "sync-2")); + const rendersBeforeAppends = renders; + await render({ code: "a\nb", language }); + await render({ code: "a\nb\nc", language }); + expect(lineColors(result)).toEqual(["sync-2", "sync-2", "sync-2"]); + // A synchronous hit is one render per append: no state update, no highlighter call. + expect(renders - rendersBeforeAppends).toBe(2); + expect(highlightCode).toHaveBeenCalledTimes(1); + + await render({ code: "a\nb\nc\nd", language }); + expect(highlightCode).toHaveBeenCalledTimes(2); + expect(lineText(result)).toBe("a\nb\nc\nd"); + // Lines before the previous code's final newline are the completed ones. + expect(lineColors(result)).toEqual(["sync-2", "sync-2", null, null]); + + await resolveAsync(1, "async-2"); + expect(lineColors(result)).toEqual(["async-2", "async-2", "async-2", "async-2"]); + }); + + it("does not show obsolete text after a non-append edit", async () => { + const language = "non-append"; + await render({ code: "a\nb", language }); + await resolveAsync(0, "async"); + readResults.set("a\nb\nc", tokensFor("a\nb\nc", "sync")); + await render({ code: "a\nb\nc", language }); + expect(lineColors(result)).toEqual(["sync", "sync", "sync"]); + + await render({ code: "x\nb\nc", language }); + expect(result).toBeNull(); + await resolveAsync(1, "async-2"); + expect(lineText(result)).toBe("x\nb\nc"); + }); + + it("never reuses colors across a language or theme change", async () => { + const language = "theme-change"; + await render({ code: "a\nb", language }); + await resolveAsync(0, "async"); + readResults.set("a\nb\nc", tokensFor("a\nb\nc", "sync")); + await render({ code: "a\nb\nc", language }); + expect(lineColors(result)).toEqual(["sync", "sync", "sync"]); + + await render({ code: "a\nb\nc\nd", language, theme: "light" }); + expect(result).toBeNull(); + await render({ code: "a\nb\nc\nd", language: `${language}-other` }); + expect(result).toBeNull(); + }); + + it("ignores an asynchronous completion that is older than the current code", async () => { + const language = "stale-async"; + await render({ code: "a\nb", language }); + await resolveAsync(0, "async"); + + await render({ code: "a\nb\nc", language }); + expect(deferred).toHaveLength(2); + expect(lineColors(result)).toEqual(["async", null, null]); + + readResults.set("a\nb\nc\nd", tokensFor("a\nb\nc\nd", "sync")); + await render({ code: "a\nb\nc\nd", language }); + expect(lineColors(result)).toEqual(["sync", "sync", "sync", "sync"]); + + await render({ code: "a\nb\nc\nd\ne", language }); + expect(deferred).toHaveLength(3); + expect(lineColors(result)).toEqual(["sync", "sync", "sync", null, null]); + + // The older request finishes after newer synchronous results exist. + await resolveAsync(1, "stale"); + expect(lineText(result)).toBe("a\nb\nc\nd\ne"); + expect(lineColors(result)).toEqual(["sync", "sync", "sync", null, null]); + + await resolveAsync(2, "async-3"); + expect(lineColors(result)).toEqual(["async-3", "async-3", "async-3", "async-3", "async-3"]); + }); +}); diff --git a/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.ts b/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.ts new file mode 100644 index 000000000000..dd86dc3af9af --- /dev/null +++ b/apps/mobile/modules/t3-markdown-text/src/useHighlightedCode.ts @@ -0,0 +1,149 @@ +import { useEffect, useMemo, useRef, useState } from "react"; + +import { pendingCodeHighlight } from "./pendingCodeHighlight"; +import type { + MarkdownCodeHighlighter, + MarkdownHighlightedToken, +} from "./SelectableMarkdownText.types"; + +export type HighlightedCode = ReadonlyArray>; + +interface HighlightedCodeResult { + readonly key: string; + readonly code: string; + readonly language: string | undefined; + readonly theme: "light" | "dark"; + readonly tokens: HighlightedCode | null; +} + +const highlightedCodeCache = new Map(); +const highlightedCodePromiseCache = new Map>(); +const HIGHLIGHTED_CODE_CACHE_LIMIT = 64; + +function codeHighlightCacheKey( + code: string, + language: string | undefined, + theme: "light" | "dark", +): string { + return `${theme}:${language ?? "text"}:${code}`; +} + +function cacheHighlightedCode(key: string, tokens: HighlightedCode): void { + highlightedCodeCache.delete(key); + highlightedCodeCache.set(key, tokens); + + while (highlightedCodeCache.size > HIGHLIGHTED_CODE_CACHE_LIMIT) { + const oldestKey = highlightedCodeCache.keys().next().value; + if (oldestKey === undefined) { + break; + } + highlightedCodeCache.delete(oldestKey); + } +} + +function loadHighlightedCode( + code: string, + language: string | undefined, + theme: "light" | "dark", + highlightCode: MarkdownCodeHighlighter, + session: object, +): Promise { + const key = codeHighlightCacheKey(code, language, theme); + const cached = highlightedCodeCache.get(key); + if (cached) { + return Promise.resolve(cached); + } + + const pending = highlightedCodePromiseCache.get(key); + if (pending) { + return pending; + } + + const promise = highlightCode({ code, language, theme, session }) + .then((tokens) => { + cacheHighlightedCode(key, tokens); + highlightedCodePromiseCache.delete(key); + return tokens; + }) + .catch((error) => { + highlightedCodePromiseCache.delete(key); + throw error; + }); + highlightedCodePromiseCache.set(key, promise); + return promise; +} + +/** + * Tokens for a code block, or null while nothing usable exists yet. A cached or + * synchronous result renders in the same pass. Otherwise the most recent + * completed result seeds `pendingCodeHighlight` so finished lines keep their + * colors while the asynchronous highlighter catches up. + */ +export function useHighlightedCode( + code: string, + language: string | undefined, + theme: "light" | "dark", + highlightCode: MarkdownCodeHighlighter, +): HighlightedCode | null { + const [session] = useState(() => ({})); + const key = codeHighlightCacheKey(code, language, theme); + const ready = useMemo( + () => highlightedCodeCache.get(key) ?? highlightCode.read?.({ code, language, theme, session }), + [code, language, theme, key, highlightCode, session], + ); + const [highlighted, setHighlighted] = useState(() => ({ + code, + language, + theme, + key, + tokens: highlightedCodeCache.get(key) ?? null, + })); + // Synchronous reads never touch state, so each append costs one render. The + // ref remembers the newest of them, and is cleared whenever an asynchronous + // result commits so `highlighted` is the baseline again from then on. + const latestRead = useRef(null); + + useEffect(() => { + if (ready) { + latestRead.current = { code, language, theme, key, tokens: ready }; + return; + } + let active = true; + const commit = (tokens: HighlightedCode | null) => { + latestRead.current = null; + setHighlighted({ code, language, theme, key, tokens }); + }; + const cached = highlightedCodeCache.get(key); + if (cached) { + cacheHighlightedCode(key, cached); + commit(cached); + return () => { + active = false; + }; + } + + void loadHighlightedCode(code, language, theme, highlightCode, session) + .then((tokens) => { + if (active) { + commit(tokens); + } + }) + .catch(() => { + if (active) { + commit(null); + } + }); + return () => { + active = false; + }; + }, [code, highlightCode, key, language, theme, ready, session]); + + if (ready) return ready; + // oxlint-disable-next-line react/refs -- Written only after commit; a discarded render never advances it. + const baseline = latestRead.current ?? highlighted; + if (baseline.key === key) return baseline.tokens; + if (baseline.tokens && baseline.language === language && baseline.theme === theme) { + return pendingCodeHighlight(baseline.code, code, baseline.tokens); + } + return null; +} diff --git a/apps/mobile/src/features/review/incrementalSnippet.test.ts b/apps/mobile/src/features/review/incrementalSnippet.test.ts new file mode 100644 index 000000000000..3e768bbcd539 --- /dev/null +++ b/apps/mobile/src/features/review/incrementalSnippet.test.ts @@ -0,0 +1,85 @@ +import { createHighlighterCore } from "@shikijs/core"; +import { createJavaScriptRegexEngine } from "@shikijs/engine-javascript"; +import typescript from "@shikijs/langs/typescript"; +import dark from "@shikijs/themes/github-dark-default"; +import { describe, expect, it } from "vite-plus/test"; + +import { createIncrementalSnippet } from "./incrementalSnippet"; +import { highlightCodeSnippet } from "./shikiReviewHighlighter"; + +const highlighter = await createHighlighterCore({ + langs: [typescript], + themes: [dark], + engine: createJavaScriptRegexEngine(), +}); +const rendered = (lines: ReturnType) => + lines.map((line) => line.map(({ content, color, fontStyle }) => ({ content, color, fontStyle }))); +const options = { lang: "typescript", theme: "github-dark-default" }; + +describe("incremental snippet highlighting", () => { + it("matches full highlighting through partial lines, multiline syntax, edits and truncation", async () => { + const highlight = createIncrementalSnippet(highlighter, options.lang, options.theme); + const source = "/* comment\nstill comment */\nconst text = `first\nsecond ${42}`;\n"; + for (let end = 1; end <= source.length; end++) { + const code = source.slice(0, end); + expect(rendered(highlight.read(code) ?? (await highlight(code)))).toEqual( + rendered(highlighter.codeToTokensBase(code, options)), + ); + } + for (const code of ['const other = "new";\n', "const other", source, source.slice(0, 20)]) { + expect(rendered(highlight.read(code) ?? (await highlight(code)))).toEqual( + rendered(highlighter.codeToTokensBase(code, options)), + ); + } + }); + + it("carries grammar state across batches and overlapping updates", async () => { + const highlight = createIncrementalSnippet(highlighter, options.lang, options.theme); + const code = "/*\n" + "still a comment\n".repeat(410) + "*/\nconst value = 42;"; + const first = highlight(code); + const next = highlight("const changed = true;\n"); + expect(rendered(await first)).toEqual(rendered(highlighter.codeToTokensBase(code, options))); + expect(rendered(await next)).toEqual( + rendered(highlighter.codeToTokensBase("const changed = true;\n", options)), + ); + const last = 'const changed = true;\nconst tail = "ok";'; + expect(rendered(await highlight(last))).toEqual( + rendered(highlighter.codeToTokensBase(last, options)), + ); + }); + + it("reuses completed token rows on warm reads and falls back for large updates", async () => { + const session = {}; + const input = { language: "ts", theme: "dark" as const, session }; + const code = "const a = 1;\nconst b = 2;"; + expect(highlightCodeSnippet.read({ ...input, code })).toBeUndefined(); + const first = await highlightCodeSnippet({ ...input, code }); + const nextCode = code + "\nconst c = 3;"; + const next = highlightCodeSnippet.read({ ...input, code: nextCode })!; + expect(next[0]).toBe(first[0]); + expect(next).toEqual( + await highlightCodeSnippet({ code: nextCode, language: "ts", theme: "dark" }), + ); + expect( + highlightCodeSnippet.read({ ...input, code: nextCode + "\n" + "const x = 1;\n".repeat(210) }), + ).toBeUndefined(); + expect(highlightCodeSnippet.read({ ...input, code: nextCode, theme: "light" })).toBeUndefined(); + }); + + it("resets session on language, theme, CRLF, long-line and empty input changes", async () => { + const session = {}; + const cases = [ + { code: "/* hello\nworld */", language: "ts", theme: "dark" as const }, + { code: "/* hello\nworld */\nconst n = 3;", language: "ts", theme: "light" as const }, + { code: "hello\nworld", language: "text", theme: "light" as const }, + { code: "const n = 2;\r\nconst m = 3;", language: "ts", theme: "dark" as const }, + { code: "a".repeat(1100), language: "ts", theme: "dark" as const }, + { code: "", language: "ts", theme: "dark" as const }, + { code: "const fresh = true;\n", language: "ts", theme: "dark" as const }, + ]; + for (const input of cases) + expect(await highlightCodeSnippet({ ...input, session })).toEqual( + await highlightCodeSnippet(input), + ); + }); +}); diff --git a/apps/mobile/src/features/review/incrementalSnippet.ts b/apps/mobile/src/features/review/incrementalSnippet.ts new file mode 100644 index 000000000000..ba8acd847b41 --- /dev/null +++ b/apps/mobile/src/features/review/incrementalSnippet.ts @@ -0,0 +1,68 @@ +import type { HighlighterCore } from "@shikijs/core"; + +/** A code block owns this session. Only completed lines survive the next update. */ +export function createIncrementalSnippet( + highlighter: HighlighterCore, + language: string, + theme: string, +) { + const options = { lang: language, theme }; + let cached: + | { + prefix: string; + tokens: ReturnType; + state: ReturnType; + } + | undefined; + let revision = 0; + + function* tokenize(code: string) { + const currentRevision = ++revision; + const previous = cached && code.startsWith(cached.prefix) ? cached : undefined; + const end = code.lastIndexOf("\n") + 1; + let state = previous?.state; + const tokens = [...(previous?.tokens ?? [])]; + const completed = code.slice(previous?.prefix.length ?? 0, end); + if (completed) { + const lines = completed.slice(0, -1).split("\n"); + for (let offset = 0; offset < lines.length; offset += 200) { + const batch = highlighter.codeToTokensBase(lines.slice(offset, offset + 200).join("\n"), { + ...options, + ...(state ? { grammarState: state } : {}), + }); + state = highlighter.getLastGrammarState(batch); + tokens.push(...batch); + if (offset + 200 < lines.length) yield; + } + } + if (currentRevision === revision) { + cached = state ? { prefix: code.slice(0, end), tokens, state } : undefined; + } + return [ + ...tokens, + ...highlighter.codeToTokensBase(code.slice(end), { + ...options, + ...(state ? { grammarState: state } : {}), + }), + ]; + } + const highlight = async (code: string) => { + const work = tokenize(code); + let next = work.next(); + while (!next.done) { + await new Promise((resolve) => setTimeout(resolve, 0)); + next = work.next(); + } + return next.value; + }; + // A small append can finish during render, avoiding a plain-text commit before + // the asynchronous effect supplies colors. Cold/large changes stay asynchronous. + highlight.read = (code: string) => { + if (!cached || !code.startsWith(cached.prefix)) return undefined; + const tail = code.slice(cached.prefix.length); + if (tail.length > 2_000 || tail.split("\n").length > 200) return undefined; + const next = tokenize(code).next(); + return next.done ? next.value : undefined; + }; + return highlight; +} diff --git a/apps/mobile/src/features/review/shikiReviewHighlighter.ts b/apps/mobile/src/features/review/shikiReviewHighlighter.ts index a2e3fbd4eb0c..52e0b83be34d 100644 --- a/apps/mobile/src/features/review/shikiReviewHighlighter.ts +++ b/apps/mobile/src/features/review/shikiReviewHighlighter.ts @@ -17,6 +17,7 @@ import { resolveReviewHighlighterEnginePreference, type ReviewHighlighterEngine, } from "./reviewHighlighterEngine"; +import { createIncrementalSnippet } from "./incrementalSnippet"; import type { ReviewRenderableLineRow } from "./reviewModel"; import { applyDiffRangesToTokens, computeWordAltDiffRanges } from "./reviewWordDiffs"; @@ -453,16 +454,23 @@ async function resolveLanguageFromPath( return candidate; } +type RawHighlightedLine = ReadonlyArray<{ content: string; color?: string; fontStyle?: number }>; +const normalizedLines = new WeakMap>(); + function normalizeHighlightedLines( - tokenLines: ReadonlyArray>, + tokenLines: ReadonlyArray, ): ReadonlyArray> { - return tokenLines.map((line) => - line.map((token) => ({ + return tokenLines.map((line) => { + const cached = normalizedLines.get(line); + if (cached) return cached; + const normalized = line.map((token) => ({ content: token.content, color: token.color ?? null, fontStyle: token.fontStyle ?? null, - })), - ); + })); + normalizedLines.set(line, normalized); + return normalized; + }); } function applyWordAltDiffHighlightsToSelectedLines(input: { @@ -594,16 +602,62 @@ async function highlightLines( return highlightedLines; } +const snippetSessions = new WeakMap< + object, + { language: string; theme: string; highlight: ReturnType } +>(); + export async function highlightCodeSnippet(input: { + readonly session?: object; readonly code: string; readonly language?: string | null; readonly theme: ReviewDiffTheme; }): Promise>> { const languageHint = input.language?.trim() || "text"; const language = await resolveLanguageFromPath(`snippet.${languageHint}`, languageHint); - return highlightLines(input.code, language, SHIKI_THEME_NAME_BY_SCHEME[input.theme]); + const theme = SHIKI_THEME_NAME_BY_SCHEME[input.theme]; + // Bound retained text and preserve the existing plain-text/long-line fallback. + if ( + !input.session || + language === "text" || + input.code.length === 0 || + input.code.length > 100_000 || + input.code.includes("\r") || + input.code.split("\n").some((line) => line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) + ) { + if (input.session) snippetSessions.delete(input.session); + return highlightLines(input.code, language, theme); + } + const highlighter = await getHighlighter(); + let session = snippetSessions.get(input.session); + if (!session || session.language !== language || session.theme !== theme) { + session = { + language, + theme, + highlight: createIncrementalSnippet(highlighter, language, theme), + }; + snippetSessions.set(input.session, session); + } + return normalizeHighlightedLines(await session.highlight(input.code)); } +highlightCodeSnippet.read = (input: Parameters[0]) => { + if (!input.session || !input.code || input.code.length > 100_000 || input.code.includes("\r")) + return undefined; + const session = snippetSessions.get(input.session); + const hint = input.language?.trim() || "text"; + const language = resolveLoadedLanguageFromPath(`snippet.${hint}`, hint); + if ( + !session || + session.language !== language || + session.theme !== SHIKI_THEME_NAME_BY_SCHEME[input.theme] || + input.code.split("\n").some((line) => line.length > REVIEW_TOKENIZE_MAX_LINE_LENGTH) + ) + return undefined; + const tokens = session.highlight.read(input.code); + return tokens ? normalizeHighlightedLines(tokens) : undefined; +}; + export async function highlightSourceFile(input: { readonly path: string; readonly contents: string; diff --git a/apps/mobile/src/lib/nativeMarkdownText.test.ts b/apps/mobile/src/lib/nativeMarkdownText.test.ts index c3951c8d81f0..bf7e90e41973 100644 --- a/apps/mobile/src/lib/nativeMarkdownText.test.ts +++ b/apps/mobile/src/lib/nativeMarkdownText.test.ts @@ -536,7 +536,7 @@ describe("nativeMarkdownDocumentChunks", () => { ).toEqual([ { kind: "rich", - key: "rich:blockquote:0:120", + key: "rich:blockquote:offset:0", node: blockquote, }, ]); @@ -663,7 +663,7 @@ describe("nativeMarkdownDocumentChunks", () => { expect(chunks[0]).toMatchObject({ kind: "selectable" }); expect(chunks[1]).toEqual({ kind: "rich", - key: "rich:code_block:11:35", + key: "rich:code_block:offset:11", node: document.children?.[1], }); expect(chunks[2]).toMatchObject({ kind: "selectable" }); @@ -700,7 +700,7 @@ describe("nativeMarkdownDocumentChunks", () => { expect(nativeMarkdownDocumentChunks(document)).toEqual([ { kind: "rich", - key: "rich:list:0:45", + key: "rich:list:offset:0", node: document.children?.[0], }, ]); @@ -728,7 +728,7 @@ describe("nativeMarkdownDocumentChunks", () => { expect(chunks[0]).toMatchObject({ kind: "selectable" }); expect(chunks[1]).toEqual({ kind: "rich", - key: "rich:horizontal_rule:1:1", + key: "rich:horizontal_rule:index:1", node: document.children?.[1], }); expect(chunks[2]).toMatchObject({ kind: "selectable" }); @@ -774,7 +774,7 @@ describe("nativeMarkdownDocumentChunks", () => { expect(chunks[0]).toMatchObject({ kind: "selectable" }); expect(chunks[1]).toEqual({ kind: "rich", - key: "rich:list:1:1", + key: "rich:list:index:1", node: document.children?.[1], }); expect(chunks[2]).toMatchObject({ kind: "selectable" }); @@ -824,6 +824,89 @@ describe("nativeMarkdownDocumentChunks", () => { }); }); + it("keys positioned and offset-free siblings in separate namespaces", () => { + const positioned: MarkdownNode = { + type: "blockquote", + beg: 1, + end: 10, + children: [{ type: "paragraph", children: [{ type: "text", content: "Positioned" }] }], + }; + const offsetFree: MarkdownNode = { + type: "blockquote", + children: [{ type: "paragraph", children: [{ type: "text", content: "Generated" }] }], + }; + const chunks = nativeMarkdownDocumentChunks({ + type: "document", + children: [ + { type: "paragraph", beg: 0, end: 0, children: [] }, + offsetFree, + positioned, + { type: "paragraph", children: [{ type: "text", content: "Tail" }] }, + ], + }); + + expect(chunks.map((chunk) => chunk.key)).toEqual([ + "selectable:offset:0", + "rich:blockquote:index:1", + "rich:blockquote:offset:1", + "selectable:index:3", + ]); + }); + + it("gives every offset-free selectable group its own key", () => { + const chunks = nativeMarkdownDocumentChunks({ + type: "document", + children: [ + { type: "paragraph", children: [{ type: "text", content: "One" }] }, + { type: "horizontal_rule" }, + { type: "paragraph", children: [{ type: "text", content: "Two" }] }, + { type: "horizontal_rule" }, + { type: "paragraph", children: [{ type: "text", content: "Three" }] }, + ], + }); + + expect(chunks.map((chunk) => chunk.key)).toEqual([ + "selectable:index:0", + "rich:horizontal_rule:index:1", + "selectable:index:2", + "rich:horizontal_rule:index:3", + "selectable:index:4", + ]); + }); + + it("keeps positioned chunk keys stable while text streams in", () => { + const before: MarkdownNode = { + type: "document", + children: [ + { type: "paragraph", beg: 0, end: 5, children: [{ type: "text", content: "Intro" }] }, + { + type: "code_block", + language: "ts", + beg: 7, + end: 20, + children: [{ type: "text", content: "const a" }], + }, + ], + }; + const after: MarkdownNode = { + type: "document", + children: [ + { type: "paragraph", beg: 0, end: 5, children: [{ type: "text", content: "Intro" }] }, + { + type: "code_block", + language: "ts", + beg: 7, + end: 40, + children: [{ type: "text", content: "const a = 1;\nconst b" }], + }, + ], + }; + + expect(nativeMarkdownDocumentChunks(after).map((chunk) => chunk.key)).toEqual( + nativeMarkdownDocumentChunks(before).map((chunk) => chunk.key), + ); + }); + it("keeps a plain list in one selectable native text container", () => { const list: MarkdownNode = { type: "list",