From 1cc2ab65db2cced8dd8bf523944cc9167d13b935 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 4 Nov 2026 14:41:22 +0000 Subject: [PATCH] Worker: upload the docs content and the Shiki grammars as text modules The main module bundled to 14.4 MB, which hosts that cap each uploaded part refuse (open-compute: 8 MiB, #24). The generated docs content (4.4 MB) and Shiki's grammars (3.2 MB, inlined by the bundler) now upload as text modules, like the Kumo islands, and are parsed once from JSON. The main module is 6.8 MB; the total upload is unchanged. Refs #24 Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_015ALxdbMU5hpZXjGpL2jhC4 --- CHANGELOG.md | 7 ++ apps/diffinery/.gitignore | 1 + .../docs/self-hosting/architecture.md | 2 + .../docs/self-hosting/configuration.md | 2 +- apps/diffinery/package.json | 4 +- apps/diffinery/scripts/build-docs.mjs | 48 ++++++++-- apps/diffinery/scripts/build-grammars.mjs | 58 ++++++++++++ apps/diffinery/src/web/highlight.ts | 91 ++++--------------- .../test/unit/main-module-size.test.ts | 27 ++++++ 9 files changed, 155 insertions(+), 85 deletions(-) create mode 100644 apps/diffinery/scripts/build-grammars.mjs create mode 100644 apps/diffinery/test/unit/main-module-size.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fa488f..ea7d73a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ How a release is made is in [docs/RELEASING.md](docs/RELEASING.md). ## [Unreleased] +### Self-hosting + +- The Worker's main module is about 7 MB instead of 14 MB: the docs, the downloads and the + syntax highlighting grammars upload as separate text modules, none larger than about + 4.5 MB. Hosts that cap each part of an upload, like open-compute (8 MiB), take it, and + a cold start parses less JavaScript. + ## [0.2.0] - 2026-10-31 Development moved to [GitHub](https://github.com/codician-team/diffinery). GitHub diff --git a/apps/diffinery/.gitignore b/apps/diffinery/.gitignore index 056cfff..4ac5930 100644 --- a/apps/diffinery/.gitignore +++ b/apps/diffinery/.gitignore @@ -1,2 +1,3 @@ build/ src/docs/content.generated.ts +src/web/grammars.generated.ts diff --git a/apps/diffinery/docs/self-hosting/architecture.md b/apps/diffinery/docs/self-hosting/architecture.md index 6a74547..22f39b3 100644 --- a/apps/diffinery/docs/self-hosting/architecture.md +++ b/apps/diffinery/docs/self-hosting/architecture.md @@ -68,6 +68,8 @@ Routes are mounted in this order, so earlier ones win: profile pictures (`/api/a Some work happens in the Worker itself: profile pictures are checked, cropped and resized to small WebP files with WebAssembly image codecs, and code in the web UI and the docs is syntax highlighted with Shiki on the server. +The Worker uploads as one JavaScript module of about 7 MB plus separate modules for the bulky content: the public docs, the downloads and the syntax highlighting grammars as text modules, which `pnpm build` writes to `apps/diffinery/build/` (`scripts/build-docs.mjs`, `scripts/build-grammars.mjs`), and the image codecs as WebAssembly. No module is larger than about 4.5 MB, so the Worker also deploys to hosts that cap each part of an upload, like open-compute's 8 MiB. The source map is larger than that: on such a host, set `upload_source_maps` to `false`. + The Worker holds no state of its own. It authenticates, checks the caller's role on the repository against D1, and then calls the repository's Durable Object over RPC. Before it calls the object, the Worker also decides whether the request is a read or a write. `GET` and `HEAD` are reads, and so are the two `POST` requests that only fetch data: git's `git-upload-pack` (clone and fetch) and a Git LFS batch request for `download`. MCP tools marked read-only count as reads too. Every other request is a write: diff --git a/apps/diffinery/docs/self-hosting/configuration.md b/apps/diffinery/docs/self-hosting/configuration.md index 6786aea..dec69a5 100644 --- a/apps/diffinery/docs/self-hosting/configuration.md +++ b/apps/diffinery/docs/self-hosting/configuration.md @@ -300,7 +300,7 @@ The Durable Object classes are declared in eight migrations (`new_sqlite_classes | `routes` | `[{ "pattern": "diffinery.com/*", "zone_name": "diffinery.com" }]` | A zone route, not a Custom Domain. The hostname's DNS record must be proxied. On deploy, `scripts/render-config.mjs` replaces it with one route, `/*` in the zone `DIFFINERY_ZONE` (by default the hostname's last two labels), so you don't edit it by hand. | | `observability` | Enabled, `head_sampling_rate: 1` | Every request is logged to Workers Logs. | | `version_metadata` | Binding `CF_VERSION_METADATA` | The deployed version's id and time. [IndexNow](/docs/admin/site-administration#search-engines) checks the site's pages once for each new version and ignores older versions still running during a rollout. Optional: without it, it checks whenever the pages' content differs from what it last submitted. | -| `upload_source_maps` | `true` | Readable stack traces in the logs. | +| `upload_source_maps` | `true` | Readable stack traces in the logs. The source map is uploaded as one part of about 12 MB: on a host that caps each part (open-compute's limit is 8 MiB), set it to `false`. | | `limits.cpu_ms` | `300000` | Up to 5 minutes of CPU time per request (large pushes, diffs and merges). | | `containers` | `RunnerContainer`: `standard-4`, `max_instances: 100`; `RunnerContainerMedium`, `RunnerContainerSmall`, `RunnerContainerTiny`: `standard-3`, `standard-1`, `basic`, `max_instances: 100` each (smaller hosted runners; remove them and their `RUNNER_*` bindings to run every job on `standard-4`); `SandboxContainer`: `standard-1`, `max_instances: 50`; `ImportContainer`: `standard-4`, `max_instances: 20`; `rollout_active_grace_period`: 23400 for runners and 25200 for imports (longer than a job or transfer can run, so a deploy never replaces a running one), 3600 for sandboxes; `rollout_step_percentage: 100`, so containers started after a deploy get the new image | All three build `containers/runner/Dockerfile` with `containers/runner` as context. Raise `max_instances` for more concurrent jobs, sandboxes or import transfers. | | `triggers.crons` | `["*/5 * * * *"]` | Token expiry reminders, email log pruning, registry cleanup, cancelling draft imports older than 7 days, and setting the workflow scheduler's alarm if it isn't set. Scheduled workflows run from that alarm, not from the cron trigger. | diff --git a/apps/diffinery/package.json b/apps/diffinery/package.json index 87077f8..07acb42 100644 --- a/apps/diffinery/package.json +++ b/apps/diffinery/package.json @@ -12,8 +12,8 @@ "cf-typegen": "wrangler types", "e2e": "node --test --test-concurrency=1 --test-timeout=600000 test/e2e/e2e.test.mjs test/e2e/passkeys.test.mjs test/e2e/accounts.test.mjs test/e2e/registry.test.mjs test/e2e/email.test.mjs test/e2e/security-txt.test.mjs test/e2e/security.test.mjs test/e2e/security-limits.test.mjs test/e2e/agentic.test.mjs test/e2e/tasks.test.mjs test/e2e/terminals.test.mjs test/e2e/review.test.mjs", "db:migrate:local": "wrangler d1 migrations apply diffinery-dev --local -e dev", - "build": "node islands/build.mjs && node scripts/build-docs.mjs", - "predev": "node islands/build.mjs && node scripts/build-docs.mjs", + "build": "node islands/build.mjs && node scripts/build-docs.mjs && node scripts/build-grammars.mjs", + "predev": "node islands/build.mjs && node scripts/build-docs.mjs && node scripts/build-grammars.mjs", "build:docs": "node scripts/build-docs.mjs" }, "dependencies": { diff --git a/apps/diffinery/scripts/build-docs.mjs b/apps/diffinery/scripts/build-docs.mjs index 6205a06..bc97cd6 100644 --- a/apps/diffinery/scripts/build-docs.mjs +++ b/apps/diffinery/scripts/build-docs.mjs @@ -1,4 +1,6 @@ -// Builds the public documentation into src/docs/content.generated.ts: +// Builds the public documentation into src/docs/content.generated.ts, with the bulky values (pages, images and +// the downloads) in build/docs/content.txt, which the Worker imports as a separate text module and parses once: +// that keeps the Worker's main module small (hosts that cap each uploaded part, like open-compute's 8 MiB, take it). // - every Markdown page under docs/ (with its frontmatter), in the order of docs/nav.json // - the public site's pages under site/ (the product, comparison and company pages around the // home page), in the order of site/nav.json @@ -39,6 +41,7 @@ const pluginSources = path.resolve(app, "../../plugins"); /** The Claude Code plugins Diffinery serves for upload to claude.ai, by name. */ const CLAUDE_PLUGIN_NAMES = new Set(["diffinery", "diffinery-ci"]); const out = path.join(app, "src/docs/content.generated.ts"); +const dataOut = path.join(app, "build/docs/content.txt"); /** Pages produced in code rather than from a Markdown file. */ const RUNTIME_PAGES = new Set(["api", "reference/mcp-tools"]); @@ -315,38 +318,63 @@ if (problems.length) { } const ordered = Object.fromEntries(navSlugs.filter((s) => pages[s]).map((s) => [s, pages[s]])); +/** The bulky values, in build/docs/content.txt (JSON). */ +const data = { + DOCS_PAGES: ordered, + SITE_PAGES: Object.fromEntries(sitePaths.filter((u) => site[u]).map((u) => [u, site[u]])), + DOCS_IMAGES: images, + CLI_BUNDLE: cli.bin, + EDITOR_EXTENSION_VSIX_B64: editor.vsix.toString("base64"), + JETBRAINS_PLUGIN_ZIP_B64: jetbrains.zip.toString("base64"), + CLAUDE_PLUGINS: claudePlugins, +}; const src = `// Generated by scripts/build-docs.mjs from docs/, site/, claude-plugins/, apps/cli, apps/vscode and apps/jetbrains/dist. Do not edit. /* eslint-disable */ +import DATA from "../../build/docs/content.txt"; + export interface DocPage { title: string; description: string; markdown: string; updated?: string } export interface DocSection { title: string; pages: string[] } -export const DOCS_NAV: DocSection[] = ${JSON.stringify(nav)}; -export const DOCS_PAGES: Record = ${JSON.stringify(ordered)}; -/** docs/images/: content type and base64 bytes by file name. */ export interface SitePage { title: string; description: string; markdown: string; updated: string; seoTitle: string | null; published?: string } export interface SiteSection { title: string; pages: string[] } + +const data = JSON.parse(DATA) as { + DOCS_PAGES: Record; + SITE_PAGES: Record; + DOCS_IMAGES: Record; + CLI_BUNDLE: string; + EDITOR_EXTENSION_VSIX_B64: string; + JETBRAINS_PLUGIN_ZIP_B64: string; + CLAUDE_PLUGINS: Record; +}; + +export const DOCS_NAV: DocSection[] = ${JSON.stringify(nav)}; +export const DOCS_PAGES = data.DOCS_PAGES; /** The public site's pages (site/), by URL path, and their sections in reading order. The home page is code (web/site.tsx). */ export const SITE_NAV: SiteSection[] = ${JSON.stringify(siteNav)}; -export const SITE_PAGES: Record = ${JSON.stringify(Object.fromEntries(sitePaths.filter((u) => site[u]).map((u) => [u, site[u]])))}; -export const DOCS_IMAGES: Record = ${JSON.stringify(images)}; +export const SITE_PAGES = data.SITE_PAGES; +/** docs/images/: content type and base64 bytes by file name. */ +export const DOCS_IMAGES = data.DOCS_IMAGES; export const CLI_VERSION = ${JSON.stringify(cli.VERSION)}; -export const CLI_BUNDLE = ${JSON.stringify(cli.bin)}; +export const CLI_BUNDLE = data.CLI_BUNDLE; export const EDITOR_EXTENSION_VERSION = ${JSON.stringify(editor.version)}; export const EDITOR_EXTENSION_SHA256 = ${JSON.stringify(createHash("sha256").update(editor.vsix).digest("hex"))}; -export const EDITOR_EXTENSION_VSIX_B64 = ${JSON.stringify(editor.vsix.toString("base64"))}; +export const EDITOR_EXTENSION_VSIX_B64 = data.EDITOR_EXTENSION_VSIX_B64; export const EDITOR_INSTALL_SH = ${JSON.stringify(await readFile(path.join(editorInstall, "install.sh"), "utf8"))}; export const EDITOR_INSTALL_PS1 = ${JSON.stringify(await readFile(path.join(editorInstall, "install.ps1"), "utf8"))}; export const JETBRAINS_PLUGIN_VERSION = ${JSON.stringify(jetbrains.version)}; export const JETBRAINS_PLUGIN_SINCE_BUILD = ${JSON.stringify(jetbrains.sinceBuild)}; export const JETBRAINS_PLUGIN_DESCRIPTION = ${JSON.stringify(jetbrains.description)}; export const JETBRAINS_PLUGIN_SHA256 = ${JSON.stringify(createHash("sha256").update(jetbrains.zip).digest("hex"))}; -export const JETBRAINS_PLUGIN_ZIP_B64 = ${JSON.stringify(jetbrains.zip.toString("base64"))}; +export const JETBRAINS_PLUGIN_ZIP_B64 = data.JETBRAINS_PLUGIN_ZIP_B64; /** claude-plugins/: the Claude Code plugins served at /-/claude/.zip, base64. */ -export const CLAUDE_PLUGINS: Record = ${JSON.stringify(claudePlugins)}; +export const CLAUDE_PLUGINS = data.CLAUDE_PLUGINS; export const CLI_INSTALL_SH = ${JSON.stringify(await readFile(path.join(cliInstall, "install.sh"), "utf8"))}; export const CLI_INSTALL_PS1 = ${JSON.stringify(await readFile(path.join(cliInstall, "install.ps1"), "utf8"))}; export const CLI_SHA256 = ${JSON.stringify(createHash("sha256").update(cli.bin).digest("hex"))}; export const AGENT_SETUP_SH = ${JSON.stringify(await readFile(path.join(cliInstall, "agent-setup.sh"), "utf8"))}; `; await mkdir(path.dirname(out), { recursive: true }); +await mkdir(path.dirname(dataOut), { recursive: true }); +await writeFile(dataOut, JSON.stringify(data)); await writeFile(out, src); console.log(`docs: ${Object.keys(ordered).length} pages (+${RUNTIME_PAGES.size} generated at runtime), ${Object.keys(site).length} site pages, ${Object.keys(images).length} images, CLI ${cli.VERSION} (${Math.round(cli.bin.length / 1024)} KiB), editor extension ${editor.version} (${Math.round(editor.vsix.length / 1024)} KiB), JetBrains plugin ${jetbrains.version} (${Math.round(jetbrains.zip.length / 1024)} KiB), Claude Code plugins ${Object.entries(claudePlugins).map(([n, p]) => `${n} ${p.version}`).join(", ")}`); diff --git a/apps/diffinery/scripts/build-grammars.mjs b/apps/diffinery/scripts/build-grammars.mjs new file mode 100644 index 0000000..84a2c56 --- /dev/null +++ b/apps/diffinery/scripts/build-grammars.mjs @@ -0,0 +1,58 @@ +// Builds the Shiki grammars the Worker highlights code with into text modules in build/grammars/, one JSON file +// per grammar (a grammar embedded by several languages, like JavaScript, is there once), and writes +// src/web/grammars.generated.ts, which imports them. As JavaScript the grammars would add about 3 MB to the Worker's +// main module; as text modules they are separate parts of the upload, parsed only when a file in that language is +// shown. That keeps the main module small (hosts that cap each uploaded part, like +// open-compute's 8 MiB, take it). +// +// Run by `pnpm build` and `pnpm dev`. A language added to LANGS needs no other change: src/web/highlight.ts +// takes its list from the generated file. +import { mkdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const app = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const dir = path.join(app, "build/grammars"); +const out = path.join(app, "src/web/grammars.generated.ts"); + +/** Shiki's names for the languages Diffinery highlights (src/web/highlight.ts maps file names and fences to them). */ +const LANGS = ["astro", "bat", "c", "clojure", "cmake", "cpp", "csharp", "css", "dart", "diff", "docker", "dotenv", "elixir", "erlang", "fish", "fsharp", "go", "graphql", "groovy", "haskell", "hcl", "html", "ini", "java", "javascript", "json", "json5", "jsonc", "jsx", "julia", "kotlin", "less", "lua", "make", "markdown", "nginx", "nix", "objective-c", "ocaml", "perl", "php", "powershell", "proto", "python", "r", "ruby", "rust", "sass", "scala", "scss", "shellscript", "sql", "svelte", "swift", "terraform", "toml", "tsx", "typescript", "vue", "xml", "yaml", "zig"]; + +/** Each grammar's JSON by name, and each language's grammars in registration order (the ones it embeds, then its own). */ +const grammars = new Map(); +const order = {}; +for (const lang of LANGS) { + const { default: regs } = await import(`shiki/langs/${lang}.mjs`); + order[lang] = []; + for (const reg of regs) { + const json = JSON.stringify(reg); + if (grammars.has(reg.name) && grammars.get(reg.name) !== json) throw new Error(`grammars: two different grammars are named "${reg.name}"`); + grammars.set(reg.name, json); + if (!order[lang].includes(reg.name)) order[lang].push(reg.name); + } +} + +await rm(dir, { recursive: true, force: true }); +await mkdir(dir, { recursive: true }); +for (const [name, json] of grammars) await writeFile(path.join(dir, `${name}.txt`), json); + +const names = [...grammars.keys()].sort(); +const imports = names.map((n, i) => `import g${i} from "../../build/grammars/${n}.txt";`); +const texts = names.map((n, i) => ` ${JSON.stringify(n)}: g${i},`); +await writeFile( + out, + `// Generated by scripts/build-grammars.mjs from Shiki's grammars. Do not edit. +/* eslint-disable */ +${imports.join("\n")} + +export const GRAMMAR_LANGS = ${JSON.stringify(LANGS)} as const; +/** Each language's grammars in registration order: the ones it embeds, then its own. */ +export const GRAMMAR_ORDER: Record<(typeof GRAMMAR_LANGS)[number], string[]> = ${JSON.stringify(order)}; +/** Each grammar's JSON, by name. */ +export const GRAMMAR_TEXT: Record = { +${texts.join("\n")} +}; +`, +); +const bytes = [...grammars.values()].reduce((n, j) => n + j.length, 0); +console.log(`grammars: ${LANGS.length} languages, ${grammars.size} grammars (${Math.round(bytes / 1024)} KiB)`); diff --git a/apps/diffinery/src/web/highlight.ts b/apps/diffinery/src/web/highlight.ts index 4df7dd4..5266058 100644 --- a/apps/diffinery/src/web/highlight.ts +++ b/apps/diffinery/src/web/highlight.ts @@ -1,82 +1,29 @@ /** * Syntax highlighting with Shiki (JavaScript regex engine, fine-grained grammars - * loaded on first use). Tokens carry both GitHub themes as CSS variables - * (--shiki-light / --shiki-dark); the site stylesheet picks one. Everything here - * fails soft: an unknown language, an oversized input or an error means plain text. + * parsed on first use from the text modules scripts/build-grammars.mjs writes). + * Tokens carry both GitHub themes as CSS variables (--shiki-light / --shiki-dark); + * the site stylesheet picks one. Everything here fails soft: an unknown language, + * an oversized input or an error means plain text. */ import { createHighlighterCore, type HighlighterCore, type LanguageRegistration, type ThemedToken, type ThemeRegistration } from "shiki/core"; import { createJavaScriptRegexEngine } from "shiki/engine/javascript"; +import { GRAMMAR_LANGS, GRAMMAR_ORDER, GRAMMAR_TEXT } from "./grammars.generated.js"; import { escapeHtml } from "./markdown.js"; -type Grammar = () => Promise<{ default: LanguageRegistration[] }>; +export type Lang = (typeof GRAMMAR_LANGS)[number]; +export const LANGS: Lang[] = [...GRAMMAR_LANGS]; +const KNOWN = new Set(GRAMMAR_LANGS); -const GRAMMARS = { - astro: () => import("shiki/langs/astro.mjs"), - bat: () => import("shiki/langs/bat.mjs"), - c: () => import("shiki/langs/c.mjs"), - clojure: () => import("shiki/langs/clojure.mjs"), - cmake: () => import("shiki/langs/cmake.mjs"), - cpp: () => import("shiki/langs/cpp.mjs"), - csharp: () => import("shiki/langs/csharp.mjs"), - css: () => import("shiki/langs/css.mjs"), - dart: () => import("shiki/langs/dart.mjs"), - diff: () => import("shiki/langs/diff.mjs"), - docker: () => import("shiki/langs/docker.mjs"), - dotenv: () => import("shiki/langs/dotenv.mjs"), - elixir: () => import("shiki/langs/elixir.mjs"), - erlang: () => import("shiki/langs/erlang.mjs"), - fish: () => import("shiki/langs/fish.mjs"), - fsharp: () => import("shiki/langs/fsharp.mjs"), - go: () => import("shiki/langs/go.mjs"), - graphql: () => import("shiki/langs/graphql.mjs"), - groovy: () => import("shiki/langs/groovy.mjs"), - haskell: () => import("shiki/langs/haskell.mjs"), - hcl: () => import("shiki/langs/hcl.mjs"), - html: () => import("shiki/langs/html.mjs"), - ini: () => import("shiki/langs/ini.mjs"), - java: () => import("shiki/langs/java.mjs"), - javascript: () => import("shiki/langs/javascript.mjs"), - json: () => import("shiki/langs/json.mjs"), - json5: () => import("shiki/langs/json5.mjs"), - jsonc: () => import("shiki/langs/jsonc.mjs"), - jsx: () => import("shiki/langs/jsx.mjs"), - julia: () => import("shiki/langs/julia.mjs"), - kotlin: () => import("shiki/langs/kotlin.mjs"), - less: () => import("shiki/langs/less.mjs"), - lua: () => import("shiki/langs/lua.mjs"), - make: () => import("shiki/langs/make.mjs"), - markdown: () => import("shiki/langs/markdown.mjs"), - nginx: () => import("shiki/langs/nginx.mjs"), - nix: () => import("shiki/langs/nix.mjs"), - "objective-c": () => import("shiki/langs/objective-c.mjs"), - ocaml: () => import("shiki/langs/ocaml.mjs"), - perl: () => import("shiki/langs/perl.mjs"), - php: () => import("shiki/langs/php.mjs"), - powershell: () => import("shiki/langs/powershell.mjs"), - proto: () => import("shiki/langs/proto.mjs"), - python: () => import("shiki/langs/python.mjs"), - r: () => import("shiki/langs/r.mjs"), - ruby: () => import("shiki/langs/ruby.mjs"), - rust: () => import("shiki/langs/rust.mjs"), - sass: () => import("shiki/langs/sass.mjs"), - scala: () => import("shiki/langs/scala.mjs"), - scss: () => import("shiki/langs/scss.mjs"), - shellscript: () => import("shiki/langs/shellscript.mjs"), - sql: () => import("shiki/langs/sql.mjs"), - svelte: () => import("shiki/langs/svelte.mjs"), - swift: () => import("shiki/langs/swift.mjs"), - terraform: () => import("shiki/langs/terraform.mjs"), - toml: () => import("shiki/langs/toml.mjs"), - tsx: () => import("shiki/langs/tsx.mjs"), - typescript: () => import("shiki/langs/typescript.mjs"), - vue: () => import("shiki/langs/vue.mjs"), - xml: () => import("shiki/langs/xml.mjs"), - yaml: () => import("shiki/langs/yaml.mjs"), - zig: () => import("shiki/langs/zig.mjs"), -} satisfies Record; +const parsed = new Map(); -export type Lang = keyof typeof GRAMMARS; -export const LANGS = Object.keys(GRAMMARS) as Lang[]; +/** A language's grammars (the ones it embeds, then its own), each parsed from its text module once. */ +function grammar(lang: Lang): LanguageRegistration[] { + return GRAMMAR_ORDER[lang].map((name) => { + let g = parsed.get(name); + if (!g) parsed.set(name, (g = JSON.parse(GRAMMAR_TEXT[name]!) as LanguageRegistration)); + return g; + }); +} const EXT: Record = { ts: "typescript", mts: "typescript", cts: "typescript", tsx: "tsx", @@ -137,7 +84,7 @@ const ALIASES: Record = { /** Shiki language for a Markdown fence info string ("ts", "yaml", "c++" …), or null. */ export function langForFence(info: string): Lang | null { const name = info.trim().toLowerCase().replace(/^\{?\.?/, "").replace(/\}$/, ""); - return name in GRAMMARS ? (name as Lang) : (ALIASES[name] ?? null); + return KNOWN.has(name) ? (name as Lang) : (ALIASES[name] ?? null); } // ---------------------------------------------------------------- highlighter @@ -207,7 +154,7 @@ export async function ready(langs: Iterable): Promise void hl.codeToTokens("x", { lang: l, theme: PAIR })) diff --git a/apps/diffinery/test/unit/main-module-size.test.ts b/apps/diffinery/test/unit/main-module-size.test.ts new file mode 100644 index 0000000..219a526 --- /dev/null +++ b/apps/diffinery/test/unit/main-module-size.test.ts @@ -0,0 +1,27 @@ +/** + * The bulky generated content stays out of the Worker's main module, in text modules that upload as parts of their + * own (scripts/build-docs.mjs, scripts/build-grammars.mjs). With it inlined the main module was 14 MB, which hosts + * that cap each uploaded part refuse (open-compute: 8 MiB). Needs `pnpm build`. + */ +import { statSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { DOCS_PAGES } from "../../src/docs/content.generated.js"; +import { GRAMMAR_LANGS, GRAMMAR_ORDER, GRAMMAR_TEXT } from "../../src/web/grammars.generated.js"; + +const size = (rel: string) => statSync(new URL(rel, import.meta.url)).size; + +describe("main module size", () => { + it("keeps the docs' pages, images and downloads in a text module", () => { + expect(size("../../src/docs/content.generated.ts")).toBeLessThan(64 * 1024); + expect(size("../../build/docs/content.txt")).toBeGreaterThan(1024 * 1024); + expect(DOCS_PAGES.quickstart?.title).toBeTruthy(); + }); + + it("keeps the highlighting grammars in text modules, each grammar once", () => { + expect(size("../../src/web/grammars.generated.ts")).toBeLessThan(32 * 1024); + for (const lang of GRAMMAR_LANGS) { + expect(GRAMMAR_ORDER[lang].length).toBeGreaterThan(0); + for (const name of GRAMMAR_ORDER[lang]) expect(JSON.parse(GRAMMAR_TEXT[name]!).name).toBe(name); + } + }); +});