From f34f0d22a9f4381334a3720bac6c029617d73b2d Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:05:04 +0100 Subject: [PATCH 1/3] computer: select worker shell commands by import Split the worker shell bundle into a core group and one optional group per heavy command, and select commands by importing their group rather than by a build-time flag. build-bundle.mjs runs esbuild with splitting and partitions the emitted modules into a core group (every always-on command plus the ShellWorker entry) and one group per optional command: curl, html-to-markdown, python, sqlite, js-exec, yq, file, xan, and jq. Each group is published on its own @cloudflare/computer/shell/* subpath. shell-modules.ts imports only the core group and exposes assembleShellModules, which folds a caller-supplied list of groups on top of core. WorkerBackend gains a commands option that takes the imported groups and assembles the Loader modules table from them. Because nothing in the package references the optional groups, a group the consumer never imports is unreachable in their module graph and the bundler drops it, along with its exclusive dependencies. Opting a command in is a single import; there is no default-on cost to opt out of and no flag to set. The package is marked sideEffects: false so the bundler is free to elide unused groups. curl runs on a SecureFetch adapter over the isolate's global fetch rather than undici, which is redirected to a throwing stub at build time and never ships. Egress stays governed by the Dynamic Worker's globalOutbound, so including curl does not by itself open the network. --- .changeset/worker-shell-opt-in-commands.md | 5 + .gitignore | 9 +- packages/computer/package.json | 41 +++ packages/computer/rolldown.config.ts | 23 ++ .../src/backends/worker-shell/entrypoint.ts | 43 ++- .../worker-shell/generated-bundle.test.ts | 58 ---- .../src/backends/worker-shell/index.ts | 31 ++- .../worker-shell/script/build-bundle.mjs | 254 +++++++++++++++--- .../worker-shell/script/undici-stub.mjs | 29 ++ .../worker-shell/shell-modules.test.ts | 154 +++++++++++ .../backends/worker-shell/shell-modules.ts | 40 +++ .../src/backends/worker-shell/worker-shell.ts | 18 +- .../test-helpers/shell-module-aliases.ts | 28 ++ .../computer/tests/worker-backend-worker.ts | 4 + .../computer/tests/worker-backend.test.ts | 16 ++ packages/computer/tsconfig.json | 16 +- packages/computer/vitest.config.ts | 8 + .../computer/vitest.config.worker-backend.ts | 10 + 18 files changed, 671 insertions(+), 116 deletions(-) create mode 100644 .changeset/worker-shell-opt-in-commands.md delete mode 100644 packages/computer/src/backends/worker-shell/generated-bundle.test.ts create mode 100644 packages/computer/src/backends/worker-shell/script/undici-stub.mjs create mode 100644 packages/computer/src/backends/worker-shell/shell-modules.test.ts create mode 100644 packages/computer/src/backends/worker-shell/shell-modules.ts create mode 100644 packages/computer/test-helpers/shell-module-aliases.ts diff --git a/.changeset/worker-shell-opt-in-commands.md b/.changeset/worker-shell-opt-in-commands.md new file mode 100644 index 00000000..19b04bfd --- /dev/null +++ b/.changeset/worker-shell-opt-in-commands.md @@ -0,0 +1,5 @@ +--- +"@cloudflare/computer": minor +--- + +Make worker-shell commands opt-in to cut the deployed bundle. The shell now ships an always-on core plus one optional group per heavy command. Import the groups you want from `@cloudflare/computer/shell/` and pass them to `WorkerShellBackend`'s new `commands` option; a group you never import is dropped from your Worker upload. This is a breaking change: commands such as `curl`, `python`, `sqlite`, `html-to-markdown`, `js-exec`, `yq`, `file`, `xan`, and `jq` no longer ship unless their group is imported. diff --git a/.gitignore b/.gitignore index 6c6f75b3..56157203 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,11 @@ PLAN.md # tracked code and must not be caught by this rule. /artifacts/ -# Generated bundle for the worker backend's ShellWorker. Built -# by packages/computer/src/backends/worker/script/build-bundle.mjs -# on prepare / pretest / pretypecheck. -packages/computer/src/backends/worker-shell/generated-bundle.ts +# Generated shell-module groups for the worker backend's +# ShellWorker (core plus one file per optional feature). Built by +# packages/computer/src/backends/worker-shell/script/build-bundle.mjs on +# prepare / pretest / pretypecheck. +packages/computer/src/backends/worker-shell/generated/ # SEA binary destinations populated at publish time from # artifacts/computerd/ via the build-bin step. The @cloudflare/computer diff --git a/packages/computer/package.json b/packages/computer/package.json index 0439a5f9..f915a58f 100644 --- a/packages/computer/package.json +++ b/packages/computer/package.json @@ -13,6 +13,7 @@ "publishConfig": { "tag": "unreleased" }, + "sideEffects": false, "exports": { ".": { "types": "./dist/index.d.ts", @@ -46,6 +47,46 @@ "types": "./dist/backends/worker-shell/index.d.ts", "import": "./dist/backends/worker-shell/index.js" }, + "./shell/core": { + "types": "./dist/backends/worker-shell/shell/core.d.ts", + "default": "./dist/backends/worker-shell/shell/core.js" + }, + "./shell/curl": { + "types": "./dist/backends/worker-shell/shell/curl.d.ts", + "default": "./dist/backends/worker-shell/shell/curl.js" + }, + "./shell/html-to-markdown": { + "types": "./dist/backends/worker-shell/shell/html-to-markdown.d.ts", + "default": "./dist/backends/worker-shell/shell/html-to-markdown.js" + }, + "./shell/python": { + "types": "./dist/backends/worker-shell/shell/python.d.ts", + "default": "./dist/backends/worker-shell/shell/python.js" + }, + "./shell/sqlite": { + "types": "./dist/backends/worker-shell/shell/sqlite.d.ts", + "default": "./dist/backends/worker-shell/shell/sqlite.js" + }, + "./shell/js-exec": { + "types": "./dist/backends/worker-shell/shell/js-exec.d.ts", + "default": "./dist/backends/worker-shell/shell/js-exec.js" + }, + "./shell/yq": { + "types": "./dist/backends/worker-shell/shell/yq.d.ts", + "default": "./dist/backends/worker-shell/shell/yq.js" + }, + "./shell/file": { + "types": "./dist/backends/worker-shell/shell/file.d.ts", + "default": "./dist/backends/worker-shell/shell/file.js" + }, + "./shell/xan": { + "types": "./dist/backends/worker-shell/shell/xan.d.ts", + "default": "./dist/backends/worker-shell/shell/xan.js" + }, + "./shell/jq": { + "types": "./dist/backends/worker-shell/shell/jq.d.ts", + "default": "./dist/backends/worker-shell/shell/jq.js" + }, "./observe/cloudflare": { "types": "./dist/observe/cloudflare.d.ts", "import": "./dist/observe/cloudflare.js" diff --git a/packages/computer/rolldown.config.ts b/packages/computer/rolldown.config.ts index 8143bb8c..7f21309e 100644 --- a/packages/computer/rolldown.config.ts +++ b/packages/computer/rolldown.config.ts @@ -34,6 +34,23 @@ export default defineConfig({ "backends/container/index": "src/backends/container/index.ts", "backends/worker-javascript/index": "src/backends/worker-javascript/index.ts", "backends/worker-shell/index": "src/backends/worker-shell/index.ts", + // The shell-module groups build-bundle.mjs emits. Each is its + // own entry so it lands at the dist path the ./shell/* package + // exports point at; shell-modules.ts imports the core group by + // subpath (kept external below) and a consumer imports the + // optional ones it wants, so the bundler tree-shakes any group + // that is never imported. + "backends/worker-shell/shell/core": "src/backends/worker-shell/generated/core.ts", + "backends/worker-shell/shell/curl": "src/backends/worker-shell/generated/curl.ts", + "backends/worker-shell/shell/html-to-markdown": + "src/backends/worker-shell/generated/html-to-markdown.ts", + "backends/worker-shell/shell/python": "src/backends/worker-shell/generated/python.ts", + "backends/worker-shell/shell/sqlite": "src/backends/worker-shell/generated/sqlite.ts", + "backends/worker-shell/shell/js-exec": "src/backends/worker-shell/generated/js-exec.ts", + "backends/worker-shell/shell/yq": "src/backends/worker-shell/generated/yq.ts", + "backends/worker-shell/shell/file": "src/backends/worker-shell/generated/file.ts", + "backends/worker-shell/shell/xan": "src/backends/worker-shell/generated/xan.ts", + "backends/worker-shell/shell/jq": "src/backends/worker-shell/generated/jq.ts", "observe/cloudflare": "src/observe/cloudflare.ts", }, external: [ @@ -44,6 +61,12 @@ export default defineConfig({ "zod", "just-bash", /^node:/, + // shell-modules.ts imports the generated groups by their + // published subpath. Keep the specifiers intact in the emitted + // bundle rather than inlining the group here so the consumer's + // bundler sees each group as its own module and can drop one it + // never imports; each group is built as its own entry above. + /^@cloudflare\/computer\/shell\//, ], resolve: { alias: { diff --git a/packages/computer/src/backends/worker-shell/entrypoint.ts b/packages/computer/src/backends/worker-shell/entrypoint.ts index 690d6a91..492d2c99 100644 --- a/packages/computer/src/backends/worker-shell/entrypoint.ts +++ b/packages/computer/src/backends/worker-shell/entrypoint.ts @@ -18,7 +18,7 @@ // workspace stub; there's no shared instance state to race. import { WorkerEntrypoint } from "cloudflare:workers"; -import { Bash, type CustomCommand } from "just-bash"; +import { Bash, type CustomCommand, type SecureFetch } from "just-bash"; import { WorkspaceFsAdapter } from "./adapter.js"; import { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js"; @@ -40,8 +40,45 @@ export interface ShellWorkerOptions { // container backend uses, so scripts that hard-code that path // keep working. cwd?: string; + // Fetch implementation backing `curl`. just-bash registers curl + // whenever a fetch is supplied and calls it directly — no undici, + // no in-isolate DNS pinning (undici is excluded from the bundle + // at build time). Defaults to defaultSecureFetch below, a thin + // wrapper over the isolate's global `fetch`, so curl is enabled + // by default. Egress is governed by the Dynamic Worker's + // globalOutbound, not by this fetch (see worker.ts). Pass a + // custom SecureFetch to add an allow-list or credential + // injection, or `null` to drop curl entirely. + fetch?: SecureFetch | null; } +// Default curl fetch: adapt the isolate's global `fetch` to +// just-bash's SecureFetch contract. No allow-list or private-range +// checks run here — the Dynamic Worker's globalOutbound is the +// egress boundary, so requests only leave the isolate once a +// consumer wires a trusted outbound gateway; policy belongs in +// that gateway, not the untrusted shell. +const defaultSecureFetch: SecureFetch = async (url, options) => { + const response = await fetch(url, { + method: options?.method, + headers: options?.headers, + body: options?.body, + redirect: options?.followRedirects === false ? "manual" : "follow", + signal: options?.signal, + }); + const headers: Record = Object.create(null); + response.headers.forEach((value, key) => { + headers[key] = value; + }); + return { + status: response.status, + statusText: response.statusText, + headers, + body: new Uint8Array(await response.arrayBuffer()), + url: response.url || url, + }; +}; + // Env shape the host Worker is expected to wire through the // Loader callback. The shell calls env.HOST.getWorkspace() on // every exec; no caching. @@ -186,6 +223,10 @@ export class ShellWorker< ConstructorParameters[0] >["fs"], cwd, + fetch: + this.shellOptions.fetch === null + ? undefined + : (this.shellOptions.fetch ?? defaultSecureFetch), customCommands, // just-bash's in-process DefenseInDepthBox activates by // registering ESM loader hooks through node:module's diff --git a/packages/computer/src/backends/worker-shell/generated-bundle.test.ts b/packages/computer/src/backends/worker-shell/generated-bundle.test.ts deleted file mode 100644 index c18f2235..00000000 --- a/packages/computer/src/backends/worker-shell/generated-bundle.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -// Tests the shape build-bundle.mjs produces in generated-bundle.ts. -// -// The bundle used to be a single ~3 MB JS string assigned to -// SHELL_BUNDLE. esbuild was inlining every dynamic import() -// just-bash makes (python3, js-exec, sqlite3, html-to-markdown, -// curl, …) into the one string, so workerd's Worker Loader -// parsed all of it on every cold start even though the default -// ShellWorker disables python, javascript, and network. -// -// build-bundle.mjs now runs esbuild with splitting: true and -// emits a record of module name → source. The host Worker -// spreads the whole record into the Loader callback's modules -// table; workerd parses each chunk on first import, so the -// dynamic ones stay cold until a script actually reaches for -// them. These tests are the contract. - -import { describe, expect, it } from "vitest"; - -import { SHELL_MODULES } from "./generated-bundle.js"; - -describe("SHELL_MODULES", () => { - it("exposes shell.js as the main module", () => { - expect(SHELL_MODULES["shell.js"]).toBeDefined(); - expect(typeof SHELL_MODULES["shell.js"].js).toBe("string"); - expect(SHELL_MODULES["shell.js"].js.length).toBeGreaterThan(0); - }); - - it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { - // Static-reachable set from entrypoint.ts measured at ~651 KB. - // Anything materially above that means esbuild stopped - // splitting and went back to inlining dynamic imports. - const mainBytes = SHELL_MODULES["shell.js"].js.length; - expect(mainBytes).toBeLessThan(1_000_000); - }); - - it("splits dynamic just-bash chunks into separate modules", () => { - // The whole point of (2): the bundle is no longer one blob. - // At least one chunk besides shell.js should be present. - const names = Object.keys(SHELL_MODULES); - expect(names.length).toBeGreaterThan(1); - expect(names).toContain("shell.js"); - }); - - it("emits chunk module names with a .js extension", () => { - // workerd's Worker Loader rejects extensionless module names - // for bare-string modules; chunks must keep their .js suffix. - for (const name of Object.keys(SHELL_MODULES)) { - expect(name.endsWith(".js")).toBe(true); - } - }); - - it("every module entry has a js source string", () => { - for (const [name, mod] of Object.entries(SHELL_MODULES)) { - expect(typeof mod.js, `module ${name}`).toBe("string"); - expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); - } - }); -}); diff --git a/packages/computer/src/backends/worker-shell/index.ts b/packages/computer/src/backends/worker-shell/index.ts index 8605014b..c4cbf561 100644 --- a/packages/computer/src/backends/worker-shell/index.ts +++ b/packages/computer/src/backends/worker-shell/index.ts @@ -10,24 +10,33 @@ // // import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; // -// The package ships SHELL_MODULES — a record of module name → -// source string covering the pre-built ShellWorker entry plus -// every dynamic chunk just-bash code-splits into — and -// SHELL_RUNTIME_MODULES — the module shims just-bash's static -// native imports need to load under workerd. The backend -// spreads both into the Loader callback's `modules` table -// internally; consumers only need to reach for them when they -// construct the Loader callback by hand (in which case they -// pass a `fetcher` factory to WorkerShellBackend instead of -// `loader` + `workspace` + `ctx`). +// The package ships the shell as feature groups: SHELL_CORE_MODULES +// (the always-on core — the pre-built ShellWorker entry plus every +// dynamic chunk just-bash code-splits into for the base command +// set) and one optional group per command at +// @cloudflare/computer/shell/. A consumer imports the +// optional groups it wants and passes them to WorkerShellBackend's +// `commands` option; groups it never imports drop out of its +// bundle. It also ships SHELL_RUNTIME_MODULES — the module shims +// just-bash's static native imports need to load under workerd. +// The backend assembles core + `commands` and spreads the runtime +// shims into the Loader callback's `modules` table internally. +// assembleShellModules is exposed for consumers that construct the +// Loader callback by hand (in which case they pass a `fetcher` +// factory to WorkerShellBackend instead of `loader` + `workspace` + +// `ctx`). export { type WorkspaceFs, WorkspaceFsAdapter } from "./adapter.js"; export { type ArtifactsCommandHost, defineArtifactsCommand } from "./artifacts-command.js"; export { type AssetsCommandHost, defineAssetsCommand } from "./assets-command.js"; export { type ExecInput, ShellWorker, type ShellWorkerOptions } from "./entrypoint.js"; -export { SHELL_MODULES } from "./generated-bundle.js"; export { defineGitCommand, type GitCommandHost } from "./git-command.js"; export { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +export { + assembleShellModules, + SHELL_CORE_MODULES, + type ShellModuleGroup, +} from "./shell-modules.js"; export { WorkerShellBackend, type WorkerShellBackendOptions, diff --git a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs index 406cbba4..4cf657f9 100644 --- a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs +++ b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs @@ -1,8 +1,8 @@ -// Bundle entrypoint.ts into a record of module-name → source -// string and write it to generated-bundle.ts as a TypeScript -// module exporting that record (SHELL_MODULES). Consumers -// spread SHELL_MODULES into the Worker Loader callback's -// `modules` field without running a build of their own. +// Bundle entrypoint.ts into a set of module-name → source +// string records and write them under generated/ as TypeScript +// modules, each default-exporting one record. Consumers spread +// the records into the Worker Loader callback's `modules` field +// without running a build of their own. // // Why a record and not a single string? esbuild's bundle: true // inlines dynamic import() targets too, which dragged the full @@ -15,7 +15,28 @@ // it, so the cold-start cost matches the static-reachable set // (~650 KB) and optional features stay free until used. // -// The output file is gitignored. It's regenerated by the +// Why several files and not one? Each optional command +// (html-to-markdown, python, sqlite, js-exec, …) carries chunks +// that are only reachable through that command — html-to-markdown's +// domino (~555 KB) dominates the bundle. Splitting them into their +// own generated module, each published at +// @cloudflare/computer/shell/, lets a consumer select +// commands by import rather than by a build flag: shell-modules.ts +// imports only the core group, so a feature group the consumer +// never imports is unreachable in its module graph and the bundler +// drops the feature's exclusive chunks — and their heavy +// dependencies — from the uploaded Worker. Opting a command in is a +// single import passed to WorkerShellBackend's `commands` option; +// there is no default-on cost and no flag to set. +// +// curl is an optional group like the rest, but its heavy undici +// dependency (~620 KB) is aliased to a throwing stub at bundle time +// (script/undici-stub.mjs) because the Worker backend runs curl on +// the plain-fetch path, never the undici DNS-pinning path — so a +// consumer that imports the curl group gets a working curl without +// undici's bytes. +// +// The generated/ output is gitignored. It's regenerated by the // computer package's prepare / pretest / pretypecheck scripts // so any install from this repo or downstream consumer sees a // fresh bundle matching the source. @@ -30,15 +51,39 @@ const here = dirname(fileURLToPath(import.meta.url)); // Script lives at .../backends/worker-shell/script/build-bundle.mjs; // the bundle target is one level up at .../backends/worker-shell/. const root = resolve(here, ".."); -const out = resolve(root, "generated-bundle.ts"); +const outDir = resolve(root, "generated"); + +// Optional commands whose chunks are worth splitting out so a +// consumer can drop them. Keyed by the feature name used in the +// package subpath (@cloudflare/computer/shell/); the +// value lists the just-bash command names that pull the feature +// in. A chunk lands in a feature group only when that feature is +// the sole optional owner of it and core can't reach it; anything +// shared with core or with a second feature stays in core so +// dropping one feature never breaks another. +const OPTIONAL_FEATURES = { + curl: ["curl"], + "html-to-markdown": ["html-to-markdown"], + python: ["python3", "python"], + sqlite: ["sqlite3"], + "js-exec": ["js-exec", "node"], + yq: ["yq"], + file: ["file"], + xan: ["xan"], + jq: ["jq"], +}; -await mkdir(dirname(out), { recursive: true }); +// Start from a clean generated/ so a group that stops being +// emitted (e.g. curl folded into core) doesn't leave a stale file +// behind. The directory is gitignored and fully regenerated here. +await rm(outDir, { recursive: true, force: true }); +await mkdir(outDir, { recursive: true }); // esbuild's code-splitting requires an outdir to compute chunk // paths against. Use a scratch directory so the build output is // transient; we read result.outputFiles and never touch disk // for the .js fragments themselves. -const outdir = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); +const scratch = await mkdtemp(resolve(tmpdir(), "shell-bundle-")); let result; try { @@ -47,7 +92,7 @@ try { bundle: true, write: false, splitting: true, - outdir, + outdir: scratch, format: "esm", target: "es2022", platform: "neutral", @@ -93,6 +138,27 @@ try { // resolves the import at module load time. "seek-bzip", ], + plugins: [ + // curl reaches undici only through just-bash's DNS-pinning + // connection owner, which the Worker backend never activates: + // ShellWorker registers curl on the plain-`fetch` path + // (entrypoint.ts passes a SecureFetch), so just-bash never + // constructs the pinned undici Agent. just-bash ships + // pre-bundled, so it imports undici as a hashed relative + // chunk (./chunks/undici-.js) rather than the bare + // "undici" specifier — redirect both to a throwing stub so + // the real ~620 KB module stays out of the upload while curl + // itself ships in core. + { + name: "exclude-undici", + setup(pluginBuild) { + pluginBuild.onResolve( + { filter: /(^undici$|[/\\]chunks[/\\]undici-[^/\\]*\.js$)/ }, + () => ({ path: resolve(here, "undici-stub.mjs") }), + ); + }, + }, + ], // Treat any stray .node native binding the resolver still // surfaces as an empty module. Belt-and-braces for transitive // deps that pull in native code through a deeper path. @@ -112,27 +178,20 @@ try { // The scratch outdir is only used to anchor relative paths. // No files were written (write: false), so removing it is // best-effort cleanup of the empty directory. - await rm(outdir, { recursive: true, force: true }); + await rm(scratch, { recursive: true, force: true }); } if (!result.outputFiles || result.outputFiles.length === 0) { throw new Error("build-bundle: esbuild returned no output"); } -// Collect each emitted file into the modules record keyed by -// its outdir-relative path. esbuild writes the entry as -// shell.js and chunks as chunk-.js per entryNames / -// chunkNames above. Worker Loader's `{ js: "..." }` module -// shape lets the keys carry the .js extension cleanly. -// -// Order is stable: sort by name so the generated file diffs -// predictably across builds when content hashes don't change. +// Collect each emitted file into a modules record keyed by its +// scratch-relative path. esbuild writes the entry as shell.js and +// chunks as chunk-.js per entryNames / chunkNames above. const modules = {}; -let totalBytes = 0; for (const file of result.outputFiles) { - const name = relative(outdir, file.path).split(/[\\/]/).join("/"); - modules[name] = { js: file.text }; - totalBytes += file.text.length; + const name = relative(scratch, file.path).split(/[\\/]/).join("/"); + modules[name] = file.text; } if (!modules["shell.js"]) { @@ -141,22 +200,141 @@ if (!modules["shell.js"]) { ); } -const ordered = {}; -for (const name of Object.keys(modules).sort()) { - ordered[name] = modules[name]; +const partition = partitionModules(modules); + +// Emit one generated file per group. shell-modules.ts imports +// each by its @cloudflare/computer/shell/ subpath and +// spreads them back together. +const groupNames = ["core", ...Object.keys(OPTIONAL_FEATURES)]; +let totalBytes = 0; +for (const group of groupNames) { + const names = (partition[group] ?? []).sort(); + const record = {}; + for (const name of names) { + record[name] = { js: modules[name] }; + totalBytes += modules[name].length; + } + const header = + `// Generated by script/build-bundle.mjs — do not edit.\n` + + `// Shell modules exclusive to the "${group}" feature group.\n`; + const body = `export default Object.freeze(${JSON.stringify( + record, + null, + 2, + )}) as Readonly>;\n`; + await writeFile(resolve(outDir, `${group}.ts`), `${header}\n${body}`); } -const header = "// Generated by script/build-bundle.mjs — do not edit.\n"; -const body = `export const SHELL_MODULES: Readonly> = Object.freeze(${JSON.stringify( - ordered, - null, - 2, -)});\n`; -const source = `${header}\n${body}`; - -await writeFile(out, source); -const chunkCount = Object.keys(ordered).length; -const mainBytes = ordered["shell.js"].js.length; +const coreCount = (partition.core ?? []).length; +const mainBytes = modules["shell.js"].length; +const featureSummary = Object.keys(OPTIONAL_FEATURES) + .map((f) => `${f} ${(partition[f] ?? []).length}`) + .join(", "); console.log( - `Wrote ${out} (${chunkCount} modules, shell.js ${mainBytes} bytes, total ${totalBytes} bytes)`, + `Wrote ${outDir} (core ${coreCount} modules, shell.js ${mainBytes} bytes, ` + + `features: ${featureSummary}, total ${totalBytes} bytes)`, ); + +// Assign every emitted module to exactly one group: "core" or one +// of the OPTIONAL_FEATURES keys. A module belongs to a feature +// only when that feature is its sole reacher and core can't reach +// it; everything else — shared chunks, chunks reachable from a +// kept command, the shell.js entry itself — stays in core. +function partitionModules(mods) { + const names = Object.keys(mods); + const staticEdges = new Map(); + const dynamicEdges = new Map(); + for (const name of names) { + staticEdges.set(name, moduleEdges(mods[name], /* dynamic */ false)); + dynamicEdges.set(name, moduleEdges(mods[name], /* dynamic */ true)); + } + + const closure = (starts, followDynamic) => { + const seen = new Set(); + const stack = [...starts]; + while (stack.length > 0) { + const cur = stack.pop(); + if (seen.has(cur) || !mods[cur]) continue; + seen.add(cur); + for (const next of staticEdges.get(cur) ?? []) stack.push(next); + if (followDynamic) for (const next of dynamicEdges.get(cur) ?? []) stack.push(next); + } + return seen; + }; + + const registry = parseCommandChunks(mods["shell.js"]); + + const optionalCommands = new Set(Object.values(OPTIONAL_FEATURES).flat()); + + // Core reach: everything statically pulled by shell.js (the + // always-parsed entry) plus the full closure of every command + // that isn't optional. Dynamic edges out of shell.js are the + // per-command import() fan-out — following them would drag every + // optional chunk into core, so core uses shell.js's static edges + // only. + const coreReach = closure(["shell.js"], /* dynamic */ false); + for (const [command, chunk] of Object.entries(registry)) { + if (!optionalCommands.has(command)) { + for (const m of closure([chunk], /* dynamic */ true)) coreReach.add(m); + } + } + + // Each feature reaches the full closure of its command entry + // chunks. + const featureReach = new Map(); + for (const [feature, commands] of Object.entries(OPTIONAL_FEATURES)) { + const roots = commands.map((c) => registry[c]).filter(Boolean); + featureReach.set(feature, closure(roots, /* dynamic */ true)); + } + + const partition = { core: [] }; + for (const feature of Object.keys(OPTIONAL_FEATURES)) partition[feature] = []; + for (const name of names) { + const owners = []; + if (coreReach.has(name)) owners.push("core"); + for (const feature of Object.keys(OPTIONAL_FEATURES)) { + if (featureReach.get(feature).has(name)) owners.push(feature); + } + const optionalOwners = owners.filter((o) => o !== "core"); + if (!owners.includes("core") && optionalOwners.length === 1) { + partition[optionalOwners[0]].push(name); + } else { + partition.core.push(name); + } + } + return partition; +} + +// Import specifiers a module references. Static edges are the +// top-level `import`/`export … from` and bare side-effect +// imports; dynamic edges are `import(...)` calls. Only relative +// chunk specifiers matter — externals resolve at runtime. +function moduleEdges(source, dynamic) { + const targets = new Set(); + if (dynamic) { + for (const m of source.matchAll(/import\("(\.\/[^"]+)"\)/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + return targets; + } + for (const m of source.matchAll(/(?:import|export)[^;]*?from\s*"(\.\/[^"]+)"/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + for (const m of source.matchAll(/import\s*"(\.\/[^"]+)"/g)) { + targets.add(m[1].replace(/^\.\//, "")); + } + return targets; +} + +// Map each just-bash command to the chunk its lazy loader +// imports. The registry entries look like +// { name: "curl", load: async () => (await import("./chunk-…js")).curlCommand } +function parseCommandChunks(shellSource) { + const registry = {}; + const re = + /\{\s*name:\s*"([^"]+)",\s*load:\s*async\s*\(\)\s*=>\s*\(await import\("(\.\/chunk-[^"]+)"\)\)/g; + for (const m of shellSource.matchAll(re)) { + registry[m[1]] = m[2].replace(/^\.\//, ""); + } + return registry; +} diff --git a/packages/computer/src/backends/worker-shell/script/undici-stub.mjs b/packages/computer/src/backends/worker-shell/script/undici-stub.mjs new file mode 100644 index 00000000..04578218 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/undici-stub.mjs @@ -0,0 +1,29 @@ +// Build-time replacement for `undici`, aliased in by +// build-bundle.mjs so the real ~620 KB network stack never enters +// the Worker shell bundle. +// +// just-bash only reaches undici through its DNS-pinning connection +// owner (network/dns-pin.ts), which runs solely when a curl/wget +// request is made with `denyPrivateRanges` enabled. The Worker +// backend registers curl on the plain-fetch path with +// `denyPrivateRanges` off (egress is governed by the Dynamic +// Worker's globalOutbound, not by in-isolate DNS pinning), so this +// code is never executed. The throwing members exist only to keep +// the dynamic `import("undici")` resolvable; reaching them means +// pinning was switched on without shipping the real dependency. + +const excluded = () => { + throw new Error( + "undici is excluded from the Worker shell bundle; curl runs on the fetch path with denyPrivateRanges disabled", + ); +}; + +export class Agent { + constructor() { + excluded(); + } +} + +export const fetch = excluded; + +export default { Agent, fetch }; diff --git a/packages/computer/src/backends/worker-shell/shell-modules.test.ts b/packages/computer/src/backends/worker-shell/shell-modules.test.ts new file mode 100644 index 00000000..445c3683 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/shell-modules.test.ts @@ -0,0 +1,154 @@ +// Tests the groups build-bundle.mjs emits under generated/ and the +// import-based assembly shell-modules.ts performs on top of them. +// +// build-bundle.mjs runs esbuild with splitting: true and partitions +// the emitted modules into a core group (every always-on command +// plus the ShellWorker entry) and one group per optional command +// (curl included), each published on its own +// @cloudflare/computer/shell/* subpath. shell-modules.ts imports +// only the core group; a consumer opts a command in by importing +// its group and passing it to assembleShellModules (or to +// WorkerBackend's `commands` option). A group the consumer never +// imports is unreachable in their bundle and drops out. These tests +// are the contract. + +import curlModules from "@cloudflare/computer/shell/curl"; +import fileModules from "@cloudflare/computer/shell/file"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; +import jqModules from "@cloudflare/computer/shell/jq"; +import jsExecModules from "@cloudflare/computer/shell/js-exec"; +import pythonModules from "@cloudflare/computer/shell/python"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; +import xanModules from "@cloudflare/computer/shell/xan"; +import yqModules from "@cloudflare/computer/shell/yq"; +import { describe, expect, it } from "vitest"; +import { assembleShellModules, SHELL_CORE_MODULES } from "./shell-modules.js"; + +const OPTIONAL_GROUPS = { + curl: curlModules, + "html-to-markdown": htmlToMarkdownModules, + python: pythonModules, + sqlite: sqliteModules, + "js-exec": jsExecModules, + yq: yqModules, + file: fileModules, + xan: xanModules, + jq: jqModules, +}; + +describe("SHELL_CORE_MODULES", () => { + it("exposes shell.js as the main module", () => { + expect(SHELL_CORE_MODULES["shell.js"]).toBeDefined(); + expect(typeof SHELL_CORE_MODULES["shell.js"].js).toBe("string"); + expect(SHELL_CORE_MODULES["shell.js"].js.length).toBeGreaterThan(0); + }); + + it("keeps the main module under 1 MB so cold start parses ~650 KB, not 3 MB", () => { + // Static-reachable set from entrypoint.ts measured at ~651 KB. + // Anything materially above that means esbuild stopped + // splitting and went back to inlining dynamic imports. + const mainBytes = SHELL_CORE_MODULES["shell.js"].js.length; + expect(mainBytes).toBeLessThan(1_000_000); + }); + + it("splits dynamic just-bash chunks into separate modules", () => { + // The whole point of splitting: the bundle is no longer one + // blob. At least one chunk besides shell.js should be present. + const names = Object.keys(SHELL_CORE_MODULES); + expect(names.length).toBeGreaterThan(1); + expect(names).toContain("shell.js"); + }); + + it("emits chunk module names with a .js extension", () => { + // workerd's Worker Loader rejects extensionless module names + // for bare-string modules; chunks must keep their .js suffix. + for (const name of Object.keys(SHELL_CORE_MODULES)) { + expect(name.endsWith(".js")).toBe(true); + } + }); + + it("every module entry has a js source string", () => { + for (const [name, mod] of Object.entries(SHELL_CORE_MODULES)) { + expect(typeof mod.js, `module ${name}`).toBe("string"); + expect(mod.js.length, `module ${name} non-empty`).toBeGreaterThan(0); + } + }); + + it("carries no optional command's chunks", () => { + // Every optional command lives in its own group; core holds + // none of them. Importing a group is the only way to ship it. + for (const [feature, groupModules] of Object.entries(OPTIONAL_GROUPS)) { + const names = Object.keys(groupModules); + expect(names.length, `${feature} group non-empty on disk`).toBeGreaterThan(0); + for (const name of names) { + expect( + SHELL_CORE_MODULES[name], + `${feature} chunk ${name} absent from core`, + ).toBeUndefined(); + } + } + }); + + it("excludes the real undici dependency, leaving only the stub", () => { + // undici is redirected to a throwing stub at bundle time, so + // even when curl is included it runs on the fetch path and the + // ~620 KB real dependency never ships. The stub marker lives in + // core (the secure-fetch seam), so it is present by default. + const marker = "undici is excluded from the Worker shell bundle"; + const stubPresent = Object.values(SHELL_CORE_MODULES).some((mod) => mod.js.includes(marker)); + expect(stubPresent).toBe(true); + }); +}); + +describe("assembleShellModules", () => { + it("returns core only when no groups are passed", () => { + const assembled = assembleShellModules(); + expect(Object.keys(assembled).sort()).toEqual(Object.keys(SHELL_CORE_MODULES).sort()); + }); + + it("folds an imported group in on top of core", () => { + const assembled = assembleShellModules([curlModules]); + // Core is still there. + expect(assembled["shell.js"]).toBeDefined(); + // curl's chunks are now present. + for (const name of Object.keys(curlModules)) { + expect(assembled[name], `curl chunk ${name} present`).toBeDefined(); + } + // A group that was not passed stays out. + for (const name of Object.keys(sqliteModules)) { + expect(assembled[name], `sqlite chunk ${name} absent`).toBeUndefined(); + } + }); + + it("folds multiple imported groups in", () => { + const assembled = assembleShellModules([curlModules, sqliteModules]); + for (const name of [...Object.keys(curlModules), ...Object.keys(sqliteModules)]) { + expect(assembled[name], `chunk ${name} present`).toBeDefined(); + } + }); +}); + +describe("shell feature groups", () => { + it("keeps curl in its own group with its arg-check message", () => { + const curlOnDisk = Object.values(curlModules).some((mod) => + mod.js.includes("curl: no URL specified"), + ); + expect(curlOnDisk).toBe(true); + }); + + it("keeps feature groups disjoint from each other", () => { + // A chunk owned by one feature must not also appear in another; + // a shared chunk belongs in core. + const entries = Object.entries(OPTIONAL_GROUPS); + for (let i = 0; i < entries.length; i++) { + for (let j = i + 1; j < entries.length; j++) { + const [aName, aMods] = entries[i]; + const [bName, bMods] = entries[j]; + const aKeys = new Set(Object.keys(aMods)); + for (const name of Object.keys(bMods)) { + expect(aKeys.has(name), `${name} in both ${aName} and ${bName}`).toBe(false); + } + } + } + }); +}); diff --git a/packages/computer/src/backends/worker-shell/shell-modules.ts b/packages/computer/src/backends/worker-shell/shell-modules.ts new file mode 100644 index 00000000..4a2ea850 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/shell-modules.ts @@ -0,0 +1,40 @@ +// Assembles the Worker Loader modules table the shell runs from. +// +// build-bundle.mjs runs esbuild with splitting: true and +// partitions the emitted modules into a core group (every +// always-on command plus the ShellWorker entry) and one group per +// optional command, each published on its own +// @cloudflare/computer/shell/ subpath. +// +// Selection is import-based, not flag-based: this module imports +// only the core group. A consumer opts a command in by importing +// its group module and passing it to WorkerBackend's `commands` +// option (or into assembleShellModules directly). Because nothing +// here references the optional groups, a group the consumer never +// imports is unreachable in their module graph and the bundler +// drops it — no build-time define, no explicit opt-out needed. + +import coreModules from "@cloudflare/computer/shell/core"; + +// One generated feature group: module name -> source string. The +// core group and every @cloudflare/computer/shell/ import +// share this shape. +export type ShellModuleGroup = Readonly>; + +// The always-on core group. Ships in every Worker shell; carries +// the ShellWorker entry (shell.js), the base command set, and the +// secure-fetch seam curl runs on when its group is included. +export const SHELL_CORE_MODULES: ShellModuleGroup = Object.freeze({ ...coreModules }); + +// Merge the core group with the optional groups the consumer +// imported and passed. Later groups win on key collisions, but the +// build keeps groups disjoint so order never matters in practice. +export function assembleShellModules( + groups: readonly ShellModuleGroup[] = [], +): Readonly> { + const modules: Record = { ...coreModules }; + for (const group of groups) { + Object.assign(modules, group); + } + return Object.freeze(modules); +} diff --git a/packages/computer/src/backends/worker-shell/worker-shell.ts b/packages/computer/src/backends/worker-shell/worker-shell.ts index e413ed42..6890ce5f 100644 --- a/packages/computer/src/backends/worker-shell/worker-shell.ts +++ b/packages/computer/src/backends/worker-shell/worker-shell.ts @@ -5,7 +5,8 @@ // caller hands the backend a Loader binding plus a {binding, id} // reference to the host DO, and the backend takes care of the // rest. It builds the Worker Loader callback's modules table -// (SHELL_MODULES plus the runtime stubs), wires a +// (core plus any opted-in command groups, plus the runtime +// stubs), wires a // WorkspaceServiceProxy loopback into the loaded Worker's env so // the shell can call env.HOST.getWorkspace() back into the host // DO, mints the Dynamic Worker stub through env.LOADER.get(...), @@ -26,8 +27,8 @@ import type { ExecEvent, ShellRPC, SyncRPC, WorkspaceRPC } from "@cloudflare/com import type { BackendHandle, WorkspaceBackend } from "../../backend.js"; import type { WorkspaceServiceProxyProps } from "../../proxy.js"; -import { SHELL_MODULES } from "./generated-bundle.js"; import { SHELL_RUNTIME_MODULES } from "./runtime-modules.js"; +import { assembleShellModules, type ShellModuleGroup } from "./shell-modules.js"; // The shape the loaded ShellWorker exposes. The host-side // implementation lives in ./entrypoint.ts; the backend consumes @@ -136,6 +137,17 @@ export interface WorkerShellBackendOptions { // workers on different loaders or with different shell // configurations). id?: string; + + // Optional shell command groups to include beyond the always-on + // core. Import the groups you want from + // @cloudflare/computer/shell/ and pass them here; the + // backend folds them into the Loader modules table on top of + // core. A group you never import is unreachable in your bundle + // and the bundler drops it, so this is how you opt a command in + // without shipping the rest. Ignored on the `fetcher` path, + // where the caller assembles the modules table itself (use + // assembleShellModules there). + commands?: readonly ShellModuleGroup[]; } const DEFAULT_COMPAT_DATE = "2026-06-17"; @@ -227,7 +239,7 @@ export class WorkerShellBackend implements WorkspaceBackend { compatibilityFlags, mainModule: "shell.js", modules: { - ...SHELL_MODULES, + ...assembleShellModules(this.#options.commands), ...SHELL_RUNTIME_MODULES, }, env: { diff --git a/packages/computer/test-helpers/shell-module-aliases.ts b/packages/computer/test-helpers/shell-module-aliases.ts new file mode 100644 index 00000000..fda19e62 --- /dev/null +++ b/packages/computer/test-helpers/shell-module-aliases.ts @@ -0,0 +1,28 @@ +// Resolve the published @cloudflare/computer/shell/* subpaths to +// their source files for the test runners. shell-modules.ts +// imports the core group by subpath and a consumer imports the +// optional ones it wants; those subpaths resolve through the +// package's dist exports, which a src-based test run doesn't +// build. Point them at the generated src files instead. + +import { resolve } from "node:path"; + +const src = resolve(import.meta.dirname, "..", "src", "backends", "worker-shell"); + +const groups = [ + "core", + "curl", + "html-to-markdown", + "python", + "sqlite", + "js-exec", + "yq", + "file", + "xan", + "jq", +] as const; + +export const shellModuleAliases = groups.map((group) => ({ + find: `@cloudflare/computer/shell/${group}`, + replacement: resolve(src, "generated", `${group}.ts`), +})); diff --git a/packages/computer/tests/worker-backend-worker.ts b/packages/computer/tests/worker-backend-worker.ts index 1e210136..a116d541 100644 --- a/packages/computer/tests/worker-backend-worker.ts +++ b/packages/computer/tests/worker-backend-worker.ts @@ -18,6 +18,7 @@ // SELF.fetch instead of holding a DO reference itself. import { DurableObject, WorkerEntrypoint } from "cloudflare:workers"; +import curlModules from "@cloudflare/computer/shell/curl"; import { WorkerShellBackend } from "../src/backends/worker-shell/index.js"; import type { DurableObjectStorageLike, WorkspaceStub } from "../src/index.js"; import { Workspace } from "../src/index.js"; @@ -42,6 +43,9 @@ export class HostDO extends DurableObject { loader: env.LOADER, workspace: { binding: "HOST", id: ctx.id.toString() }, ctx, + // Opt curl in by importing its group and passing it; the + // fetch-path curl integration test exercises the wiring. + commands: [curlModules], }), ], }); diff --git a/packages/computer/tests/worker-backend.test.ts b/packages/computer/tests/worker-backend.test.ts index 6030d352..6bc42510 100644 --- a/packages/computer/tests/worker-backend.test.ts +++ b/packages/computer/tests/worker-backend.test.ts @@ -120,6 +120,22 @@ describe("WorkerShellBackend end-to-end", () => { expect(result.stdout).toMatch(/done/); }); + it("registers curl on the fetch path when its group is imported", async () => { + // The harness opts curl in by passing the imported curl group + // to WorkerBackend's `commands`. ShellWorker wires curl to a + // SecureFetch over the isolate's global fetch (no undici). A + // registered curl with no URL fails its own arg check ("curl: + // no URL specified"); an unregistered command would instead be + // reported as not found. This proves the fetch-path curl is + // wired without depending on egress, which globalOutbound keeps + // closed. + const id = freshId(); + const result = await exec(id, "curl 2>&1; echo done"); + expect(result.stdout).toMatch(/done/); + expect(result.stdout).toMatch(/curl: no URL specified/); + expect(result.stdout).not.toMatch(/command not found/); + }); + it("isolates state between separate workspace ids", async () => { // Two host-DO names → two distinct workspaces, two distinct // Dynamic Worker isolates (the loader caches by diff --git a/packages/computer/tsconfig.json b/packages/computer/tsconfig.json index 0d9b0289..39d2c4e4 100644 --- a/packages/computer/tsconfig.json +++ b/packages/computer/tsconfig.json @@ -10,7 +10,21 @@ "esModuleInterop": true, "resolveJsonModule": true, "isolatedModules": true, - "types": ["@cloudflare/workers-types", "vitest/globals", "node"] + "types": ["@cloudflare/workers-types", "vitest/globals", "node"], + "paths": { + "@cloudflare/computer/shell/core": ["./src/backends/worker-shell/generated/core.ts"], + "@cloudflare/computer/shell/curl": ["./src/backends/worker-shell/generated/curl.ts"], + "@cloudflare/computer/shell/html-to-markdown": [ + "./src/backends/worker-shell/generated/html-to-markdown.ts" + ], + "@cloudflare/computer/shell/python": ["./src/backends/worker-shell/generated/python.ts"], + "@cloudflare/computer/shell/sqlite": ["./src/backends/worker-shell/generated/sqlite.ts"], + "@cloudflare/computer/shell/js-exec": ["./src/backends/worker-shell/generated/js-exec.ts"], + "@cloudflare/computer/shell/yq": ["./src/backends/worker-shell/generated/yq.ts"], + "@cloudflare/computer/shell/file": ["./src/backends/worker-shell/generated/file.ts"], + "@cloudflare/computer/shell/xan": ["./src/backends/worker-shell/generated/xan.ts"], + "@cloudflare/computer/shell/jq": ["./src/backends/worker-shell/generated/jq.ts"] + } }, "include": ["src/**/*.ts", "*.ts"] } diff --git a/packages/computer/vitest.config.ts b/packages/computer/vitest.config.ts index d8927922..54742b25 100644 --- a/packages/computer/vitest.config.ts +++ b/packages/computer/vitest.config.ts @@ -2,6 +2,8 @@ import { resolve } from "node:path"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ resolve: { alias: [ @@ -27,6 +29,12 @@ export default defineConfig({ find: "pako", replacement: resolve(__dirname, "src/git/pako-zlib-shim.ts"), }, + // shell-modules.ts imports the generated groups by their + // published @cloudflare/computer/shell/* subpath. Those + // subpaths resolve through the package's dist exports, which + // don't exist under the src test runner — point them at the + // generated src files instead. + ...shellModuleAliases, ], }, test: { diff --git a/packages/computer/vitest.config.worker-backend.ts b/packages/computer/vitest.config.worker-backend.ts index ce1a53ab..3721d2c3 100644 --- a/packages/computer/vitest.config.worker-backend.ts +++ b/packages/computer/vitest.config.worker-backend.ts @@ -8,12 +8,22 @@ import { cloudflareTest } from "@cloudflare/vitest-pool-workers"; import { defineConfig } from "vitest/config"; +import { shellModuleAliases } from "./test-helpers/shell-module-aliases.js"; + export default defineConfig({ plugins: [ cloudflareTest({ wrangler: { configPath: "./tests/wrangler.worker-backend.jsonc" }, }), ], + resolve: { + // The backend imports the core shell group, and the harness + // imports the curl group it opts in, both by their + // @cloudflare/computer/shell/* subpath. Those resolve through + // dist exports that this src test run doesn't build; map them + // to the generated src files. + alias: shellModuleAliases, + }, test: { globals: true, include: ["tests/worker-backend.test.ts"], From 5ef0fd267b78a53494974b6e9fb22f7baa228e9c Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:05:04 +0100 Subject: [PATCH 2/3] examples/worker, docs: opt shell commands in by import Update the worker example and the worker-backend docs to the import-based command selection. The example imports the curl and sqlite groups and passes them to WorkerBackend's commands option, demonstrating that opting a command in is a single import and opting out is deleting it. The package README, the example README, and docs/12_worker_backend.md describe the always-on core, the per-command groups published at @cloudflare/computer/shell/*, the commands option, and the assembleShellModules helper for callers that build the Loader callback by hand. --- docs/12_worker_backend.md | 52 +++++++++++++++++++++++++----- examples/worker-shell/README.md | 34 ++++++++++++++----- examples/worker-shell/src/index.ts | 12 +++++++ packages/computer/README.md | 12 +++++++ 4 files changed, 94 insertions(+), 16 deletions(-) diff --git a/docs/12_worker_backend.md b/docs/12_worker_backend.md index a207794a..d434d221 100644 --- a/docs/12_worker_backend.md +++ b/docs/12_worker_backend.md @@ -249,22 +249,58 @@ network-bound `git` subcommands do. See - `src/index.ts` holds the DO and the HTTP surface (the `/c//file/...` and `/c//exec` routes the container example also exposes). -- No Dockerfile, no build script. The shell bundle ships with - `@cloudflare/computer/backends/worker-shell` as `SHELL_MODULES` - (a record of module name → source covering the entry plus - every code-split chunk); the backend hands the whole record - to the Loader callback itself. +- No Dockerfile, no build script. The shell ships with + `@cloudflare/computer/backends/worker-shell` as feature groups: an + always-on core (`SHELL_CORE_MODULES`) plus one optional group per + command at `@cloudflare/computer/shell/`. The backend + assembles core with whatever groups you opt into and hands the + result to the Loader callback itself. -The DO's backend wiring fits in three lines: +The DO's backend wiring: ```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; + new WorkerShellBackend({ loader: env.LOADER, workspace: { binding: "ContainerExample", id: ctx.id.toString() }, ctx, + commands: [curlModules, sqliteModules], }) ``` Run with `npm run dev --workspace @example/computer-worker`. -The same `curl` recipes from the container example work without -changes. +The same `curl` recipes from the container example work once +`curlModules` is passed to `commands`. + +## Optional shell commands + +Core carries the always-on command set (`cat`, `ls`, `grep`, `sed`, +`awk`, `sort`, …). The heavier commands are split into optional +groups that are opt-in by import: import a group from +`@cloudflare/computer/shell/` and pass it to the +`commands` option, and only then does its code enter your bundle. + +```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import htmlToMarkdownModules from "@cloudflare/computer/shell/html-to-markdown"; + +// commands: [curlModules, htmlToMarkdownModules] +``` + +A group you never import is unreachable in your module graph, so +the bundler drops it — there is no build-time flag to set and no +default-on cost to opt out of. The full set of optional groups is +`curl`, `html-to-markdown`, `python`, `sqlite`, `js-exec`, `yq`, +`file`, `xan`, and `jq`. + +`curl` runs on a `SecureFetch` adapter over the isolate's global +`fetch` — `undici` is redirected to a throwing stub at build time +and never ships. Egress stays governed by the Dynamic Worker's +`globalOutbound` (left `null`, i.e. closed), not by the shell, so +enabling `curl` does not by itself open the network. + +Consumers that build the Loader callback by hand (the `fetcher` +path) assemble the modules table themselves with +`assembleShellModules([...groups])` from the same package. diff --git a/examples/worker-shell/README.md b/examples/worker-shell/README.md index 76231733..fd7dcb71 100644 --- a/examples/worker-shell/README.md +++ b/examples/worker-shell/README.md @@ -101,14 +101,32 @@ POST /c//exec { command | argv, cwd?, encoding? } ## Run it locally -No Docker, no extra build step. The shell ships as a record of -pre-bundled modules (`SHELL_MODULES`) inside -`@cloudflare/computer/backends/worker-shell`; `WorkerShellBackend` spreads -the whole record into the Loader callback internally so the DO -constructor stays a three-line backend invocation. The entry -module parses on cold start; the dynamic chunks (python, js-exec, -sqlite, curl, html-to-markdown) stay cold until a script reaches -for them. +No Docker, no extra build step. The shell ships as pre-bundled +feature groups inside `@cloudflare/computer/backends/worker-shell`: an +always-on core plus one optional group per command at +`@cloudflare/computer/shell/`. `WorkerShellBackend` assembles +core with whatever groups you pass to its `commands` option and +spreads the result into the Loader callback internally. This +example opts `curl` and `sqlite` in: + +```ts +import curlModules from "@cloudflare/computer/shell/curl"; +import sqliteModules from "@cloudflare/computer/shell/sqlite"; + +new WorkerShellBackend({ + loader: env.LOADER, + workspace: { binding: "ContainerExample", id: ctx.id.toString() }, + ctx, + commands: [curlModules, sqliteModules], +}); +``` + +A group you never import (`html-to-markdown`, `python`, `js-exec`, +`yq`, `file`, `xan`, `jq`, or either of the two above) is +unreachable in the bundle and the bundler drops it — opting a +command in is a single import, and opting out is deleting it. The +core entry module parses on cold start; each opted-in group's +chunks stay cold until a script reaches for them. ```sh npm run dev --workspace @example/computer-worker-shell diff --git a/examples/worker-shell/src/index.ts b/examples/worker-shell/src/index.ts index 6099511f..c112714e 100644 --- a/examples/worker-shell/src/index.ts +++ b/examples/worker-shell/src/index.ts @@ -36,6 +36,14 @@ import { withWorkspace, } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; +// Opt-in shell commands. Each import pulls one command group into +// this Worker's bundle; a group you do not import is unreachable +// and the bundler drops it. Pass the ones you want to the +// WorkerShellBackend `commands` option below. Other importable groups: +// @cloudflare/computer/shell/{html-to-markdown,python,js-exec,yq, +// file,xan,jq}. +import curl from "@cloudflare/computer/shell/curl"; +import jq from "@cloudflare/computer/shell/jq"; // Re-export so the runtime can wrap WorkspaceServiceProxy into a // loopback Fetcher binding. The DO reaches the wrapped class @@ -60,6 +68,10 @@ export class ContainerExample extends withWorkspace(class extends DurableObject< loader: env.LOADER, workspace: { binding: "ContainerExample", id: ctx.id.toString() }, ctx, + // Only the groups listed here ship. Core (cat, ls, grep, + // sed, …) is always included; drop an import above to + // shrink the bundle by that command's cost. + commands: [curl, jq], }), ], // Mount the Bucket binding at /workspace/r2. Seed it with diff --git a/packages/computer/README.md b/packages/computer/README.md index 1f8a2c47..bffba952 100644 --- a/packages/computer/README.md +++ b/packages/computer/README.md @@ -100,6 +100,7 @@ quickest way to get `exec` working: ```ts import { withWorkspace, getWorkspace } from "@cloudflare/computer"; import { WorkerShellBackend } from "@cloudflare/computer/backends/worker-shell"; +import curlModules from "@cloudflare/computer/shell/curl"; import { DurableObject } from "cloudflare:workers"; export class Agent extends withWorkspace( @@ -111,6 +112,7 @@ export class Agent extends withWorkspace( loader: self.env.LOADER, workspace: { binding: "Agent", id: self.ctx.id.toString() }, ctx: self.ctx, + commands: [curlModules], }), ], }), @@ -126,6 +128,16 @@ Add the loader binding and the `experimental` flag to `wrangler.jsonc`: } ``` +The worker shell ships as feature groups: an always-on core plus +one optional group per command at +`@cloudflare/computer/shell/`. Import the groups you want +and pass them to `WorkerShellBackend`'s `commands` option; a group you +never import is unreachable in your bundle and the bundler drops +it. The optional groups are `curl`, `html-to-markdown`, `python`, +`sqlite`, `js-exec`, `yq`, `file`, `xan`, and `jq`. `curl` runs on +the isolate's global `fetch` (no `undici` in the bundle); egress +stays governed by the Dynamic Worker's `globalOutbound`. + Now `exec` runs against the same files your `fs` calls wrote: ```ts From 13b4eb108ca07c36651565a62a43955ddcc30193 Mon Sep 17 00:00:00 2001 From: aron <263346377+aron-cf@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:10:39 +0000 Subject: [PATCH 3/3] computer: derive shell bundle graph from esbuild metafile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bundle partitioner read the module graph by scraping the emitted shell.js and chunk sources with regexes: one pass matched import()/from specifiers to recover edges, another matched just-bash's { name, load } registry to map commands to chunks. The edge scrape was fragile — a change in esbuild's codegen or a minification pass would silently return no edges, and an empty graph folds every optional command's heavy dependency back into the core bundle, the exact regression the split exists to prevent. Turn on esbuild's metafile and read the graph from it. The metafile is esbuild's own structured account of every output and its imports, each tagged static or dynamic, so the edge set no longer depends on the shape of the generated source. Command identity still comes from the { name, load } registry in shell.js, because that is the only signal that separates a real command from an internal diagnostic such as flag-coverage, whose dynamic fan-out reaches every command and must not be followed into core. Move the partitioning logic into partition.mjs as pure functions and cover them with unit tests over synthetic graphs, including the diagnostic-fan-out case. resolveFeatureRoots now throws when an optional feature resolves to no command chunk or to a chunk missing from the output, so a broken registry parse fails the build loudly instead of quietly shipping a fat core. The emitted groups are unchanged, byte for byte. --- .changeset/worker-shell-opt-in-commands.md | 2 +- .../worker-shell/script/build-bundle.mjs | 130 ++--------- .../worker-shell/script/partition.mjs | 179 +++++++++++++++ .../worker-shell/script/partition.test.ts | 206 ++++++++++++++++++ 4 files changed, 406 insertions(+), 111 deletions(-) create mode 100644 packages/computer/src/backends/worker-shell/script/partition.mjs create mode 100644 packages/computer/src/backends/worker-shell/script/partition.test.ts diff --git a/.changeset/worker-shell-opt-in-commands.md b/.changeset/worker-shell-opt-in-commands.md index 19b04bfd..93483072 100644 --- a/.changeset/worker-shell-opt-in-commands.md +++ b/.changeset/worker-shell-opt-in-commands.md @@ -2,4 +2,4 @@ "@cloudflare/computer": minor --- -Make worker-shell commands opt-in to cut the deployed bundle. The shell now ships an always-on core plus one optional group per heavy command. Import the groups you want from `@cloudflare/computer/shell/` and pass them to `WorkerShellBackend`'s new `commands` option; a group you never import is dropped from your Worker upload. This is a breaking change: commands such as `curl`, `python`, `sqlite`, `html-to-markdown`, `js-exec`, `yq`, `file`, `xan`, and `jq` no longer ship unless their group is imported. +Heavy worker-shell commands are now opt-in to reduce the final bundle size. See the [documentation](/docs/12_worker_backend.md) for details. diff --git a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs index 4cf657f9..ef396c79 100644 --- a/packages/computer/src/backends/worker-shell/script/build-bundle.mjs +++ b/packages/computer/src/backends/worker-shell/script/build-bundle.mjs @@ -43,10 +43,12 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, relative, resolve } from "node:path"; +import { basename, dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { build } from "esbuild"; +import { buildModuleGraph, parseCommandRegistry, partitionModules } from "./partition.mjs"; + const here = dirname(fileURLToPath(import.meta.url)); // Script lives at .../backends/worker-shell/script/build-bundle.mjs; // the bundle target is one level up at .../backends/worker-shell/. @@ -173,6 +175,10 @@ try { define: { "import.meta.url": JSON.stringify("file:///shell.js"), }, + // Emit the module graph as structured data so the partitioner + // reads import edges (and their static/dynamic kind) from + // esbuild rather than scraping the emitted source with regexes. + metafile: true, }); } finally { // The scratch outdir is only used to anchor relative paths. @@ -186,12 +192,13 @@ if (!result.outputFiles || result.outputFiles.length === 0) { } // Collect each emitted file into a modules record keyed by its -// scratch-relative path. esbuild writes the entry as shell.js and -// chunks as chunk-.js per entryNames / chunkNames above. +// output basename. esbuild writes the entry as shell.js and chunks +// as chunk-.js per entryNames / chunkNames above; the +// basenames are unique and match the keys the module graph and +// partitioner use. const modules = {}; for (const file of result.outputFiles) { - const name = relative(scratch, file.path).split(/[\\/]/).join("/"); - modules[name] = file.text; + modules[basename(file.path)] = file.text; } if (!modules["shell.js"]) { @@ -200,7 +207,14 @@ if (!modules["shell.js"]) { ); } -const partition = partitionModules(modules); +// The graph (import edges) comes from esbuild's metafile; command +// identity (which chunks are just-bash commands vs internal +// diagnostics) comes from parsing shell.js's { name, load } +// registry. partitionModules combines the two and throws if any +// optional feature resolves to no command chunk. +const graph = buildModuleGraph(result.metafile.outputs); +const registry = parseCommandRegistry(modules["shell.js"]); +const partition = partitionModules({ graph, registry, optionalFeatures: OPTIONAL_FEATURES }); // Emit one generated file per group. shell-modules.ts imports // each by its @cloudflare/computer/shell/ subpath and @@ -234,107 +248,3 @@ console.log( `Wrote ${outDir} (core ${coreCount} modules, shell.js ${mainBytes} bytes, ` + `features: ${featureSummary}, total ${totalBytes} bytes)`, ); - -// Assign every emitted module to exactly one group: "core" or one -// of the OPTIONAL_FEATURES keys. A module belongs to a feature -// only when that feature is its sole reacher and core can't reach -// it; everything else — shared chunks, chunks reachable from a -// kept command, the shell.js entry itself — stays in core. -function partitionModules(mods) { - const names = Object.keys(mods); - const staticEdges = new Map(); - const dynamicEdges = new Map(); - for (const name of names) { - staticEdges.set(name, moduleEdges(mods[name], /* dynamic */ false)); - dynamicEdges.set(name, moduleEdges(mods[name], /* dynamic */ true)); - } - - const closure = (starts, followDynamic) => { - const seen = new Set(); - const stack = [...starts]; - while (stack.length > 0) { - const cur = stack.pop(); - if (seen.has(cur) || !mods[cur]) continue; - seen.add(cur); - for (const next of staticEdges.get(cur) ?? []) stack.push(next); - if (followDynamic) for (const next of dynamicEdges.get(cur) ?? []) stack.push(next); - } - return seen; - }; - - const registry = parseCommandChunks(mods["shell.js"]); - - const optionalCommands = new Set(Object.values(OPTIONAL_FEATURES).flat()); - - // Core reach: everything statically pulled by shell.js (the - // always-parsed entry) plus the full closure of every command - // that isn't optional. Dynamic edges out of shell.js are the - // per-command import() fan-out — following them would drag every - // optional chunk into core, so core uses shell.js's static edges - // only. - const coreReach = closure(["shell.js"], /* dynamic */ false); - for (const [command, chunk] of Object.entries(registry)) { - if (!optionalCommands.has(command)) { - for (const m of closure([chunk], /* dynamic */ true)) coreReach.add(m); - } - } - - // Each feature reaches the full closure of its command entry - // chunks. - const featureReach = new Map(); - for (const [feature, commands] of Object.entries(OPTIONAL_FEATURES)) { - const roots = commands.map((c) => registry[c]).filter(Boolean); - featureReach.set(feature, closure(roots, /* dynamic */ true)); - } - - const partition = { core: [] }; - for (const feature of Object.keys(OPTIONAL_FEATURES)) partition[feature] = []; - for (const name of names) { - const owners = []; - if (coreReach.has(name)) owners.push("core"); - for (const feature of Object.keys(OPTIONAL_FEATURES)) { - if (featureReach.get(feature).has(name)) owners.push(feature); - } - const optionalOwners = owners.filter((o) => o !== "core"); - if (!owners.includes("core") && optionalOwners.length === 1) { - partition[optionalOwners[0]].push(name); - } else { - partition.core.push(name); - } - } - return partition; -} - -// Import specifiers a module references. Static edges are the -// top-level `import`/`export … from` and bare side-effect -// imports; dynamic edges are `import(...)` calls. Only relative -// chunk specifiers matter — externals resolve at runtime. -function moduleEdges(source, dynamic) { - const targets = new Set(); - if (dynamic) { - for (const m of source.matchAll(/import\("(\.\/[^"]+)"\)/g)) { - targets.add(m[1].replace(/^\.\//, "")); - } - return targets; - } - for (const m of source.matchAll(/(?:import|export)[^;]*?from\s*"(\.\/[^"]+)"/g)) { - targets.add(m[1].replace(/^\.\//, "")); - } - for (const m of source.matchAll(/import\s*"(\.\/[^"]+)"/g)) { - targets.add(m[1].replace(/^\.\//, "")); - } - return targets; -} - -// Map each just-bash command to the chunk its lazy loader -// imports. The registry entries look like -// { name: "curl", load: async () => (await import("./chunk-…js")).curlCommand } -function parseCommandChunks(shellSource) { - const registry = {}; - const re = - /\{\s*name:\s*"([^"]+)",\s*load:\s*async\s*\(\)\s*=>\s*\(await import\("(\.\/chunk-[^"]+)"\)\)/g; - for (const m of shellSource.matchAll(re)) { - registry[m[1]] = m[2].replace(/^\.\//, ""); - } - return registry; -} diff --git a/packages/computer/src/backends/worker-shell/script/partition.mjs b/packages/computer/src/backends/worker-shell/script/partition.mjs new file mode 100644 index 00000000..0a8b1d45 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/partition.mjs @@ -0,0 +1,179 @@ +// Pure partitioning logic for build-bundle.mjs, split out so it can +// be unit-tested against synthetic module graphs without running a +// real esbuild bundle. +// +// The problem: esbuild emits shell.js plus a flat set of +// chunk-.js code-split fragments. We assign each fragment to +// exactly one group — "core" (always shipped) or one optional +// command feature — so a consumer that never imports a feature's +// group drops the feature's exclusive chunks from its bundle. +// +// Two independent facts drive the assignment, and they come from +// two different sources on purpose: +// +// 1. The module graph — which chunk imports which, and whether +// the edge is a static `import` or a lazy `import()`. This +// comes from esbuild's metafile (imports[].kind), not from +// scraping the emitted source. The metafile is esbuild's own +// structured account of the graph, so it survives codegen and +// minification changes that would break a text scrape. +// +// 2. Command identity — which chunks are just-bash *commands* +// (curl, python3, …) versus internal diagnostics that also +// lazy-load chunks (e.g. flag-coverage, which fans out to +// every command). Only just-bash's `{ name, load }` registry +// in shell.js carries this: a diagnostic's dynamic edges must +// not drag every command's heavy dependency into core, so we +// follow dynamic edges only out of genuine commands. The +// metafile can't distinguish the two — both look like +// code-split entry points — so the registry parse stays. + +import { basename } from "node:path"; + +// Build the module graph from esbuild's metafile `outputs`. Keys +// are output basenames (shell.js, chunk-.js) to match the +// modules record build-bundle.mjs assembles from result.outputFiles. +// Static and dynamic edges are kept apart so the partitioner can +// follow them selectively; external imports (node:*, workerd +// built-ins) are dropped since they resolve at runtime, not from +// the modules table. +export function buildModuleGraph(metafileOutputs) { + const modules = new Set(); + const staticEdges = new Map(); + const dynamicEdges = new Map(); + const entryPointOf = new Map(); + for (const [output, meta] of Object.entries(metafileOutputs)) { + const name = basename(output); + modules.add(name); + const staticTargets = new Set(); + const dynamicTargets = new Set(); + for (const edge of meta.imports ?? []) { + if (edge.external) continue; + const target = basename(edge.path); + if (edge.kind === "dynamic-import") dynamicTargets.add(target); + else if (edge.kind === "import-statement") staticTargets.add(target); + } + staticEdges.set(name, staticTargets); + dynamicEdges.set(name, dynamicTargets); + if (meta.entryPoint) entryPointOf.set(name, basename(meta.entryPoint)); + } + return { modules, staticEdges, dynamicEdges, entryPointOf }; +} + +// Parse just-bash's lazy command registry out of shell.js. Each +// entry looks like +// { name: "curl", load: async () => (await import("./chunk-…js")).curlCommand } +// Returns a command-name -> chunk-basename map. Diagnostics that +// lazy-load chunks through a different shape (flag-coverage's +// `{ …FlagCoverage } = await import(…)`) are deliberately not +// matched: they are not commands, and following their fan-out would +// pull every command's dependency into core. +export function parseCommandRegistry(shellSource) { + const registry = {}; + const re = + /\{\s*name:\s*"([^"]+)",\s*load:\s*async\s*\(\)\s*=>\s*\(await import\("(\.\/chunk-[^"]+)"\)\)/g; + for (const match of shellSource.matchAll(re)) { + registry[match[1]] = basename(match[2]); + } + return registry; +} + +// Resolve each optional feature to the chunk(s) its command entries +// load. A feature may list several command names (aliases such as +// python/python3 or node/js-exec) that resolve to the same chunk; +// duplicates collapse. Throws when a feature resolves to nothing: +// that means the registry parse came back empty or the command +// names drifted, and silently continuing would fold the feature's +// heavy chunk into core — the exact regression the split exists to +// prevent. Also throws if a resolved chunk is missing from the +// module graph, which signals the registry and the emitted output +// have drifted apart. +export function resolveFeatureRoots(registry, optionalFeatures, modules) { + const roots = new Map(); + for (const [feature, commands] of Object.entries(optionalFeatures)) { + const chunks = new Set(); + for (const command of commands) { + const chunk = registry[command]; + if (chunk !== undefined) chunks.add(chunk); + } + if (chunks.size === 0) { + throw new Error( + `partition: optional feature "${feature}" resolved no command chunk ` + + `from the shell registry (commands: ${commands.join(", ")}). The ` + + `registry parse likely broke or just-bash's command names changed; ` + + `refusing to fold the feature's chunks into core silently.`, + ); + } + for (const chunk of chunks) { + if (modules !== undefined && !modules.has(chunk)) { + throw new Error( + `partition: feature "${feature}" command chunk "${chunk}" is not in ` + + `the emitted module set. The shell registry and the bundle output ` + + `have drifted apart.`, + ); + } + } + roots.set(feature, [...chunks]); + } + return roots; +} + +// Assign every emitted module to exactly one group: "core" or one +// optional feature. A module belongs to a feature only when that +// feature is its sole reacher and core can't reach it; everything +// else — shared chunks, chunks a non-optional command reaches, the +// shell.js entry — stays in core. That invariant is what makes +// dropping one feature safe: no core code and no other feature can +// depend on a feature's exclusive chunks. +export function partitionModules({ graph, registry, optionalFeatures }) { + const { modules, staticEdges, dynamicEdges } = graph; + + const closure = (starts, followDynamic) => { + const seen = new Set(); + const stack = [...starts]; + while (stack.length > 0) { + const current = stack.pop(); + if (seen.has(current) || !modules.has(current)) continue; + seen.add(current); + for (const next of staticEdges.get(current) ?? []) stack.push(next); + if (followDynamic) for (const next of dynamicEdges.get(current) ?? []) stack.push(next); + } + return seen; + }; + + const featureRoots = resolveFeatureRoots(registry, optionalFeatures, modules); + const optionalCommands = new Set(Object.values(optionalFeatures).flat()); + + // Core reach: everything statically pulled by shell.js (the + // always-parsed entry) plus the full closure of every command + // that isn't optional. Dynamic edges out of shell.js are the + // per-command import() fan-out — following them would drag every + // optional chunk into core, so core uses shell.js's static edges + // only, then adds the closure of each kept command explicitly. + const coreReach = closure(["shell.js"], /* followDynamic */ false); + for (const [command, chunk] of Object.entries(registry)) { + if (!optionalCommands.has(command)) { + for (const name of closure([chunk], /* followDynamic */ true)) coreReach.add(name); + } + } + + const featureReach = new Map(); + for (const [feature, roots] of featureRoots) { + featureReach.set(feature, closure(roots, /* followDynamic */ true)); + } + + const partition = { core: [] }; + for (const feature of Object.keys(optionalFeatures)) partition[feature] = []; + for (const name of modules) { + const optionalOwners = []; + for (const feature of Object.keys(optionalFeatures)) { + if (featureReach.get(feature).has(name)) optionalOwners.push(feature); + } + if (!coreReach.has(name) && optionalOwners.length === 1) { + partition[optionalOwners[0]].push(name); + } else { + partition.core.push(name); + } + } + return partition; +} diff --git a/packages/computer/src/backends/worker-shell/script/partition.test.ts b/packages/computer/src/backends/worker-shell/script/partition.test.ts new file mode 100644 index 00000000..7c90de00 --- /dev/null +++ b/packages/computer/src/backends/worker-shell/script/partition.test.ts @@ -0,0 +1,206 @@ +// Unit tests for the bundle partitioner. These drive synthetic +// module graphs through the pure functions so the assignment rules +// are pinned without running a real esbuild bundle — the real bundle +// is exercised end-to-end by shell-modules.test.ts against the +// generated output. + +import { describe, expect, it } from "vitest"; + +import { + buildModuleGraph, + parseCommandRegistry, + partitionModules, + resolveFeatureRoots, +} from "./partition.mjs"; + +describe("buildModuleGraph", () => { + it("keys modules by output basename and splits static from dynamic edges", () => { + const graph = buildModuleGraph({ + "out/shell.js": { + imports: [ + { path: "out/chunk-core.js", kind: "import-statement" }, + { path: "out/chunk-curl.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-core.js": { imports: [] }, + "out/chunk-curl.js": { imports: [], entryPoint: "vendor/curl-ABCD1234.js" }, + }); + expect([...graph.modules].sort()).toEqual(["chunk-core.js", "chunk-curl.js", "shell.js"]); + expect([...graph.staticEdges.get("shell.js")]).toEqual(["chunk-core.js"]); + expect([...graph.dynamicEdges.get("shell.js")]).toEqual(["chunk-curl.js"]); + expect(graph.entryPointOf.get("chunk-curl.js")).toBe("curl-ABCD1234.js"); + }); + + it("drops external imports, which resolve at runtime not from the modules table", () => { + const graph = buildModuleGraph({ + "out/shell.js": { + imports: [ + { path: "node:fs", kind: "import-statement", external: true }, + { path: "out/chunk-core.js", kind: "import-statement" }, + ], + }, + "out/chunk-core.js": { imports: [] }, + }); + expect([...graph.staticEdges.get("shell.js")]).toEqual(["chunk-core.js"]); + }); +}); + +describe("parseCommandRegistry", () => { + it("maps command names to the chunk each lazy loader imports", () => { + const source = ` + const commands = [ + { name: "curl", load: async () => (await import("./chunk-CURL1234.js")).curlCommand }, + { name: "jq", load: async () => (await import("./chunk-JQAB5678.js")).jqCommand }, + ]; + `; + expect(parseCommandRegistry(source)).toEqual({ + curl: "chunk-CURL1234.js", + jq: "chunk-JQAB5678.js", + }); + }); + + it("ignores diagnostics that lazy-load through a different shape", () => { + // flag-coverage loads a chunk but is not a { name, load } command; + // matching it would drag every command's dependency into core. + const source = ` + const { instrumentFlagCoverage } = await import("./chunks/flag-coverage-THYQHOT3.js"); + const commands = [ + { name: "curl", load: async () => (await import("./chunk-CURL1234.js")).curlCommand }, + ]; + `; + expect(parseCommandRegistry(source)).toEqual({ curl: "chunk-CURL1234.js" }); + }); +}); + +describe("resolveFeatureRoots", () => { + const modules = new Set(["shell.js", "chunk-curl.js", "chunk-python.js"]); + + it("collapses command aliases that resolve to the same chunk", () => { + const registry = { python3: "chunk-python.js", python: "chunk-python.js" }; + const roots = resolveFeatureRoots(registry, { python: ["python3", "python"] }, modules); + expect(roots.get("python")).toEqual(["chunk-python.js"]); + }); + + it("throws when a feature resolves to no chunk", () => { + // An empty or drifted registry would otherwise fold the feature's + // heavy chunk into core silently — the regression the split + // exists to prevent. + expect(() => resolveFeatureRoots({}, { curl: ["curl"] }, modules)).toThrow( + /optional feature "curl" resolved no command chunk/, + ); + }); + + it("throws when a resolved chunk is absent from the module graph", () => { + const registry = { curl: "chunk-missing.js" }; + expect(() => resolveFeatureRoots(registry, { curl: ["curl"] }, modules)).toThrow( + /chunk "chunk-missing.js" is not in the emitted module set/, + ); + }); +}); + +describe("partitionModules", () => { + // A synthetic graph shaped like the real one: shell.js statically + // pulls its core, and lazy-loads commands. `diag` is a diagnostic + // (not a registry command) that fans out to the optional curl + // command; `req` is a required (non-optional) command. `shared` is + // reachable from the required command, so it must stay in core even + // though curl reaches it too. + const outputs = { + "out/shell.js": { + imports: [ + { path: "out/chunk-corelib.js", kind: "import-statement" }, + { path: "out/chunk-req.js", kind: "dynamic-import" }, + { path: "out/chunk-curl.js", kind: "dynamic-import" }, + { path: "out/chunk-diag.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-corelib.js": { imports: [] }, + "out/chunk-req.js": { + imports: [{ path: "out/chunk-shared.js", kind: "import-statement" }], + entryPoint: "vendor/cat-REQ00000.js", + }, + "out/chunk-curl.js": { + imports: [ + { path: "out/chunk-shared.js", kind: "import-statement" }, + { path: "out/chunk-curlonly.js", kind: "import-statement" }, + ], + entryPoint: "vendor/curl-CURL0000.js", + }, + "out/chunk-curlonly.js": { imports: [] }, + "out/chunk-shared.js": { imports: [] }, + "out/chunk-diag.js": { + imports: [{ path: "out/chunk-curl.js", kind: "dynamic-import" }], + entryPoint: "vendor/flag-coverage-DIAG0000.js", + }, + }; + const registry = { + cat: "chunk-req.js", + curl: "chunk-curl.js", + }; + const optionalFeatures = { curl: ["curl"] }; + + it("assigns a feature's exclusive chunk to that feature", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + expect(part.curl).toContain("chunk-curlonly.js"); + expect(part.curl).toContain("chunk-curl.js"); + }); + + it("keeps a chunk a non-optional command reaches in core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + // shared.js is reached by the required `cat` command, so it must + // ship in core even though curl reaches it too. + expect(part.core).toContain("chunk-shared.js"); + expect(part.curl).not.toContain("chunk-shared.js"); + }); + + it("does not follow a diagnostic's dynamic fan-out into core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + // diag lazy-loads curl, but diag is not a command; its fan-out + // must not pull curl's exclusive chunk into core. + expect(part.core).not.toContain("chunk-curlonly.js"); + // The diagnostic chunk itself has no single optional owner, so it + // lands in core. + expect(part.core).toContain("chunk-diag.js"); + }); + + it("keeps shell.js and statically-reachable core in core", () => { + const graph = buildModuleGraph(outputs); + const part = partitionModules({ graph, registry, optionalFeatures }); + expect(part.core).toContain("shell.js"); + expect(part.core).toContain("chunk-corelib.js"); + }); + + it("keeps a chunk two features share in core", () => { + // fa and fb are both optional; a chunk both reach has more than + // one optional owner, so it belongs in core, not in either group. + const twoFeatureOutputs = { + "out/shell.js": { + imports: [ + { path: "out/chunk-fa.js", kind: "dynamic-import" }, + { path: "out/chunk-fb.js", kind: "dynamic-import" }, + ], + }, + "out/chunk-fa.js": { + imports: [{ path: "out/chunk-both.js", kind: "import-statement" }], + entryPoint: "vendor/xan-FA000000.js", + }, + "out/chunk-fb.js": { + imports: [{ path: "out/chunk-both.js", kind: "import-statement" }], + entryPoint: "vendor/jq-FB000000.js", + }, + "out/chunk-both.js": { imports: [] }, + }; + const graph = buildModuleGraph(twoFeatureOutputs); + const part = partitionModules({ + graph, + registry: { xan: "chunk-fa.js", jq: "chunk-fb.js" }, + optionalFeatures: { xan: ["xan"], jq: ["jq"] }, + }); + expect(part.core).toContain("chunk-both.js"); + expect(part.xan).toEqual(["chunk-fa.js"]); + expect(part.jq).toEqual(["chunk-fb.js"]); + }); +});