feat: Download migrations and emulate bundles at runtime

.gitignore

@@ -24,6 +24,7 @@ obj/
 src/version.ts
 src/generated/skills-manifest.ts
 src/generated/agent-sdk-manifest.ts
+src/generated/runtime-deps-manifest.ts
 
 .eslintcache
 .vscode/launch.json

CLAUDE.md

@@ -20,7 +20,7 @@ WorkOS CLI for installing AuthKit integrations and managing WorkOS resources (or
 ## Tech Constraints
 
 - **Bun** only; the shipped CLI is a Bun-compiled standalone binary
-- Runtime assets must be statically imported or materialized from the compiled binary. Exception: the Agent SDK `claude` executable is downloaded on first agent use — pinned by version + sha256 in the generated manifest — and cached under `~/.workos/cache/agent-sdk/`
+- Runtime assets must be statically imported or materialized from the compiled binary. Two exceptions: (1) the Agent SDK `claude` executable is downloaded on first agent use — pinned by version + sha256 in the generated manifest — and cached under `~/.workos/cache/agent-sdk/`; (2) runtime dep bundles (`@workos/migrations`, `@workos/emulate`) are resolved from the npm registry against semver ranges baked in the generated manifest, integrity-verified, cached under `~/.workos/cache/<dep>/`, and always fall back to the compiled-in module (`WORKOS_RUNTIME_DEPS=0` kill switch) — see `src/lib/runtime-assets.ts`
 
 ## Commit Conventions
 

package.json

@@ -68,7 +68,7 @@
   "packageManager": "bun@1.3.14",
   "scripts": {
     "clean": "bun -e \"await import('node:fs/promises').then((fs) => fs.rm('./dist', { recursive: true, force: true }))\"",
-    "generate": "bun run ./scripts/gen-integration-manifest.ts && bun run ./scripts/gen-skills-manifest.ts && bun run ./scripts/gen-agent-sdk-manifest.ts",
+    "generate": "bun run ./scripts/gen-integration-manifest.ts && bun run ./scripts/gen-skills-manifest.ts && bun run ./scripts/gen-agent-sdk-manifest.ts && bun run ./scripts/gen-runtime-deps-manifest.ts",
     "postinstall": "bun run generate",
     "prebuild": "bun run clean && bun run generate",
     "build": "bun run ./scripts/build.ts",

scripts/gen-runtime-deps-manifest.ts

@@ -0,0 +1,69 @@
+import { mkdir, readFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+
+// Dependencies the compiled CLI can update independently of its own releases.
+// Each package publishes a self-contained ESM bundle at dist/bundle.js
+// (exports subpath `./bundle`), plus any sidecar files the bundle resolves
+// relative to itself at runtime (migrations' worker.js is loaded via
+// `path.join(__dirname, 'worker.js')`, so it must be co-located). The CLI
+// downloads the newest registry version inside the range baked here and
+// imports it in place of the compiled-in module (see src/lib/runtime-assets.ts).
+// Ranges come straight from package.json so a runtime download can never
+// drift past what the compiled-in types were written against.
+//
+// `files` are tarball paths inside `package/`; the FIRST one is the ESM
+// entrypoint the CLI imports.
+const RUNTIME_DEP_PACKAGES: Record<string, { npmPackage: string; files: string[] }> = {
+  migrations: { npmPackage: '@workos/migrations', files: ['dist/bundle.js', 'dist/worker.js'] },
+  emulate: { npmPackage: '@workos/emulate', files: ['dist/bundle.js'] },
+};
+
+const projectRoot = join(import.meta.dirname, '..');
+const manifestPath = join(projectRoot, 'src', 'generated', 'runtime-deps-manifest.ts');
+
+const packageJson = JSON.parse(await readFile(join(projectRoot, 'package.json'), 'utf8')) as {
+  dependencies?: Record<string, string>;
+};
+
+const entries: string[] = [];
+for (const [name, { npmPackage, files }] of Object.entries(RUNTIME_DEP_PACKAGES)) {
+  const range = packageJson.dependencies?.[npmPackage];
+  if (typeof range !== 'string') {
+    throw new Error(`${npmPackage} must be a package.json dependency to be a runtime dep`);
+  }
+  entries.push(
+    `  ${JSON.stringify(name)}: {`,
+    `    npmPackage: ${JSON.stringify(npmPackage)},`,
+    `    range: ${JSON.stringify(range)},`,
+    `    files: [${files.map((file) => JSON.stringify(file)).join(', ')}],`,
+    `  },`,
+  );
+}
+
+const manifest = `// Generated by scripts/gen-runtime-deps-manifest.ts. Do not edit manually.
+// Runtime-downloadable dependency bundles: resolved against these baked semver
+// ranges from the npm registry on use, integrity-verified, cached under
+// ~/.workos/cache/<name>/, and imported in place of the compiled-in module.
+// See src/lib/runtime-assets.ts.
+export type RuntimeDepManifestEntry = {
+  /** npm package that publishes the bundle. */
+  npmPackage: string;
+  /** Semver range baked at CLI build time (mirrors package.json). */
+  range: string;
+  /**
+   * Files to extract from the tarball, as paths inside \`package/\`. The FIRST
+   * is the ESM entrypoint to import; the rest are sidecars the bundle resolves
+   * relative to itself and are installed alongside it.
+   */
+  files: readonly string[];
+};
+
+export const RUNTIME_DEPS = {
+${entries.join('\n')}
+} as const satisfies Record<string, RuntimeDepManifestEntry>;
+
+export type RuntimeDepName = keyof typeof RUNTIME_DEPS;
+`;
+
+await mkdir(dirname(manifestPath), { recursive: true });
+await Bun.write(manifestPath, manifest);

src/commands/debug.ts

@@ -401,6 +401,11 @@ export const ENV_VAR_CATALOG: { name: string; effect: string }[] = [
   // Development
   { name: 'WORKOS_DEV', effect: 'Enables dev mode — loads .env.local at startup' },
   { name: 'WORKOS_DISABLE_PROXY', effect: 'Disables the credential proxy for gateway auth' },
+  {
+    name: 'WORKOS_RUNTIME_DEPS',
+    effect:
+      'Set to "0" to disable runtime-downloaded dependency bundles (compiled-in modules only); "1" forces them on when running from source',
+  },
 ];
 
 export async function runDebugEnv(): Promise<void> {

src/commands/dev.ts

@@ -1,5 +1,8 @@
-import { createEmulator, type EmulatorSeedConfig } from '@workos/emulate';
+// Type-only: the compiled-in package stays the compile-time contract even when
+// a runtime-downloaded bundle provides the implementation (emulate-loader.ts).
+import type { EmulatorSeedConfig } from '@workos/emulate';
 import { resolveDevCommand } from '../lib/dev-command.js';
+import { resolveCreateEmulator } from '../lib/emulate-loader.js';
 import { spawn, type ChildProcess } from 'node:child_process';
 import { readFileSync, existsSync } from 'node:fs';
 import { resolve } from 'node:path';
@@ -89,6 +92,7 @@ export async function runDev(argv: DevArgs): Promise<void> {
   const seedConfig = userSeed ?? DEFAULT_DEV_SEED;
 
   // 1. Start emulator
+  const createEmulator = await resolveCreateEmulator();
   const emulator = await createEmulator({
     port: argv.port,
     seed: seedConfig,

src/commands/emulate.ts

@@ -1,8 +1,11 @@
-import { createEmulator, type Emulator, type EmulatorSeedConfig } from '@workos/emulate';
+// Type-only: the compiled-in package stays the compile-time contract even when
+// a runtime-downloaded bundle provides the implementation (emulate-loader.ts).
+import type { Emulator, EmulatorSeedConfig } from '@workos/emulate';
 import { readFileSync, existsSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { parse as parseYaml } from 'yaml';
 import chalk from 'chalk';
+import { resolveCreateEmulator } from '../lib/emulate-loader.js';
 import { IS_WINDOWS } from '../utils/platform.js';
 import { exitWithError } from '../utils/output.js';
 
@@ -53,6 +56,7 @@ function printBanner(emulator: Pick<Emulator, 'url' | 'apiKey'>): void {
 export async function runEmulate(argv: EmulateArgs): Promise<void> {
   const seedConfig = argv.seed ? loadSeedFile(argv.seed) : autoDetectSeedFile();
 
+  const createEmulator = await resolveCreateEmulator();
   const emulator = await createEmulator({
     port: argv.port,
     seed: seedConfig ?? undefined,

src/commands/migrations.spec.ts

@@ -2,16 +2,22 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
 
 const mockParseAsync = vi.fn();
 const mockName = vi.fn();
+const mockLoadRuntimeBundle = vi.fn();
 
 vi.mock('@workos/migrations/dist/cli/index.js', () => ({
   program: { parseAsync: mockParseAsync, name: mockName },
 }));
 
+vi.mock('../lib/runtime-assets.js', () => ({
+  loadRuntimeBundle: mockLoadRuntimeBundle,
+}));
+
 const { getMigrationsPassthroughArgs, runMigrations } = await import('./migrations.js');
 
 describe('runMigrations', () => {
   beforeEach(() => {
     vi.clearAllMocks();
+    mockLoadRuntimeBundle.mockResolvedValue(null);
     delete process.env.WORKOS_SECRET_KEY;
     delete process.env.WORKOS_API_URL;
   });
@@ -74,6 +80,27 @@ describe('runMigrations', () => {
     });
   });
 
+  it('uses the runtime bundle program when it exposes the commander surface', async () => {
+    const bundleParseAsync = vi.fn();
+    const bundleName = vi.fn();
+    mockLoadRuntimeBundle.mockResolvedValue({ program: { parseAsync: bundleParseAsync, name: bundleName } });
+
+    await runMigrations(['wizard']);
+
+    expect(mockLoadRuntimeBundle).toHaveBeenCalledWith('migrations');
+    expect(bundleName).toHaveBeenCalledTimes(1);
+    expect(bundleParseAsync).toHaveBeenCalledWith(['wizard'], { from: 'user' });
+    expect(mockParseAsync).not.toHaveBeenCalled();
+  });
+
+  it('falls back to the compiled-in program when the bundle lacks the expected export', async () => {
+    mockLoadRuntimeBundle.mockResolvedValue({ program: { notCommander: true } });
+
+    await runMigrations(['wizard']);
+
+    expect(mockParseAsync).toHaveBeenCalledWith(['wizard'], { from: 'user' });
+  });
+
   it('sets WORKOS_API_URL when apiBaseUrl is provided', async () => {
     await runMigrations(['import', '--csv', 'users.csv'], 'sk_test_123', 'https://api.staging.workos.com');
     expect(process.env.WORKOS_API_URL).toBe('https://api.staging.workos.com');

src/commands/migrations.ts

@@ -1,5 +1,27 @@
+import { loadRuntimeBundle } from '../lib/runtime-assets.js';
 import { getWorkOSCommand } from '../utils/command-invocation.js';
 
+/** The commander surface the CLI drives; the compiled-in package is the compile-time contract. */
+type MigrationsProgram = {
+  name(str: string): unknown;
+  parseAsync(argv: string[], options?: { from: 'user' }): Promise<unknown>;
+};
+
+/**
+ * Prefer the runtime-downloaded @workos/migrations bundle (exports `program`;
+ * see lib/runtime-assets.ts), falling back to the compiled-in package when no
+ * bundle is available or it lacks the expected export shape.
+ */
+async function resolveMigrationsProgram(): Promise<MigrationsProgram> {
+  const bundle = await loadRuntimeBundle('migrations');
+  const candidate = bundle?.program as Partial<MigrationsProgram> | undefined;
+  if (typeof candidate?.name === 'function' && typeof candidate?.parseAsync === 'function') {
+    return candidate as MigrationsProgram;
+  }
+  const compiledIn = (await import('@workos/migrations/dist/cli/index.js')) as { program: MigrationsProgram };
+  return compiledIn.program;
+}
+
 const workosOnlyMigrationsFlags = new Map([
   ['--api-key', true],
   ['--insecure-storage', false],
@@ -53,12 +75,7 @@ export async function runMigrations(args: string[], apiKey?: string, apiBaseUrl?
     process.env.WORKOS_API_URL = apiBaseUrl;
   }
 
-  const { program } = (await import('@workos/migrations/dist/cli/index.js')) as {
-    program: {
-      name(str: string): unknown;
-      parseAsync(argv: string[], options?: { from: 'user' }): Promise<unknown>;
-    };
-  };
+  const program = await resolveMigrationsProgram();
 
   program.name(`${getWorkOSCommand()} migrations`);
   await program.parseAsync(args, { from: 'user' });

src/lib/__test-helpers__/npm-tarball-fixtures.ts

@@ -0,0 +1,26 @@
+import { gzipSync } from 'node:zlib';
+
+/** Build a minimal ustar header for a regular-file entry. */
+function tarHeader(name: string, size: number): Buffer {
+  const header = Buffer.alloc(512);
+  header.write(name, 0, 'utf8');
+  header.write(`${size.toString(8).padStart(11, '0')}\0`, 124, 'utf8');
+  header.write('0', 156, 'utf8'); // typeflag: regular file
+  header.write('ustar\0', 257, 'utf8');
+  header.write('00', 263, 'utf8');
+  header.fill(' ', 148, 156); // checksum field counts as spaces while summing
+  let sum = 0;
+  for (const byte of header) sum += byte;
+  header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 'utf8');
+  return header;
+}
+
+/** Build a gzipped npm-style tarball from `[entryName, content]` pairs. */
+export function makeTarball(entries: Array<[string, Buffer]>): Buffer {
+  const parts: Buffer[] = [];
+  for (const [name, content] of entries) {
+    parts.push(tarHeader(name, content.length), content, Buffer.alloc((512 - (content.length % 512)) % 512));
+  }
+  parts.push(Buffer.alloc(1024)); // end-of-archive
+  return gzipSync(Buffer.concat(parts));
+}

src/lib/agent-sdk-assets.spec.ts

@@ -1,32 +1,9 @@
 import { existsSync } from 'node:fs';
-import { gzipSync } from 'node:zlib';
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import { AGENT_SDK_TARGET, AGENT_SDK_VERSION } from '../generated/agent-sdk-manifest.js';
+import { makeTarball } from './__test-helpers__/npm-tarball-fixtures.js';
 import { downloadTarball, ensureClaudeCodeExecutable, extractTarEntry, isBunVirtualFsUrl } from './agent-sdk-assets.js';
 
-function tarHeader(name: string, size: number): Buffer {
-  const header = Buffer.alloc(512);
-  header.write(name, 0, 'utf8');
-  header.write(`${size.toString(8).padStart(11, '0')}\0`, 124, 'utf8');
-  header.write('0', 156, 'utf8'); // typeflag: regular file
-  header.write('ustar\0', 257, 'utf8');
-  header.write('00', 263, 'utf8');
-  header.fill(' ', 148, 156); // checksum field counts as spaces while summing
-  let sum = 0;
-  for (const byte of header) sum += byte;
-  header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 'utf8');
-  return header;
-}
-
-function makeTarball(entries: Array<[string, Buffer]>): Buffer {
-  const parts: Buffer[] = [];
-  for (const [name, content] of entries) {
-    parts.push(tarHeader(name, content.length), content, Buffer.alloc((512 - (content.length % 512)) % 512));
-  }
-  parts.push(Buffer.alloc(1024)); // end-of-archive
-  return gzipSync(Buffer.concat(parts));
-}
-
 describe('isBunVirtualFsUrl', () => {
   it('detects the POSIX compiled-binary virtual filesystem', () => {
     expect(isBunVirtualFsUrl('file:///$bunfs/root/workos')).toBe(true);
@@ -57,6 +34,8 @@ describe('ensureClaudeCodeExecutable', () => {
 });
 
 describe('extractTarEntry', () => {
+  // Extraction itself is covered by npm-tarball.spec.ts; this pins the
+  // size-capped wrapper the Agent SDK download path uses.
   it('extracts the named entry from a gzipped tarball', () => {
     const content = Buffer.from('#!/bin/sh\necho claude\n');
     const tarball = makeTarball([
@@ -65,20 +44,6 @@ describe('extractTarEntry', () => {
     ]);
     expect(extractTarEntry(tarball, 'package/claude').equals(content)).toBe(true);
   });
-
-  it('handles entries whose size is an exact block multiple', () => {
-    const content = Buffer.alloc(1024, 7);
-    const tarball = makeTarball([
-      ['package/claude', content],
-      ['package/LICENSE.md', Buffer.from('license')],
-    ]);
-    expect(extractTarEntry(tarball, 'package/LICENSE.md').toString()).toBe('license');
-  });
-
-  it('throws when the entry is missing', () => {
-    const tarball = makeTarball([['package/package.json', Buffer.from('{}')]]);
-    expect(() => extractTarEntry(tarball, 'package/claude')).toThrow(/not found/);
-  });
 });
 
 describe('downloadTarball', () => {

src/lib/agent-sdk-assets.ts

@@ -13,7 +13,7 @@ import {
 import { homedir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
-import { gunzipSync } from 'node:zlib';
+import { extractTarEntry as extractNpmTarEntry } from './npm-tarball.js';
 import {
   AGENT_SDK_EXECUTABLE_NAME,
   AGENT_SDK_EXECUTABLE_SHA256,
@@ -64,8 +64,12 @@ export function isBunVirtualFsUrl(url: string): boolean {
   return url.includes('$bunfs') || url.includes('~BUN') || url.includes('%7EBUN');
 }
 
-/** Running from a compiled binary: the module graph lives in Bun's virtual filesystem. */
-function isCompiledBinary(): boolean {
+/**
+ * Running from a compiled binary: the module graph lives in Bun's virtual
+ * filesystem. Shared with runtime-assets.ts, which gates runtime bundle
+ * downloads the same way.
+ */
+export function isCompiledBinary(): boolean {
   return isBunVirtualFsUrl(import.meta.url);
 }
 
@@ -98,35 +102,11 @@ function resolveFromNodeModules(): string {
 const MAX_TARBALL_UNCOMPRESSED_BYTES = AGENT_SDK_EXECUTABLE_SIZE + 64 * 1024 * 1024;
 
 /**
- * Extract a single entry from a gzipped npm tarball. Exported for tests.
- *
- * Minimal ustar reader: npm tarballs are flat `package/…` archives well within
- * ustar limits, so pax/GNU long-name extensions never apply to the entry we
- * want — unknown entry types are skipped by the generic size-based walk.
+ * Extract a single entry from the gzipped Agent SDK tarball. Thin wrapper over
+ * the shared ustar reader in npm-tarball.ts. Exported for tests.
  */
 export function extractTarEntry(tarGz: Buffer, entryName: string): Buffer {
-  const raw = gunzipSync(tarGz, { maxOutputLength: MAX_TARBALL_UNCOMPRESSED_BYTES });
-  let offset = 0;
-  while (offset + 512 <= raw.length) {
-    const header = raw.subarray(offset, offset + 512);
-    if (header.every((byte) => byte === 0)) break;
-    const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, '');
-    const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, '');
-    const fullName = prefix ? `${prefix}/${name}` : name;
-    const size = Number.parseInt(header.subarray(124, 136).toString('utf8').replace(/\0.*$/s, '').trim(), 8);
-    if (Number.isNaN(size) || size < 0) {
-      throw new Error(`Malformed tar header at offset ${offset}`);
-    }
-    const dataStart = offset + 512;
-    if (fullName === entryName) {
-      if (dataStart + size > raw.length) {
-        throw new Error(`Truncated tar entry ${entryName}`);
-      }
-      return Buffer.from(raw.subarray(dataStart, dataStart + size));
-    }
-    offset = dataStart + Math.ceil(size / 512) * 512;
-  }
-  throw new Error(`Entry ${entryName} not found in tarball`);
+  return extractNpmTarEntry(tarGz, entryName, MAX_TARBALL_UNCOMPRESSED_BYTES);
 }
 
 /** Abort a download that goes this long without a data chunk, so a stalled connection can't hang forever. */