diff --git a/.gitignore b/.gitignore index 725c1b92..f3e5d0d2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ obj/ src/version.ts src/generated/skills-manifest.ts src/generated/agent-sdk-manifest.ts +src/generated/runtime-deps-manifest.ts .eslintcache .vscode/launch.json diff --git a/CLAUDE.md b/CLAUDE.md index b8c0c405..891b2a29 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,7 +20,7 @@ WorkOS CLI for installing AuthKit integrations and managing WorkOS resources (or ## Tech Constraints - **Bun** only; the shipped CLI is a Bun-compiled standalone binary -- Runtime assets must be statically imported or materialized from the compiled binary. Exception: the Agent SDK `claude` executable is downloaded on first agent use — pinned by version + sha256 in the generated manifest — and cached under `~/.workos/cache/agent-sdk/` +- Runtime assets must be statically imported or materialized from the compiled binary. Two exceptions: (1) the Agent SDK `claude` executable is downloaded on first agent use — pinned by version + sha256 in the generated manifest — and cached under `~/.workos/cache/agent-sdk/`; (2) runtime dep bundles (`@workos/migrations`, `@workos/emulate`) are resolved from the npm registry against semver ranges baked in the generated manifest, integrity-verified, cached under `~/.workos/cache//`, and always fall back to the compiled-in module (`WORKOS_RUNTIME_DEPS=0` kill switch) — see `src/lib/runtime-assets.ts` ## Commit Conventions diff --git a/package.json b/package.json index cd0a4651..d22903f6 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,7 @@ "packageManager": "bun@1.3.14", "scripts": { "clean": "bun -e \"await import('node:fs/promises').then((fs) => fs.rm('./dist', { recursive: true, force: true }))\"", - "generate": "bun run ./scripts/gen-integration-manifest.ts && bun run ./scripts/gen-skills-manifest.ts && bun run ./scripts/gen-agent-sdk-manifest.ts", + "generate": "bun run ./scripts/gen-integration-manifest.ts && bun run ./scripts/gen-skills-manifest.ts && bun run ./scripts/gen-agent-sdk-manifest.ts && bun run ./scripts/gen-runtime-deps-manifest.ts", "postinstall": "bun run generate", "prebuild": "bun run clean && bun run generate", "build": "bun run ./scripts/build.ts", diff --git a/scripts/gen-runtime-deps-manifest.ts b/scripts/gen-runtime-deps-manifest.ts new file mode 100644 index 00000000..b4b0c9fc --- /dev/null +++ b/scripts/gen-runtime-deps-manifest.ts @@ -0,0 +1,69 @@ +import { mkdir, readFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; + +// Dependencies the compiled CLI can update independently of its own releases. +// Each package publishes a self-contained ESM bundle at dist/bundle.js +// (exports subpath `./bundle`), plus any sidecar files the bundle resolves +// relative to itself at runtime (migrations' worker.js is loaded via +// `path.join(__dirname, 'worker.js')`, so it must be co-located). The CLI +// downloads the newest registry version inside the range baked here and +// imports it in place of the compiled-in module (see src/lib/runtime-assets.ts). +// Ranges come straight from package.json so a runtime download can never +// drift past what the compiled-in types were written against. +// +// `files` are tarball paths inside `package/`; the FIRST one is the ESM +// entrypoint the CLI imports. +const RUNTIME_DEP_PACKAGES: Record = { + migrations: { npmPackage: '@workos/migrations', files: ['dist/bundle.js', 'dist/worker.js'] }, + emulate: { npmPackage: '@workos/emulate', files: ['dist/bundle.js'] }, +}; + +const projectRoot = join(import.meta.dirname, '..'); +const manifestPath = join(projectRoot, 'src', 'generated', 'runtime-deps-manifest.ts'); + +const packageJson = JSON.parse(await readFile(join(projectRoot, 'package.json'), 'utf8')) as { + dependencies?: Record; +}; + +const entries: string[] = []; +for (const [name, { npmPackage, files }] of Object.entries(RUNTIME_DEP_PACKAGES)) { + const range = packageJson.dependencies?.[npmPackage]; + if (typeof range !== 'string') { + throw new Error(`${npmPackage} must be a package.json dependency to be a runtime dep`); + } + entries.push( + ` ${JSON.stringify(name)}: {`, + ` npmPackage: ${JSON.stringify(npmPackage)},`, + ` range: ${JSON.stringify(range)},`, + ` files: [${files.map((file) => JSON.stringify(file)).join(', ')}],`, + ` },`, + ); +} + +const manifest = `// Generated by scripts/gen-runtime-deps-manifest.ts. Do not edit manually. +// Runtime-downloadable dependency bundles: resolved against these baked semver +// ranges from the npm registry on use, integrity-verified, cached under +// ~/.workos/cache//, and imported in place of the compiled-in module. +// See src/lib/runtime-assets.ts. +export type RuntimeDepManifestEntry = { + /** npm package that publishes the bundle. */ + npmPackage: string; + /** Semver range baked at CLI build time (mirrors package.json). */ + range: string; + /** + * Files to extract from the tarball, as paths inside \`package/\`. The FIRST + * is the ESM entrypoint to import; the rest are sidecars the bundle resolves + * relative to itself and are installed alongside it. + */ + files: readonly string[]; +}; + +export const RUNTIME_DEPS = { +${entries.join('\n')} +} as const satisfies Record; + +export type RuntimeDepName = keyof typeof RUNTIME_DEPS; +`; + +await mkdir(dirname(manifestPath), { recursive: true }); +await Bun.write(manifestPath, manifest); diff --git a/src/commands/debug.ts b/src/commands/debug.ts index dc7abe51..8bded8d5 100644 --- a/src/commands/debug.ts +++ b/src/commands/debug.ts @@ -401,6 +401,11 @@ export const ENV_VAR_CATALOG: { name: string; effect: string }[] = [ // Development { name: 'WORKOS_DEV', effect: 'Enables dev mode — loads .env.local at startup' }, { name: 'WORKOS_DISABLE_PROXY', effect: 'Disables the credential proxy for gateway auth' }, + { + name: 'WORKOS_RUNTIME_DEPS', + effect: + 'Set to "0" to disable runtime-downloaded dependency bundles (compiled-in modules only); "1" forces them on when running from source', + }, ]; export async function runDebugEnv(): Promise { diff --git a/src/commands/dev.ts b/src/commands/dev.ts index e5772fc9..307af7dd 100644 --- a/src/commands/dev.ts +++ b/src/commands/dev.ts @@ -1,5 +1,8 @@ -import { createEmulator, type EmulatorSeedConfig } from '@workos/emulate'; +// Type-only: the compiled-in package stays the compile-time contract even when +// a runtime-downloaded bundle provides the implementation (emulate-loader.ts). +import type { EmulatorSeedConfig } from '@workos/emulate'; import { resolveDevCommand } from '../lib/dev-command.js'; +import { resolveCreateEmulator } from '../lib/emulate-loader.js'; import { spawn, type ChildProcess } from 'node:child_process'; import { readFileSync, existsSync } from 'node:fs'; import { resolve } from 'node:path'; @@ -89,6 +92,7 @@ export async function runDev(argv: DevArgs): Promise { const seedConfig = userSeed ?? DEFAULT_DEV_SEED; // 1. Start emulator + const createEmulator = await resolveCreateEmulator(); const emulator = await createEmulator({ port: argv.port, seed: seedConfig, diff --git a/src/commands/emulate.ts b/src/commands/emulate.ts index c6a11c6c..a49de855 100644 --- a/src/commands/emulate.ts +++ b/src/commands/emulate.ts @@ -1,8 +1,11 @@ -import { createEmulator, type Emulator, type EmulatorSeedConfig } from '@workos/emulate'; +// Type-only: the compiled-in package stays the compile-time contract even when +// a runtime-downloaded bundle provides the implementation (emulate-loader.ts). +import type { Emulator, EmulatorSeedConfig } from '@workos/emulate'; import { readFileSync, existsSync } from 'node:fs'; import { resolve } from 'node:path'; import { parse as parseYaml } from 'yaml'; import chalk from 'chalk'; +import { resolveCreateEmulator } from '../lib/emulate-loader.js'; import { IS_WINDOWS } from '../utils/platform.js'; import { exitWithError } from '../utils/output.js'; @@ -53,6 +56,7 @@ function printBanner(emulator: Pick): void { export async function runEmulate(argv: EmulateArgs): Promise { const seedConfig = argv.seed ? loadSeedFile(argv.seed) : autoDetectSeedFile(); + const createEmulator = await resolveCreateEmulator(); const emulator = await createEmulator({ port: argv.port, seed: seedConfig ?? undefined, diff --git a/src/commands/migrations.spec.ts b/src/commands/migrations.spec.ts index 9b3f6e44..871aa9bd 100644 --- a/src/commands/migrations.spec.ts +++ b/src/commands/migrations.spec.ts @@ -2,16 +2,22 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; const mockParseAsync = vi.fn(); const mockName = vi.fn(); +const mockLoadRuntimeBundle = vi.fn(); vi.mock('@workos/migrations/dist/cli/index.js', () => ({ program: { parseAsync: mockParseAsync, name: mockName }, })); +vi.mock('../lib/runtime-assets.js', () => ({ + loadRuntimeBundle: mockLoadRuntimeBundle, +})); + const { getMigrationsPassthroughArgs, runMigrations } = await import('./migrations.js'); describe('runMigrations', () => { beforeEach(() => { vi.clearAllMocks(); + mockLoadRuntimeBundle.mockResolvedValue(null); delete process.env.WORKOS_SECRET_KEY; delete process.env.WORKOS_API_URL; }); @@ -74,6 +80,27 @@ describe('runMigrations', () => { }); }); + it('uses the runtime bundle program when it exposes the commander surface', async () => { + const bundleParseAsync = vi.fn(); + const bundleName = vi.fn(); + mockLoadRuntimeBundle.mockResolvedValue({ program: { parseAsync: bundleParseAsync, name: bundleName } }); + + await runMigrations(['wizard']); + + expect(mockLoadRuntimeBundle).toHaveBeenCalledWith('migrations'); + expect(bundleName).toHaveBeenCalledTimes(1); + expect(bundleParseAsync).toHaveBeenCalledWith(['wizard'], { from: 'user' }); + expect(mockParseAsync).not.toHaveBeenCalled(); + }); + + it('falls back to the compiled-in program when the bundle lacks the expected export', async () => { + mockLoadRuntimeBundle.mockResolvedValue({ program: { notCommander: true } }); + + await runMigrations(['wizard']); + + expect(mockParseAsync).toHaveBeenCalledWith(['wizard'], { from: 'user' }); + }); + it('sets WORKOS_API_URL when apiBaseUrl is provided', async () => { await runMigrations(['import', '--csv', 'users.csv'], 'sk_test_123', 'https://api.staging.workos.com'); expect(process.env.WORKOS_API_URL).toBe('https://api.staging.workos.com'); diff --git a/src/commands/migrations.ts b/src/commands/migrations.ts index e84742c5..20f7fb37 100644 --- a/src/commands/migrations.ts +++ b/src/commands/migrations.ts @@ -1,5 +1,27 @@ +import { loadRuntimeBundle } from '../lib/runtime-assets.js'; import { getWorkOSCommand } from '../utils/command-invocation.js'; +/** The commander surface the CLI drives; the compiled-in package is the compile-time contract. */ +type MigrationsProgram = { + name(str: string): unknown; + parseAsync(argv: string[], options?: { from: 'user' }): Promise; +}; + +/** + * Prefer the runtime-downloaded @workos/migrations bundle (exports `program`; + * see lib/runtime-assets.ts), falling back to the compiled-in package when no + * bundle is available or it lacks the expected export shape. + */ +async function resolveMigrationsProgram(): Promise { + const bundle = await loadRuntimeBundle('migrations'); + const candidate = bundle?.program as Partial | undefined; + if (typeof candidate?.name === 'function' && typeof candidate?.parseAsync === 'function') { + return candidate as MigrationsProgram; + } + const compiledIn = (await import('@workos/migrations/dist/cli/index.js')) as { program: MigrationsProgram }; + return compiledIn.program; +} + const workosOnlyMigrationsFlags = new Map([ ['--api-key', true], ['--insecure-storage', false], @@ -53,12 +75,7 @@ export async function runMigrations(args: string[], apiKey?: string, apiBaseUrl? process.env.WORKOS_API_URL = apiBaseUrl; } - const { program } = (await import('@workos/migrations/dist/cli/index.js')) as { - program: { - name(str: string): unknown; - parseAsync(argv: string[], options?: { from: 'user' }): Promise; - }; - }; + const program = await resolveMigrationsProgram(); program.name(`${getWorkOSCommand()} migrations`); await program.parseAsync(args, { from: 'user' }); diff --git a/src/lib/__test-helpers__/npm-tarball-fixtures.ts b/src/lib/__test-helpers__/npm-tarball-fixtures.ts new file mode 100644 index 00000000..03198d5d --- /dev/null +++ b/src/lib/__test-helpers__/npm-tarball-fixtures.ts @@ -0,0 +1,26 @@ +import { gzipSync } from 'node:zlib'; + +/** Build a minimal ustar header for a regular-file entry. */ +function tarHeader(name: string, size: number): Buffer { + const header = Buffer.alloc(512); + header.write(name, 0, 'utf8'); + header.write(`${size.toString(8).padStart(11, '0')}\0`, 124, 'utf8'); + header.write('0', 156, 'utf8'); // typeflag: regular file + header.write('ustar\0', 257, 'utf8'); + header.write('00', 263, 'utf8'); + header.fill(' ', 148, 156); // checksum field counts as spaces while summing + let sum = 0; + for (const byte of header) sum += byte; + header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 'utf8'); + return header; +} + +/** Build a gzipped npm-style tarball from `[entryName, content]` pairs. */ +export function makeTarball(entries: Array<[string, Buffer]>): Buffer { + const parts: Buffer[] = []; + for (const [name, content] of entries) { + parts.push(tarHeader(name, content.length), content, Buffer.alloc((512 - (content.length % 512)) % 512)); + } + parts.push(Buffer.alloc(1024)); // end-of-archive + return gzipSync(Buffer.concat(parts)); +} diff --git a/src/lib/agent-sdk-assets.spec.ts b/src/lib/agent-sdk-assets.spec.ts index e6004462..750a8dd0 100644 --- a/src/lib/agent-sdk-assets.spec.ts +++ b/src/lib/agent-sdk-assets.spec.ts @@ -1,32 +1,9 @@ import { existsSync } from 'node:fs'; -import { gzipSync } from 'node:zlib'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { AGENT_SDK_TARGET, AGENT_SDK_VERSION } from '../generated/agent-sdk-manifest.js'; +import { makeTarball } from './__test-helpers__/npm-tarball-fixtures.js'; import { downloadTarball, ensureClaudeCodeExecutable, extractTarEntry, isBunVirtualFsUrl } from './agent-sdk-assets.js'; -function tarHeader(name: string, size: number): Buffer { - const header = Buffer.alloc(512); - header.write(name, 0, 'utf8'); - header.write(`${size.toString(8).padStart(11, '0')}\0`, 124, 'utf8'); - header.write('0', 156, 'utf8'); // typeflag: regular file - header.write('ustar\0', 257, 'utf8'); - header.write('00', 263, 'utf8'); - header.fill(' ', 148, 156); // checksum field counts as spaces while summing - let sum = 0; - for (const byte of header) sum += byte; - header.write(`${sum.toString(8).padStart(6, '0')}\0 `, 148, 'utf8'); - return header; -} - -function makeTarball(entries: Array<[string, Buffer]>): Buffer { - const parts: Buffer[] = []; - for (const [name, content] of entries) { - parts.push(tarHeader(name, content.length), content, Buffer.alloc((512 - (content.length % 512)) % 512)); - } - parts.push(Buffer.alloc(1024)); // end-of-archive - return gzipSync(Buffer.concat(parts)); -} - describe('isBunVirtualFsUrl', () => { it('detects the POSIX compiled-binary virtual filesystem', () => { expect(isBunVirtualFsUrl('file:///$bunfs/root/workos')).toBe(true); @@ -57,6 +34,8 @@ describe('ensureClaudeCodeExecutable', () => { }); describe('extractTarEntry', () => { + // Extraction itself is covered by npm-tarball.spec.ts; this pins the + // size-capped wrapper the Agent SDK download path uses. it('extracts the named entry from a gzipped tarball', () => { const content = Buffer.from('#!/bin/sh\necho claude\n'); const tarball = makeTarball([ @@ -65,20 +44,6 @@ describe('extractTarEntry', () => { ]); expect(extractTarEntry(tarball, 'package/claude').equals(content)).toBe(true); }); - - it('handles entries whose size is an exact block multiple', () => { - const content = Buffer.alloc(1024, 7); - const tarball = makeTarball([ - ['package/claude', content], - ['package/LICENSE.md', Buffer.from('license')], - ]); - expect(extractTarEntry(tarball, 'package/LICENSE.md').toString()).toBe('license'); - }); - - it('throws when the entry is missing', () => { - const tarball = makeTarball([['package/package.json', Buffer.from('{}')]]); - expect(() => extractTarEntry(tarball, 'package/claude')).toThrow(/not found/); - }); }); describe('downloadTarball', () => { diff --git a/src/lib/agent-sdk-assets.ts b/src/lib/agent-sdk-assets.ts index 29a5eccf..95a5f6b6 100644 --- a/src/lib/agent-sdk-assets.ts +++ b/src/lib/agent-sdk-assets.ts @@ -13,7 +13,7 @@ import { import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { gunzipSync } from 'node:zlib'; +import { extractTarEntry as extractNpmTarEntry } from './npm-tarball.js'; import { AGENT_SDK_EXECUTABLE_NAME, AGENT_SDK_EXECUTABLE_SHA256, @@ -64,8 +64,12 @@ export function isBunVirtualFsUrl(url: string): boolean { return url.includes('$bunfs') || url.includes('~BUN') || url.includes('%7EBUN'); } -/** Running from a compiled binary: the module graph lives in Bun's virtual filesystem. */ -function isCompiledBinary(): boolean { +/** + * Running from a compiled binary: the module graph lives in Bun's virtual + * filesystem. Shared with runtime-assets.ts, which gates runtime bundle + * downloads the same way. + */ +export function isCompiledBinary(): boolean { return isBunVirtualFsUrl(import.meta.url); } @@ -98,35 +102,11 @@ function resolveFromNodeModules(): string { const MAX_TARBALL_UNCOMPRESSED_BYTES = AGENT_SDK_EXECUTABLE_SIZE + 64 * 1024 * 1024; /** - * Extract a single entry from a gzipped npm tarball. Exported for tests. - * - * Minimal ustar reader: npm tarballs are flat `package/…` archives well within - * ustar limits, so pax/GNU long-name extensions never apply to the entry we - * want — unknown entry types are skipped by the generic size-based walk. + * Extract a single entry from the gzipped Agent SDK tarball. Thin wrapper over + * the shared ustar reader in npm-tarball.ts. Exported for tests. */ export function extractTarEntry(tarGz: Buffer, entryName: string): Buffer { - const raw = gunzipSync(tarGz, { maxOutputLength: MAX_TARBALL_UNCOMPRESSED_BYTES }); - let offset = 0; - while (offset + 512 <= raw.length) { - const header = raw.subarray(offset, offset + 512); - if (header.every((byte) => byte === 0)) break; - const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, ''); - const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, ''); - const fullName = prefix ? `${prefix}/${name}` : name; - const size = Number.parseInt(header.subarray(124, 136).toString('utf8').replace(/\0.*$/s, '').trim(), 8); - if (Number.isNaN(size) || size < 0) { - throw new Error(`Malformed tar header at offset ${offset}`); - } - const dataStart = offset + 512; - if (fullName === entryName) { - if (dataStart + size > raw.length) { - throw new Error(`Truncated tar entry ${entryName}`); - } - return Buffer.from(raw.subarray(dataStart, dataStart + size)); - } - offset = dataStart + Math.ceil(size / 512) * 512; - } - throw new Error(`Entry ${entryName} not found in tarball`); + return extractNpmTarEntry(tarGz, entryName, MAX_TARBALL_UNCOMPRESSED_BYTES); } /** Abort a download that goes this long without a data chunk, so a stalled connection can't hang forever. */ diff --git a/src/lib/emulate-loader.spec.ts b/src/lib/emulate-loader.spec.ts new file mode 100644 index 00000000..2104e561 --- /dev/null +++ b/src/lib/emulate-loader.spec.ts @@ -0,0 +1,42 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => ({ + loadRuntimeBundle: vi.fn(), + compiledCreateEmulator: vi.fn(), +})); + +vi.mock('./runtime-assets.js', () => ({ + loadRuntimeBundle: mocks.loadRuntimeBundle, +})); + +vi.mock('@workos/emulate', () => ({ + createEmulator: mocks.compiledCreateEmulator, +})); + +const { resolveCreateEmulator } = await import('./emulate-loader.js'); + +describe('resolveCreateEmulator', () => { + beforeEach(() => { + mocks.loadRuntimeBundle.mockReset(); + }); + + it('uses the runtime bundle createEmulator when present', async () => { + const runtimeCreateEmulator = vi.fn(); + mocks.loadRuntimeBundle.mockResolvedValue({ createEmulator: runtimeCreateEmulator }); + + expect(await resolveCreateEmulator()).toBe(runtimeCreateEmulator); + expect(mocks.loadRuntimeBundle).toHaveBeenCalledWith('emulate'); + }); + + it('falls back to the compiled-in module when no bundle is available', async () => { + mocks.loadRuntimeBundle.mockResolvedValue(null); + + expect(await resolveCreateEmulator()).toBe(mocks.compiledCreateEmulator); + }); + + it('falls back when the bundle lacks a createEmulator function', async () => { + mocks.loadRuntimeBundle.mockResolvedValue({ createEmulator: 'not-a-function' }); + + expect(await resolveCreateEmulator()).toBe(mocks.compiledCreateEmulator); + }); +}); diff --git a/src/lib/emulate-loader.ts b/src/lib/emulate-loader.ts new file mode 100644 index 00000000..b92fc9b6 --- /dev/null +++ b/src/lib/emulate-loader.ts @@ -0,0 +1,22 @@ +import { loadRuntimeBundle } from './runtime-assets.js'; + +/** + * The compiled-in signature stays the compile-time contract: a runtime bundle + * is only ever cast to it after a shape check, never trusted for new API. + */ +export type CreateEmulatorFn = (typeof import('@workos/emulate'))['createEmulator']; + +/** + * Resolve `createEmulator`, preferring the runtime-downloaded @workos/emulate + * bundle (see runtime-assets.ts) and falling back to the compiled-in package + * when no bundle is available or it lacks the expected export. + */ +export async function resolveCreateEmulator(): Promise { + const bundle = await loadRuntimeBundle('emulate'); + const candidate = bundle?.createEmulator; + if (typeof candidate === 'function') { + return candidate as CreateEmulatorFn; + } + const { createEmulator } = await import('@workos/emulate'); + return createEmulator; +} diff --git a/src/lib/npm-tarball.spec.ts b/src/lib/npm-tarball.spec.ts new file mode 100644 index 00000000..af4a2afe --- /dev/null +++ b/src/lib/npm-tarball.spec.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest'; +import { makeTarball } from './__test-helpers__/npm-tarball-fixtures.js'; +import { extractTarEntry } from './npm-tarball.js'; + +const MAX_BYTES = 64 * 1024 * 1024; + +describe('extractTarEntry', () => { + it('extracts the named entry from a gzipped tarball', () => { + const content = Buffer.from('#!/bin/sh\necho claude\n'); + const tarball = makeTarball([ + ['package/package.json', Buffer.from('{}')], + ['package/claude', content], + ]); + expect(extractTarEntry(tarball, 'package/claude', MAX_BYTES).equals(content)).toBe(true); + }); + + it('handles entries whose size is an exact block multiple', () => { + const content = Buffer.alloc(1024, 7); + const tarball = makeTarball([ + ['package/claude', content], + ['package/LICENSE.md', Buffer.from('license')], + ]); + expect(extractTarEntry(tarball, 'package/LICENSE.md', MAX_BYTES).toString()).toBe('license'); + }); + + it('throws when the entry is missing', () => { + const tarball = makeTarball([['package/package.json', Buffer.from('{}')]]); + expect(() => extractTarEntry(tarball, 'package/claude', MAX_BYTES)).toThrow(/not found/); + }); + + it('refuses to decompress past the gzip-bomb cap', () => { + const tarball = makeTarball([['package/big', Buffer.alloc(64 * 1024)]]); + expect(() => extractTarEntry(tarball, 'package/big', 4 * 1024)).toThrow(); + }); +}); diff --git a/src/lib/npm-tarball.ts b/src/lib/npm-tarball.ts new file mode 100644 index 00000000..eee8a12c --- /dev/null +++ b/src/lib/npm-tarball.ts @@ -0,0 +1,38 @@ +import { gunzipSync } from 'node:zlib'; + +/** + * Extract a single entry from a gzipped npm package tarball. Shared by the two + * runtime-download paths (agent-sdk-assets.ts and runtime-assets.ts). + * + * Minimal ustar reader: npm tarballs are flat `package/…` archives well within + * ustar limits, so pax/GNU long-name extensions never apply to the entries we + * want — unknown entry types are skipped by the generic size-based walk. + * + * `maxUncompressedBytes` caps gunzip output so a compromised or corrupted + * response can't expand a gzip bomb in memory before the caller's checksum + * gate gets a chance to reject it. + */ +export function extractTarEntry(tarGz: Buffer, entryName: string, maxUncompressedBytes: number): Buffer { + const raw = gunzipSync(tarGz, { maxOutputLength: maxUncompressedBytes }); + let offset = 0; + while (offset + 512 <= raw.length) { + const header = raw.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const name = header.subarray(0, 100).toString('utf8').replace(/\0.*$/s, ''); + const prefix = header.subarray(345, 500).toString('utf8').replace(/\0.*$/s, ''); + const fullName = prefix ? `${prefix}/${name}` : name; + const size = Number.parseInt(header.subarray(124, 136).toString('utf8').replace(/\0.*$/s, '').trim(), 8); + if (Number.isNaN(size) || size < 0) { + throw new Error(`Malformed tar header at offset ${offset}`); + } + const dataStart = offset + 512; + if (fullName === entryName) { + if (dataStart + size > raw.length) { + throw new Error(`Truncated tar entry ${entryName}`); + } + return Buffer.from(raw.subarray(dataStart, dataStart + size)); + } + offset = dataStart + Math.ceil(size / 512) * 512; + } + throw new Error(`Entry ${entryName} not found in tarball`); +} diff --git a/src/lib/runtime-assets.spec.ts b/src/lib/runtime-assets.spec.ts new file mode 100644 index 00000000..c0361427 --- /dev/null +++ b/src/lib/runtime-assets.spec.ts @@ -0,0 +1,296 @@ +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { makeTarball } from './__test-helpers__/npm-tarball-fixtures.js'; +import { + loadRuntimeDep, + pickHighestSatisfying, + verifySriIntegrity, + type AbbreviatedPackument, + type RuntimeDep, +} from './runtime-assets.js'; + +const DEP: RuntimeDep = { + name: 'test-dep', + npmPackage: '@workos/test-dep', + range: '^1.0.0', + files: ['dist/bundle.js'], +}; + +function sri(data: Buffer): string { + return `sha512-${createHash('sha512').update(data).digest('base64')}`; +} + +function jsonResponse(body: unknown): Response { + return { ok: true, status: 200, json: async () => body } as unknown as Response; +} + +function bytesResponse(body: Buffer): Response { + return { + ok: true, + status: 200, + arrayBuffer: async () => body.buffer.slice(body.byteOffset, body.byteOffset + body.byteLength), + } as unknown as Response; +} + +/** Abbreviated packument with per-version dist info defaulted. */ +function metadata( + versions: Record, +): AbbreviatedPackument { + return { + versions: Object.fromEntries( + Object.entries(versions).map(([version, info]) => [ + version, + { + ...(info.deprecated === undefined ? {} : { deprecated: info.deprecated }), + dist: { + tarball: info.tarballUrl ?? `https://registry.npmjs.org/@workos/test-dep/-/test-dep-${version}.tgz`, + integrity: info.integrity ?? `sha512-${'A'.repeat(86)}==`, + }, + }, + ]), + ), + }; +} + +describe('pickHighestSatisfying', () => { + it('picks the highest version inside the range, ignoring newer out-of-range ones', () => { + const resolved = pickHighestSatisfying(metadata({ '1.0.0': {}, '1.4.0': {}, '2.0.0': {} }), '^1.0.0'); + expect(resolved?.version).toBe('1.4.0'); + expect(resolved?.tarballUrl).toContain('test-dep-1.4.0.tgz'); + }); + + it('skips deprecated versions', () => { + const resolved = pickHighestSatisfying( + metadata({ '1.0.0': {}, '1.4.0': { deprecated: 'broken release' } }), + '^1.0.0', + ); + expect(resolved?.version).toBe('1.0.0'); + }); + + it('skips versions without a sha512 integrity or tarball URL', () => { + const withoutDist: AbbreviatedPackument = { + versions: { + '1.0.0': { dist: { tarball: 'https://example.invalid/1.0.0.tgz', integrity: 'sha512-x' } }, + '1.5.0': { dist: {} }, + '1.6.0': { + dist: { tarball: 'https://example.invalid/1.6.0.tgz', integrity: 'sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=' }, + }, + }, + }; + expect(pickHighestSatisfying(withoutDist, '^1.0.0')?.version).toBe('1.0.0'); + }); + + it('returns null when nothing satisfies the range', () => { + expect(pickHighestSatisfying(metadata({ '2.0.0': {} }), '^1.0.0')).toBeNull(); + }); +}); + +describe('verifySriIntegrity', () => { + const data = Buffer.from('runtime bundle bytes'); + + it('accepts a matching sha512 digest', () => { + expect(() => verifySriIntegrity(data, sri(data))).not.toThrow(); + }); + + it('throws on a digest mismatch', () => { + expect(() => verifySriIntegrity(data, sri(Buffer.from('other bytes')))).toThrow(/Integrity mismatch/); + }); + + it('rejects integrity strings without a sha512 hash', () => { + expect(() => verifySriIntegrity(data, 'sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=')).toThrow(/No sha512/); + }); +}); + +describe('loadRuntimeDep', () => { + let cacheRoot: string; + + beforeEach(() => { + cacheRoot = mkdtempSync(join(tmpdir(), 'workos-runtime-assets-')); + // Vitest runs from source, where the mechanism is off by default; force it + // on the way a compiled binary has it. + vi.stubEnv('WORKOS_RUNTIME_DEPS', '1'); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + rmSync(cacheRoot, { recursive: true, force: true }); + }); + + function seedInstalledVersion(version: string, marker: string): void { + const versionDir = join(cacheRoot, DEP.name, version); + mkdirSync(versionDir, { recursive: true }); + writeFileSync(join(versionDir, 'bundle.mjs'), `export const marker = ${JSON.stringify(marker)};\n`); + } + + function seedResolution(resolution: Record): void { + const depDir = join(cacheRoot, DEP.name); + mkdirSync(depDir, { recursive: true }); + writeFileSync(join(depDir, 'resolution.json'), JSON.stringify(resolution)); + } + + it('resolves the highest in-range version, downloads, verifies, and imports the bundle', async () => { + const tarball = makeTarball([['package/dist/bundle.js', Buffer.from('export const marker = "v1.4.0";\n')]]); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.0.0': {}, '1.4.0': { integrity: sri(tarball) }, '2.0.0': {} }))) + .mockResolvedValueOnce(bytesResponse(tarball)); + vi.stubGlobal('fetch', fetchMock); + + const mod = await loadRuntimeDep(DEP, { cacheRoot }); + + expect(mod?.marker).toBe('v1.4.0'); + expect(fetchMock).toHaveBeenCalledTimes(2); + const [metadataUrl, metadataInit] = fetchMock.mock.calls[0]; + expect(metadataUrl).toBe('https://registry.npmjs.org/%40workos%2Ftest-dep'); + expect(metadataInit.headers.accept).toBe('application/vnd.npm.install-v1+json'); + expect(existsSync(join(cacheRoot, 'test-dep', '1.4.0', 'bundle.mjs'))).toBe(true); + const resolution = JSON.parse(readFileSync(join(cacheRoot, 'test-dep', 'resolution.json'), 'utf8')); + expect(resolution.version).toBe('1.4.0'); + }); + + it('installs sidecar files next to the entrypoint (migrations worker contract)', async () => { + const twoFileDep: RuntimeDep = { ...DEP, files: ['dist/bundle.js', 'dist/worker.js'] }; + const tarball = makeTarball([ + ['package/dist/bundle.js', Buffer.from('export const marker = "with-worker";\n')], + ['package/dist/worker.js', Buffer.from('// worker thread code\n')], + ]); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.4.0': { integrity: sri(tarball) } }))) + .mockResolvedValueOnce(bytesResponse(tarball)); + vi.stubGlobal('fetch', fetchMock); + + const mod = await loadRuntimeDep(twoFileDep, { cacheRoot }); + + expect(mod?.marker).toBe('with-worker'); + const versionDir = join(cacheRoot, 'test-dep', '1.4.0'); + // The sidecar keeps its exact basename, co-located with the imported entrypoint. + expect(existsSync(join(versionDir, 'worker.js'))).toBe(true); + expect(existsSync(join(versionDir, 'bundle.mjs'))).toBe(true); + }); + + it('rejects a tarball that fails integrity verification and does not install it', async () => { + const tarball = makeTarball([['package/dist/bundle.js', Buffer.from('export const marker = "tampered";\n')]]); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.4.0': { integrity: `sha512-${'A'.repeat(86)}==` } }))) + .mockResolvedValueOnce(bytesResponse(tarball)); + vi.stubGlobal('fetch', fetchMock); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + expect(existsSync(join(cacheRoot, 'test-dep', '1.4.0'))).toBe(false); + }); + + it('honors the resolution TTL: no metadata refetch while the cache is fresh', async () => { + seedInstalledVersion('1.2.3', 'cached'); + seedResolution({ + version: '1.2.3', + tarballUrl: 'https://example.invalid/never-fetched.tgz', + integrity: 'sha512-never-checked', + fetchedAt: Date.now(), + }); + const fetchMock = vi.fn().mockRejectedValue(new Error('network must not be touched')); + vi.stubGlobal('fetch', fetchMock); + + const mod = await loadRuntimeDep(DEP, { cacheRoot }); + + expect(mod?.marker).toBe('cached'); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('refetches metadata once the resolution cache is past its TTL', async () => { + seedInstalledVersion('1.2.3', 'stale'); + seedResolution({ + version: '1.2.3', + tarballUrl: 'https://example.invalid/old.tgz', + integrity: 'sha512-old', + fetchedAt: Date.now() - 25 * 60 * 60 * 1000, + }); + const tarball = makeTarball([['package/dist/bundle.js', Buffer.from('export const marker = "v1.5.0";\n')]]); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.5.0': { integrity: sri(tarball) } }))) + .mockResolvedValueOnce(bytesResponse(tarball)); + vi.stubGlobal('fetch', fetchMock); + + const mod = await loadRuntimeDep(DEP, { cacheRoot }); + + expect(mod?.marker).toBe('v1.5.0'); + expect(fetchMock).toHaveBeenCalledTimes(2); + const resolution = JSON.parse(readFileSync(join(cacheRoot, 'test-dep', 'resolution.json'), 'utf8')); + expect(resolution.version).toBe('1.5.0'); + }); + + it('falls back to the newest downloaded in-range version when the registry is unreachable', async () => { + seedInstalledVersion('1.1.0', 'older'); + seedInstalledVersion('1.3.0', 'newest-in-range'); + seedInstalledVersion('2.0.0', 'out-of-range'); + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))); + + const mod = await loadRuntimeDep(DEP, { cacheRoot }); + + expect(mod?.marker).toBe('newest-in-range'); + }); + + it('returns null when offline with nothing cached', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('offline'))); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + }); + + it('falls back to a previously downloaded version when the new download fails', async () => { + seedInstalledVersion('1.1.0', 'previously-verified'); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.6.0': {} }))) + .mockRejectedValueOnce(new Error('tarball download failed')); + vi.stubGlobal('fetch', fetchMock); + + const mod = await loadRuntimeDep(DEP, { cacheRoot }); + + expect(mod?.marker).toBe('previously-verified'); + }); + + it('does not retry the download within the TTL after an install failure', async () => { + // Transition-period shape: the published tarball has no bundle entry yet. + const tarball = makeTarball([['package/README.md', Buffer.from('no bundle here\n')]]); + const fetchMock = vi + .fn() + .mockResolvedValueOnce(jsonResponse(metadata({ '1.6.0': { integrity: sri(tarball) } }))) + .mockResolvedValueOnce(bytesResponse(tarball)); + vi.stubGlobal('fetch', fetchMock); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + expect(fetchMock).toHaveBeenCalledTimes(2); + + // Next invocation inside the TTL: no metadata refetch, no tarball retry. + const secondFetch = vi.fn().mockRejectedValue(new Error('network must not be touched')); + vi.stubGlobal('fetch', secondFetch); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + expect(secondFetch).not.toHaveBeenCalled(); + }); + + it('does nothing at all when WORKOS_RUNTIME_DEPS=0', async () => { + vi.stubEnv('WORKOS_RUNTIME_DEPS', '0'); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + seedInstalledVersion('1.3.0', 'must-not-load'); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('stays on the compiled-in module by default when running from source', async () => { + vi.stubEnv('WORKOS_RUNTIME_DEPS', undefined); + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + + expect(await loadRuntimeDep(DEP, { cacheRoot })).toBeNull(); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/runtime-assets.ts b/src/lib/runtime-assets.ts new file mode 100644 index 00000000..4ec5277d --- /dev/null +++ b/src/lib/runtime-assets.ts @@ -0,0 +1,372 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { basename, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { maxSatisfying, satisfies, valid } from 'semver'; +import { RUNTIME_DEPS, type RuntimeDepManifestEntry, type RuntimeDepName } from '../generated/runtime-deps-manifest.js'; +import { logWarn } from '../utils/debug.js'; +import { isCompiledBinary } from './agent-sdk-assets.js'; +import { extractTarEntry } from './npm-tarball.js'; + +/** + * Runtime-downloadable dependency bundles. + * + * Like the Agent SDK executable (agent-sdk-assets.ts), this is a deliberate + * exception to the "runtime assets must be statically imported or materialized + * from the compiled binary" rule: packages such as @workos/migrations and + * @workos/emulate ship fixes independently of CLI releases, so the compiled + * binary prefers a downloaded, integrity-verified bundle of the newest version + * inside a baked semver range and falls back to the compiled-in module + * whenever anything goes wrong. + * + * Flow per dependency (see loadRuntimeBundle): + * 1. resolve the newest non-deprecated version in range from the npm registry + * (abbreviated metadata, cached on disk for 24h), + * 2. download the version's tarball, verify its SRI sha512 integrity BEFORE + * extracting anything, extract the manifest's files (the ESM bundle plus + * any sidecars it resolves relative to itself, like migrations' worker.js), + * and install them atomically under ~/.workos/cache///, + * 3. dynamic-import the cached bundle entrypoint. + * + * Failure at any stage is silent-but-debuggable (logged via logWarn, like + * version-check.ts) and leaves the caller on the compiled-in module. + * WORKOS_RUNTIME_DEPS=0 is the kill switch: compiled-in only, no network and + * no cache reads. When running from source the mechanism is off by default — + * node_modules already has the packages — and WORKOS_RUNTIME_DEPS=1 forces it + * on for testing. + */ + +const REGISTRY_BASE_URL = 'https://registry.npmjs.org'; +/** Registry metadata is a small JSON document; keep the budget tight so cold starts never hang. */ +const METADATA_TIMEOUT_MS = 3_000; +/** Bundle tarballs can be tens of MB; downloaded at most once per version. */ +const TARBALL_TIMEOUT_MS = 30_000; +const RESOLUTION_TTL_MS = 24 * 60 * 60 * 1000; +/** Generous gzip-bomb cap for JS bundle files (see npm-tarball.ts). */ +const MAX_BUNDLE_UNCOMPRESSED_BYTES = 256 * 1024 * 1024; +const RESOLUTION_FILENAME = 'resolution.json'; + +/** A manifest entry plus its cache-directory name. Exported for tests. */ +export type RuntimeDep = RuntimeDepManifestEntry & { name: string }; + +type ResolvedVersion = { + version: string; + tarballUrl: string; + integrity: string; +}; + +type ResolutionCache = ResolvedVersion & { + fetchedAt: number; + /** A download/install of this version failed; don't retry until the TTL expires. */ + installFailed?: boolean; +}; + +/** Shape of the npm registry's abbreviated ("install") package metadata. */ +export type AbbreviatedPackument = { + versions?: Record< + string, + | { + deprecated?: unknown; + dist?: { tarball?: unknown; integrity?: unknown }; + } + | undefined + >; +}; + +export type LoadRuntimeBundleOptions = { + /** Cache root override for tests. Defaults to ~/.workos/cache. @internal */ + cacheRoot?: string; +}; + +/** + * Pick the highest non-deprecated version satisfying `range` from abbreviated + * registry metadata. Returns null when nothing usable satisfies the range + * (including versions missing a tarball URL or sha512 integrity — those could + * never be verified, so they are never candidates). Exported for tests. + */ +export function pickHighestSatisfying(metadata: AbbreviatedPackument, range: string): ResolvedVersion | null { + const versions = metadata.versions ?? {}; + const candidates = Object.keys(versions).filter((version) => { + const info = versions[version]; + return ( + valid(version) !== null && + !info?.deprecated && + typeof info?.dist?.tarball === 'string' && + typeof info?.dist?.integrity === 'string' && + sha512Digests(info.dist.integrity).length > 0 + ); + }); + const version = maxSatisfying(candidates, range); + if (!version) return null; + const dist = versions[version]?.dist as { tarball: string; integrity: string }; + return { version, tarballUrl: dist.tarball, integrity: dist.integrity }; +} + +/** The sha512 digests in an SRI integrity string — the only algorithm we verify. */ +function sha512Digests(integrity: string): string[] { + return integrity + .split(/\s+/) + .map((entry) => /^sha512-([A-Za-z0-9+/=]+)$/.exec(entry)?.[1]) + .filter((digest): digest is string => digest !== undefined); +} + +/** + * Verify npm's SRI integrity string (sha512 over the raw tarball bytes). + * Throws on mismatch or when no sha512 hash is present — weaker algorithms + * (old sha1-only packages) are treated as unverifiable. Exported for tests. + */ +export function verifySriIntegrity(data: Buffer, integrity: string): void { + const digests = sha512Digests(integrity); + if (digests.length === 0) { + throw new Error(`No sha512 hash in integrity string ${JSON.stringify(integrity)}`); + } + const actual = createHash('sha512').update(data).digest('base64'); + if (!digests.includes(actual)) { + throw new Error(`Integrity mismatch: expected ${integrity}, got sha512-${actual}`); + } +} + +function defaultCacheRoot(): string { + return join(homedir(), '.workos', 'cache'); +} + +/** + * On-disk name for an extracted file. Files are flattened to their basename — + * all of a dep's files ship in the same tarball directory, so `__dirname` + * -relative sidecar resolution (migrations' worker.js) keeps working. The + * entrypoint is renamed .js → .mjs so both Bun (compiled binary) and Node + * (vitest) unambiguously parse it as ESM without a package.json in the cache + * dir; sidecars keep their exact basename because the bundle looks them up by + * that name at runtime. + */ +function installedFileName(tarballPath: string, isEntry: boolean): string { + const base = basename(tarballPath); + return isEntry ? base.replace(/\.js$/, '.mjs') : base; +} + +function entryPath(cacheDir: string, version: string, dep: RuntimeDep): string { + return join(cacheDir, version, installedFileName(dep.files[0], true)); +} + +function readResolutionCache(cacheDir: string, range: string): (ResolvedVersion & { installFailed?: boolean }) | null { + try { + const parsed = JSON.parse(readFileSync(join(cacheDir, RESOLUTION_FILENAME), 'utf8')) as Partial; + if ( + typeof parsed.version !== 'string' || + typeof parsed.tarballUrl !== 'string' || + typeof parsed.integrity !== 'string' || + typeof parsed.fetchedAt !== 'number' + ) { + return null; + } + const age = Date.now() - parsed.fetchedAt; + // A negative age means the clock rolled back under a future timestamp; + // treat it as stale rather than trusting it forever. + if (age < 0 || age >= RESOLUTION_TTL_MS) return null; + // The baked range may have moved since the resolution was cached (CLI update). + if (valid(parsed.version) === null || !satisfies(parsed.version, range)) return null; + return { + version: parsed.version, + tarballUrl: parsed.tarballUrl, + integrity: parsed.integrity, + ...(parsed.installFailed === true ? { installFailed: true } : {}), + }; + } catch { + return null; + } +} + +function writeResolutionCache(cacheDir: string, resolved: ResolvedVersion & { installFailed?: boolean }): void { + // Best-effort: a failed cache write only costs a refetch next run. + try { + mkdirSync(cacheDir, { recursive: true, mode: 0o700 }); + const path = join(cacheDir, RESOLUTION_FILENAME); + const temporary = `${path}.tmp.${process.pid}.${randomUUID()}`; + writeFileSync(temporary, JSON.stringify({ ...resolved, fetchedAt: Date.now() } satisfies ResolutionCache)); + renameSync(temporary, path); + } catch { + // Ignored. + } +} + +async function fetchResolution(dep: RuntimeDep): Promise { + const response = await fetch(`${REGISTRY_BASE_URL}/${encodeURIComponent(dep.npmPackage)}`, { + headers: { accept: 'application/vnd.npm.install-v1+json' }, + signal: AbortSignal.timeout(METADATA_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`HTTP ${response.status} fetching ${dep.npmPackage} metadata`); + } + const metadata = (await response.json()) as AbbreviatedPackument; + const resolved = pickHighestSatisfying(metadata, dep.range); + if (!resolved) { + throw new Error(`No non-deprecated ${dep.npmPackage} version satisfies ${dep.range}`); + } + return resolved; +} + +/** Write-to-temp + rename; a concurrent winner is accepted (its bytes passed the same verification). */ +function atomicWriteFile(path: string, bytes: Buffer): void { + const temporary = `${path}.tmp.${process.pid}.${randomUUID()}`; + try { + writeFileSync(temporary, bytes); + renameSync(temporary, path); + } catch (error) { + rmSync(temporary, { force: true }); + if (!existsSync(path)) throw error; + } +} + +/** + * Download the resolved tarball, verify its integrity before extracting, and + * install the dep's files into the version dir. The entrypoint is written + * LAST: its presence marks the version dir complete, so a crash mid-install + * can never leave an importable entrypoint missing its sidecars. + */ +async function downloadBundle(dep: RuntimeDep, resolved: ResolvedVersion, versionDir: string): Promise { + const response = await fetch(resolved.tarballUrl, { signal: AbortSignal.timeout(TARBALL_TIMEOUT_MS) }); + if (!response.ok) { + throw new Error(`HTTP ${response.status} downloading ${resolved.tarballUrl}`); + } + const tarball = Buffer.from(await response.arrayBuffer()); + // Verify BEFORE extracting; nothing unverified is ever written to the cache. + // The tarball-wide sha512 covers every extracted file in one check. + verifySriIntegrity(tarball, resolved.integrity); + // Extract everything up front so a missing entry aborts before any writes. + const [entry, ...sidecars] = dep.files.map((file, index) => ({ + // npm tarballs prefix all entries with `package/`. + bytes: extractTarEntry(tarball, `package/${file}`, MAX_BUNDLE_UNCOMPRESSED_BYTES), + installedName: installedFileName(file, index === 0), + })); + + mkdirSync(versionDir, { recursive: true, mode: 0o700 }); + for (const file of [...sidecars, entry]) { + atomicWriteFile(join(versionDir, file.installedName), file.bytes); + } +} + +// Only reap version dirs untouched for this long — a fresh sibling may belong +// to a concurrently running other-version CLI. Mirrors agent-sdk-assets.ts. +const STALE_VERSION_MS = 24 * 60 * 60 * 1000; + +/** Best-effort reap of superseded version dirs after a successful install. */ +function cleanupStaleVersions(cacheDir: string, currentVersion: string): void { + let entries: string[]; + try { + entries = readdirSync(cacheDir); + } catch { + return; + } + const cutoff = Date.now() - STALE_VERSION_MS; + for (const entry of entries) { + // Only semver-named version dirs; never resolution.json or temp files. + if (entry === currentVersion || valid(entry) === null) continue; + const path = join(cacheDir, entry); + try { + if (statSync(path).mtimeMs >= cutoff) continue; + rmSync(path, { recursive: true, force: true }); + } catch { + // In use, already gone, or unreadable — skip it. + } + } +} + +/** Newest already-downloaded version inside the baked range, for offline runs. */ +function newestDownloadedVersion(cacheDir: string, dep: RuntimeDep): string | null { + let entries: string[]; + try { + entries = readdirSync(cacheDir); + } catch { + return null; + } + const candidates = entries.filter( + (entry) => valid(entry) !== null && satisfies(entry, dep.range) && existsSync(entryPath(cacheDir, entry, dep)), + ); + return maxSatisfying(candidates, dep.range); +} + +async function importBundle(bundlePath: string): Promise> { + // Runtime-computed dynamic import: Bun leaves this as a real runtime import + // in compiled binaries (verified on Bun 1.3.11), so the cached bundle loads + // from outside the binary's virtual filesystem. + return (await import(pathToFileURL(bundlePath).href)) as Record; +} + +/** + * Core loader, parameterized by dep so tests can drive it with fixtures. + * Returns the imported bundle's module namespace, or null when the caller + * should use the compiled-in module instead. Never throws. Exported for tests + * — production callers go through loadRuntimeBundle. + */ +export async function loadRuntimeDep( + dep: RuntimeDep, + options: LoadRuntimeBundleOptions = {}, +): Promise | null> { + const setting = process.env.WORKOS_RUNTIME_DEPS; + // Kill switch: compiled-in only, no network, no cache reads. + if (setting === '0') return null; + // Running from source (dev, tests, evals): node_modules already provides the + // packages, so stay off the network unless explicitly forced on. + if (!isCompiledBinary() && setting !== '1') return null; + + const cacheDir = join(options.cacheRoot ?? defaultCacheRoot(), dep.name); + try { + let resolved = readResolutionCache(cacheDir, dep.range); + if (!resolved) { + try { + resolved = await fetchResolution(dep); + writeResolutionCache(cacheDir, resolved); + } catch (error) { + logWarn(`Version resolution for runtime dep ${dep.npmPackage} failed:`, error); + resolved = null; + } + } + + if (resolved) { + try { + const bundlePath = entryPath(cacheDir, resolved.version, dep); + if (existsSync(bundlePath)) return await importBundle(bundlePath); + if (!resolved.installFailed) { + await downloadBundle(dep, resolved, join(cacheDir, resolved.version)); + cleanupStaleVersions(cacheDir, resolved.version); + return await importBundle(bundlePath); + } + } catch (error) { + logWarn(`Runtime bundle install for ${dep.npmPackage}@${resolved.version} failed:`, error); + // Remember the failure so every invocation inside the TTL window + // doesn't re-download a tarball that can't install (e.g. the package + // hasn't published a bundle yet). Retried after the TTL expires. + writeResolutionCache(cacheDir, { ...resolved, installFailed: true }); + } + } + + // Resolution or install failed — the newest previously downloaded version + // (verified when it was installed) still works offline. + const fallback = newestDownloadedVersion(cacheDir, dep); + if (!fallback) return null; + return await importBundle(entryPath(cacheDir, fallback, dep)); + } catch (error) { + logWarn(`Runtime bundle for ${dep.npmPackage} unavailable; using the compiled-in module:`, error); + return null; + } +} + +const moduleCache = new Map | null>(); + +/** + * Load the runtime-downloaded bundle for a manifest dep, memoized per process. + * Returns null whenever the compiled-in module should be used instead; callers + * validate the export shape they need and fall back themselves, keeping the + * compiled-in types as the compile-time contract. + */ +export async function loadRuntimeBundle( + name: RuntimeDepName, + options: LoadRuntimeBundleOptions = {}, +): Promise | null> { + const cached = moduleCache.get(name); + if (cached !== undefined) return cached; + const loaded = await loadRuntimeDep({ ...RUNTIME_DEPS[name], name }, options); + moduleCache.set(name, loaded); + return loaded; +}