From 45c7bcdf12aea8911fd7e39c12e46d5bc365df4f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:19:46 -0700 Subject: [PATCH 01/30] perf(core): retain workflow VM across inline steps Combines the retained-session architecture from #2984 with the env kill switch and loop-level single-VM test from #2966. - executeWorkflow with discriminated request/result types and a WorkflowSession state machine (running/suspended/failed/replay/completed) - EventsConsumer.append: only newly durable events feed the live VM - WORKFLOW_RETAINED_VM=0 kill switch (default on) - retained-vm-loop.test.ts: proves one VM per run and byte-identical output vs the from-scratch replay path --- .changeset/retain-workflow-vm.md | 6 + .../docs/v5/configuration/runtime-tuning.mdx | 7 + packages/core/src/events-consumer.test.ts | 27 ++ packages/core/src/events-consumer.ts | 7 +- packages/core/src/retained-vm-loop.test.ts | 178 +++++++++ packages/core/src/runtime.test.ts | 9 + packages/core/src/runtime.ts | 120 ++++-- packages/core/src/runtime/constants.ts | 17 + packages/core/src/telemetry.ts | 20 +- .../src/telemetry/semantic-conventions.ts | 5 + .../src/workflow-session-telemetry.test.ts | 69 ++++ packages/core/src/workflow.test.ts | 342 ++++++++++++++++- packages/core/src/workflow.ts | 356 +++++++++++++++--- 13 files changed, 1069 insertions(+), 94 deletions(-) create mode 100644 .changeset/retain-workflow-vm.md create mode 100644 packages/core/src/retained-vm-loop.test.ts create mode 100644 packages/core/src/workflow-session-telemetry.test.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md new file mode 100644 index 0000000000..37b874325e --- /dev/null +++ b/.changeset/retain-workflow-vm.md @@ -0,0 +1,6 @@ +--- +'@workflow/core': patch +'workflow': patch +--- + +Retain workflow execution across inline steps within one invocation. `WORKFLOW_RETAINED_VM=0` disables retention. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 34603add1e..f2f1a98e53 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -75,6 +75,13 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Use only when step side effects are idempotent. - Set `0` or `false` to force it off, including the first-delivery fast path used by `WORKFLOW_TURBO`. +### `WORKFLOW_RETAINED_VM` + +- Default: enabled +- Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. +- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Set `0` or `false` to replay from scratch in a fresh VM on every iteration. + ### `WORKFLOW_INLINE_OWNERSHIP` - Default: enabled diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index ecf828f728..3e320b16c7 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -45,6 +45,15 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(0); expect(consumer.callbacks).toEqual([]); }); + + it('should own its event array', () => { + const events = [createMockEvent()]; + const consumer = new EventsConsumer(events, defaultOptions); + + events.push(createMockEvent({ id: 'event-2' })); + + expect(consumer.events).toHaveLength(1); + }); }); describe('subscribe', () => { @@ -82,6 +91,24 @@ describe('EventsConsumer', () => { expect(callback).toHaveBeenCalledWith(event); expect(callback).toHaveBeenCalledTimes(1); }); + + it('should consume appended events', async () => { + const event = createMockEvent(); + const consumer = new EventsConsumer([], defaultOptions); + const callback = vi + .fn() + .mockImplementation((value: Event | null) => + value ? EventConsumerResult.Finished : EventConsumerResult.NotConsumed + ); + consumer.subscribe(callback); + await waitForNextTick(); + + consumer.append([event]); + await waitForNextTick(); + + expect(callback).toHaveBeenLastCalledWith(event); + expect(consumer.eventIndex).toBe(1); + }); }); describe('consume (implicit)', () => { diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index 4b7cf0742e..7b4afd6a9a 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -79,13 +79,18 @@ export class EventsConsumer { private unconsumedCheckVersion = 0; constructor(events: Event[], options: EventsConsumerOptions) { - this.events = events; + this.events = [...events]; this.eventIndex = 0; this.onConsumedEvent = options.onConsumedEvent; this.onUnconsumedEvent = options.onUnconsumedEvent; this.getPromiseQueue = options.getPromiseQueue; } + append(events: Event[]): void { + this.events.push(...events); + process.nextTick(this.consume); + } + /** * Registers a callback function to be called after an event has been consumed * by a different callback. The callback can return: diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts new file mode 100644 index 0000000000..d30740ba69 --- /dev/null +++ b/packages/core/src/retained-vm-loop.test.ts @@ -0,0 +1,178 @@ +import { + type Event, + SPEC_VERSION_CURRENT, + type WorkflowRun, +} from '@workflow/world'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +// Spy on VM-context construction while preserving the real implementation, so +// we can prove the retained path builds ONE VM for a whole run instead of one +// per replay iteration. workflow.ts imports `createContext` from this same +// module, so the spy observes its constructions too. +vi.mock('./vm/index.js', async (importActual) => { + const actual = await importActual(); + return { ...actual, createContext: vi.fn(actual.createContext) }; +}); + +const { createContext } = await import('./vm/index.js'); +const { registerStepFunction } = await import('./private.js'); +const { setWorld } = await import('./runtime/world.js'); +const { workflowEntrypoint } = await import('./runtime.js'); +const { dehydrateWorkflowArguments } = await import('./serialization.js'); + +vi.mock('@vercel/functions', () => ({ + waitUntil: vi.fn((p: Promise) => { + p.catch(() => {}); + }), +})); + +const createContextSpy = createContext as unknown as ReturnType; + +// Sequential two-step workflow: two replay-advancing suspensions before it +// completes — so a from-scratch replay builds the VM three times while the +// retained path builds it once. +const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +registerStepFunction('r_s1', async () => 10); +registerStepFunction('r_s2', async () => 20); + +// Drive the full workflow handler over a stateful (dynamic) event log so the +// inline loop makes real progress across its own writes, exactly like a World. +// Non-turbo (no runInput, attempt 2) to keep the path simple and deterministic. +async function drive(runId: string) { + const run: WorkflowRun = { + runId, + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments([], runId, undefined, []), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const events: Event[] = []; + const createdEvents: any[] = []; + let seq = 0; + + const eventsCreate = vi.fn(async (_runId: string, data: any) => { + createdEvents.push(data); + if (data.eventType === 'run_started') { + return { run, events }; + } + const event = { + eventId: `e-${++seq}`, + runId, + createdAt: new Date(), + ...data, + } as Event; + events.push(event); + // step_started returns a running step entity so executeStep proceeds to + // run the body and write step_completed. + if (data.eventType === 'step_started') { + const d = data.eventData as { stepName?: string; input?: unknown }; + return { + event, + step: { + runId, + stepId: data.correlationId, + stepName: d?.stepName, + status: 'running' as const, + attempt: 1, + input: d?.input, + startedAt: new Date(), + createdAt: new Date(), + updatedAt: new Date(), + }, + ...(d?.input !== undefined ? { stepCreated: true } : {}), + }; + } + return { event }; + }); + + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + createQueueHandler: vi.fn( + (_p: string, handler: (m: unknown, md: unknown) => Promise) => + async () => { + await handler( + { runId, requestedAt: new Date('2024-01-01T00:00:00.000Z') }, + { + requestId: 'req_retained', + attempt: 2, + queueName: '__wkf_workflow_workflow', + messageId: 'msg_retained', + } + ); + return new Response(null, { status: 204 }); + } + ), + events: { + create: eventsCreate, + list: vi.fn(async () => ({ + data: [...events], + hasMore: false, + cursor: 'cursor_retained', + })), + }, + runs: { get: vi.fn(async () => run) }, + queue: vi.fn(async () => ({ messageId: null })), + getEncryptionKeyForRun: vi.fn(async () => undefined), + } as any); + + await workflowEntrypoint(twoStepWorkflow)( + new Request('https://example.test') + ); + + const runCompleted = createdEvents.find( + (e) => e.eventType === 'run_completed' + ); + return { + vmBuilds: createContextSpy.mock.calls.length, + output: runCompleted?.eventData?.output as Uint8Array | undefined, + }; +} + +describe('retained VM through the inline replay loop', () => { + beforeEach(() => { + createContextSpy.mockClear(); + }); + afterEach(() => { + delete process.env.WORKFLOW_RETAINED_VM; + setWorld(undefined); + vi.clearAllMocks(); + }); + + it('rebuilds the VM once per replay under the kill switch (WORKFLOW_RETAINED_VM=0)', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const { vmBuilds, output } = await drive('wrun_retained_off'); + expect(output).toBeInstanceOf(Uint8Array); + // A 2-step sequential workflow suspends twice then completes → >1 replay, + // each building a fresh VM. + expect(vmBuilds).toBeGreaterThan(1); + }); + + it('builds the VM once and resumes it when retention is ON (the default)', async () => { + // Baseline via the kill switch, to compare the dehydrated result against. + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive('wrun_retained_baseline_off'); + createContextSpy.mockClear(); + + // Unset → retention is ON by default. + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive('wrun_retained_on'); + + expect(on.output).toBeInstanceOf(Uint8Array); + // The whole run is served by a single VM: built once on the first pass, + // resumed (not rebuilt) for every subsequent step. + expect(on.vmBuilds).toBe(1); + // And it produces the identical dehydrated result as the replay path. + expect(on.output).toEqual(off.output); + }); +}); diff --git a/packages/core/src/runtime.test.ts b/packages/core/src/runtime.test.ts index aafec00075..1e229e2fec 100644 --- a/packages/core/src/runtime.test.ts +++ b/packages/core/src/runtime.test.ts @@ -11,6 +11,7 @@ import { } from '@workflow/world'; import { ulid } from 'ulid'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runtimeLogger } from './logger.js'; import { registerStepFunction } from './private.js'; import { REPLAY_DIVERGENCE_MAX_RETRIES } from './runtime/constants.js'; import { setWorld } from './runtime/world.js'; @@ -702,6 +703,9 @@ describe('workflowEntrypoint replay guards', () => { }); it('replays attribute events before executing a step that loses the same race', async () => { + const debug = vi + .spyOn(runtimeLogger, 'debug') + .mockImplementation(() => undefined); const ops: Promise[] = []; const workflowRun: WorkflowRun = { runId: 'wrun_attribute_step_race', @@ -760,6 +764,11 @@ describe('workflowEntrypoint replay guards', () => { expect(createdEvents).not.toContainEqual( expect.objectContaining({ eventType: 'step_started' }) ); + const executionModes = debug.mock.calls + .filter(([message]) => message === 'Starting workflow execution') + .map(([, context]) => context?.executionMode); + expect(executionModes).toEqual(['replay', 'replay']); + debug.mockRestore(); }); it('fails the run when the World rejects an attr_set event as invalid', async () => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 8780818421..a085c44884 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -40,6 +40,7 @@ import { getReplayDivergenceMaxRetries, isInlineOwnershipEnabled, isTurboEnabled, + isVmRetentionEnabled, } from './runtime/constants.js'; import { getQueueOverhead, @@ -91,7 +92,11 @@ import { } from './telemetry.js'; import { getErrorName, getErrorStack, normalizeUnknownError } from './types.js'; import { buildWorkflowSuspensionMessage } from './util.js'; -import { runWorkflow } from './workflow.js'; +import { + executeWorkflow, + type WorkflowExecutionResult, + type WorkflowSession, +} from './workflow.js'; export type { Event, WorkflowRun }; export { WorkflowSuspension } from './global.js'; @@ -311,6 +316,27 @@ function openHookAndWaitState(events: Event[]): { return { openHook, openWait }; } +/** + * Retain only pure step boundaries with no out-of-band continuation source. + * Attributes require replay; hooks and waits can wake another invocation. + * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + */ +function canRetainWorkflowSession( + suspension: WorkflowSuspension, + events: Event[] +): boolean { + return ( + isVmRetentionEnabled() && + suspension.stepCount > 0 && + suspension.hookCount === 0 && + suspension.waitCount === 0 && + suspension.attributeCount === 0 && + suspension.hookDisposedCount === 0 && + suspension.abortCount === 0 && + !hasOpenHookOrWait(events) + ); +} + /** * Creates a single route which handles workflow execution requests, * executing steps inline when possible to reduce function invocations @@ -1105,6 +1131,13 @@ export function workflowEntrypoint( encryptionKey ); + let workflowExecution: + | { readonly type: 'replay' } + | { + readonly type: 'retained'; + readonly session: WorkflowSession; + } = { type: 'replay' }; + // Main replay loop // biome-ignore lint/correctness/noConstantCondition: intentional loop while (true) { @@ -1419,37 +1452,76 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; - // Replay workflow - runtimeLogger.debug('Starting workflow replay', { + let executionMode = workflowExecution.type; + runtimeLogger.debug('Starting workflow execution', { workflowRunId: runId, loopIteration, eventCount: events.length, + executionMode, }); replayStart = Date.now(); - // Start every missing decrypt/decompress operation before - // VM setup. Web Crypto work can overlap bundle evaluation; - // consumers still deserialize and resolve in event order. - const payloadPrewarm = replayPayloadCache.prewarm( - workflowRun, - events - ); - const result = await runWorkflow( - workflowCode, - workflowRun, - events, - encryptionKey, - replayPayloadCache, - // Turbo: the end-of-run drain inside runWorkflow commits - // fire-and-forget `*_created` events before the terminal - // `awaitRunReady()` below, so gate those writes on the - // backgrounded run_started too. Undefined outside turbo. - runReadyBarrier - ); - await payloadPrewarm; - runtimeLogger.debug('Workflow replay completed', { + let workflowResult: WorkflowExecutionResult = { + type: 'replay', + }; + if (workflowExecution.type === 'retained') { + workflowResult = await executeWorkflow({ + type: 'resume', + session: workflowExecution.session, + events, + }); + } + + if (workflowResult.type === 'replay') { + executionMode = 'replay'; + workflowExecution = { type: 'replay' }; + // Start every missing decrypt/decompress operation + // before VM setup. Web Crypto work can overlap bundle + // evaluation; consumers still deserialize and resolve + // in event order. + const payloadPrewarm = replayPayloadCache.prewarm( + workflowRun, + events + ); + workflowResult = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + // Turbo: the end-of-run drain inside workflow + // execution commits fire-and-forget `*_created` + // events before the terminal `awaitRunReady()` below. + runReadyBarrier, + }); + await payloadPrewarm; + } + + if (workflowResult.type === 'suspended') { + workflowExecution = canRetainWorkflowSession( + workflowResult.suspension, + events + ) + ? { + type: 'retained', + session: workflowResult.session, + } + : { type: 'replay' }; + throw workflowResult.suspension; + } + + if (workflowResult.type === 'replay') { + throw new Error( + 'Invariant violation: fresh workflow execution requested another replay' + ); + } + + const result = workflowResult.output; + runtimeLogger.debug('Workflow execution completed', { workflowRunId: runId, loopIteration, replayMs: Date.now() - replayStart, + executionMode, }); // Workflow completed. Send the snapshot but do NOT diff --git a/packages/core/src/runtime/constants.ts b/packages/core/src/runtime/constants.ts index 8db7a3f723..b72bed773b 100644 --- a/packages/core/src/runtime/constants.ts +++ b/packages/core/src/runtime/constants.ts @@ -275,6 +275,23 @@ export function isTurboEnabled(): boolean { return !(raw === '0' || raw.toLowerCase() === 'false'); } +/** + * Whether the inline loop retains a suspended workflow VM across inline steps + * within one invocation (default ON). When on, a step-only suspension keeps + * the live VM, event consumer, and hydrated state alive, and the next loop + * iteration appends only the newly durable events instead of rebuilding the + * `vm.Context` and replaying the whole event log. Non-step suspensions and + * replay divergence always fall back to the ordinary durable replay path. + * + * `WORKFLOW_RETAINED_VM=0` (or `false`) is the kill switch: every iteration + * replays from scratch in a fresh VM, matching the pre-retention behavior. + */ +export function isVmRetentionEnabled(): boolean { + const raw = process.env.WORKFLOW_RETAINED_VM; + if (raw === undefined || raw === '') return true; + return !(raw === '0' || raw.toLowerCase() === 'false'); +} + /** * Whether inline step ownership is enabled (default ON). When on, the lazy * `step_started` that creates an inline step records the owning queue diff --git a/packages/core/src/telemetry.ts b/packages/core/src/telemetry.ts index da69d2756d..0c3a32f9d5 100644 --- a/packages/core/src/telemetry.ts +++ b/packages/core/src/telemetry.ts @@ -249,7 +249,10 @@ export async function trace( code: otel.SpanStatusCode.ERROR, message: (e as Error).message, }); - applyWorkflowSuspensionToSpan(e, otel, span); + if (WorkflowSuspension.is(e)) { + span.setStatus({ code: otel.SpanStatusCode.OK }); + applyWorkflowSuspensionToSpan(e, span); + } throw e; } finally { span.end(); @@ -275,19 +278,12 @@ export async function recordElapsedSpan( } /** - * Applies workflow suspension attributes to the given span if the error is a WorkflowSuspension - * which is technically not an error, but an algebraic effect indicating suspension. + * Applies the workflow suspension algebraic effect to an active span. */ -function applyWorkflowSuspensionToSpan( - error: unknown, - otel: typeof api, +export function applyWorkflowSuspensionToSpan( + error: WorkflowSuspension, span: api.Span -) { - if (!error || !WorkflowSuspension.is(error)) { - return; - } - - span.setStatus({ code: otel.SpanStatusCode.OK }); +): void { span.setAttributes({ ...Attr.WorkflowSuspensionState('suspended'), ...Attr.WorkflowSuspensionStepCount(error.stepCount), diff --git a/packages/core/src/telemetry/semantic-conventions.ts b/packages/core/src/telemetry/semantic-conventions.ts index bcc6d1dabf..0793b26d87 100644 --- a/packages/core/src/telemetry/semantic-conventions.ts +++ b/packages/core/src/telemetry/semantic-conventions.ts @@ -77,6 +77,11 @@ export const WorkflowEventsCount = SemanticConvention( 'workflow.events.count' ); +/** Whether workflow execution starts with replay or resumes a retained VM */ +export const WorkflowExecutionMode = SemanticConvention<'replay' | 'retained'>( + 'workflow.execution.mode' +); + /** Number of arguments passed to the workflow */ export const WorkflowArgumentsCount = SemanticConvention( 'workflow.arguments.count' diff --git a/packages/core/src/workflow-session-telemetry.test.ts b/packages/core/src/workflow-session-telemetry.test.ts new file mode 100644 index 0000000000..49058a4585 --- /dev/null +++ b/packages/core/src/workflow-session-telemetry.test.ts @@ -0,0 +1,69 @@ +import { trace as otelTrace } from '@opentelemetry/api'; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import type { WorkflowRun } from '@workflow/world'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { ReplayPayloadCache } from './replay-payload-cache.js'; +import { dehydrateWorkflowArguments } from './serialization.js'; +import { executeWorkflow } from './workflow.js'; + +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider(); + +beforeAll(() => { + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + otelTrace.setGlobalTracerProvider(provider); +}); + +afterAll(async () => { + await provider.shutdown(); + otelTrace.disable(); +}); + +afterEach(() => { + exporter.reset(); +}); + +describe('retained workflow telemetry', () => { + it('records suspension attributes on workflow.run spans', async () => { + const runId = 'wrun_retained_telemetry'; + const workflowRun: WorkflowRun = { + runId, + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments([], runId, undefined, []), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + + const result = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events: [], + encryptionKey: undefined, + replayPayloadCache: new ReplayPayloadCache(undefined), + }); + expect(result.type).toBe('suspended'); + + const span = exporter + .getFinishedSpans() + .find((candidate) => candidate.name === 'workflow.run workflow'); + expect(span?.attributes).toMatchObject({ + 'workflow.execution.mode': 'replay', + 'workflow.suspension.state': 'suspended', + 'workflow.suspension.step_count': 1, + 'workflow.suspension.hook_count': 0, + 'workflow.suspension.wait_count': 0, + }); + }); +}); diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index fc21423518..66cc295e27 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -6,6 +6,7 @@ import { monotonicFactory } from 'ulid'; import { afterEach, assert, describe, expect, it, vi } from 'vitest'; import { DEFERRED_CHECK_DELAY_MS } from './events-consumer.js'; import type { WorkflowSuspension } from './global.js'; +import { ReplayPayloadCache } from './replay-payload-cache.js'; import { setWorld } from './runtime/world.js'; import { dehydrateStepReturnValue, @@ -13,7 +14,7 @@ import { hydrateWorkflowReturnValue, } from './serialization.js'; import { createContext } from './vm/index.js'; -import { runWorkflow } from './workflow.js'; +import { executeWorkflow, runWorkflow } from './workflow.js'; // No encryption key = encryption disabled const noEncryptionKey = undefined; @@ -221,6 +222,345 @@ describe('runWorkflow', () => { ).toEqual(3); }); + it('should retain workflow execution across sequential step suspensions', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const add = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("add"); + async function workflow() { + console.log("retained:entered"); + const first = await add(1, 2); + console.log("retained:continued"); + const second = await add(first, 3); + return second; + } + ${getWorkflowTransformCode('workflow')}`; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + const events: Event[] = []; + let eventNumber = 0; + const appendStepEvents = async ( + correlationId: string, + result: number + ): Promise => { + const createdAt = new Date(`2024-01-01T00:00:0${eventNumber + 1}.000Z`); + events.push( + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_created', + correlationId, + eventData: { stepName: 'add' }, + createdAt, + }, + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_started', + correlationId, + eventData: { stepName: 'add' }, + createdAt, + }, + { + eventId: `event-${eventNumber++}`, + runId: workflowRun.runId, + eventType: 'step_completed', + correlationId, + eventData: { + stepName: 'add', + result: await dehydrateStepReturnValue( + result, + workflowRun.runId, + noEncryptionKey, + ops + ), + }, + createdAt, + } + ); + }; + + const first = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(first.type === 'suspended'); + const firstStep = first.suspension.steps[0]; + assert(firstStep?.type === 'step'); + + await appendStepEvents(firstStep.correlationId, 3); + + const second = await executeWorkflow({ + type: 'resume', + session: first.session, + events, + }); + assert(second.type === 'suspended'); + expect(second.session).toBe(first.session); + const secondStep = second.suspension.steps[0]; + assert(secondStep?.type === 'step'); + + await appendStepEvents(secondStep.correlationId, 6); + + const completed = await executeWorkflow({ + type: 'resume', + session: second.session, + events, + }); + assert(completed.type === 'completed'); + + expect( + await hydrateWorkflowReturnValue( + completed.output as any, + workflowRun.runId, + noEncryptionKey, + ops + ) + ).toBe(6); + expect( + log.mock.calls + .map(([message]) => message) + .filter((message) => String(message).startsWith('retained:')) + ).toEqual(['retained:entered', 'retained:continued']); + log.mockRestore(); + }); + + it('returns a workflow that completes while its retained session is suspended', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_async_completion', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_async_completion', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function digestRepeatedly() { + const bytes = new Uint8Array(8 * 1024 * 1024); + for (let index = 0; index < 8; index++) { + await crypto.subtle.digest("SHA-256", bytes); + } + } + async function workflow() { + await Promise.race([step(), digestRepeatedly()]); + console.log("retained:digest-completed"); + return "digest completed"; + } + ${getWorkflowTransformCode('workflow')}`; + const events: Event[] = [ + { + eventId: 'event-run-created', + runId: workflowRun.runId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + ]; + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + const pendingStep = suspended.suspension.steps[0]; + assert(pendingStep?.type === 'step'); + + const rewrittenSession = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(rewrittenSession.type === 'suspended'); + + await vi.waitFor( + () => { + expect(log).toHaveBeenCalledTimes(2); + }, + { timeout: 10_000, interval: 10 } + ); + const rewrittenEvents = [ + { ...events[0], eventId: 'rewritten-event' } as Event, + { ...events[0], eventId: 'new-event' } as Event, + ]; + expect( + await executeWorkflow({ + type: 'resume', + session: rewrittenSession.session, + events: rewrittenEvents, + }) + ).toEqual({ type: 'replay' }); + + const createdAt = new Date('2024-01-01T00:00:01.000Z'); + events.push( + { + eventId: 'event-step-created', + runId: workflowRun.runId, + eventType: 'step_created', + correlationId: pendingStep.correlationId, + eventData: { stepName: pendingStep.stepName }, + createdAt, + }, + { + eventId: 'event-step-started', + runId: workflowRun.runId, + eventType: 'step_started', + correlationId: pendingStep.correlationId, + eventData: { stepName: pendingStep.stepName }, + createdAt, + }, + { + eventId: 'event-step-completed', + runId: workflowRun.runId, + eventType: 'step_completed', + correlationId: pendingStep.correlationId, + eventData: { + stepName: pendingStep.stepName, + result: await dehydrateStepReturnValue( + undefined, + workflowRun.runId, + noEncryptionKey, + ops + ), + }, + createdAt, + } + ); + + const completed = await executeWorkflow({ + type: 'resume', + session: suspended.session, + events, + }); + assert(completed.type === 'completed'); + expect( + await executeWorkflow({ + type: 'resume', + session: rewrittenSession.session, + events, + }) + ).toEqual({ type: 'replay' }); + expect( + await hydrateWorkflowReturnValue( + completed.output as any, + workflowRun.runId, + noEncryptionKey, + ops + ) + ).toBe('digest completed'); + log.mockRestore(); + }); + + it('does not drain operations from a discarded retained session', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_discarded', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_discarded', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; + async function digestRepeatedly() { + const bytes = new Uint8Array(8 * 1024 * 1024); + for (let index = 0; index < 8; index++) { + await crypto.subtle.digest("SHA-256", bytes); + } + } + async function workflow() { + await Promise.race([step(), digestRepeatedly()]); + void setAttributes([{ key: "stale", value: true }]); + console.log("retained:discarded-completed"); + return "discarded"; + } + ${getWorkflowTransformCode('workflow')}`; + const create = vi.fn(); + const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); + setWorld({ + specVersion: SPEC_VERSION_CURRENT, + events: { create }, + streams: { write: vi.fn(), close: vi.fn() }, + } as any); + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events: [], + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + + await vi.waitFor( + () => { + expect(log).toHaveBeenCalledWith('retained:discarded-completed'); + }, + { timeout: 10_000, interval: 10 } + ); + await new Promise((resolve) => setTimeout(resolve, 250)); + + expect( + await executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [ + { + eventId: 'event-run-created', + runId: workflowRun.runId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + ], + }) + ).toEqual({ type: 'replay' }); + expect(create).not.toHaveBeenCalled(); + setWorld(undefined); + log.mockRestore(); + }); + it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { // Turbo's first delivery synthesizes `startedAt` from the local clock, // while later (non-turbo) deliveries load the server-canonical `startedAt`. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4595f6a17c..1fe6668470 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -4,7 +4,11 @@ import { WorkflowNotRegisteredError, WorkflowRuntimeError, } from '@workflow/errors'; -import { createWorkflowBaseUrl, withResolvers } from '@workflow/utils'; +import { + createWorkflowBaseUrl, + type PromiseWithResolvers, + withResolvers, +} from '@workflow/utils'; import { parseWorkflowName } from '@workflow/utils/parse-name'; import type { Event, WorkflowRun } from '@workflow/world'; import { SPEC_VERSION_SUPPORTS_COMPRESSION } from '@workflow/world'; @@ -36,7 +40,7 @@ import { WORKFLOW_USE_STEP, } from './symbols.js'; import * as Attribute from './telemetry/semantic-conventions.js'; -import { trace } from './telemetry.js'; +import { applyWorkflowSuspensionToSpan, trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; import { createContext } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; @@ -133,6 +137,142 @@ async function drainPendingQueueItems( } } +interface WorkflowCompletion { + readonly output: unknown; + readonly resultType: string; +} + +type WorkflowSessionState = + | { + readonly type: 'running'; + readonly interruption: PromiseWithResolvers; + } + | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } + | { readonly type: 'failed'; readonly error: Error } + | { readonly type: 'replay' } + | { readonly type: 'completed' }; + +function isSameSuspensionBoundary( + previous: WorkflowSuspension, + next: WorkflowSuspension +): boolean { + return ( + previous.stepCount === next.stepCount && + previous.hookCount === next.hookCount && + previous.waitCount === next.waitCount && + previous.attributeCount === next.attributeCount && + previous.hookDisposedCount === next.hookDisposedCount && + previous.abortCount === next.abortCount && + previous.steps.length === next.steps.length && + previous.steps.every((item, index) => item === next.steps[index]) + ); +} + +function updateSuspendedSession( + suspension: WorkflowSuspension, + error: Error +): WorkflowSessionState { + if (!WorkflowSuspension.is(error)) return { type: 'failed', error }; + return isSameSuspensionBoundary(suspension, error) + ? { type: 'suspended', suspension } + : { type: 'replay' }; +} + +export interface WorkflowSession { + readonly workflowRun: WorkflowRun; + readonly argumentCount: number; + resume(events: Event[]): WorkflowSessionResumeResult; +} + +type WorkflowSessionResumeResult = + | { + readonly type: 'resumed'; + readonly execution: Promise; + } + | { readonly type: 'replay' }; + +interface WorkflowSessionOptions { + readonly workflowCode: string; + readonly workflowRun: WorkflowRun; + readonly events: Event[]; + readonly encryptionKey: CryptoKey | undefined; + readonly replayPayloadCache: ReplayPayloadCache; + readonly runReadyBarrier?: Promise; +} + +type WorkflowExecutionRequest = + | ({ readonly type: 'replay' } & WorkflowSessionOptions) + | { + readonly type: 'resume'; + readonly session: WorkflowSession; + readonly events: Event[]; + }; + +export type WorkflowExecutionResult = + | { readonly type: 'replay' } + | { readonly type: 'completed'; readonly output: unknown } + | { + readonly type: 'suspended'; + readonly suspension: WorkflowSuspension; + readonly session: WorkflowSession; + }; + +export async function executeWorkflow( + request: WorkflowExecutionRequest +): Promise { + const workflowRun = + request.type === 'replay' + ? request.workflowRun + : request.session.workflowRun; + const mode = request.type === 'replay' ? 'replay' : 'retained'; + + return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { + span?.setAttributes({ + ...Attribute.WorkflowName(workflowRun.workflowName), + ...Attribute.WorkflowRunId(workflowRun.runId), + ...Attribute.WorkflowRunStatus(workflowRun.status), + ...Attribute.WorkflowEventsCount(request.events.length), + ...Attribute.WorkflowExecutionMode(mode), + }); + + let session: WorkflowSession; + let execution: Promise; + switch (request.type) { + case 'replay': { + const started = await createWorkflowSession(request); + session = started.session; + execution = started.execution; + break; + } + case 'resume': { + session = request.session; + const resumed = session.resume(request.events); + if (resumed.type === 'replay') return resumed; + execution = resumed.execution; + break; + } + } + + span?.setAttributes({ + ...Attribute.WorkflowArgumentsCount(session.argumentCount), + }); + + try { + const completed = await execution; + span?.setAttributes({ + ...Attribute.WorkflowResultType(completed.resultType), + }); + return { type: 'completed', output: completed.output }; + } catch (error) { + if (WorkflowSuspension.is(error)) { + if (span) applyWorkflowSuspensionToSpan(error, span); + return { type: 'suspended', suspension: error, session }; + } + throw error; + } + }); +} + export async function runWorkflow( workflowCode: string, workflowRun: WorkflowRun, @@ -140,8 +280,7 @@ export async function runWorkflow( encryptionKey: CryptoKey | undefined, /** * Optional per-run cache for replay payload preparation and immutable final - * values. Owned by the inline replay loop so it survives fresh VM contexts - * created by successive iterations of this invocation. + * values. Owned by the inline execution loop for this invocation. */ replayPayloadCache: ReplayPayloadCache = new ReplayPayloadCache( encryptionKey @@ -154,14 +293,36 @@ export async function runWorkflow( */ runReadyBarrier?: Promise ): Promise { - return trace(`workflow.run ${workflowRun.workflowName}`, async (span) => { - span?.setAttributes({ - ...Attribute.WorkflowName(workflowRun.workflowName), - ...Attribute.WorkflowRunId(workflowRun.runId), - ...Attribute.WorkflowRunStatus(workflowRun.status), - ...Attribute.WorkflowEventsCount(events.length), - }); + const result = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + runReadyBarrier, + }); + if (result.type === 'replay') { + throw new WorkflowRuntimeError( + `Fresh workflow "${workflowRun.runId}" unexpectedly requested replay` + ); + } + if (result.type === 'suspended') throw result.suspension; + return result.output; +} +function createWorkflowSession({ + workflowCode, + workflowRun, + events, + encryptionKey, + replayPayloadCache, + runReadyBarrier, +}: WorkflowSessionOptions): Promise<{ + session: WorkflowSession; + execution: Promise; +}> { + return (async () => { const startedAt = workflowRun.startedAt; if (!startedAt) { throw new WorkflowRuntimeError( @@ -209,7 +370,32 @@ export async function runWorkflow( fixedTimestamp, }); - const workflowDiscontinuation = withResolvers(); + const initialInterruption = withResolvers(); + let state: WorkflowSessionState = { + type: 'running', + interruption: initialInterruption, + }; + + const onWorkflowError = (error: Error): void => { + switch (state.type) { + case 'running': { + const { interruption } = state; + state = WorkflowSuspension.is(error) + ? { type: 'suspended', suspension: error } + : { type: 'failed', error }; + interruption.reject(error); + return; + } + case 'suspended': + state = updateSuspendedSession(state.suspension, error); + return; + case 'failed': + case 'replay': + case 'completed': + return; + } + state satisfies never; + }; const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => @@ -226,7 +412,7 @@ export async function runWorkflow( updateTimestamp(+event.createdAt); }, onUnconsumedEvent: (event) => { - workflowDiscontinuation.reject( + onWorkflowError( new ReplayDivergenceError( `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, { eventId: event.eventId } @@ -240,7 +426,7 @@ export async function runWorkflow( runId: workflowRun.runId, encryptionKey, globalThis: vmGlobalThis, - onWorkflowError: workflowDiscontinuation.reject, + onWorkflowError, eventsConsumer, // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so // the time prefix fed to `ulid()` MUST be replay-stable across every @@ -873,48 +1059,16 @@ export async function runWorkflow( ); await workflowContext.promiseQueue; - span?.setAttributes({ - ...Attribute.WorkflowArgumentsCount(args.length), - }); - - // Invoke user workflow - try { - const result = await Promise.race([ - workflowFn(...args), - workflowDiscontinuation.promise, - ]); - - const dehydrated = await dehydrateWorkflowReturnValue( - result, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - false, - // Gate payload compression on the run's specVersion: only runs - // marked as possibly containing compressed payloads (spec >= 5) - // get gzip data. - (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION - ); + const workflowExecution = (async (): Promise => { + return await workflowFn(...args); + })(); - span?.setAttributes({ - ...Attribute.WorkflowResultType(typeof result), - }); - - await drainPendingQueueItems( - workflowRun.runId, - workflowContext.invocationsQueue, - vmGlobalThis, - workflowRun, - 'completed', - runReadyBarrier - ); - - return dehydrated; - } catch (err) { + const failWorkflow = async (error: unknown): Promise => { + state = { type: 'completed' }; // Control-flow signals are handled by the runtime and do not mean the // workflow has terminally failed. - if (WorkflowSuspension.is(err) || ReplayDivergenceError.is(err)) { - throw err; + if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + throw error; } await drainPendingQueueItems( @@ -926,7 +1080,97 @@ export async function runWorkflow( runReadyBarrier ); - throw err; - } - }); + throw error; + }; + + const completeWorkflow = async ( + result: unknown + ): Promise => { + state = { type: 'completed' }; + try { + const output = await dehydrateWorkflowReturnValue( + result, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + false, + // Gate payload compression on the run's specVersion: only runs + // marked as possibly containing compressed payloads (spec >= 5) + // get gzip data. + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION + ); + + await drainPendingQueueItems( + workflowRun.runId, + workflowContext.invocationsQueue, + vmGlobalThis, + workflowRun, + 'completed', + runReadyBarrier + ); + + return { output, resultType: typeof result }; + } catch (error) { + return failWorkflow(error); + } + }; + + const waitForExecution = async ( + interruption: PromiseWithResolvers + ): Promise => { + let result: unknown; + try { + result = await Promise.race([workflowExecution, interruption.promise]); + } catch (error) { + if (state.type === 'suspended' && error === state.suspension) { + throw error; + } + return failWorkflow(error); + } + return completeWorkflow(result); + }; + + const session: WorkflowSession = { + workflowRun, + argumentCount: args.length, + resume(nextEvents) { + switch (state.type) { + case 'suspended': { + const consumedEvents = eventsConsumer.events; + const isStrictExtension = + nextEvents.length > consumedEvents.length && + consumedEvents.every( + (event, index) => event.eventId === nextEvents[index]?.eventId + ); + if (!isStrictExtension) { + state = { type: 'replay' }; + return { type: 'replay' }; + } + const interruption = withResolvers(); + state = { type: 'running', interruption }; + eventsConsumer.append(nextEvents.slice(consumedEvents.length)); + return { + type: 'resumed', + execution: waitForExecution(interruption), + }; + } + case 'failed': + return { type: 'resumed', execution: failWorkflow(state.error) }; + case 'replay': + return { type: 'replay' }; + case 'completed': + case 'running': + throw new WorkflowRuntimeError( + `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + ); + } + state satisfies never; + }, + }; + + return { + session, + execution: waitForExecution(initialInterruption), + }; + })(); } From bab996c703e93e51675755f664aa6588a24444e7 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 11:45:06 -0700 Subject: [PATCH 02/30] refactor(core): simplify retained-session control flow - executeWorkflow overloads: a fresh replay request can no longer return { type: 'replay' }, deleting the runtime invariant throw and runWorkflow's dead branch - isSameSuspensionBoundary reduced to the steps-array comparison (all suspension counts are derived from steps in the constructor) - runtime loop initializes workflowResult with a ternary --- packages/core/src/runtime.ts | 24 ++++++++---------------- packages/core/src/workflow.ts | 28 +++++++++++++++------------- 2 files changed, 23 insertions(+), 29 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index a085c44884..a5b6c52d41 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1460,16 +1460,14 @@ export function workflowEntrypoint( executionMode, }); replayStart = Date.now(); - let workflowResult: WorkflowExecutionResult = { - type: 'replay', - }; - if (workflowExecution.type === 'retained') { - workflowResult = await executeWorkflow({ - type: 'resume', - session: workflowExecution.session, - events, - }); - } + let workflowResult: WorkflowExecutionResult = + workflowExecution.type === 'retained' + ? await executeWorkflow({ + type: 'resume', + session: workflowExecution.session, + events, + }) + : { type: 'replay' }; if (workflowResult.type === 'replay') { executionMode = 'replay'; @@ -1510,12 +1508,6 @@ export function workflowEntrypoint( throw workflowResult.suspension; } - if (workflowResult.type === 'replay') { - throw new Error( - 'Invariant violation: fresh workflow execution requested another replay' - ); - } - const result = workflowResult.output; runtimeLogger.debug('Workflow execution completed', { workflowRunId: runId, diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 1fe6668470..b7517bc3a9 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -152,17 +152,13 @@ type WorkflowSessionState = | { readonly type: 'replay' } | { readonly type: 'completed' }; +// The suspension counts are all derived from `steps` in the WorkflowSuspension +// constructor, so identical items mean an identical boundary. function isSameSuspensionBoundary( previous: WorkflowSuspension, next: WorkflowSuspension ): boolean { return ( - previous.stepCount === next.stepCount && - previous.hookCount === next.hookCount && - previous.waitCount === next.waitCount && - previous.attributeCount === next.attributeCount && - previous.hookDisposedCount === next.hookDisposedCount && - previous.abortCount === next.abortCount && previous.steps.length === next.steps.length && previous.steps.every((item, index) => item === next.steps[index]) ); @@ -208,8 +204,7 @@ type WorkflowExecutionRequest = readonly events: Event[]; }; -export type WorkflowExecutionResult = - | { readonly type: 'replay' } +type WorkflowReplayResult = | { readonly type: 'completed'; readonly output: unknown } | { readonly type: 'suspended'; @@ -217,6 +212,18 @@ export type WorkflowExecutionResult = readonly session: WorkflowSession; }; +export type WorkflowExecutionResult = + | { readonly type: 'replay' } + | WorkflowReplayResult; + +// A fresh replay always completes or suspends; only resuming a session can +// request another replay (stale session or diverged event prefix). +export async function executeWorkflow( + request: { readonly type: 'replay' } & WorkflowSessionOptions +): Promise; +export async function executeWorkflow( + request: WorkflowExecutionRequest +): Promise; export async function executeWorkflow( request: WorkflowExecutionRequest ): Promise { @@ -302,11 +309,6 @@ export async function runWorkflow( replayPayloadCache, runReadyBarrier, }); - if (result.type === 'replay') { - throw new WorkflowRuntimeError( - `Fresh workflow "${workflowRun.runId}" unexpectedly requested replay` - ); - } if (result.type === 'suspended') throw result.suspension; return result.output; } From bc5a318b70af4bc2c3570ce8058da9abedec3a5f Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:13:50 -0700 Subject: [PATCH 03/30] fix(core): decline retention for VMs that ran host-timed async work crypto.subtle.digest is the only sandbox API whose promise resolves on host timing rather than from the event log, so a workflow racing it against a step can advance while suspended and diverge from what replay reconstructs. A sticky usedHostAsync bit on the VM context makes canRetainWorkflowSession fall back to ordinary replay for such VMs; a quiescent step-only VM remains a pure function of the consumed event prefix and stays retainable. --- packages/core/src/retained-vm-loop.test.ts | 29 +++++++++++++++++++--- packages/core/src/runtime.ts | 7 +++++- packages/core/src/vm/index.ts | 13 +++++++++- packages/core/src/workflow.ts | 8 ++++++ 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index d30740ba69..6f54e0fcd9 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,13 +40,25 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// `crypto.subtle.digest` resolves on host timing, not from the event log, so +// this VM can advance while suspended — the loop must decline retention. +const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + await crypto.subtle.digest("SHA-256", new Uint8Array(8)); + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + registerStepFunction('r_s1', async () => 10); registerStepFunction('r_s2', async () => 20); // Drive the full workflow handler over a stateful (dynamic) event log so the // inline loop makes real progress across its own writes, exactly like a World. // Non-turbo (no runInput, attempt 2) to keep the path simple and deterministic. -async function drive(runId: string) { +async function drive(runId: string, workflowCode = twoStepWorkflow) { const run: WorkflowRun = { runId, workflowName: 'workflow', @@ -126,9 +138,7 @@ async function drive(runId: string) { getEncryptionKeyForRun: vi.fn(async () => undefined), } as any); - await workflowEntrypoint(twoStepWorkflow)( - new Request('https://example.test') - ); + await workflowEntrypoint(workflowCode)(new Request('https://example.test')); const runCompleted = createdEvents.find( (e) => e.eventType === 'run_completed' @@ -175,4 +185,15 @@ describe('retained VM through the inline replay loop', () => { // And it produces the identical dehydrated result as the replay path. expect(on.output).toEqual(off.output); }); + + it('declines retention for a VM that ran host-timed async work', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_host_async', + hostAsyncWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + // The digest marks the VM as non-quiescent, so every iteration replays + // in a fresh VM even though retention is on by default. + expect(vmBuilds).toBeGreaterThan(1); + }); }); diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index a5b6c52d41..1e629c5292 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -318,11 +318,14 @@ function openHookAndWaitState(events: Event[]): { /** * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation. + * Attributes require replay; hooks and waits can wake another invocation; a + * VM that ran host-timed async work can advance while suspended, so its live + * state may not match what a replay of the event log reconstructs. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, + session: WorkflowSession, events: Event[] ): boolean { return ( @@ -333,6 +336,7 @@ function canRetainWorkflowSession( suspension.attributeCount === 0 && suspension.hookDisposedCount === 0 && suspension.abortCount === 0 && + !session.usedHostAsync() && !hasOpenHookOrWait(events) ); } @@ -1498,6 +1502,7 @@ export function workflowEntrypoint( if (workflowResult.type === 'suspended') { workflowExecution = canRetainWorkflowSession( workflowResult.suspension, + workflowResult.session, events ) ? { diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index c0003bdc39..d0b868e1df 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -58,7 +58,17 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); + // `crypto.subtle.digest` is the only API in the sandbox whose promise + // resolves on host timing rather than from the event log, so it is the only + // way a suspended workflow can make progress the event log cannot replay. + // Record its use so the runtime falls back to ordinary replay (see + // `canRetainWorkflowSession`). + let usedHostAsync = false; const boundDigest = originalSubtle.digest.bind(originalSubtle); + const digest: typeof boundDigest = (...args) => { + usedHostAsync = true; + return boundDigest(...args); + }; g.crypto = new Proxy(originalCrypto, { get(target, prop) { @@ -78,7 +88,7 @@ export function createContext(options: CreateContextOptions) { ); }; } else if (prop === 'digest') { - return boundDigest; + return digest; } return target[prop as keyof typeof originalSubtle]; }, @@ -122,5 +132,6 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, + usedHostAsync: () => usedHostAsync, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index b7517bc3a9..6989647953 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -177,6 +177,12 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; + /** + * Whether the VM ran host-timed async work (`crypto.subtle.digest`). Such a + * VM can advance while suspended, so its live state may diverge from what a + * replay of the durable event log reconstructs — it must not be retained. + */ + usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -367,6 +373,7 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, + usedHostAsync, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1135,6 +1142,7 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, + usedHostAsync, resume(nextEvents) { switch (state.type) { case 'suspended': { From 106936a0686d793738fc1ca6febf0c105b67df46 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:27:24 -0700 Subject: [PATCH 04/30] fix(core): track all host-timed async VM APIs for retention Atomics.waitAsync (a wall-clock timer via SharedArrayBuffer) and the async WebAssembly compilation entry points resolve on host timing just like crypto.subtle.digest. Wrap every such intrinsic in createContext so usedHostAsync covers the complete set; dynamic import() settles within a microtask and cannot advance a suspended VM. --- packages/core/src/vm/index.test.ts | 53 ++++++++++++++++++++++++++++++ packages/core/src/vm/index.ts | 31 ++++++++++++++--- packages/core/src/workflow.ts | 7 ++-- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 0381b60f63..1fad27b36e 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -257,3 +257,56 @@ describe('createContext', () => { expect(result).toBe('undefined'); }); }); + +describe('usedHostAsync', () => { + it('starts false and ignores log-driven and deterministic APIs', () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + vm.runInContext( + 'Math.random(); Date.now(); crypto.randomUUID(); crypto.getRandomValues(new Uint8Array(4))', + context + ); + expect(usedHostAsync()).toBe(false); + }); + + it('flips on crypto.subtle.digest', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + await vm.runInContext( + 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on Atomics.waitAsync', () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + vm.runInContext( + 'Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1)', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on WebAssembly.compile', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + // Minimal valid empty module: magic + version. + await vm.runInContext( + 'WebAssembly.compile(new Uint8Array([0,97,115,109,1,0,0,0]))', + context + ); + expect(usedHostAsync()).toBe(true); + }); + + it('flips on WebAssembly.instantiate', async () => { + const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); + + await vm.runInContext( + 'WebAssembly.instantiate(new Uint8Array([0,97,115,109,1,0,0,0]))', + context + ); + expect(usedHostAsync()).toBe(true); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index d0b868e1df..af61842895 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -58,12 +58,33 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - // `crypto.subtle.digest` is the only API in the sandbox whose promise - // resolves on host timing rather than from the event log, so it is the only - // way a suspended workflow can make progress the event log cannot replay. - // Record its use so the runtime falls back to ordinary replay (see - // `canRetainWorkflowSession`). + // Track every sandbox API whose promise resolves on host timing rather + // than from the event log: `crypto.subtle.digest`, `Atomics.waitAsync` + // (a wall-clock timer via SharedArrayBuffer), and the async `WebAssembly` + // compilation entry points. These are the only ways a suspended workflow + // can make progress the event log cannot replay, so the runtime declines + // to retain a VM that used any of them (see `canRetainWorkflowSession`). + // Dynamic `import()` settles within a microtask (rejected: no + // `importModuleDynamically`), so it cannot advance a suspended VM. let usedHostAsync = false; + const trackHostAsync = ( + target: Record, + method: string + ): void => { + const original = target[method]; + if (typeof original !== 'function') return; + target[method] = (...args: unknown[]) => { + usedHostAsync = true; + return Reflect.apply(original, target, args); + }; + }; + const intrinsics = g as unknown as Record>; + trackHostAsync(intrinsics.Atomics, 'waitAsync'); + trackHostAsync(intrinsics.WebAssembly, 'compile'); + trackHostAsync(intrinsics.WebAssembly, 'instantiate'); + trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); + trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); + const boundDigest = originalSubtle.digest.bind(originalSubtle); const digest: typeof boundDigest = (...args) => { usedHostAsync = true; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 6989647953..152cd0a6a5 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -178,9 +178,10 @@ export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; /** - * Whether the VM ran host-timed async work (`crypto.subtle.digest`). Such a - * VM can advance while suspended, so its live state may diverge from what a - * replay of the durable event log reconstructs — it must not be retained. + * Whether the VM ran host-timed async work (`crypto.subtle.digest`, + * `Atomics.waitAsync`, async `WebAssembly` compilation). Such a VM can + * advance while suspended, so its live state may diverge from what a replay + * of the durable event log reconstructs — it must not be retained. */ usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; From 9001a31da7c26166b519908eb17ec0e3e355055d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:33:55 -0700 Subject: [PATCH 05/30] feat(core): compute crypto.subtle.digest synchronously in the sandbox node:crypto createHash produces byte-identical values to WebCrypto and settles the digest promise on a deterministic microtask instead of host threadpool timing. A digest can therefore never advance a suspended workflow, so digest-using VMs stay retainable; only Atomics.waitAsync and async WebAssembly compilation remain host-timed. createHash is stable and undeprecated on Node 18-26 (DEP0179 only removed the direct Hash constructor). --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 27 +++++++- packages/core/src/vm/index.test.ts | 62 ++++++++++++++++++- packages/core/src/vm/index.ts | 55 ++++++++++++---- packages/core/src/workflow.test.ts | 26 ++++---- packages/core/src/workflow.ts | 8 +-- 8 files changed, 148 insertions(+), 36 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 37b874325e..d6b73c8ab6 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation. `WORKFLOW_RETAINED_VM=0` disables retention. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 6ec32b6b0b..a9824c91d6 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -26,7 +26,7 @@ These APIs are available but are **seeded or fixed** to ensure deterministic beh | [`Date`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date) / `Date.now()` / `new Date()` | Returns a fixed timestamp that advances with the workflow's logical clock | | [`crypto.getRandomValues()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/getRandomValues) | Seeded — produces deterministic output for a given workflow run | | [`crypto.randomUUID()`](https://developer.mozilla.org/en-US/docs/Web/API/Crypto/randomUUID) | Seeded — produces deterministic UUIDs for a given workflow run | -| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Passes through to the real implementation (SHA-256, etc. are deterministic by nature) | +| [`crypto.subtle.digest()`](https://developer.mozilla.org/en-US/docs/Web/API/SubtleCrypto/digest) | Computed synchronously via `node:crypto` (values are byte-identical to WebCrypto), so the promise settles at a deterministic point during replay | You can safely use `Math.random()`, `Date.now()`, and `crypto.randomUUID()` in workflow functions. The framework ensures these return the same values across replays. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index f2f1a98e53..e5b65c5eaa 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -79,7 +79,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. -- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Suspensions involving hooks, waits, or attributes, workflows that used host-timed async APIs (`Atomics.waitAsync`, async `WebAssembly` compilation), and any replay divergence always fall back to a full replay. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 6f54e0fcd9..ea94abd6ea 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,9 +40,21 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// `crypto.subtle.digest` resolves on host timing, not from the event log, so +// `Atomics.waitAsync` resolves on host timing, not from the event log, so // this VM can advance while suspended — the loop must decline retention. const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + await Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1).value; + const a = await s1(); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +// `crypto.subtle.digest` computes synchronously via node:crypto, so a +// digest-using VM stays quiescent at suspension and remains retainable. +const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); async function workflow() { await crypto.subtle.digest("SHA-256", new Uint8Array(8)); @@ -192,8 +204,17 @@ describe('retained VM through the inline replay loop', () => { hostAsyncWorkflow ); expect(output).toBeInstanceOf(Uint8Array); - // The digest marks the VM as non-quiescent, so every iteration replays - // in a fresh VM even though retention is on by default. + // Atomics.waitAsync marks the VM as non-quiescent, so every iteration + // replays in a fresh VM even though retention is on by default. expect(vmBuilds).toBeGreaterThan(1); }); + + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_digest', + digestWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBe(1); + }); }); diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 1fad27b36e..dfb42e6089 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -269,14 +269,14 @@ describe('usedHostAsync', () => { expect(usedHostAsync()).toBe(false); }); - it('flips on crypto.subtle.digest', async () => { + it('does not flip on the synchronous crypto.subtle.digest', async () => { const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); await vm.runInContext( 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', context ); - expect(usedHostAsync()).toBe(true); + expect(usedHostAsync()).toBe(false); }); it('flips on Atomics.waitAsync', () => { @@ -310,3 +310,61 @@ describe('usedHostAsync', () => { expect(usedHostAsync()).toBe(true); }); }); + +describe('crypto.subtle.digest', () => { + it.each([ + 'SHA-1', + 'SHA-256', + 'SHA-384', + 'SHA-512', + ])('matches WebCrypto for %s', async (algorithm) => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `crypto.subtle.digest(${JSON.stringify(algorithm)}, new Uint8Array([1,2,3,4,5,255,0,128]))`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + algorithm, + new Uint8Array([1, 2, 3, 4, 5, 255, 0, 128]) + ); + expect(result).toBeInstanceOf(ArrayBuffer); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('accepts the { name } algorithm form and ArrayBuffer input', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'crypto.subtle.digest({ name: "sha-256" }, new ArrayBuffer(4))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new ArrayBuffer(4) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('respects typed-array subviews', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + 'const bytes = new Uint8Array([9, 1, 2, 9]); crypto.subtle.digest("SHA-256", bytes.subarray(1, 3))', + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); + + it('rejects unsupported algorithms like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext('crypto.subtle.digest("MD5", new Uint8Array(1))', context) + ).rejects.toMatchObject({ name: 'NotSupportedError' }); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index af61842895..f10fcff956 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -1,3 +1,4 @@ +import { createHash } from 'node:crypto'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; @@ -10,6 +11,14 @@ export interface CreateContextOptions { fixedTimestamp: number; } +// WebCrypto digest algorithm names → node:crypto (OpenSSL) names. +const DIGEST_ALGORITHMS: Record = { + 'SHA-1': 'sha1', + 'SHA-256': 'sha256', + 'SHA-384': 'sha384', + 'SHA-512': 'sha512', +}; + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. @@ -59,13 +68,13 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); // Track every sandbox API whose promise resolves on host timing rather - // than from the event log: `crypto.subtle.digest`, `Atomics.waitAsync` - // (a wall-clock timer via SharedArrayBuffer), and the async `WebAssembly` - // compilation entry points. These are the only ways a suspended workflow - // can make progress the event log cannot replay, so the runtime declines - // to retain a VM that used any of them (see `canRetainWorkflowSession`). - // Dynamic `import()` settles within a microtask (rejected: no - // `importModuleDynamically`), so it cannot advance a suspended VM. + // than from the event log: `Atomics.waitAsync` (a wall-clock timer via + // SharedArrayBuffer) and the async `WebAssembly` compilation entry points. + // These are the only ways a suspended workflow can make progress the event + // log cannot replay, so the runtime declines to retain a VM that used any + // of them (see `canRetainWorkflowSession`). Dynamic `import()` settles + // within a microtask (rejected: no `importModuleDynamically`), so it + // cannot advance a suspended VM. let usedHostAsync = false; const trackHostAsync = ( target: Record, @@ -85,10 +94,34 @@ export function createContext(options: CreateContextOptions) { trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); - const boundDigest = originalSubtle.digest.bind(originalSubtle); - const digest: typeof boundDigest = (...args) => { - usedHostAsync = true; - return boundDigest(...args); + // `crypto.subtle.digest` computes synchronously via node:crypto, so its + // promise settles on a deterministic microtask instead of host threadpool + // timing — a digest can never advance a suspended workflow, and + // digest-using VMs stay retainable. Values are byte-identical to WebCrypto. + const digest = ( + algorithm: string | { name: string }, + data: ArrayBuffer | ArrayBufferView + ): Promise => { + try { + const name = typeof algorithm === 'string' ? algorithm : algorithm.name; + const ossl = DIGEST_ALGORITHMS[name.toUpperCase()]; + if (!ossl) { + throw new DOMException( + `Unrecognized algorithm name: ${name}`, + 'NotSupportedError' + ); + } + const bytes = ArrayBuffer.isView(data) + ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) + : new Uint8Array(data); + const hash = createHash(ossl).update(bytes).digest(); + // Copy out: small Buffers share the internal pool allocation. + const out = new ArrayBuffer(hash.byteLength); + new Uint8Array(out).set(hash); + return Promise.resolve(out); + } catch (error) { + return Promise.reject(error); + } }; g.crypto = new Proxy(originalCrypto, { diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 66cc295e27..2af9c6de2b 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -361,16 +361,16 @@ describe('runWorkflow', () => { }; const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function digestRepeatedly() { - const bytes = new Uint8Array(8 * 1024 * 1024); - for (let index = 0; index < 8; index++) { - await crypto.subtle.digest("SHA-256", bytes); + async function waitRepeatedly() { + const shared = new Int32Array(new SharedArrayBuffer(4)); + for (let index = 0; index < 20; index++) { + await Atomics.waitAsync(shared, 0, 0, 25).value; } } async function workflow() { - await Promise.race([step(), digestRepeatedly()]); - console.log("retained:digest-completed"); - return "digest completed"; + await Promise.race([step(), waitRepeatedly()]); + console.log("retained:background-completed"); + return "background completed"; } ${getWorkflowTransformCode('workflow')}`; const events: Event[] = [ @@ -479,7 +479,7 @@ describe('runWorkflow', () => { noEncryptionKey, ops ) - ).toBe('digest completed'); + ).toBe('background completed'); log.mockRestore(); }); @@ -503,14 +503,14 @@ describe('runWorkflow', () => { const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; - async function digestRepeatedly() { - const bytes = new Uint8Array(8 * 1024 * 1024); - for (let index = 0; index < 8; index++) { - await crypto.subtle.digest("SHA-256", bytes); + async function waitRepeatedly() { + const shared = new Int32Array(new SharedArrayBuffer(4)); + for (let index = 0; index < 20; index++) { + await Atomics.waitAsync(shared, 0, 0, 25).value; } } async function workflow() { - await Promise.race([step(), digestRepeatedly()]); + await Promise.race([step(), waitRepeatedly()]); void setAttributes([{ key: "stale", value: true }]); console.log("retained:discarded-completed"); return "discarded"; diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 152cd0a6a5..5ff5991c41 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -178,10 +178,10 @@ export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; /** - * Whether the VM ran host-timed async work (`crypto.subtle.digest`, - * `Atomics.waitAsync`, async `WebAssembly` compilation). Such a VM can - * advance while suspended, so its live state may diverge from what a replay - * of the durable event log reconstructs — it must not be retained. + * Whether the VM ran host-timed async work (`Atomics.waitAsync`, async + * `WebAssembly` compilation). Such a VM can advance while suspended, so its + * live state may diverge from what a replay of the durable event log + * reconstructs — it must not be retained. */ usedHostAsync(): boolean; resume(events: Event[]): WorkflowSessionResumeResult; From 2abf8b8e44ffefe3a14315dd0511c9e568035f92 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:40:48 -0700 Subject: [PATCH 06/30] fix(core): remove WeakRef and FinalizationRegistry from the sandbox GC observation depends on host GC timing that neither replay nor a retained VM can reconstruct from the event log. WeakMap/WeakSet stay available (they do not expose GC state). --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/api-reference/workflow-globals.mdx | 1 + packages/core/src/vm/index.test.ts | 14 ++++++++++++++ packages/core/src/vm/index.ts | 6 ++++++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index d6b73c8ab6..61cb327ee9 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef` and `FinalizationRegistry` are no longer exposed in workflow functions (GC observation cannot be replayed). diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index a9824c91d6..4f7eedf1d9 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -100,3 +100,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **Global `fetch`**: Use [`import { fetch } from "workflow"`](/docs/api-reference/workflow/fetch) instead. See [fetch-in-workflow](/docs/errors/fetch-in-workflow). - **Timers**: `setTimeout`, `setInterval`, `setImmediate`, and their `clear*` counterparts. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. See [timeout-in-workflow](/docs/errors/timeout-in-workflow). - **`Buffer`**: Node.js-specific API. Use `Uint8Array` with `toBase64()` / `fromBase64()` / `toHex()` / `fromHex()` for binary data encoding, or `atob()` / `btoa()` for string-based base64. +- **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index dfb42e6089..a29a2e146f 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -368,3 +368,17 @@ describe('crypto.subtle.digest', () => { ).rejects.toMatchObject({ name: 'NotSupportedError' }); }); }); + +describe('GC observation', () => { + it('does not expose WeakRef or FinalizationRegistry', () => { + const { context } = createContext({ seed, fixedTimestamp }); + + expect(vm.runInContext('typeof WeakRef', context)).toBe('undefined'); + expect(vm.runInContext('typeof FinalizationRegistry', context)).toBe( + 'undefined' + ); + // WeakMap/WeakSet do not expose GC state and stay available. + expect(vm.runInContext('typeof WeakMap', context)).toBe('function'); + expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index f10fcff956..fdd0d10fbf 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -94,6 +94,12 @@ export function createContext(options: CreateContextOptions) { trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); + // GC observation (`WeakRef.deref()`, finalizer callbacks) depends on host + // GC timing that neither replay nor a retained VM can reconstruct from the + // event log, so the sandbox does not expose it at all. + delete intrinsics.WeakRef; + delete intrinsics.FinalizationRegistry; + // `crypto.subtle.digest` computes synchronously via node:crypto, so its // promise settles on a deterministic microtask instead of host threadpool // timing — a digest can never advance a suspended workflow, and From 4905577767e060b03c61521601db073ffeb3bc75 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 12:56:49 -0700 Subject: [PATCH 07/30] fix(core): enforce the BufferSource contract in the sandbox digest Reject non-BufferSource digest input with TypeError like WebCrypto does, via the native ArrayBuffer.prototype.byteLength brand check (works across vm realms). Previously a plain number was treated as a Uint8Array length, turning a small input into a giant allocation. --- packages/core/src/vm/index.test.ts | 15 +++++++++++++++ packages/core/src/vm/index.ts | 23 +++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index a29a2e146f..b2d694c5cd 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -382,3 +382,18 @@ describe('GC observation', () => { expect(vm.runInContext('typeof WeakSet', context)).toBe('function'); }); }); + +describe('crypto.subtle.digest input validation', () => { + it.each([ + ['a number', '2000000000'], + ['a plain object', '({})'], + ['a string', '"data"'], + ['null', 'null'], + ])('rejects %s with TypeError', async (_label, expression) => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext(`crypto.subtle.digest("SHA-256", ${expression})`, context) + ).rejects.toThrow(TypeError); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index fdd0d10fbf..ba67168173 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -19,6 +19,24 @@ const DIGEST_ALGORITHMS: Record = { 'SHA-512': 'sha512', }; +// biome-ignore lint/style/noNonNullAssertion: byteLength always exists on ArrayBuffer.prototype +const arrayBufferByteLength = Object.getOwnPropertyDescriptor( + ArrayBuffer.prototype, + 'byteLength' +)!.get!; + +// WebCrypto BufferSource conversion: views pass through; anything else must +// be a real ArrayBuffer (the native byteLength getter is a brand check that +// works across vm realms) so that e.g. a plain number is rejected with +// TypeError instead of allocating a Uint8Array of that length. +function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { + if (ArrayBuffer.isView(data)) { + return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + } + arrayBufferByteLength.call(data); + return new Uint8Array(data); +} + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. @@ -117,10 +135,7 @@ export function createContext(options: CreateContextOptions) { 'NotSupportedError' ); } - const bytes = ArrayBuffer.isView(data) - ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) - : new Uint8Array(data); - const hash = createHash(ossl).update(bytes).digest(); + const hash = createHash(ossl).update(toDigestBytes(data)).digest(); // Copy out: small Buffers share the internal pool allocation. const out = new ArrayBuffer(hash.byteLength); new Uint8Array(out).set(hash); From 73ee4ae9a064aaddbfecfe3ae83bfbe78ad0d29d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:15:31 -0700 Subject: [PATCH 08/30] fix(core): demote retention when suspension serialization draws randomness handleSuspension dehydrates step arguments with the live VM, and that serialization can execute user code (getters, WORKFLOW_SERIALIZE hooks). Randomness drawn there would desync the retained VM's future correlation IDs from what a fresh replay regenerates. Count every draw from the seeded stream at its single source in createContext and fall back to ordinary replay if handleSuspension consumed any. --- .../workflow-serde/workflow-serialize.mdx | 1 + packages/core/src/retained-vm-loop.test.ts | 21 +++++++++++++++++++ packages/core/src/runtime.ts | 15 +++++++++++++ packages/core/src/vm/index.ts | 13 +++++++++++- packages/core/src/workflow.ts | 10 +++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx b/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx index 85c5eb8acf..a5380c3188 100644 --- a/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx +++ b/docs/content/docs/v5/api-reference/workflow-serde/workflow-serialize.mdx @@ -69,6 +69,7 @@ This method runs inside the workflow context and is subject to the same constrai - No Node.js-specific APIs (like `fs`, `path`, `crypto`, etc.) - No non-deterministic operations (like `Math.random()` or `Date.now()`) - No external network calls +- No side effects on workflow state — the method may run outside deterministic replay, so mutations would not be reconstructed Keep this method simple and focused on extracting data from the instance. diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index ea94abd6ea..a76f58f8aa 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -52,6 +52,18 @@ const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP") } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// Serializing this step's arguments executes the getter, which draws from +// the VM's seeded random stream after the suspension — the loop must demote +// the session so future correlation IDs stay replayable. +const impureArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1({ get x() { Math.random(); return 1; } }); + const b = await s2(); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -209,6 +221,15 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); + it('demotes retention when argument serialization draws randomness', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_impure_args', + impureArgsWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBeGreaterThan(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 1e629c5292..bd29c8e93e 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1140,6 +1140,7 @@ export function workflowEntrypoint( | { readonly type: 'retained'; readonly session: WorkflowSession; + readonly drawCountAtSuspension: number; } = { type: 'replay' }; // Main replay loop @@ -1508,6 +1509,8 @@ export function workflowEntrypoint( ? { type: 'retained', session: workflowResult.session, + drawCountAtSuspension: + workflowResult.session.randomDrawCount(), } : { type: 'replay' }; throw workflowResult.suspension; @@ -1644,6 +1647,18 @@ export function workflowEntrypoint( eventLog: suspensionLog, runReadyBarrier, }); + // Argument serialization above can execute user code + // (getters, WORKFLOW_SERIALIZE hooks) in the live VM. + // Any randomness it drew would desync future + // correlation IDs from a fresh replay's, so demote + // the session to ordinary replay. + if ( + workflowExecution.type === 'retained' && + workflowExecution.session.randomDrawCount() !== + workflowExecution.drawCountAtSuspension + ) { + workflowExecution = { type: 'replay' }; + } } catch (suspensionError) { // A suspension create whose stale (412) rejection // survived the in-guard reload retries: schedule an diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index ba67168173..3b71fe03f5 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -47,7 +47,17 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { export function createContext(options: CreateContextOptions) { let { fixedTimestamp } = options; const { seed } = options; - const rng = seedrandom(seed); + // Count every draw from the seeded stream (Math.random, getRandomValues, + // randomUUID, and the ULIDs/nanoids derived from them all flow through this + // one closure). The runtime compares the count across the suspension + // boundary to detect serialization side effects (see + // `canRetainWorkflowSession`). + const baseRng = seedrandom(seed); + let randomDrawCount = 0; + const rng = () => { + randomDrawCount++; + return baseRng(); + }; const context = vmCreateContext(); const g: typeof globalThis = runInContext('globalThis', context); @@ -208,5 +218,6 @@ export function createContext(options: CreateContextOptions) { fixedTimestamp = timestamp; }, usedHostAsync: () => usedHostAsync, + randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 5ff5991c41..c158f006d5 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -184,6 +184,14 @@ export interface WorkflowSession { * reconstructs — it must not be retained. */ usedHostAsync(): boolean; + /** + * Draws from the VM's seeded random stream so far. The runtime compares + * this across the suspension boundary: if post-suspension argument + * serialization (getters, `WORKFLOW_SERIALIZE` hooks) drew randomness, the + * retained VM's future correlation IDs would desync from what a fresh + * replay regenerates, so the session must not be retained. + */ + randomDrawCount(): number; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -375,6 +383,7 @@ function createWorkflowSession({ globalThis: vmGlobalThis, updateTimestamp, usedHostAsync, + randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1144,6 +1153,7 @@ function createWorkflowSession({ workflowRun, argumentCount: args.length, usedHostAsync, + randomDrawCount, resume(nextEvents) { switch (state.type) { case 'suspended': { From d83b651ad6dda2cb09c66f9afcccdfe56517f48b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:23:40 -0700 Subject: [PATCH 09/30] refactor(core): make VM quiescence unconditional, cut tracking machinery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Delete Atomics.waitAsync and the async WebAssembly entry points from the sandbox instead of tracking their use — with digest synchronous and GC intrinsics removed, no sandbox API settles a promise on host timing, so a suspended VM provably cannot advance. This deletes the trackHostAsync wrapper, the usedHostAsync bit and session method, the runtime gate clause, the session 'failed' state (unreachable), and the background-progress test scenarios (impossible by construction). --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 + .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 23 --- packages/core/src/runtime.ts | 7 +- packages/core/src/vm/index.test.ts | 72 +++---- packages/core/src/vm/index.ts | 47 ++--- packages/core/src/workflow.test.ts | 176 +----------------- packages/core/src/workflow.ts | 22 +-- 9 files changed, 68 insertions(+), 285 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 61cb327ee9..b86e3d918c 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef` and `FinalizationRegistry` are no longer exposed in workflow functions (GC observation cannot be replayed). +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 4f7eedf1d9..6faa7e3df6 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -101,3 +101,5 @@ The following are **not available** in workflow functions. Move this logic to [s - **Timers**: `setTimeout`, `setInterval`, `setImmediate`, and their `clear*` counterparts. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. See [timeout-in-workflow](/docs/errors/timeout-in-workflow). - **`Buffer`**: Node.js-specific API. Use `Uint8Array` with `toBase64()` / `fromBase64()` / `toHex()` / `fromHex()` for binary data encoding, or `atob()` / `btoa()` for string-based base64. - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) +- **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. +- **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index e5b65c5eaa..f2f1a98e53 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -79,7 +79,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. -- Suspensions involving hooks, waits, or attributes, workflows that used host-timed async APIs (`Atomics.waitAsync`, async `WebAssembly` compilation), and any replay divergence always fall back to a full replay. +- Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index a76f58f8aa..84886896fb 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -40,18 +40,6 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// `Atomics.waitAsync` resolves on host timing, not from the event log, so -// this VM can advance while suspended — the loop must decline retention. -const hostAsyncWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); - async function workflow() { - await Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1).value; - const a = await s1(); - const b = await s2(); - return a + b; - } - globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - // Serializing this step's arguments executes the getter, which draws from // the VM's seeded random stream after the suspension — the loop must demote // the session so future correlation IDs stay replayable. @@ -210,17 +198,6 @@ describe('retained VM through the inline replay loop', () => { expect(on.output).toEqual(off.output); }); - it('declines retention for a VM that ran host-timed async work', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_host_async', - hostAsyncWorkflow - ); - expect(output).toBeInstanceOf(Uint8Array); - // Atomics.waitAsync marks the VM as non-quiescent, so every iteration - // replays in a fresh VM even though retention is on by default. - expect(vmBuilds).toBeGreaterThan(1); - }); - it('demotes retention when argument serialization draws randomness', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_impure_args', diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index bd29c8e93e..448cfd6fce 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -318,14 +318,11 @@ function openHookAndWaitState(events: Event[]): { /** * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation; a - * VM that ran host-timed async work can advance while suspended, so its live - * state may not match what a replay of the event log reconstructs. + * Attributes require replay; hooks and waits can wake another invocation. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, - session: WorkflowSession, events: Event[] ): boolean { return ( @@ -336,7 +333,6 @@ function canRetainWorkflowSession( suspension.attributeCount === 0 && suspension.hookDisposedCount === 0 && suspension.abortCount === 0 && - !session.usedHostAsync() && !hasOpenHookOrWait(events) ); } @@ -1503,7 +1499,6 @@ export function workflowEntrypoint( if (workflowResult.type === 'suspended') { workflowExecution = canRetainWorkflowSession( workflowResult.suspension, - workflowResult.session, events ) ? { diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index b2d694c5cd..ffd23eae66 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -258,56 +258,44 @@ describe('createContext', () => { }); }); -describe('usedHostAsync', () => { - it('starts false and ignores log-driven and deterministic APIs', () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); +describe('host-timed async', () => { + it('does not expose any API that settles on host timing', () => { + const { context } = createContext({ seed, fixedTimestamp }); - vm.runInContext( - 'Math.random(); Date.now(); crypto.randomUUID(); crypto.getRandomValues(new Uint8Array(4))', - context + expect(vm.runInContext('typeof Atomics.waitAsync', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(false); - }); - - it('does not flip on the synchronous crypto.subtle.digest', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - await vm.runInContext( - 'crypto.subtle.digest("SHA-256", new Uint8Array(1))', - context + expect(vm.runInContext('typeof WebAssembly.compile', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(false); - }); - - it('flips on Atomics.waitAsync', () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - vm.runInContext( - 'Atomics.waitAsync(new Int32Array(new SharedArrayBuffer(4)), 0, 1)', - context + expect(vm.runInContext('typeof WebAssembly.instantiate', context)).toBe( + 'undefined' ); - expect(usedHostAsync()).toBe(true); - }); - - it('flips on WebAssembly.compile', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); - - // Minimal valid empty module: magic + version. - await vm.runInContext( - 'WebAssembly.compile(new Uint8Array([0,97,115,109,1,0,0,0]))', - context + expect( + vm.runInContext('typeof WebAssembly.compileStreaming', context) + ).toBe('undefined'); + expect( + vm.runInContext('typeof WebAssembly.instantiateStreaming', context) + ).toBe('undefined'); + // Deterministic synchronous WebAssembly stays available. + expect(vm.runInContext('typeof WebAssembly.Module', context)).toBe( + 'function' ); - expect(usedHostAsync()).toBe(true); }); +}); - it('flips on WebAssembly.instantiate', async () => { - const { context, usedHostAsync } = createContext({ seed, fixedTimestamp }); +describe('randomDrawCount', () => { + it('counts every draw from the seeded stream', () => { + const { context, randomDrawCount } = createContext({ + seed, + fixedTimestamp, + }); - await vm.runInContext( - 'WebAssembly.instantiate(new Uint8Array([0,97,115,109,1,0,0,0]))', - context - ); - expect(usedHostAsync()).toBe(true); + expect(randomDrawCount()).toBe(0); + vm.runInContext('Math.random()', context); + expect(randomDrawCount()).toBe(1); + vm.runInContext('crypto.getRandomValues(new Uint8Array(4))', context); + expect(randomDrawCount()).toBe(5); }); }); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 3b71fe03f5..325862c5f8 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -95,36 +95,24 @@ export function createContext(options: CreateContextOptions) { const randomUUID = createRandomUUID(rng); - // Track every sandbox API whose promise resolves on host timing rather - // than from the event log: `Atomics.waitAsync` (a wall-clock timer via - // SharedArrayBuffer) and the async `WebAssembly` compilation entry points. - // These are the only ways a suspended workflow can make progress the event - // log cannot replay, so the runtime declines to retain a VM that used any - // of them (see `canRetainWorkflowSession`). Dynamic `import()` settles - // within a microtask (rejected: no `importModuleDynamically`), so it - // cannot advance a suspended VM. - let usedHostAsync = false; - const trackHostAsync = ( - target: Record, - method: string - ): void => { - const original = target[method]; - if (typeof original !== 'function') return; - target[method] = (...args: unknown[]) => { - usedHostAsync = true; - return Reflect.apply(original, target, args); - }; - }; + // The sandbox must not expose any way for workflow code to observe host + // timing or host state: after this block, every promise a workflow can + // create settles either from the event log or within its own microtask + // cascade, so a suspended VM is fully quiescent and can be retained across + // inline steps (see `canRetainWorkflowSession`). + // + // - `Atomics.waitAsync` is a wall-clock timer (via SharedArrayBuffer). + // - The async `WebAssembly` entry points resolve on compile-thread timing; + // the synchronous `new WebAssembly.Module()` / `Instance()` remain. + // - `WeakRef.deref()` and finalizer callbacks observe GC timing. + // - Dynamic `import()` settles within a microtask (rejected: no + // `importModuleDynamically`), so it needs no handling. const intrinsics = g as unknown as Record>; - trackHostAsync(intrinsics.Atomics, 'waitAsync'); - trackHostAsync(intrinsics.WebAssembly, 'compile'); - trackHostAsync(intrinsics.WebAssembly, 'instantiate'); - trackHostAsync(intrinsics.WebAssembly, 'compileStreaming'); - trackHostAsync(intrinsics.WebAssembly, 'instantiateStreaming'); - - // GC observation (`WeakRef.deref()`, finalizer callbacks) depends on host - // GC timing that neither replay nor a retained VM can reconstruct from the - // event log, so the sandbox does not expose it at all. + delete intrinsics.Atomics.waitAsync; + delete intrinsics.WebAssembly.compile; + delete intrinsics.WebAssembly.instantiate; + delete intrinsics.WebAssembly.compileStreaming; + delete intrinsics.WebAssembly.instantiateStreaming; delete intrinsics.WeakRef; delete intrinsics.FinalizationRegistry; @@ -217,7 +205,6 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, - usedHostAsync: () => usedHostAsync, randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 2af9c6de2b..d7dfe39ab5 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -342,15 +342,15 @@ describe('runWorkflow', () => { log.mockRestore(); }); - it('returns a workflow that completes while its retained session is suspended', async () => { + it('falls back to replay permanently when the event prefix diverges', async () => { const ops: Promise[] = []; const workflowRun: WorkflowRun = { - runId: 'wrun_retained_async_completion', + runId: 'wrun_retained_divergence', workflowName: 'workflow', status: 'running', input: await dehydrateWorkflowArguments( [], - 'wrun_retained_async_completion', + 'wrun_retained_divergence', noEncryptionKey, ops ), @@ -361,17 +361,7 @@ describe('runWorkflow', () => { }; const workflowCode = ` const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - async function waitRepeatedly() { - const shared = new Int32Array(new SharedArrayBuffer(4)); - for (let index = 0; index < 20; index++) { - await Atomics.waitAsync(shared, 0, 0, 25).value; - } - } - async function workflow() { - await Promise.race([step(), waitRepeatedly()]); - console.log("retained:background-completed"); - return "background completed"; - } + async function workflow() { await step(); } ${getWorkflowTransformCode('workflow')}`; const events: Event[] = [ { @@ -381,7 +371,6 @@ describe('runWorkflow', () => { createdAt: new Date('2024-01-01T00:00:00.000Z'), }, ]; - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); const suspended = await executeWorkflow({ type: 'replay', @@ -392,173 +381,28 @@ describe('runWorkflow', () => { replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), }); assert(suspended.type === 'suspended'); - const pendingStep = suspended.suspension.steps[0]; - assert(pendingStep?.type === 'step'); - - const rewrittenSession = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events, - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(rewrittenSession.type === 'suspended'); - await vi.waitFor( - () => { - expect(log).toHaveBeenCalledTimes(2); - }, - { timeout: 10_000, interval: 10 } - ); - const rewrittenEvents = [ + // A log whose consumed prefix was rewritten is not a strict extension. + const rewritten = [ { ...events[0], eventId: 'rewritten-event' } as Event, { ...events[0], eventId: 'new-event' } as Event, ]; expect( await executeWorkflow({ type: 'resume', - session: rewrittenSession.session, - events: rewrittenEvents, - }) - ).toEqual({ type: 'replay' }); - - const createdAt = new Date('2024-01-01T00:00:01.000Z'); - events.push( - { - eventId: 'event-step-created', - runId: workflowRun.runId, - eventType: 'step_created', - correlationId: pendingStep.correlationId, - eventData: { stepName: pendingStep.stepName }, - createdAt, - }, - { - eventId: 'event-step-started', - runId: workflowRun.runId, - eventType: 'step_started', - correlationId: pendingStep.correlationId, - eventData: { stepName: pendingStep.stepName }, - createdAt, - }, - { - eventId: 'event-step-completed', - runId: workflowRun.runId, - eventType: 'step_completed', - correlationId: pendingStep.correlationId, - eventData: { - stepName: pendingStep.stepName, - result: await dehydrateStepReturnValue( - undefined, - workflowRun.runId, - noEncryptionKey, - ops - ), - }, - createdAt, - } - ); - - const completed = await executeWorkflow({ - type: 'resume', - session: suspended.session, - events, - }); - assert(completed.type === 'completed'); - expect( - await executeWorkflow({ - type: 'resume', - session: rewrittenSession.session, - events, + session: suspended.session, + events: rewritten, }) ).toEqual({ type: 'replay' }); - expect( - await hydrateWorkflowReturnValue( - completed.output as any, - workflowRun.runId, - noEncryptionKey, - ops - ) - ).toBe('background completed'); - log.mockRestore(); - }); - - it('does not drain operations from a discarded retained session', async () => { - const ops: Promise[] = []; - const workflowRun: WorkflowRun = { - runId: 'wrun_retained_discarded', - workflowName: 'workflow', - status: 'running', - input: await dehydrateWorkflowArguments( - [], - 'wrun_retained_discarded', - noEncryptionKey, - ops - ), - createdAt: new Date('2024-01-01T00:00:00.000Z'), - updatedAt: new Date('2024-01-01T00:00:00.000Z'), - startedAt: new Date('2024-01-01T00:00:00.000Z'), - deploymentId: 'test-deployment', - }; - const workflowCode = ` - const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); - const setAttributes = globalThis[Symbol.for("WORKFLOW_SET_ATTRIBUTES")]; - async function waitRepeatedly() { - const shared = new Int32Array(new SharedArrayBuffer(4)); - for (let index = 0; index < 20; index++) { - await Atomics.waitAsync(shared, 0, 0, 25).value; - } - } - async function workflow() { - await Promise.race([step(), waitRepeatedly()]); - void setAttributes([{ key: "stale", value: true }]); - console.log("retained:discarded-completed"); - return "discarded"; - } - ${getWorkflowTransformCode('workflow')}`; - const create = vi.fn(); - const log = vi.spyOn(console, 'log').mockImplementation(() => undefined); - setWorld({ - specVersion: SPEC_VERSION_CURRENT, - events: { create }, - streams: { write: vi.fn(), close: vi.fn() }, - } as any); - - const suspended = await executeWorkflow({ - type: 'replay', - workflowCode, - workflowRun, - events: [], - encryptionKey: noEncryptionKey, - replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), - }); - assert(suspended.type === 'suspended'); - - await vi.waitFor( - () => { - expect(log).toHaveBeenCalledWith('retained:discarded-completed'); - }, - { timeout: 10_000, interval: 10 } - ); - await new Promise((resolve) => setTimeout(resolve, 250)); + // The fallback is permanent, even for a well-formed extension. expect( await executeWorkflow({ type: 'resume', session: suspended.session, - events: [ - { - eventId: 'event-run-created', - runId: workflowRun.runId, - eventType: 'run_created', - createdAt: new Date('2024-01-01T00:00:00.000Z'), - }, - ], + events, }) ).toEqual({ type: 'replay' }); - expect(create).not.toHaveBeenCalled(); - setWorld(undefined); - log.mockRestore(); }); it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index c158f006d5..35ee9313a3 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -148,7 +148,6 @@ type WorkflowSessionState = readonly interruption: PromiseWithResolvers; } | { readonly type: 'suspended'; readonly suspension: WorkflowSuspension } - | { readonly type: 'failed'; readonly error: Error } | { readonly type: 'replay' } | { readonly type: 'completed' }; @@ -164,12 +163,15 @@ function isSameSuspensionBoundary( ); } +// A suspended VM is quiescent, but each parked step consumer signals its own +// (identical) suspension via scheduleWhenIdle — absorb those duplicates and +// treat anything else as unretainable. function updateSuspendedSession( suspension: WorkflowSuspension, error: Error ): WorkflowSessionState { - if (!WorkflowSuspension.is(error)) return { type: 'failed', error }; - return isSameSuspensionBoundary(suspension, error) + return WorkflowSuspension.is(error) && + isSameSuspensionBoundary(suspension, error) ? { type: 'suspended', suspension } : { type: 'replay' }; } @@ -177,13 +179,6 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; - /** - * Whether the VM ran host-timed async work (`Atomics.waitAsync`, async - * `WebAssembly` compilation). Such a VM can advance while suspended, so its - * live state may diverge from what a replay of the durable event log - * reconstructs — it must not be retained. - */ - usedHostAsync(): boolean; /** * Draws from the VM's seeded random stream so far. The runtime compares * this across the suspension boundary: if post-suspension argument @@ -382,7 +377,6 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, - usedHostAsync, randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, @@ -401,14 +395,13 @@ function createWorkflowSession({ const { interruption } = state; state = WorkflowSuspension.is(error) ? { type: 'suspended', suspension: error } - : { type: 'failed', error }; + : { type: 'replay' }; interruption.reject(error); return; } case 'suspended': state = updateSuspendedSession(state.suspension, error); return; - case 'failed': case 'replay': case 'completed': return; @@ -1152,7 +1145,6 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - usedHostAsync, randomDrawCount, resume(nextEvents) { switch (state.type) { @@ -1175,8 +1167,6 @@ function createWorkflowSession({ execution: waitForExecution(interruption), }; } - case 'failed': - return { type: 'resumed', execution: failWorkflow(state.error) }; case 'replay': return { type: 'replay' }; case 'completed': From b2c33d2645aa75f2eddc59d30db5c363fcbd3ec8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 13:54:39 -0700 Subject: [PATCH 10/30] refactor(core): gate retention on passively cloneable step inputs Replace the RNG draw-counter demotion with prevention: when a session is a retention candidate, new step inputs take a passive descriptor walk (never invoking getters; proxies, accessors, functions, custom classes, and platform wrappers decline) and safe values are structuredClone'd into the host realm before dehydration, so serialization never executes workflow-owned code against a retained VM. Unsafe inputs serialize the old way and the session falls back to ordinary replay. --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 1 + packages/core/src/retained-vm-loop.test.ts | 116 +++++++++-- packages/core/src/runtime.ts | 13 +- .../src/runtime/retained-step-input.test.ts | 140 +++++++++++++ .../core/src/runtime/retained-step-input.ts | 193 ++++++++++++++++++ .../core/src/runtime/suspension-handler.ts | 53 ++++- packages/core/src/vm/index.test.ts | 15 -- packages/core/src/vm/index.ts | 13 +- packages/core/src/workflow.ts | 10 - 10 files changed, 489 insertions(+), 67 deletions(-) create mode 100644 packages/core/src/runtime/retained-step-input.test.ts create mode 100644 packages/core/src/runtime/retained-step-input.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index b86e3d918c..9949f21f78 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index f2f1a98e53..9c275ef8f0 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,6 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. +- Step inputs containing accessors, proxies, functions, custom class serializers, or other values whose serialization could execute workflow code also fall back to replay. Plain data and standard in-memory collections remain retainable. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 84886896fb..532f981f79 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -15,10 +15,13 @@ vi.mock('./vm/index.js', async (importActual) => { }); const { createContext } = await import('./vm/index.js'); +const { registerSerializationClass } = await import('./class-serialization.js'); const { registerStepFunction } = await import('./private.js'); const { setWorld } = await import('./runtime/world.js'); const { workflowEntrypoint } = await import('./runtime.js'); -const { dehydrateWorkflowArguments } = await import('./serialization.js'); +const { dehydrateWorkflowArguments, hydrateWorkflowReturnValue } = await import( + './serialization.js' +); vi.mock('@vercel/functions', () => ({ waitUntil: vi.fn((p: Promise) => { @@ -40,18 +43,50 @@ const twoStepWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]( } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// Serializing this step's arguments executes the getter, which draws from -// the VM's seeded random stream after the suspension — the loop must demote -// the session so future correlation IDs stay replayable. +// Serializing this step's arguments executes the getter after suspension. Its +// state mutation is not reconstructed by a cold replay, so this boundary must +// demote even though it does not draw randomness. const impureArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + const echo = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_echo"); async function workflow() { - const a = await s1({ get x() { Math.random(); return 1; } }); - const b = await s2(); - return a + b; + let counter = 0; + await s1({ get x() { counter++; return 1; } }); + return await echo(counter); + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + +const impureSerializerWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const echo = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_echo"); + class Value { + static classId = "test/RetainedSerializerValue"; + static [Symbol.for("workflow-serialize")](instance) { + instance.onSerialize(); + return { value: instance.value }; + } + constructor(value, onSerialize) { + this.value = value; + this.onSerialize = onSerialize; + } + } + async function workflow() { + let counter = 0; + await s1(new Value(1, () => counter++)); + return await echo(counter); } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +class RetainedSerializerValue { + constructor(readonly value: number) {} + + static [Symbol.for('workflow-deserialize')](data: { value: number }) { + return new RetainedSerializerValue(data.value); + } +} +registerSerializationClass( + 'test/RetainedSerializerValue', + RetainedSerializerValue +); + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -66,6 +101,7 @@ const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")](" registerStepFunction('r_s1', async () => 10); registerStepFunction('r_s2', async () => 20); +registerStepFunction('r_echo', async (value) => value); // Drive the full workflow handler over a stateful (dynamic) event log so the // inline loop makes real progress across its own writes, exactly like a World. @@ -198,13 +234,67 @@ describe('retained VM through the inline replay loop', () => { expect(on.output).toEqual(off.output); }); - it('demotes retention when argument serialization draws randomness', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_impure_args', + it('matches cold replay when argument serialization mutates workflow state', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive( + 'wrun_retained_impure_args_off', impureArgsWorkflow ); - expect(output).toBeInstanceOf(Uint8Array); - expect(vmBuilds).toBeGreaterThan(1); + createContextSpy.mockClear(); + + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive('wrun_retained_impure_args_on', impureArgsWorkflow); + + expect(on.vmBuilds).toBeGreaterThan(1); + expect( + await hydrateWorkflowReturnValue( + off.output, + 'wrun_retained_impure_args_off', + undefined, + [] + ) + ).toBe(0); + expect( + await hydrateWorkflowReturnValue( + on.output, + 'wrun_retained_impure_args_on', + undefined, + [] + ) + ).toBe(0); + }); + + it('matches cold replay when a custom serializer mutates workflow state', async () => { + process.env.WORKFLOW_RETAINED_VM = '0'; + const off = await drive( + 'wrun_retained_impure_serializer_off', + impureSerializerWorkflow + ); + createContextSpy.mockClear(); + + delete process.env.WORKFLOW_RETAINED_VM; + const on = await drive( + 'wrun_retained_impure_serializer_on', + impureSerializerWorkflow + ); + + expect(on.vmBuilds).toBeGreaterThan(1); + expect( + await hydrateWorkflowReturnValue( + off.output, + 'wrun_retained_impure_serializer_off', + undefined, + [] + ) + ).toBe(0); + expect( + await hydrateWorkflowReturnValue( + on.output, + 'wrun_retained_impure_serializer_on', + undefined, + [] + ) + ).toBe(0); }); it('retains a VM that used the synchronous crypto.subtle.digest', async () => { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 448cfd6fce..05f8f09795 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -1136,7 +1136,6 @@ export function workflowEntrypoint( | { readonly type: 'retained'; readonly session: WorkflowSession; - readonly drawCountAtSuspension: number; } = { type: 'replay' }; // Main replay loop @@ -1504,8 +1503,6 @@ export function workflowEntrypoint( ? { type: 'retained', session: workflowResult.session, - drawCountAtSuspension: - workflowResult.session.randomDrawCount(), } : { type: 'replay' }; throw workflowResult.suspension; @@ -1641,16 +1638,12 @@ export function workflowEntrypoint( requestId, eventLog: suspensionLog, runReadyBarrier, + prepareForRetention: + workflowExecution.type === 'retained', }); - // Argument serialization above can execute user code - // (getters, WORKFLOW_SERIALIZE hooks) in the live VM. - // Any randomness it drew would desync future - // correlation IDs from a fresh replay's, so demote - // the session to ordinary replay. if ( workflowExecution.type === 'retained' && - workflowExecution.session.randomDrawCount() !== - workflowExecution.drawCountAtSuspension + !suspensionResult.retainedStepInputsSafe ) { workflowExecution = { type: 'replay' }; } diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts new file mode 100644 index 0000000000..0b8e892cd1 --- /dev/null +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -0,0 +1,140 @@ +import vm from 'node:vm'; +import { describe, expect, it } from 'vitest'; +import * as stepSerialization from '../serialization/step.js'; +import { createContext } from '../vm/index.js'; +import { prepareRetainedStepInput } from './retained-step-input.js'; + +const seed = 'retained-step-input'; +const fixedTimestamp = 1_700_000_000_000; + +describe('prepareRetainedStepInput', () => { + it('clones passive cross-realm data into the host realm', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const buffer = new ArrayBuffer(8); + return { + nested: [{ ok: true }], + map: new Map([["key", new Set([1, 2])]]), + bytes: new Uint8Array(buffer, 2, 4), + }; + })()`, + context + ); + + const prepared = prepareRetainedStepInput(value, workflowGlobal); + + expect(prepared.retainable).toBe(true); + if (!prepared.retainable) return; + expect(prepared.value).toEqual({ + nested: [{ ok: true }], + map: new Map([['key', new Set([1, 2])]]), + bytes: new Uint8Array(4), + }); + expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); + }); + + it('produces the same serialized bytes as ordinary VM traversal', async () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `({ + nested: [{ ok: true, missing: undefined }], + map: new Map([["key", new Set([1, 2])]]), + bytes: new Uint8Array([1, 2, 3, 4]), + date: new Date(1234), + regexp: /workflow/gi, + })`, + context + ); + const prepared = prepareRetainedStepInput(value, workflowGlobal); + expect(prepared.retainable).toBe(true); + if (!prepared.retainable) return; + + const original = await stepSerialization.serialize(value, undefined, { + global: workflowGlobal, + }); + const cloned = await stepSerialization.serialize( + prepared.value, + undefined, + { global: globalThis } + ); + + expect(cloned).toEqual(original); + }); + + it('declines accessors without invoking them', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + return { get value() { globalThis.__retainedTestCalls++; return 1; } }; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); + + it('declines proxies without invoking their traps', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + return new Proxy({ value: 1 }, { + ownKeys(target) { + globalThis.__retainedTestCalls++; + return Reflect.ownKeys(target); + } + }); + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); + + it('declines custom class serializers without invoking them', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + class Value { + static classId = "test/Value"; + static [Symbol.for("workflow-serialize")](instance) { + globalThis.__retainedTestCalls++; + return { value: instance.value }; + } + constructor(value) { this.value = value; } + } + return new Value(1); + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts new file mode 100644 index 0000000000..1440dd4728 --- /dev/null +++ b/packages/core/src/runtime/retained-step-input.ts @@ -0,0 +1,193 @@ +import { types } from 'node:util'; + +export type RetainedStepInputPreparation = + | { readonly retainable: true; readonly value: unknown } + | { readonly retainable: false }; + +const typedArrayNames = [ + 'BigInt64Array', + 'BigUint64Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'Int8Array', + 'Int16Array', + 'Int32Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Uint16Array', + 'Uint32Array', +] as const; + +function hasAllowedPrototype( + value: object, + workflowGlobal: Record, + constructorName: string +): boolean { + const prototype = Object.getPrototypeOf(value); + return ( + prototype === + globalThis[constructorName as keyof typeof globalThis]?.prototype || + prototype === workflowGlobal[constructorName]?.prototype + ); +} + +function isArrayIndex(key: string): boolean { + const index = Number(key); + return ( + Number.isInteger(index) && + index >= 0 && + index < 2 ** 32 - 1 && + String(index) === key + ); +} + +/** + * Whether structured cloning `value` can avoid executing workflow-owned code. + * + * The retained VM must not observe serialization side effects that a later + * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom + * classes, and platform wrappers can all execute workflow code while devalue + * traverses them, so they deliberately decline the fast path. Plain data and + * standard in-memory collections are cloned into the host realm before + * serialization, keeping even mutable VM prototypes out of the write path. + */ +function isPassivelyCloneable( + value: unknown, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if ( + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'bigint' || + typeof value === 'number' || + typeof value === 'string' + ) { + return true; + } + if (typeof value !== 'object' || types.isProxy(value)) return false; + if (seen.has(value)) return true; + seen.add(value); + + if (Array.isArray(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Array')) return false; + for (const key of Reflect.ownKeys(value)) { + if (key === 'length') continue; + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor) return false; + if (key === 'then' || key === Symbol.toStringTag) return false; + if (typeof key !== 'string' || !isArrayIndex(key)) { + if (descriptor.enumerable) return false; + continue; + } + if (!('value' in descriptor)) return false; + if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { + return false; + } + } + return true; + } + + if (types.isMap(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + retainable &&= + isPassivelyCloneable(key, workflowGlobal, seen) && + isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; + } + + if (types.isSet(value)) { + if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; + } + + if (types.isDate(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'Date') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isRegExp(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'RegExp') && + Reflect.ownKeys(value).every((key) => key === 'lastIndex') + ); + } + if (types.isArrayBuffer(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'ArrayBuffer') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isSharedArrayBuffer(value)) return false; + if (types.isDataView(value)) { + return ( + hasAllowedPrototype(value, workflowGlobal, 'DataView') && + Reflect.ownKeys(value).length === 0 + ); + } + if (types.isTypedArray(value)) { + return ( + Reflect.ownKeys(value).every( + (key) => typeof key === 'string' && isArrayIndex(key) + ) && + typedArrayNames.some( + (name) => + workflowGlobal[name] !== undefined && + hasAllowedPrototype(value, workflowGlobal, name) + ) + ); + } + + const prototype = Object.getPrototypeOf(value); + if ( + prototype !== Object.prototype && + prototype !== workflowGlobal.Object?.prototype + ) { + return false; + } + + for (const key of Reflect.ownKeys(value)) { + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (!descriptor) return false; + if (typeof key === 'symbol') { + if (key === Symbol.toStringTag) return false; + if (descriptor.enumerable) return false; + continue; + } + if (!descriptor.enumerable) { + if (key === 'constructor' || key === 'then') return false; + continue; + } + if (key === '__proto__' || !('value' in descriptor)) return false; + if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { + return false; + } + } + return true; +} + +export function prepareRetainedStepInput( + value: unknown, + workflowGlobal: Record +): RetainedStepInputPreparation { + if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { + return { retainable: false }; + } + try { + return { retainable: true, value: structuredClone(value) }; + } catch { + return { retainable: false }; + } +} diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 291a45f57d..e0c35f63ef 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,6 +31,7 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; +import { prepareRetainedStepInput } from './retained-step-input.js'; export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -56,6 +57,11 @@ export interface SuspensionHandlerParams { * where `run_started` was already awaited up front. */ runReadyBarrier?: Promise; + /** + * Prepare new step inputs without traversing workflow-owned objects during + * serialization. Enabled only while the caller is holding a retained VM. + */ + prepareForRetention?: boolean; } /** @@ -121,6 +127,8 @@ export interface SuspensionHandlerResult { * durably creating the user's hooks doesn't count as runtime overhead. */ hookCreationMs: number; + /** Whether every newly serialized step input was passive retained-VM data. */ + retainedStepInputsSafe: boolean; } async function createHookEvent({ @@ -205,6 +213,7 @@ export async function handleSuspension({ requestId, eventLog, runReadyBarrier, + prepareForRetention = false, }: SuspensionHandlerParams): Promise { const runId = run.runId; @@ -488,6 +497,35 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); + let retainedStepInputsSafe = true; + const inputsByCorrelationId = new Map< + string, + { value: unknown; global: Record } + >(); + for (const queueItem of stepItems) { + if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; + const value = { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }; + if (prepareForRetention) { + const prepared = prepareRetainedStepInput(value, suspension.globalThis); + if (prepared.retainable) { + inputsByCorrelationId.set(queueItem.correlationId, { + value: prepared.value, + global: globalThis, + }); + continue; + } + retainedStepInputsSafe = false; + } + inputsByCorrelationId.set(queueItem.correlationId, { + value, + global: suspension.globalThis, + }); + } + // Lazy inline start: defer the step_created write for up to // `getMaxInlineSteps()` steps the caller will run inline (in parallel). Each // step is created on the fly by the lazy `step_started` executeStep sends @@ -521,15 +559,17 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { + const input = inputsByCorrelationId.get(queueItem.correlationId); + if (!input) { + throw new Error( + `Missing prepared input for step ${queueItem.correlationId}` + ); + } const dehydratedInput = await dehydrateStepArguments( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, + input.value, runId, encryptionKey, - suspension.globalThis, + input.global, false, compression ); @@ -716,6 +756,7 @@ export async function handleSuspension({ hasAttributeEvents: attributeItems.length > 0, hasHookEvents: hooksNeedingCreation.length > 0, hookCreationMs, + retainedStepInputsSafe, }; } diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index ffd23eae66..677a5ba1f1 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -284,21 +284,6 @@ describe('host-timed async', () => { }); }); -describe('randomDrawCount', () => { - it('counts every draw from the seeded stream', () => { - const { context, randomDrawCount } = createContext({ - seed, - fixedTimestamp, - }); - - expect(randomDrawCount()).toBe(0); - vm.runInContext('Math.random()', context); - expect(randomDrawCount()).toBe(1); - vm.runInContext('crypto.getRandomValues(new Uint8Array(4))', context); - expect(randomDrawCount()).toBe(5); - }); -}); - describe('crypto.subtle.digest', () => { it.each([ 'SHA-1', diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 325862c5f8..03bc34d2d7 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -47,17 +47,7 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { export function createContext(options: CreateContextOptions) { let { fixedTimestamp } = options; const { seed } = options; - // Count every draw from the seeded stream (Math.random, getRandomValues, - // randomUUID, and the ULIDs/nanoids derived from them all flow through this - // one closure). The runtime compares the count across the suspension - // boundary to detect serialization side effects (see - // `canRetainWorkflowSession`). - const baseRng = seedrandom(seed); - let randomDrawCount = 0; - const rng = () => { - randomDrawCount++; - return baseRng(); - }; + const rng = seedrandom(seed); const context = vmCreateContext(); const g: typeof globalThis = runInContext('globalThis', context); @@ -205,6 +195,5 @@ export function createContext(options: CreateContextOptions) { updateTimestamp: (timestamp: number) => { fixedTimestamp = timestamp; }, - randomDrawCount: () => randomDrawCount, }; } diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 35ee9313a3..83f25823b7 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -179,14 +179,6 @@ function updateSuspendedSession( export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; - /** - * Draws from the VM's seeded random stream so far. The runtime compares - * this across the suspension boundary: if post-suspension argument - * serialization (getters, `WORKFLOW_SERIALIZE` hooks) drew randomness, the - * retained VM's future correlation IDs would desync from what a fresh - * replay regenerates, so the session must not be retained. - */ - randomDrawCount(): number; resume(events: Event[]): WorkflowSessionResumeResult; } @@ -377,7 +369,6 @@ function createWorkflowSession({ context, globalThis: vmGlobalThis, updateTimestamp, - randomDrawCount, } = createContext({ seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, fixedTimestamp, @@ -1145,7 +1136,6 @@ function createWorkflowSession({ const session: WorkflowSession = { workflowRun, argumentCount: args.length, - randomDrawCount, resume(nextEvents) { switch (state.type) { case 'suspended': { From 744f0b7c1078648fa2e56f6ecbc775b88d1460b0 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:08:42 -0700 Subject: [PATCH 11/30] fix(core): harden the passive step-input walker - require enumerable on array index descriptors: structuredClone drops non-enumerable indices that devalue persists - read workflow globals and constructor prototypes via own-property descriptors only, so validation can never execute workflow-owned accessors on redefined globals --- .../src/runtime/retained-step-input.test.ts | 48 ++++ .../core/src/runtime/retained-step-input.ts | 237 +++++++++++------- .../core/src/runtime/suspension-handler.ts | 21 +- 3 files changed, 208 insertions(+), 98 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 0b8e892cd1..c316a42992 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -138,3 +138,51 @@ describe('prepareRetainedStepInput', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('prepareRetainedStepInput review-hardening', () => { + it('declines non-enumerable array indices (structuredClone would drop them)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1, 2]; + Object.defineProperty(arr, "0", { value: 7, enumerable: false }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); + + it('never performs a property Get on redefined workflow globals', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + const arr = [1]; + for (const name of ["Array", "Object"]) { + Object.defineProperty(globalThis, name, { + get() { globalThis.__retainedTestCalls++; throw new Error("boom"); }, + }); + } + return { arr }; + })()`, + context + ); + + // The vandalized globals make the VM-realm prototypes unverifiable, so + // validation declines — without throwing or invoking the getters. + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 1440dd4728..129b1922be 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -19,17 +19,28 @@ const typedArrayNames = [ 'Uint32Array', ] as const; +// Own data-property read that never performs a property Get — workflow code +// can redefine its globals (or their `prototype` slots) with accessors, and +// validation must not execute workflow-owned code. +function ownDataProperty(target: object, key: string): unknown { + const descriptor = Object.getOwnPropertyDescriptor(target, key); + return descriptor && 'value' in descriptor ? descriptor.value : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, constructorName: string ): boolean { const prototype = Object.getPrototypeOf(value); - return ( - prototype === - globalThis[constructorName as keyof typeof globalThis]?.prototype || - prototype === workflowGlobal[constructorName]?.prototype - ); + const hostConstructor = globalThis[ + constructorName as keyof typeof globalThis + ] as { prototype?: object } | undefined; + if (hostConstructor !== undefined && prototype === hostConstructor.prototype) + return true; + const vmConstructor = ownDataProperty(workflowGlobal, constructorName); + if (typeof vmConstructor !== 'function') return false; + return prototype === ownDataProperty(vmConstructor, 'prototype'); } function isArrayIndex(key: string): boolean { @@ -42,76 +53,75 @@ function isArrayIndex(key: string): boolean { ); } -/** - * Whether structured cloning `value` can avoid executing workflow-owned code. - * - * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom - * classes, and platform wrappers can all execute workflow code while devalue - * traverses them, so they deliberately decline the fast path. Plain data and - * standard in-memory collections are cloned into the host realm before - * serialization, keeping even mutable VM prototypes out of the write path. - */ -function isPassivelyCloneable( - value: unknown, +function isPassiveArrayProperty( + array: unknown[], + key: string | symbol, workflowGlobal: Record, seen: WeakSet ): boolean { - if ( - value === null || - value === undefined || - typeof value === 'boolean' || - typeof value === 'bigint' || - typeof value === 'number' || - typeof value === 'string' - ) { - return true; + if (key === 'length') return true; + const descriptor = Object.getOwnPropertyDescriptor(array, key); + if (!descriptor) return false; + if (key === 'then' || key === Symbol.toStringTag) return false; + if (typeof key !== 'string' || !isArrayIndex(key)) { + return !descriptor.enumerable; } - if (typeof value !== 'object' || types.isProxy(value)) return false; - if (seen.has(value)) return true; - seen.add(value); + // Non-enumerable indices are dropped by structuredClone but persisted by + // devalue, so they must decline the fast path. + return ( + descriptor.enumerable === true && + 'value' in descriptor && + isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + ); +} - if (Array.isArray(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Array')) return false; - for (const key of Reflect.ownKeys(value)) { - if (key === 'length') continue; - const descriptor = Object.getOwnPropertyDescriptor(value, key); - if (!descriptor) return false; - if (key === 'then' || key === Symbol.toStringTag) return false; - if (typeof key !== 'string' || !isArrayIndex(key)) { - if (descriptor.enumerable) return false; - continue; - } - if (!('value' in descriptor)) return false; - if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { - return false; - } - } - return true; - } +function isPassiveArray( + value: unknown[], + workflowGlobal: Record, + seen: WeakSet +): boolean { + return ( + hasAllowedPrototype(value, workflowGlobal, 'Array') && + Reflect.ownKeys(value).every((key) => + isPassiveArrayProperty(value, key, workflowGlobal, seen) + ) + ); +} - if (types.isMap(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { - retainable &&= - isPassivelyCloneable(key, workflowGlobal, seen) && - isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; - } +function isPassiveMap( + value: Map, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + retainable &&= + isPassivelyCloneable(key, workflowGlobal, seen) && + isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; +} - if (types.isSet(value)) { - if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { - retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; - } +function isPassiveSet( + value: Set, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; + if (Reflect.ownKeys(value).length > 0) return false; + let retainable = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); + }); + return retainable; +} +function isPassiveBuiltIn( + value: object, + workflowGlobal: Record +): boolean | undefined { if (types.isDate(value)) { return ( hasAllowedPrototype(value, workflowGlobal, 'Date') && @@ -142,40 +152,83 @@ function isPassivelyCloneable( Reflect.ownKeys(value).every( (key) => typeof key === 'string' && isArrayIndex(key) ) && - typedArrayNames.some( - (name) => - workflowGlobal[name] !== undefined && - hasAllowedPrototype(value, workflowGlobal, name) + typedArrayNames.some((name) => + hasAllowedPrototype(value, workflowGlobal, name) ) ); } + return undefined; +} - const prototype = Object.getPrototypeOf(value); +function isPassiveObjectProperty( + object: object, + key: string | symbol, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const descriptor = Object.getOwnPropertyDescriptor(object, key); + if (!descriptor) return false; + if (typeof key === 'symbol') { + return key !== Symbol.toStringTag && !descriptor.enumerable; + } + if (!descriptor.enumerable) { + return key !== 'constructor' && key !== 'then'; + } + return ( + key !== '__proto__' && + 'value' in descriptor && + isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + ); +} + +function isPassivePlainObject( + value: object, + workflowGlobal: Record, + seen: WeakSet +): boolean { + if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; + return Reflect.ownKeys(value).every((key) => + isPassiveObjectProperty(value, key, workflowGlobal, seen) + ); +} + +/** + * Whether structured cloning `value` can avoid executing workflow-owned code. + * + * The retained VM must not observe serialization side effects that a later + * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom + * classes, and platform wrappers can all execute workflow code while devalue + * traverses them, so they deliberately decline the fast path. Plain data and + * standard in-memory collections are cloned into the host realm before + * serialization, keeping even mutable VM prototypes out of the write path. + */ +function isPassivelyCloneable( + value: unknown, + workflowGlobal: Record, + seen: WeakSet +): boolean { if ( - prototype !== Object.prototype && - prototype !== workflowGlobal.Object?.prototype + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'bigint' || + typeof value === 'number' || + typeof value === 'string' ) { - return false; + return true; } + if (typeof value !== 'object' || types.isProxy(value)) return false; + if (seen.has(value)) return true; + seen.add(value); - for (const key of Reflect.ownKeys(value)) { - const descriptor = Object.getOwnPropertyDescriptor(value, key); - if (!descriptor) return false; - if (typeof key === 'symbol') { - if (key === Symbol.toStringTag) return false; - if (descriptor.enumerable) return false; - continue; - } - if (!descriptor.enumerable) { - if (key === 'constructor' || key === 'then') return false; - continue; - } - if (key === '__proto__' || !('value' in descriptor)) return false; - if (!isPassivelyCloneable(descriptor.value, workflowGlobal, seen)) { - return false; - } + if (Array.isArray(value)) { + return isPassiveArray(value, workflowGlobal, seen); } - return true; + if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); + if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); + + const builtIn = isPassiveBuiltIn(value, workflowGlobal); + return builtIn ?? isPassivePlainObject(value, workflowGlobal, seen); } export function prepareRetainedStepInput( diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index e0c35f63ef..7fb976c9c0 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -131,6 +131,17 @@ export interface SuspensionHandlerResult { retainedStepInputsSafe: boolean; } +function getStepInput( + inputs: Map }>, + correlationId: string +): { value: unknown; global: Record } { + const input = inputs.get(correlationId); + if (!input) { + throw new Error(`Missing prepared input for step ${correlationId}`); + } + return input; +} + async function createHookEvent({ runId, hookEvent, @@ -559,12 +570,10 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const input = inputsByCorrelationId.get(queueItem.correlationId); - if (!input) { - throw new Error( - `Missing prepared input for step ${queueItem.correlationId}` - ); - } + const input = getStepInput( + inputsByCorrelationId, + queueItem.correlationId + ); const dehydratedInput = await dehydrateStepArguments( input.value, runId, From 8d96711611d05a497d54968d1030021823b2ce0d Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:09:49 -0700 Subject: [PATCH 12/30] fix(core): guard proxied constructors in the passive-input walker constructorPrototype reads both realms' constructors via own-property descriptors only and refuses proxies before any descriptor read, so a proxied redefined global can never observe validation. --- .../src/runtime/retained-step-input.test.ts | 26 ++++++++++++++ .../core/src/runtime/retained-step-input.ts | 35 ++++++++++++++----- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index c316a42992..02c69e5410 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -185,4 +185,30 @@ describe('prepareRetainedStepInput review-hardening', () => { }); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); + + it('never inspects a proxied workflow constructor', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1]; + globalThis.__retainedTestCalls = 0; + globalThis.Array = new Proxy(function Array() {}, { + getOwnPropertyDescriptor(target, key) { + globalThis.__retainedTestCalls++; + return Reflect.getOwnPropertyDescriptor(target, key); + }, + }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 129b1922be..5e83af11d7 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -27,20 +27,39 @@ function ownDataProperty(target: object, key: string): unknown { return descriptor && 'value' in descriptor ? descriptor.value : undefined; } +function constructorPrototype( + realmGlobal: Record, + constructorName: string +): object | undefined { + const ctor = ownDataProperty(realmGlobal, constructorName); + if ( + (typeof ctor !== 'function' && typeof ctor !== 'object') || + ctor === null || + types.isProxy(ctor) + ) { + return undefined; + } + const prototype = ownDataProperty(ctor, 'prototype'); + return typeof prototype === 'object' && + prototype !== null && + !types.isProxy(prototype) + ? prototype + : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, constructorName: string ): boolean { const prototype = Object.getPrototypeOf(value); - const hostConstructor = globalThis[ - constructorName as keyof typeof globalThis - ] as { prototype?: object } | undefined; - if (hostConstructor !== undefined && prototype === hostConstructor.prototype) - return true; - const vmConstructor = ownDataProperty(workflowGlobal, constructorName); - if (typeof vmConstructor !== 'function') return false; - return prototype === ownDataProperty(vmConstructor, 'prototype'); + return ( + prototype === + constructorPrototype( + globalThis as unknown as Record, + constructorName + ) || prototype === constructorPrototype(workflowGlobal, constructorName) + ); } function isArrayIndex(key: string): boolean { From fd73574c4b0ee3a53ebfaa0be73c93478910c73b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:19:00 -0700 Subject: [PATCH 13/30] fix(core): preserve retention gate after rebase --- packages/core/src/runtime.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 05f8f09795..fea67705c9 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -325,16 +325,19 @@ function canRetainWorkflowSession( suspension: WorkflowSuspension, events: Event[] ): boolean { - return ( - isVmRetentionEnabled() && - suspension.stepCount > 0 && - suspension.hookCount === 0 && - suspension.waitCount === 0 && - suspension.attributeCount === 0 && - suspension.hookDisposedCount === 0 && - suspension.abortCount === 0 && - !hasOpenHookOrWait(events) - ); + if ( + !isVmRetentionEnabled() || + suspension.stepCount === 0 || + suspension.hookCount > 0 || + suspension.waitCount > 0 || + suspension.attributeCount > 0 || + suspension.hookDisposedCount > 0 || + suspension.abortCount > 0 + ) { + return false; + } + const { openHook, openWait } = openHookAndWaitState(events); + return !openHook && !openWait; } /** From bd07e118d6278e6edb9a070e6b808e2d5888713e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:23:16 -0700 Subject: [PATCH 14/30] fix(core): all-or-nothing clone batches; reject SAB views in digest - A mixed step batch (one unsafe sibling input) now serializes every input through the ordinary VM path: a clone snapshotted before an unsafe sibling's serialization runs its getters could otherwise durably capture stale sibling state. - crypto.subtle.digest rejects SharedArrayBuffer-backed views with TypeError, matching WebCrypto's BufferSource contract. --- packages/core/src/retained-vm-loop.test.ts | 22 ++++++ .../core/src/runtime/suspension-handler.ts | 70 ++++++++----------- packages/core/src/vm/index.test.ts | 13 ++++ packages/core/src/vm/index.ts | 7 ++ 4 files changed, 71 insertions(+), 41 deletions(-) diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 532f981f79..d24b59bcc2 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -87,6 +87,19 @@ registerSerializationClass( RetainedSerializerValue ); +// A parallel batch where one sibling's input is unsafe must serialize the +// WHOLE batch through the ordinary VM path (all-or-nothing) and demote. +const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const [a, b] = await Promise.all([ + s1({ get x() { return 1; } }), + s2({ plain: true }), + ]); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -297,6 +310,15 @@ describe('retained VM through the inline replay loop', () => { ).toBe(0); }); + it('demotes retention when any input in a parallel batch is unsafe', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_mixed_batch', + mixedBatchWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBeGreaterThan(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 7fb976c9c0..ad980e1ce7 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -131,17 +131,6 @@ export interface SuspensionHandlerResult { retainedStepInputsSafe: boolean; } -function getStepInput( - inputs: Map }>, - correlationId: string -): { value: unknown; global: Record } { - const input = inputs.get(correlationId); - if (!input) { - throw new Error(`Missing prepared input for step ${correlationId}`); - } - return input; -} - async function createHookEvent({ runId, hookEvent, @@ -508,33 +497,31 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); + // All-or-nothing: clones are only used when EVERY new step input in the + // batch is passive. A mixed batch would let an unsafe sibling's + // serialization (which can run getters) mutate objects a clone already + // snapshotted, changing what the other step durably receives relative to + // the ordinary in-order VM traversal. let retainedStepInputsSafe = true; - const inputsByCorrelationId = new Map< - string, - { value: unknown; global: Record } - >(); - for (const queueItem of stepItems) { - if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - const value = { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }; - if (prepareForRetention) { - const prepared = prepareRetainedStepInput(value, suspension.globalThis); - if (prepared.retainable) { - inputsByCorrelationId.set(queueItem.correlationId, { - value: prepared.value, - global: globalThis, - }); - continue; + const clonesByCorrelationId = new Map(); + if (prepareForRetention) { + for (const queueItem of stepItems) { + if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; + const prepared = prepareRetainedStepInput( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, + suspension.globalThis + ); + if (!prepared.retainable) { + retainedStepInputsSafe = false; + clonesByCorrelationId.clear(); + break; } - retainedStepInputsSafe = false; + clonesByCorrelationId.set(queueItem.correlationId, prepared.value); } - inputsByCorrelationId.set(queueItem.correlationId, { - value, - global: suspension.globalThis, - }); } // Lazy inline start: defer the step_created write for up to @@ -570,15 +557,16 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const input = getStepInput( - inputsByCorrelationId, - queueItem.correlationId - ); + const clone = clonesByCorrelationId.get(queueItem.correlationId); const dehydratedInput = await dehydrateStepArguments( - input.value, + clone ?? { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, runId, encryptionKey, - input.global, + clone ? globalThis : suspension.globalThis, false, compression ); diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 677a5ba1f1..4ff51153ae 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -370,3 +370,16 @@ describe('crypto.subtle.digest input validation', () => { ).rejects.toThrow(TypeError); }); }); + +describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { + it('rejects SharedArrayBuffer-backed views like WebCrypto does', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + await expect( + vm.runInContext( + 'crypto.subtle.digest("SHA-256", new Uint8Array(new SharedArrayBuffer(4)))', + context + ) + ).rejects.toThrow(TypeError); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 03bc34d2d7..05874cc5c9 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto'; +import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; @@ -31,6 +32,12 @@ const arrayBufferByteLength = Object.getOwnPropertyDescriptor( // TypeError instead of allocating a Uint8Array of that length. function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { if (ArrayBuffer.isView(data)) { + // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. + if (types.isSharedArrayBuffer(data.buffer)) { + throw new TypeError( + 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' + ); + } return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); } arrayBufferByteLength.call(data); From fe425e6e9f98eef6be3447a606442e91abf44b43 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:43:19 -0700 Subject: [PATCH 15/30] fix(core): narrow the fast path to prototype-independent types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit devalue serializes Map/Set through the realm's iterator protocol and Date/RegExp/typed arrays through prototype getters, all of which workflow code can mutate — so their serialization is not provably passive and their bytes could differ between retained and cold modes. The fast path now accepts only primitives, plain objects, and plain arrays, which devalue traverses exclusively via own-property reads. Slot-bearing exotics decline even with a swapped prototype. The sandbox digest now reads view metadata (buffer/byteOffset/ byteLength) through captured intrinsic getters, so own properties shadowing them cannot change which bytes are hashed or bypass the SharedArrayBuffer rejection. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- .../src/runtime/retained-step-input.test.ts | 49 ++++-- .../core/src/runtime/retained-step-input.ts | 141 +++++------------- packages/core/src/vm/index.test.ts | 21 +++ packages/core/src/vm/index.ts | 53 +++++-- 5 files changed, 136 insertions(+), 130 deletions(-) diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 9c275ef8f0..f9ee1b7eb9 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs containing accessors, proxies, functions, custom class serializers, or other values whose serialization could execute workflow code also fall back to replay. Plain data and standard in-memory collections remain retainable. +- Step inputs made of plain data (objects, arrays, and primitives) remain retainable. Anything whose serialization could execute or observe workflow code — accessors, proxies, functions, custom class serializers, and built-ins such as `Map`, `Set`, `Date`, or typed arrays — falls back to replay for that boundary. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 02c69e5410..cc74ab54ab 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -14,14 +14,11 @@ describe('prepareRetainedStepInput', () => { fixedTimestamp, }); const value = vm.runInContext( - `(() => { - const buffer = new ArrayBuffer(8); - return { - nested: [{ ok: true }], - map: new Map([["key", new Set([1, 2])]]), - bytes: new Uint8Array(buffer, 2, 4), - }; - })()`, + `({ + nested: [{ ok: true }, "text", 42n], + sparse: [1, , 3], + flag: false, + })`, context ); @@ -30,9 +27,9 @@ describe('prepareRetainedStepInput', () => { expect(prepared.retainable).toBe(true); if (!prepared.retainable) return; expect(prepared.value).toEqual({ - nested: [{ ok: true }], - map: new Map([['key', new Set([1, 2])]]), - bytes: new Uint8Array(4), + nested: [{ ok: true }, 'text', 42n], + sparse: [1, undefined, 3], + flag: false, }); expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); }); @@ -45,10 +42,9 @@ describe('prepareRetainedStepInput', () => { const value = vm.runInContext( `({ nested: [{ ok: true, missing: undefined }], - map: new Map([["key", new Set([1, 2])]]), - bytes: new Uint8Array([1, 2, 3, 4]), - date: new Date(1234), - regexp: /workflow/gi, + sparse: [1, , 3], + big: 42n, + text: "workflow", })`, context ); @@ -68,6 +64,29 @@ describe('prepareRetainedStepInput', () => { expect(cloned).toEqual(original); }); + it('declines built-ins whose serialization consults realm prototypes', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + for (const expression of [ + 'new Map([["k", 1]])', + 'new Set([1, 2])', + 'new Date(1234)', + '/workflow/gi', + 'new Uint8Array([1, 2])', + 'new ArrayBuffer(8)', + 'new DataView(new ArrayBuffer(8))', + 'new Error("boom")', + 'Object.assign(Object.create(Object.prototype), { m: new Map() })', + ]) { + const value = vm.runInContext(expression, context); + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + } + }); + it('declines accessors without invoking them', () => { const { context, globalThis: workflowGlobal } = createContext({ seed, diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 5e83af11d7..976c8ac611 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -4,21 +4,6 @@ export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } | { readonly retainable: false }; -const typedArrayNames = [ - 'BigInt64Array', - 'BigUint64Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'Int8Array', - 'Int16Array', - 'Int32Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Uint16Array', - 'Uint32Array', -] as const; - // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -31,15 +16,15 @@ function constructorPrototype( realmGlobal: Record, constructorName: string ): object | undefined { - const ctor = ownDataProperty(realmGlobal, constructorName); + const constructor = ownDataProperty(realmGlobal, constructorName); if ( - (typeof ctor !== 'function' && typeof ctor !== 'object') || - ctor === null || - types.isProxy(ctor) + (typeof constructor !== 'function' && typeof constructor !== 'object') || + constructor === null || + types.isProxy(constructor) ) { return undefined; } - const prototype = ownDataProperty(ctor, 'prototype'); + const prototype = ownDataProperty(constructor, 'prototype'); return typeof prototype === 'object' && prototype !== null && !types.isProxy(prototype) @@ -62,6 +47,27 @@ function hasAllowedPrototype( ); } +// Cloneable exotics (per the structured-clone spec) whose devalue/reducer +// serialization reads prototype methods, getters, or iterators the workflow +// realm can mutate — serializing them is not provably passive, so they +// decline the fast path even when their prototype identity looks intact. +function isSlotBearingExotic(value: object): boolean { + return ( + types.isMap(value) || + types.isSet(value) || + types.isDate(value) || + types.isRegExp(value) || + types.isArrayBuffer(value) || + types.isSharedArrayBuffer(value) || + types.isDataView(value) || + types.isTypedArray(value) || + types.isBoxedPrimitive(value) || + types.isNativeError(value) || + types.isPromise(value) || + types.isArgumentsObject(value) + ); +} + function isArrayIndex(key: string): boolean { const index = Number(key); return ( @@ -107,78 +113,6 @@ function isPassiveArray( ); } -function isPassiveMap( - value: Map, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Map')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { - retainable &&= - isPassivelyCloneable(key, workflowGlobal, seen) && - isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; -} - -function isPassiveSet( - value: Set, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Set')) return false; - if (Reflect.ownKeys(value).length > 0) return false; - let retainable = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { - retainable &&= isPassivelyCloneable(entryValue, workflowGlobal, seen); - }); - return retainable; -} - -function isPassiveBuiltIn( - value: object, - workflowGlobal: Record -): boolean | undefined { - if (types.isDate(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'Date') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isRegExp(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'RegExp') && - Reflect.ownKeys(value).every((key) => key === 'lastIndex') - ); - } - if (types.isArrayBuffer(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'ArrayBuffer') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isSharedArrayBuffer(value)) return false; - if (types.isDataView(value)) { - return ( - hasAllowedPrototype(value, workflowGlobal, 'DataView') && - Reflect.ownKeys(value).length === 0 - ); - } - if (types.isTypedArray(value)) { - return ( - Reflect.ownKeys(value).every( - (key) => typeof key === 'string' && isArrayIndex(key) - ) && - typedArrayNames.some((name) => - hasAllowedPrototype(value, workflowGlobal, name) - ) - ); - } - return undefined; -} - function isPassiveObjectProperty( object: object, key: string | symbol, @@ -212,14 +146,18 @@ function isPassivePlainObject( } /** - * Whether structured cloning `value` can avoid executing workflow-owned code. + * Whether structured cloning `value` is provably byte-equivalent to the + * ordinary VM serialization AND cannot execute workflow-owned code. * * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct. Proxy traps, accessors, functions, custom - * classes, and platform wrappers can all execute workflow code while devalue - * traverses them, so they deliberately decline the fast path. Plain data and - * standard in-memory collections are cloned into the host realm before - * serialization, keeping even mutable VM prototypes out of the write path. + * cold replay cannot reconstruct, and the durable bytes must not depend on + * `WORKFLOW_RETAINED_VM`. Only primitives, plain objects, and plain arrays + * qualify: devalue traverses them exclusively through own-property reads + * (`Object.keys`, `Object.hasOwn` + indices), so no workflow-realm prototype + * state can influence the output. Everything else — proxies, accessors, + * functions, custom classes, and built-ins like Map/Set/Date whose + * serialization consults mutable realm prototypes (iterators, getters) — + * declines the fast path and serializes through the ordinary VM traversal. */ function isPassivelyCloneable( value: unknown, @@ -243,11 +181,8 @@ function isPassivelyCloneable( if (Array.isArray(value)) { return isPassiveArray(value, workflowGlobal, seen); } - if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); - if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); - - const builtIn = isPassiveBuiltIn(value, workflowGlobal); - return builtIn ?? isPassivePlainObject(value, workflowGlobal, seen); + if (isSlotBearingExotic(value)) return false; + return isPassivePlainObject(value, workflowGlobal, seen); } export function prepareRetainedStepInput( diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 4ff51153ae..7e778e10e1 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -383,3 +383,24 @@ describe('crypto.subtle.digest SharedArrayBuffer rejection', () => { ).rejects.toThrow(TypeError); }); }); + +describe('crypto.subtle.digest view metadata', () => { + it('reads view ranges from internal slots, ignoring shadowed properties', async () => { + const { context } = createContext({ seed, fixedTimestamp }); + + const result = await vm.runInContext( + `(() => { + const view = new Uint8Array([1, 2, 3, 4]); + Object.defineProperty(view, "byteLength", { value: 0 }); + Object.defineProperty(view, "byteOffset", { value: 2 }); + return crypto.subtle.digest("SHA-256", view); + })()`, + context + ); + const expected = await globalThis.crypto.subtle.digest( + 'SHA-256', + new Uint8Array([1, 2, 3, 4]) + ); + expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 05874cc5c9..39386d0378 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -20,28 +20,59 @@ const DIGEST_ALGORITHMS: Record = { 'SHA-512': 'sha512', }; -// biome-ignore lint/style/noNonNullAssertion: byteLength always exists on ArrayBuffer.prototype -const arrayBufferByteLength = Object.getOwnPropertyDescriptor( +// Intrinsic prototype getters, captured so view metadata is read from +// internal slots like WebCrypto's BufferSource conversion — own properties +// shadowing `buffer`/`byteOffset`/`byteLength` on a view must not change +// which bytes are hashed. +function intrinsicGetter(prototype: object, name: string) { + // biome-ignore lint/style/noNonNullAssertion: intrinsic accessors always exist + return Object.getOwnPropertyDescriptor(prototype, name)!.get!; +} +const arrayBufferByteLength = intrinsicGetter( ArrayBuffer.prototype, 'byteLength' -)!.get!; +); +const typedArrayPrototype = Object.getPrototypeOf( + Uint8Array.prototype +) as object; +const viewGetters = { + typedArray: { + buffer: intrinsicGetter(typedArrayPrototype, 'buffer'), + byteOffset: intrinsicGetter(typedArrayPrototype, 'byteOffset'), + byteLength: intrinsicGetter(typedArrayPrototype, 'byteLength'), + }, + dataView: { + buffer: intrinsicGetter(DataView.prototype, 'buffer'), + byteOffset: intrinsicGetter(DataView.prototype, 'byteOffset'), + byteLength: intrinsicGetter(DataView.prototype, 'byteLength'), + }, +}; -// WebCrypto BufferSource conversion: views pass through; anything else must -// be a real ArrayBuffer (the native byteLength getter is a brand check that -// works across vm realms) so that e.g. a plain number is rejected with -// TypeError instead of allocating a Uint8Array of that length. +// WebCrypto BufferSource conversion: typed-array/DataView views are read via +// internal slots; anything else must be a real ArrayBuffer (the native +// byteLength getter is a brand check that works across vm realms) so that +// e.g. a plain number is rejected with TypeError instead of allocating a +// Uint8Array of that length. function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { - if (ArrayBuffer.isView(data)) { + if (types.isTypedArray(data) || types.isDataView(data)) { + const getters = types.isDataView(data) + ? viewGetters.dataView + : viewGetters.typedArray; + const buffer = getters.buffer.call(data) as ArrayBuffer; // WebCrypto's BufferSource excludes SharedArrayBuffer-backed views. - if (types.isSharedArrayBuffer(data.buffer)) { + if (types.isSharedArrayBuffer(buffer)) { throw new TypeError( 'crypto.subtle.digest does not accept SharedArrayBuffer-backed views' ); } - return new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + return new Uint8Array( + buffer, + getters.byteOffset.call(data) as number, + getters.byteLength.call(data) as number + ); } arrayBufferByteLength.call(data); - return new Uint8Array(data); + return new Uint8Array(data as ArrayBuffer); } /** From 9a5f0705d2527b9b2b221e9223aed1790c353141 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:08:10 -0700 Subject: [PATCH 16/30] fix(core): freeze serialization-consulted sandbox intrinsics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instanceof dispatch (Symbol.hasInstance via the constructor, Function.prototype, and Object.prototype), the class reducer's value.constructor walk, and devalue's Object/Array traversal all consult intrinsics workflow code could redefine — legally and deterministically — which would make the durable step input depend on WORKFLOW_RETAINED_VM (spoofed values serialize as e.g. Maps on the cold path but as plain clones on the retained path). Freeze Object/Array/Function (constructors and prototypes), the VM collection constructors, and every reducer-referenced global binding (absent ones pinned to undefined) right before the workflow bundle evaluates, so the retained-input equivalence holds by construction. Host-realm constructor escapes (e.g. TextEncoder.constructor) remain out of the determinism contract: code scheduling host timers was never deterministic under ordinary replay either; documented on canRetainWorkflowSession. --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 1 + packages/core/src/runtime.ts | 6 ++ packages/core/src/vm/index.test.ts | 49 ++++++++++- packages/core/src/vm/index.ts | 83 +++++++++++++++++++ packages/core/src/workflow.ts | 6 +- 6 files changed, 144 insertions(+), 3 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 9949f21f78..e1853096a9 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed). +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function` and reducer-referenced global bindings) are frozen in the workflow sandbox. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 6faa7e3df6..59e8458b22 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -103,3 +103,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) - **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. - **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. +- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), and the global bindings for serialization-relevant built-ins (`Map`, `Set`, `Date`, `RegExp`, typed arrays, `Headers`, `URL`, and similar) are frozen. Serialization dispatches on these, so redefining them would make replay unfaithful. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index fea67705c9..02ce5b93ee 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -320,6 +320,12 @@ function openHookAndWaitState(events: Event[]): { * Retain only pure step boundaries with no out-of-band continuation source. * Attributes require replay; hooks and waits can wake another invocation. * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + * + * Quiescence assumes workflow code stays inside the sandbox's determinism + * contract. Escaping to the host realm (e.g. recovering the host `Function` + * constructor from an exposed host class to schedule real timers) makes a + * workflow nondeterministic under ordinary replay too, and is not defended + * here. */ function canRetainWorkflowSession( suspension: WorkflowSuspension, diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 7e778e10e1..92a5db593d 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -1,6 +1,6 @@ import * as vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import { createContext } from './index.js'; +import { createContext, freezeSerializationIntrinsics } from './index.js'; const seed = 'entropy seed'; const fixedTimestamp = 1234567890000; @@ -404,3 +404,50 @@ describe('crypto.subtle.digest view metadata', () => { expect(new Uint8Array(result)).toEqual(new Uint8Array(expected)); }); }); + +describe('freezeSerializationIntrinsics', () => { + it('pins serialization-consulted intrinsics against workflow mutation', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + // Statics and prototypes reject strict-mode definition attempts. + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Map, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext(`"use strict"; Object.prototype.polluted = 1`, context) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Function.prototype[Symbol.hasInstance] = () => true`, + context + ) + ).toThrow(/read only|not extensible/); + + // Bindings cannot be replaced, including intentionally absent ones. + vm.runInContext( + 'try { globalThis.Map = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Map.prototype.get', context)).toBe( + 'function' + ); + vm.runInContext( + 'try { globalThis.Request = function () {} } catch {}', + context + ); + expect(vm.runInContext('typeof Request', context)).toBe('undefined'); + }); + + it('leaves ordinary workflow globals writable', () => { + const { context, globalThis: g } = createContext({ seed, fixedTimestamp }); + freezeSerializationIntrinsics(g); + + expect( + vm.runInContext('globalThis.myState = { ok: true }; myState.ok', context) + ).toBe(true); + }); +}); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 39386d0378..8963898f71 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -75,6 +75,89 @@ function toDigestBytes(data: ArrayBuffer | ArrayBufferView): Uint8Array { return new Uint8Array(data as ArrayBuffer); } +// Global bindings the serialization reducers dispatch on (`value instanceof +// global.X`) plus the intrinsics devalue's own traversal consults. Frozen so +// workflow code cannot make serialization classify a value differently +// between the retained-input fast path (host-realm clone) and the ordinary +// VM traversal — e.g. via a spoofed `Map[Symbol.hasInstance]` or a replaced +// global binding. +const SERIALIZATION_BINDINGS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +/** + * Freeze the sandbox intrinsics that serialization consults. Called after + * ALL SDK globals are installed, immediately before the workflow bundle + * evaluates. + * + * `instanceof` dispatch resolves `Symbol.hasInstance` through the value's + * constructor, `Function.prototype`, and `Object.prototype`; the class + * reducer walks `value.constructor`; devalue reads `Object`/`Array` + * prototypes. Freezing these (and pinning every reducer-referenced global + * binding, including ones that are intentionally absent) makes the + * retained-input equivalence check in `retained-step-input.ts` hold by + * construction instead of by validation. + */ +export function freezeSerializationIntrinsics(g: typeof globalThis): void { + const vmIntrinsics = g as unknown as Record; + // VM-realm intrinsic objects: freeze the objects themselves so statics + // (e.g. `Map[Symbol.hasInstance]`) and prototype members cannot be added. + // Host-realm classes exposed into the sandbox are NOT frozen — only their + // bindings are pinned below. + Object.freeze(g.Object); + Object.freeze(g.Object.prototype); + Object.freeze(g.Array); + Object.freeze(g.Array.prototype); + Object.freeze(g.Function); + Object.freeze(g.Function.prototype); + for (const name of ['Map', 'Set', 'RegExp', 'ArrayBuffer', 'DataView']) { + Object.freeze(vmIntrinsics[name]); + } + for (const name of SERIALIZATION_BINDINGS) { + const descriptor = Object.getOwnPropertyDescriptor(g, name); + Object.defineProperty(g, name, { + // Absent bindings are pinned to undefined so workflow code cannot + // define a spoofing global the reducers would then dispatch on. + value: descriptor && 'value' in descriptor ? descriptor.value : undefined, + writable: false, + configurable: false, + enumerable: descriptor?.enumerable ?? false, + }); + } +} + /** * Creates a Node.js `vm.Context` configured to be usable for * executing workflow logic in a deterministic environment. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 83f25823b7..83a8154ca0 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -42,7 +42,7 @@ import { import * as Attribute from './telemetry/semantic-conventions.js'; import { applyWorkflowSuspensionToSpan, trace } from './telemetry.js'; import { getWorkflowRunStreamId } from './util.js'; -import { createContext } from './vm/index.js'; +import { createContext, freezeSerializationIntrinsics } from './vm/index.js'; import { runCachedWorkflowScript } from './vm/script-cache.js'; import { createAbortSignalStatics, @@ -1030,6 +1030,10 @@ function createWorkflowSession({ // Script rather than the line just past the end of the bundle. That path // is rare (it requires the lookup `?.get(...)` expression to throw) and // does not affect the workflow function or replay determinism. + // All SDK globals are installed; pin the serialization-consulted + // intrinsics before any workflow code can run. + freezeSerializationIntrinsics(vmGlobalThis); + runCachedWorkflowScript(workflowCode, filename, context); const workflowFn = runCachedWorkflowScript( `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, From 3bda890e8ddd836162e60645015a2b6548f54882 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:21:08 -0700 Subject: [PATCH 17/30] fix(core): freeze every non-shared serialization constructor Typed-array constructors (and their shared %TypedArray% parent), the Date wrapper, and the session-local AbortController/AbortSignal/ Request/Response bindings were pinned but not frozen, so workflow code could still add Symbol.hasInstance statics that diverge reducer dispatch between the retained clone (host constructors) and ordinary VM serialization. Freeze every binding value that is not the shared host intrinsic; shared host objects are dispatched identically by both paths, so mutations there cannot cause mode divergence. --- packages/core/src/vm/index.test.ts | 12 ++++++++++++ packages/core/src/vm/index.ts | 28 +++++++++++++++++++--------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 92a5db593d..5896198788 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -426,6 +426,18 @@ describe('freezeSerializationIntrinsics', () => { context ) ).toThrow(/read only|not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Uint8Array, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Date, Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); // Bindings cannot be replaced, including intentionally absent ones. vm.runInContext( diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 8963898f71..a6c94f09c8 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -131,26 +131,36 @@ const SERIALIZATION_BINDINGS = [ * construction instead of by validation. */ export function freezeSerializationIntrinsics(g: typeof globalThis): void { - const vmIntrinsics = g as unknown as Record; - // VM-realm intrinsic objects: freeze the objects themselves so statics - // (e.g. `Map[Symbol.hasInstance]`) and prototype members cannot be added. - // Host-realm classes exposed into the sandbox are NOT frozen — only their - // bindings are pinned below. Object.freeze(g.Object); Object.freeze(g.Object.prototype); Object.freeze(g.Array); Object.freeze(g.Array.prototype); Object.freeze(g.Function); Object.freeze(g.Function.prototype); - for (const name of ['Map', 'Set', 'RegExp', 'ArrayBuffer', 'DataView']) { - Object.freeze(vmIntrinsics[name]); - } + // The typed-array constructors dispatch `instanceof` through their shared + // %TypedArray% parent. + Object.freeze(Object.getPrototypeOf(g.Uint8Array)); + const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); + const value = + descriptor && 'value' in descriptor ? descriptor.value : undefined; + // Freeze VM-realm and session-local constructor objects so statics like + // `Symbol.hasInstance` cannot be added. A binding that references the + // shared host intrinsic stays unfrozen: both serialization paths + // dispatch through that same object, so a mutation cannot make them + // disagree (and freezing it would affect the whole process). + if ( + (typeof value === 'function' || + (typeof value === 'object' && value !== null)) && + value !== hostGlobal[name] + ) { + Object.freeze(value); + } Object.defineProperty(g, name, { // Absent bindings are pinned to undefined so workflow code cannot // define a spoofing global the reducers would then dispatch on. - value: descriptor && 'value' in descriptor ? descriptor.value : undefined, + value, writable: false, configurable: false, enumerable: descriptor?.enumerable ?? false, From 43788b8c69ce087af39cddd8fc1f367f122ac359 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:40:00 -0700 Subject: [PATCH 18/30] fix(core): build retained clones in a pristine realm Replace structuredClone with an explicit deep copy into an SDK-private realm: clones previously inherited host prototypes, which workflow code can reach (e.g. via structuredClone's return values) and vandalize with Symbol.toStringTag or constructor overrides, shifting devalue's classification of the clone relative to the ordinary VM path. The pristine realm is unreachable by any user code, and the explicit copy serializes exactly what devalue traverses (own indices, own enumerable string props). Arrays also now decline own constructor properties, which the class reducer reads even when non-enumerable. --- .../src/runtime/retained-step-input.test.ts | 28 ++++++++- .../core/src/runtime/retained-step-input.ts | 62 +++++++++++++++++-- 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index cc74ab54ab..0d151e9398 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -31,7 +31,11 @@ describe('prepareRetainedStepInput', () => { sparse: [1, undefined, 3], flag: false, }); - expect(Object.getPrototypeOf(prepared.value)).toBe(Object.prototype); + // The clone's prototype chain lives in a pristine realm: not the + // workflow realm, not the host realm. + const cloneProto = Object.getPrototypeOf(prepared.value); + expect(cloneProto).not.toBe(Object.prototype); + expect(Object.getPrototypeOf(cloneProto)).toBe(null); }); it('produces the same serialized bytes as ordinary VM traversal', async () => { @@ -64,6 +68,28 @@ describe('prepareRetainedStepInput', () => { expect(cloned).toEqual(original); }); + it('declines arrays with an own constructor property', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const arr = [1]; + Object.defineProperty(arr, "constructor", { + value: class Fake { static classId = "fake"; }, + enumerable: false, + }); + return arr; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); + it('declines built-ins whose serialization consults realm prototypes', () => { const { context, globalThis: workflowGlobal } = createContext({ seed, diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 976c8ac611..c8bff7a28a 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,9 +1,20 @@ import { types } from 'node:util'; +import { runInContext, createContext as vmCreateContext } from 'node:vm'; export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } | { readonly retainable: false }; +// A pristine realm nothing else can reach: clones are built from its Object +// and Array so their entire prototype chain is immune to intrinsic mutation +// in both the workflow realm and the host realm (workflow code can obtain +// host-realm objects through APIs like `structuredClone` and vandalize their +// prototypes). +const pristineRealm = runInContext( + '({ Object, Array })', + vmCreateContext() +) as { Object: ObjectConstructor; Array: ArrayConstructor }; + // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -87,7 +98,11 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - if (key === 'then' || key === Symbol.toStringTag) return false; + // `then` and `constructor` are read by serialization dispatch even when + // non-enumerable (thenable assimilation, the class reducer). + if (key === 'then' || key === 'constructor' || key === Symbol.toStringTag) { + return false; + } if (typeof key !== 'string' || !isArrayIndex(key)) { return !descriptor.enumerable; } @@ -185,6 +200,42 @@ function isPassivelyCloneable( return isPassivePlainObject(value, workflowGlobal, seen); } +// Deep-copy walker-validated plain data into the pristine realm, copying +// exactly what devalue serializes: dense/sparse own indices for arrays and +// own enumerable string properties for objects. All reads are data-property +// reads — the walker already rejected everything else. +function cloneIntoPristineRealm( + value: unknown, + copies: WeakMap +): unknown { + if (value === null || typeof value !== 'object') return value; + const existing = copies.get(value); + if (existing !== undefined) return existing; + + if (Array.isArray(value)) { + const copy = new pristineRealm.Array(value.length) as unknown[]; + copies.set(value, copy); + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== 'string' || !isArrayIndex(key)) continue; + copy[Number(key)] = cloneIntoPristineRealm( + (value as unknown as Record)[key], + copies + ); + } + return copy; + } + + const copy = new pristineRealm.Object() as Record; + copies.set(value, copy); + for (const key of Object.keys(value)) { + copy[key] = cloneIntoPristineRealm( + (value as Record)[key], + copies + ); + } + return copy; +} + export function prepareRetainedStepInput( value: unknown, workflowGlobal: Record @@ -192,9 +243,8 @@ export function prepareRetainedStepInput( if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { return { retainable: false }; } - try { - return { retainable: true, value: structuredClone(value) }; - } catch { - return { retainable: false }; - } + return { + retainable: true, + value: cloneIntoPristineRealm(value, new WeakMap()), + }; } From 2b736768446e9fd0f5ec0e03916e79f81ff57a0e Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:59:15 -0700 Subject: [PATCH 19/30] fix(core): verify host dispatch pristineness before retained cloning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Host intrinsics are shared with the whole process and cannot be frozen, but workflow code can reach them (structuredClone results, exposed host classes) and install Symbol.hasInstance predicates that distinguish the original from its clone — or WORKFLOW_SERIALIZE statics on host Object/Array that the class reducer reads for host-prototype originals (hydrated step results). prepareRetainedStepInput now verifies, via own-descriptor reads only, that every host dispatch point is pristine and declines retention before any clone exists — so a spoofed predicate can never observe or capture a pristine-realm object. --- .../src/runtime/retained-step-input.test.ts | 30 +++++++ .../core/src/runtime/retained-step-input.ts | 84 ++++++++++++++++++- 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 0d151e9398..e4021e04a2 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -257,3 +257,33 @@ describe('prepareRetainedStepInput review-hardening', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('host dispatch pristineness', () => { + it('declines retention while a host dispatch constructor is spoofed', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext('({ plain: true })', context); + + expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( + true + ); + + Object.defineProperty(Headers, Symbol.hasInstance, { + value: () => false, + configurable: true, + }); + try { + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + } finally { + delete (Headers as any)[Symbol.hasInstance]; + } + + expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( + true + ); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index c8bff7a28a..d28b9be4a9 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,5 +1,6 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; +import { WORKFLOW_SERIALIZE } from '@workflow/serde'; export type RetainedStepInputPreparation = | { readonly retainable: true; readonly value: unknown } @@ -15,6 +16,84 @@ const pristineRealm = runInContext( vmCreateContext() ) as { Object: ObjectConstructor; Array: ArrayConstructor }; +// Host constructors that serialization dispatches on when the clone is +// serialized under the host global — plus Object/Array, whose statics and +// prototypes the class reducer and devalue's tag lookup read for +// host-prototype originals (hydrated step results are host-realm objects). +// The workflow VM's own intrinsics are frozen (see vm/index.ts), but host +// intrinsics are shared with the whole process and cannot be frozen, so +// dispatch-pristineness is verified instead. Any dirt declines retention +// BEFORE a clone exists, so a spoofed predicate can never observe (or +// capture) a pristine-realm object. +const HOST_DISPATCH_CONSTRUCTORS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +function hasOwn(target: object, key: string | symbol): boolean { + return Object.getOwnPropertyDescriptor(target, key) !== undefined; +} + +function isHostDispatchPristine(): boolean { + const hostGlobal = globalThis as unknown as Record; + for (const name of HOST_DISPATCH_CONSTRUCTORS) { + const constructor = hostGlobal[name]; + if (constructor === undefined) continue; + if (hasOwn(constructor as object, Symbol.hasInstance)) return false; + } + for (const constructor of [Object, Array]) { + if ( + hasOwn(constructor, WORKFLOW_SERIALIZE) || + hasOwn(constructor, 'classId') + ) { + return false; + } + } + for (const [prototype, constructor] of [ + [Object.prototype, Object], + [Array.prototype, Array], + ] as const) { + if ( + hasOwn(prototype, Symbol.toStringTag) || + ownDataProperty(prototype, 'constructor') !== constructor + ) { + return false; + } + } + return true; +} + // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. @@ -240,7 +319,10 @@ export function prepareRetainedStepInput( value: unknown, workflowGlobal: Record ): RetainedStepInputPreparation { - if (!isPassivelyCloneable(value, workflowGlobal, new WeakSet())) { + if ( + !isHostDispatchPristine() || + !isPassivelyCloneable(value, workflowGlobal, new WeakSet()) + ) { return { retainable: false }; } return { From caf6a1a5673828a160880a95f56d774cb165ccab Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:16:30 -0700 Subject: [PATCH 20/30] fix(core): reject symbol properties from retained step inputs Reducers dispatch on symbol tags (e.g. the workflow abort-signal markers) that are non-enumerable and dropped by the pristine-realm copy, so a tagged object would serialize as an abort descriptor on the cold path but as plain data on the retained path. --- .../src/runtime/retained-step-input.test.ts | 24 +++++++++++++++++++ .../core/src/runtime/retained-step-input.ts | 15 ++++++------ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index e4021e04a2..4fe7a95646 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -287,3 +287,27 @@ describe('host dispatch pristineness', () => { ); }); }); + +describe('symbol-tagged inputs', () => { + it('declines objects carrying symbol properties (serialization dispatch tags)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const signal = { aborted: false }; + Object.defineProperty(signal, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { + value: "abort-stream", + enumerable: false, + }); + return { signal }; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index d28b9be4a9..23ecf26bd2 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -177,12 +177,13 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - // `then` and `constructor` are read by serialization dispatch even when - // non-enumerable (thenable assimilation, the class reducer). - if (key === 'then' || key === 'constructor' || key === Symbol.toStringTag) { + // Symbol properties can be serialization dispatch tags (e.g. the workflow + // abort-signal markers) that the clone would drop; `then` and + // `constructor` are read by dispatch even when non-enumerable. + if (typeof key === 'symbol' || key === 'then' || key === 'constructor') { return false; } - if (typeof key !== 'string' || !isArrayIndex(key)) { + if (!isArrayIndex(key)) { return !descriptor.enumerable; } // Non-enumerable indices are dropped by structuredClone but persisted by @@ -215,9 +216,9 @@ function isPassiveObjectProperty( ): boolean { const descriptor = Object.getOwnPropertyDescriptor(object, key); if (!descriptor) return false; - if (typeof key === 'symbol') { - return key !== Symbol.toStringTag && !descriptor.enumerable; - } + // Symbol properties can be serialization dispatch tags (e.g. the workflow + // abort-signal markers) that the clone would drop. + if (typeof key === 'symbol') return false; if (!descriptor.enumerable) { return key !== 'constructor' && key !== 'then'; } From e961466c8c20e1901652235047a5488876d9cc75 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:28:40 -0700 Subject: [PATCH 21/30] fix(core): retained inputs accept only own enumerable data properties MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hidden own keys of any kind — non-enumerable properties, accessors, symbols — can be observed by serialization dispatch (reducer probes like .signal, thenable checks, the class reducer) while the pristine clone drops them. With no hidden own keys, every probe on an accepted object resolves deterministically through validated data or pristine prototypes. --- .../src/runtime/retained-step-input.test.ts | 24 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 29 ++++++++----------- 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 4fe7a95646..aaff8608ea 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -311,3 +311,27 @@ describe('symbol-tagged inputs', () => { }); }); }); + +describe('hidden properties', () => { + it('declines objects with non-enumerable properties (reducer probes read them)', () => { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + const value = vm.runInContext( + `(() => { + const controllerish = { plain: true }; + Object.defineProperty(controllerish, "signal", { + get() { return { aborted: false }; }, + enumerable: false, + }); + return controllerish; + })()`, + context + ); + + expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ + retainable: false, + }); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 23ecf26bd2..9daf8105c5 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -177,18 +177,13 @@ function isPassiveArrayProperty( if (key === 'length') return true; const descriptor = Object.getOwnPropertyDescriptor(array, key); if (!descriptor) return false; - // Symbol properties can be serialization dispatch tags (e.g. the workflow - // abort-signal markers) that the clone would drop; `then` and - // `constructor` are read by dispatch even when non-enumerable. - if (typeof key === 'symbol' || key === 'then' || key === 'constructor') { - return false; - } - if (!isArrayIndex(key)) { - return !descriptor.enumerable; - } - // Non-enumerable indices are dropped by structuredClone but persisted by - // devalue, so they must decline the fast path. + // Only own enumerable data indices are passive. Anything hidden — symbol + // tags, non-enumerable properties, accessors — can be observed by + // serialization dispatch (reducer probes, thenable checks, the class + // reducer) while being dropped by the clone. return ( + typeof key === 'string' && + isArrayIndex(key) && descriptor.enumerable === true && 'value' in descriptor && isPassivelyCloneable(descriptor.value, workflowGlobal, seen) @@ -216,14 +211,14 @@ function isPassiveObjectProperty( ): boolean { const descriptor = Object.getOwnPropertyDescriptor(object, key); if (!descriptor) return false; - // Symbol properties can be serialization dispatch tags (e.g. the workflow - // abort-signal markers) that the clone would drop. - if (typeof key === 'symbol') return false; - if (!descriptor.enumerable) { - return key !== 'constructor' && key !== 'then'; - } + // Only own enumerable string-keyed data properties are passive. Anything + // hidden — symbol tags, non-enumerable properties, accessors — can be + // observed by serialization dispatch (reducer probes like `.signal`, + // thenable checks, the class reducer) while being dropped by the clone. return ( + typeof key === 'string' && key !== '__proto__' && + descriptor.enumerable === true && 'value' in descriptor && isPassivelyCloneable(descriptor.value, workflowGlobal, seen) ); From 3498706a6f8ddda48ba3273126d3a83f7382d35b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 17 Jul 2026 16:43:37 -0700 Subject: [PATCH 22/30] fix(core): freeze binding prototype chains for hasInstance lookup Symbol.hasInstance dispatch walks the constructor's prototype chain, so the frozen Date wrapper still exposed the unfrozen original VM Date it delegates statics to. Freeze each non-shared binding's full chain (stopping at host Function/Object prototypes) and verify host Object.prototype carries no added hasInstance on the detection side. --- .../core/src/runtime/retained-step-input.ts | 4 +++ packages/core/src/vm/index.test.ts | 8 ++++++ packages/core/src/vm/index.ts | 25 ++++++++++++++----- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 9daf8105c5..5fb77b1362 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -91,6 +91,10 @@ function isHostDispatchPristine(): boolean { return false; } } + // Constructor prototype chains end at host Object.prototype, where an + // added @@hasInstance would be found by dispatch lookup. (Host + // Function.prototype's @@hasInstance is spec non-configurable.) + if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; return true; } diff --git a/packages/core/src/vm/index.test.ts b/packages/core/src/vm/index.test.ts index 5896198788..e5127de31c 100644 --- a/packages/core/src/vm/index.test.ts +++ b/packages/core/src/vm/index.test.ts @@ -438,6 +438,14 @@ describe('freezeSerializationIntrinsics', () => { context ) ).toThrow(/not extensible/); + // @@hasInstance lookup walks the wrapper's prototype chain to the + // original VM Date, which must be frozen too. + expect(() => + vm.runInContext( + `"use strict"; Object.defineProperty(Object.getPrototypeOf(Date), Symbol.hasInstance, { value: () => true })`, + context + ) + ).toThrow(/not extensible/); // Bindings cannot be replaced, including intentionally absent ones. vm.runInContext( diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index a6c94f09c8..ef190f64e1 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -145,17 +145,30 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { const descriptor = Object.getOwnPropertyDescriptor(g, name); const value = descriptor && 'value' in descriptor ? descriptor.value : undefined; - // Freeze VM-realm and session-local constructor objects so statics like - // `Symbol.hasInstance` cannot be added. A binding that references the - // shared host intrinsic stays unfrozen: both serialization paths - // dispatch through that same object, so a mutation cannot make them - // disagree (and freezing it would affect the whole process). + // Freeze VM-realm and session-local constructor objects — including + // their prototype chains, which `Symbol.hasInstance` lookup walks (the + // Date wrapper delegates statics to the unfrozen original VM Date via + // setPrototypeOf). A binding that references the shared host intrinsic + // stays unfrozen: both serialization paths dispatch through that same + // object, so a mutation cannot make them disagree (and freezing it + // would affect the whole process). Host Function.prototype's + // @@hasInstance is spec-immutable; host Object.prototype is verified + // separately (see isHostDispatchPristine). if ( (typeof value === 'function' || (typeof value === 'object' && value !== null)) && value !== hostGlobal[name] ) { - Object.freeze(value); + let chain: object | null = value as object; + while ( + chain !== null && + chain !== Function.prototype && + chain !== Object.prototype && + !Object.isFrozen(chain) + ) { + Object.freeze(chain); + chain = Object.getPrototypeOf(chain); + } } Object.defineProperty(g, name, { // Absent bindings are pinned to undefined so workflow code cannot From 59aaebb2dd464b9937e8fbdb4d4d52dddde15b29 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:31:14 -0700 Subject: [PATCH 23/30] refactor(core): single-path retained serialization via pinned members (v2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serialize step inputs for retained boundaries through the one ordinary pipeline (original value, workflow global) instead of cloning into a pristine realm and serializing under the host global. With a single serialization event shared by every mode, durable bytes structurally cannot depend on WORKFLOW_RETAINED_VM; the only property retention needs is that serialization executes no workflow code, established by: - the passive walker (descriptor-only, unchanged in spirit), now also accepting Map/Set/Date/typed arrays/ArrayBuffer — the common built-in step arguments — via prototype-identity checks - vm/serialization-pins.ts: the 10 prototype members serialization executes for those built-ins (measured empirically), captured at context creation and identity-verified at each retained boundary; the 'touches only pinned members' test instruments every member and locks the list against serde drift - host-realm instances (hydrated step results) accepted without member verification: host members run host code, which cannot touch retained VM state Deletes the pristine clone realm, the host-dispatch pristineness checks, and the batch clone bookkeeping. --- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 20 + .../src/runtime/retained-step-input.test.ts | 445 +++++++++--------- .../core/src/runtime/retained-step-input.ts | 348 +++++++------- .../core/src/runtime/suspension-handler.ts | 41 +- packages/core/src/vm/index.ts | 16 +- packages/core/src/vm/serialization-pins.ts | 115 +++++ 7 files changed, 551 insertions(+), 436 deletions(-) create mode 100644 packages/core/src/vm/serialization-pins.ts diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 79389b4ed0..17322d7ca9 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, and primitives) remain retainable. Anything whose serialization could execute or observe workflow code — accessors, proxies, functions, custom class serializers, and built-ins such as `Map`, `Set`, `Date`, or typed arrays — falls back to replay for that boundary. +- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary, as does any input if the prototype members serialization relies on have been modified. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index d24b59bcc2..2085b1edad 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -100,6 +100,17 @@ const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP" } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; +// Map/Date/typed-array arguments serialize through pinned prototype members +// (see vm/serialization-pins.ts), so these boundaries stay retainable. +const builtinArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); + const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); + async function workflow() { + const a = await s1({ index: new Map([["k", 1]]), when: new Date(1234) }); + const b = await s2(new Uint8Array([1, 2, 3])); + return a + b; + } + globalThis.__private_workflows = new Map([["workflow", workflow]]);`; + // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -319,6 +330,15 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); + it('retains boundaries whose args are supported built-ins', async () => { + const { vmBuilds, output } = await drive( + 'wrun_retained_builtins', + builtinArgsWorkflow + ); + expect(output).toBeInstanceOf(Uint8Array); + expect(vmBuilds).toBe(1); + }); + it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index aaff8608ea..43c0ab46df 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -1,18 +1,38 @@ import vm from 'node:vm'; import { describe, expect, it } from 'vitest'; -import * as stepSerialization from '../serialization/step.js'; +import { dehydrateStepArguments } from '../serialization.js'; import { createContext } from '../vm/index.js'; -import { prepareRetainedStepInput } from './retained-step-input.js'; +import { isRetainedSerializationPassive } from './retained-step-input.js'; const seed = 'retained-step-input'; const fixedTimestamp = 1_700_000_000_000; -describe('prepareRetainedStepInput', () => { - it('clones passive cross-realm data into the host realm', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); +function makeContext() { + const { context, globalThis: workflowGlobal } = createContext({ + seed, + fixedTimestamp, + }); + // Mimic the globals workflow.ts installs before any serialization happens + // (the stream/request reducers dispatch on them unguarded). + for (const name of [ + 'ReadableStream', + 'WritableStream', + 'TransformStream', + 'Request', + 'Response', + 'AbortController', + 'AbortSignal', + ]) { + if ((workflowGlobal as any)[name] === undefined) { + (workflowGlobal as any)[name] = (globalThis as any)[name]; + } + } + return { context, workflowGlobal }; +} + +describe('isRetainedSerializationPassive', () => { + it('accepts plain cross-realm data', () => { + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `({ nested: [{ ok: true }, "text", 42n], @@ -22,102 +42,55 @@ describe('prepareRetainedStepInput', () => { context ); - const prepared = prepareRetainedStepInput(value, workflowGlobal); - - expect(prepared.retainable).toBe(true); - if (!prepared.retainable) return; - expect(prepared.value).toEqual({ - nested: [{ ok: true }, 'text', 42n], - sparse: [1, undefined, 3], - flag: false, - }); - // The clone's prototype chain lives in a pristine realm: not the - // workflow realm, not the host realm. - const cloneProto = Object.getPrototypeOf(prepared.value); - expect(cloneProto).not.toBe(Object.prototype); - expect(Object.getPrototypeOf(cloneProto)).toBe(null); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('produces the same serialized bytes as ordinary VM traversal', async () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `({ - nested: [{ ok: true, missing: undefined }], - sparse: [1, , 3], - big: 42n, - text: "workflow", - })`, - context - ); - const prepared = prepareRetainedStepInput(value, workflowGlobal); - expect(prepared.retainable).toBe(true); - if (!prepared.retainable) return; - - const original = await stepSerialization.serialize(value, undefined, { - global: workflowGlobal, - }); - const cloned = await stepSerialization.serialize( - prepared.value, - undefined, - { global: globalThis } - ); - - expect(cloned).toEqual(original); + it('accepts the supported built-ins with pristine pins', () => { + const { context, workflowGlobal } = makeContext(); + for (const expression of [ + 'new Map([["k", { ok: true }]])', + 'new Set([1, "two"])', + 'new Date(1234)', + 'new Uint8Array([1, 2, 3])', + 'new Float32Array([1.5])', + 'new ArrayBuffer(8)', + '({ when: new Date(0), bytes: new Uint8Array(2), index: new Map() })', + ]) { + const value = vm.runInContext(expression, context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + } }); - it('declines arrays with an own constructor property', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const arr = [1]; - Object.defineProperty(arr, "constructor", { - value: class Fake { static classId = "fake"; }, - enumerable: false, - }); - return arr; - })()`, - context - ); + it('accepts host-realm instances (hydrated step results)', () => { + const { workflowGlobal } = makeContext(); + const value = { + map: new Map([['k', 1]]), + date: new Date(1234), + bytes: new Uint8Array([9]), + }; - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('declines built-ins whose serialization consults realm prototypes', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + it('declines types whose serialization is not pinned', () => { + const { context, workflowGlobal } = makeContext(); for (const expression of [ - 'new Map([["k", 1]])', - 'new Set([1, 2])', - 'new Date(1234)', '/workflow/gi', - 'new Uint8Array([1, 2])', - 'new ArrayBuffer(8)', 'new DataView(new ArrayBuffer(8))', + 'new SharedArrayBuffer(8)', + 'new Uint8Array(new SharedArrayBuffer(4))', 'new Error("boom")', - 'Object.assign(Object.create(Object.prototype), { m: new Map() })', + 'new (class Sub extends Map {})()', + 'Object.assign(new Map(), { expando: 1 })', + 'Object.create(null)', ]) { const value = vm.runInContext(expression, context); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); } }); it('declines accessors without invoking them', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -126,17 +99,12 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); it('declines proxies without invoking their traps', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -150,17 +118,12 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); it('declines custom class serializers without invoking them', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { globalThis.__retainedTestCalls = 0; @@ -177,161 +140,197 @@ describe('prepareRetainedStepInput', () => { context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); -}); -describe('prepareRetainedStepInput review-hardening', () => { - it('declines non-enumerable array indices (structuredClone would drop them)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( + it('declines hidden own keys (symbols, non-enumerables, constructor)', () => { + const { context, workflowGlobal } = makeContext(); + for (const expression of [ + `(() => { + const tagged = { plain: true }; + Object.defineProperty(tagged, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { + value: "abort-stream", enumerable: false, + }); + return tagged; + })()`, + `(() => { + const hidden = { plain: true }; + Object.defineProperty(hidden, "signal", { + get() { return { aborted: false }; }, enumerable: false, + }); + return hidden; + })()`, + `(() => { + const arr = [1]; + Object.defineProperty(arr, "constructor", { + value: class Fake { static classId = "fake"; }, enumerable: false, + }); + return arr; + })()`, `(() => { const arr = [1, 2]; Object.defineProperty(arr, "0", { value: 7, enumerable: false }); return arr; })()`, + ]) { + const value = vm.runInContext(expression, context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + } + }); + + it('declines everything once a pinned member is patched', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + vm.runInContext( + `globalThis.__origIterator = Map.prototype[Symbol.iterator]; + Map.prototype[Symbol.iterator] = function () { return globalThis.__origIterator.call(this); };`, context ); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - }); - - it('never performs a property Get on redefined workflow globals', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - const arr = [1]; - for (const name of ["Array", "Object"]) { - Object.defineProperty(globalThis, name, { - get() { globalThis.__retainedTestCalls++; throw new Error("boom"); }, - }); - } - return { arr }; - })()`, + vm.runInContext( + 'Map.prototype[Symbol.iterator] = globalThis.__origIterator;', context ); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + }); - // The vandalized globals make the VM-realm prototypes unverifiable, so - // validation declines — without throwing or invoking the getters. - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + it('declines per pinned member: iterator next, Date methods', () => { + for (const patch of [ + 'Object.getPrototypeOf((new Map())[Symbol.iterator]()).next = function () {};', + 'Object.getPrototypeOf((new Set())[Symbol.iterator]()).next = function () {};', + 'Date.prototype.toISOString = function () { return "spoofed"; };', + 'Date.prototype.getDate = function () { return 1; };', + ]) { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + vm.runInContext(patch, context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } }); - it('never inspects a proxied workflow constructor', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); + it('declines typed arrays whose subclass prototype shadows a pinned getter', () => { + const { context, workflowGlobal } = makeContext(); const value = vm.runInContext( `(() => { - const arr = [1]; - globalThis.__retainedTestCalls = 0; - globalThis.Array = new Proxy(function Array() {}, { - getOwnPropertyDescriptor(target, key) { - globalThis.__retainedTestCalls++; - return Reflect.getOwnPropertyDescriptor(target, key); - }, + Object.defineProperty(Float32Array.prototype, "buffer", { + get() { return new ArrayBuffer(0); }, configurable: true, }); - return arr; + return new Float32Array([1.5]); })()`, context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + // Pins themselves are intact, so unrelated values stay retainable. + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); }); -describe('host dispatch pristineness', () => { - it('declines retention while a host dispatch constructor is spoofed', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext('({ plain: true })', context); +describe('serialization touches only pinned members', () => { + // THE coupling test: the pin list in vm/serialization-pins.ts is only sound + // if it covers every prototype member `dehydrateStepArguments` executes for + // the supported built-ins. Wrap every configurable member on the relevant + // prototypes with a recorder and assert the serializer hits nothing beyond + // the pinned set. If serde changes what it touches, this fails loudly — + // update the pin list (and this list) together. + const PINNED = new Set([ + 'Map.prototype.Symbol(Symbol.iterator)', + '%MapIteratorPrototype%.next', + 'Set.prototype.Symbol(Symbol.iterator)', + '%SetIteratorPrototype%.next', + 'Date.prototype.getDate', + 'Date.prototype.toISOString', + '%TypedArray%.prototype.buffer', + '%TypedArray%.prototype.byteOffset', + '%TypedArray%.prototype.byteLength', + 'ArrayBuffer.prototype.byteLength', + ]); - expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( - true - ); - - Object.defineProperty(Headers, Symbol.hasInstance, { - value: () => false, - configurable: true, - }); - try { - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - } finally { - delete (Headers as any)[Symbol.hasInstance]; - } + it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { + const { context, workflowGlobal } = makeContext(); + const g = workflowGlobal as any; + const touched = new Set(); - expect(prepareRetainedStepInput(value, workflowGlobal).retainable).toBe( - true - ); - }); -}); + const wrapPrototype = (prototype: object, label: string) => { + for (const key of Reflect.ownKeys(prototype)) { + if (key === 'constructor') continue; + const descriptor = Object.getOwnPropertyDescriptor(prototype, key); + if (!descriptor || !descriptor.configurable) continue; + const name = `${label}.${String(key)}`; + if (typeof descriptor.value === 'function') { + const original = descriptor.value; + Object.defineProperty(prototype, key, { + ...descriptor, + value: function (this: unknown, ...args: unknown[]) { + touched.add(name); + return original.apply(this, args); + }, + }); + } else if (descriptor.get) { + const originalGet = descriptor.get; + Object.defineProperty(prototype, key, { + ...descriptor, + get() { + touched.add(name); + return originalGet.call(this); + }, + set: descriptor.set, + }); + } + } + }; -describe('symbol-tagged inputs', () => { - it('declines objects carrying symbol properties (serialization dispatch tags)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const signal = { aborted: false }; - Object.defineProperty(signal, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { - value: "abort-stream", - enumerable: false, - }); - return { signal }; - })()`, + const values = vm.runInContext( + `({ + map: new Map([["k", 1]]), + set: new Set([1, 2]), + date: new Date(1234), + f32: new Float32Array([1.5, 2.5]), + u8: new Uint8Array([1, 2, 3]), + ab: new ArrayBuffer(8), + })`, context ); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); - }); -}); - -describe('hidden properties', () => { - it('declines objects with non-enumerable properties (reducer probes read them)', () => { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - const value = vm.runInContext( - `(() => { - const controllerish = { plain: true }; - Object.defineProperty(controllerish, "signal", { - get() { return { aborted: false }; }, - enumerable: false, - }); - return controllerish; - })()`, - context + wrapPrototype(g.Map.prototype, 'Map.prototype'); + wrapPrototype( + Object.getPrototypeOf(new g.Map()[Symbol.iterator]()), + '%MapIteratorPrototype%' + ); + wrapPrototype(g.Set.prototype, 'Set.prototype'); + wrapPrototype( + Object.getPrototypeOf(new g.Set()[Symbol.iterator]()), + '%SetIteratorPrototype%' ); + const datePrototype = Object.getOwnPropertyDescriptor(g.Date, 'prototype')! + .value as object; + wrapPrototype(datePrototype, 'Date.prototype'); + wrapPrototype( + Object.getPrototypeOf(g.Uint8Array.prototype), + '%TypedArray%.prototype' + ); + wrapPrototype(g.Uint8Array.prototype, 'Uint8Array.prototype'); + wrapPrototype(g.Float32Array.prototype, 'Float32Array.prototype'); + wrapPrototype(g.ArrayBuffer.prototype, 'ArrayBuffer.prototype'); - expect(prepareRetainedStepInput(value, workflowGlobal)).toEqual({ - retainable: false, - }); + for (const value of Object.values(values as Record)) { + touched.clear(); + await dehydrateStepArguments( + { args: [value], closureVars: undefined, thisVal: undefined }, + 'wrun_pin_coverage', + undefined, + g, + false, + false + ); + for (const name of touched) { + expect(PINNED, `unpinned member executed: ${name}`).toContain(name); + } + } }); }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 5fb77b1362..0e478cd8a2 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,102 +1,8 @@ import { types } from 'node:util'; -import { runInContext, createContext as vmCreateContext } from 'node:vm'; -import { WORKFLOW_SERIALIZE } from '@workflow/serde'; - -export type RetainedStepInputPreparation = - | { readonly retainable: true; readonly value: unknown } - | { readonly retainable: false }; - -// A pristine realm nothing else can reach: clones are built from its Object -// and Array so their entire prototype chain is immune to intrinsic mutation -// in both the workflow realm and the host realm (workflow code can obtain -// host-realm objects through APIs like `structuredClone` and vandalize their -// prototypes). -const pristineRealm = runInContext( - '({ Object, Array })', - vmCreateContext() -) as { Object: ObjectConstructor; Array: ArrayConstructor }; - -// Host constructors that serialization dispatches on when the clone is -// serialized under the host global — plus Object/Array, whose statics and -// prototypes the class reducer and devalue's tag lookup read for -// host-prototype originals (hydrated step results are host-realm objects). -// The workflow VM's own intrinsics are frozen (see vm/index.ts), but host -// intrinsics are shared with the whole process and cannot be frozen, so -// dispatch-pristineness is verified instead. Any dirt declines retention -// BEFORE a clone exists, so a spoofed predicate can never observe (or -// capture) a pristine-realm object. -const HOST_DISPATCH_CONSTRUCTORS = [ - 'Object', - 'Array', - 'Function', - 'Map', - 'Set', - 'Date', - 'RegExp', - 'ArrayBuffer', - 'SharedArrayBuffer', - 'DataView', - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', - 'Headers', - 'URL', - 'URLSearchParams', - 'DOMException', - 'AbortController', - 'AbortSignal', - 'Request', - 'Response', - 'ReadableStream', - 'WritableStream', - 'TransformStream', -] as const; - -function hasOwn(target: object, key: string | symbol): boolean { - return Object.getOwnPropertyDescriptor(target, key) !== undefined; -} - -function isHostDispatchPristine(): boolean { - const hostGlobal = globalThis as unknown as Record; - for (const name of HOST_DISPATCH_CONSTRUCTORS) { - const constructor = hostGlobal[name]; - if (constructor === undefined) continue; - if (hasOwn(constructor as object, Symbol.hasInstance)) return false; - } - for (const constructor of [Object, Array]) { - if ( - hasOwn(constructor, WORKFLOW_SERIALIZE) || - hasOwn(constructor, 'classId') - ) { - return false; - } - } - for (const [prototype, constructor] of [ - [Object.prototype, Object], - [Array.prototype, Array], - ] as const) { - if ( - hasOwn(prototype, Symbol.toStringTag) || - ownDataProperty(prototype, 'constructor') !== constructor - ) { - return false; - } - } - // Constructor prototype chains end at host Object.prototype, where an - // added @@hasInstance would be found by dispatch lookup. (Host - // Function.prototype's @@hasInstance is spec non-configurable.) - if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; - return true; -} +import { + getSerializationPins, + verifySerializationPins, +} from '../vm/serialization-pins.js'; // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and @@ -141,27 +47,6 @@ function hasAllowedPrototype( ); } -// Cloneable exotics (per the structured-clone spec) whose devalue/reducer -// serialization reads prototype methods, getters, or iterators the workflow -// realm can mutate — serializing them is not provably passive, so they -// decline the fast path even when their prototype identity looks intact. -function isSlotBearingExotic(value: object): boolean { - return ( - types.isMap(value) || - types.isSet(value) || - types.isDate(value) || - types.isRegExp(value) || - types.isArrayBuffer(value) || - types.isSharedArrayBuffer(value) || - types.isDataView(value) || - types.isTypedArray(value) || - types.isBoxedPrimitive(value) || - types.isNativeError(value) || - types.isPromise(value) || - types.isArgumentsObject(value) - ); -} - function isArrayIndex(key: string): boolean { const index = Number(key); return ( @@ -184,13 +69,13 @@ function isPassiveArrayProperty( // Only own enumerable data indices are passive. Anything hidden — symbol // tags, non-enumerable properties, accessors — can be observed by // serialization dispatch (reducer probes, thenable checks, the class - // reducer) while being dropped by the clone. + // reducer) and can execute workflow code when read. return ( typeof key === 'string' && isArrayIndex(key) && descriptor.enumerable === true && 'value' in descriptor && - isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + isPassive(descriptor.value, workflowGlobal, seen) ); } @@ -207,6 +92,119 @@ function isPassiveArray( ); } +// Instances of the supported built-ins must carry no own properties at all: +// serialization never reads own properties on them, but an own accessor or +// symbol could still be observed through other dispatch lookups, and clean +// instances are the overwhelmingly common case anyway. +function hasNoOwnProperties(value: object): boolean { + return Reflect.ownKeys(value).length === 0; +} + +function isPassiveMap( + value: Map, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype !== pins?.mapPrototype && prototype !== Map.prototype) { + return false; + } + if (!hasNoOwnProperties(value)) return false; + let passive = true; + // Host forEach iterates via internal slots — no realm members execute. + Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + passive &&= + isPassive(key, workflowGlobal, seen) && + isPassive(entryValue, workflowGlobal, seen); + }); + return passive; +} + +function isPassiveSet( + value: Set, + workflowGlobal: Record, + seen: WeakSet +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype !== pins?.setPrototype && prototype !== Set.prototype) { + return false; + } + if (!hasNoOwnProperties(value)) return false; + let passive = true; + Set.prototype.forEach.call(value, (entryValue: unknown) => { + passive &&= isPassive(entryValue, workflowGlobal, seen); + }); + return passive; +} + +function isPassiveDate( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + return ( + (prototype === pins?.datePrototype || prototype === Date.prototype) && + hasNoOwnProperties(value) + ); +} + +function isPassiveTypedArray( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + if (prototype === null || types.isProxy(prototype)) return false; + const parent = Object.getPrototypeOf(prototype); + const hostTypedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype); + if ( + parent !== pins?.typedArrayPrototype && + parent !== hostTypedArrayPrototype + ) { + return false; + } + // The measured getters resolve on %TypedArray%.prototype; a shadow on the + // subclass prototype (e.g. Float32Array.prototype) would intercept them. + for (const name of ['buffer', 'byteOffset', 'byteLength']) { + if (Object.getOwnPropertyDescriptor(prototype, name)) return false; + } + // Own keys on a typed array are exactly its canonical indices. + if ( + !Reflect.ownKeys(value).every( + (key) => typeof key === 'string' && isArrayIndex(key) + ) + ) { + return false; + } + // The pinned/native getter is safe to invoke; reject SharedArrayBuffer + // backing (cross-thread mutation is unserializable either way). + const bufferGetter = ( + pins !== undefined && parent === pins.typedArrayPrototype + ? pins.typedArrayBuffer + : Object.getOwnPropertyDescriptor(hostTypedArrayPrototype, 'buffer')?.get + ) as (() => unknown) | undefined; + return ( + bufferGetter !== undefined && + !types.isSharedArrayBuffer(bufferGetter.call(value)) + ); +} + +function isPassiveArrayBuffer( + value: object, + workflowGlobal: Record +): boolean { + const pins = getSerializationPins(workflowGlobal); + const prototype = Object.getPrototypeOf(value); + return ( + (prototype === pins?.arrayBufferPrototype || + prototype === ArrayBuffer.prototype) && + hasNoOwnProperties(value) + ); +} + function isPassiveObjectProperty( object: object, key: string | symbol, @@ -218,13 +216,13 @@ function isPassiveObjectProperty( // Only own enumerable string-keyed data properties are passive. Anything // hidden — symbol tags, non-enumerable properties, accessors — can be // observed by serialization dispatch (reducer probes like `.signal`, - // thenable checks, the class reducer) while being dropped by the clone. + // thenable checks, the class reducer) and can execute workflow code. return ( typeof key === 'string' && key !== '__proto__' && descriptor.enumerable === true && 'value' in descriptor && - isPassivelyCloneable(descriptor.value, workflowGlobal, seen) + isPassive(descriptor.value, workflowGlobal, seen) ); } @@ -240,20 +238,25 @@ function isPassivePlainObject( } /** - * Whether structured cloning `value` is provably byte-equivalent to the - * ordinary VM serialization AND cannot execute workflow-owned code. + * Whether serializing `value` through the ordinary pipeline provably executes + * no workflow code and draws no workflow-realm randomness. * - * The retained VM must not observe serialization side effects that a later - * cold replay cannot reconstruct, and the durable bytes must not depend on - * `WORKFLOW_RETAINED_VM`. Only primitives, plain objects, and plain arrays - * qualify: devalue traverses them exclusively through own-property reads - * (`Object.keys`, `Object.hasOwn` + indices), so no workflow-realm prototype - * state can influence the output. Everything else — proxies, accessors, - * functions, custom classes, and built-ins like Map/Set/Date whose - * serialization consults mutable realm prototypes (iterators, getters) — - * declines the fast path and serializes through the ordinary VM traversal. + * Retained sessions keep running after suspension, so serialization side + * effects there would desync the live VM from what a cold replay + * reconstructs (replay never re-serializes an already-created step). The + * bytes themselves cannot differ by mode — serialization happens once and + * every mode reads the same `step_created` event — so passivity is the only + * property retention needs. + * + * Passive values: primitives; plain objects and arrays (own enumerable + * string-keyed data properties only — devalue traverses these purely via own + * reads); and Map/Set/Date/typed arrays/ArrayBuffer instances whose measured + * prototype members are verified pinned (see vm/serialization-pins.ts). + * Everything else — proxies, accessors, functions, custom classes, RegExp, + * hidden keys — declines, serializes exactly the same way, and the session + * falls back to ordinary replay for that boundary. */ -function isPassivelyCloneable( +function isPassive( value: unknown, workflowGlobal: Record, seen: WeakSet @@ -272,61 +275,36 @@ function isPassivelyCloneable( if (seen.has(value)) return true; seen.add(value); - if (Array.isArray(value)) { - return isPassiveArray(value, workflowGlobal, seen); + if (Array.isArray(value)) return isPassiveArray(value, workflowGlobal, seen); + if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); + if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); + if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); + if (types.isTypedArray(value)) { + return isPassiveTypedArray(value, workflowGlobal); } - if (isSlotBearingExotic(value)) return false; - return isPassivePlainObject(value, workflowGlobal, seen); -} - -// Deep-copy walker-validated plain data into the pristine realm, copying -// exactly what devalue serializes: dense/sparse own indices for arrays and -// own enumerable string properties for objects. All reads are data-property -// reads — the walker already rejected everything else. -function cloneIntoPristineRealm( - value: unknown, - copies: WeakMap -): unknown { - if (value === null || typeof value !== 'object') return value; - const existing = copies.get(value); - if (existing !== undefined) return existing; - - if (Array.isArray(value)) { - const copy = new pristineRealm.Array(value.length) as unknown[]; - copies.set(value, copy); - for (const key of Reflect.ownKeys(value)) { - if (typeof key !== 'string' || !isArrayIndex(key)) continue; - copy[Number(key)] = cloneIntoPristineRealm( - (value as unknown as Record)[key], - copies - ); - } - return copy; + if (types.isArrayBuffer(value)) { + return isPassiveArrayBuffer(value, workflowGlobal); } - - const copy = new pristineRealm.Object() as Record; - copies.set(value, copy); - for (const key of Object.keys(value)) { - copy[key] = cloneIntoPristineRealm( - (value as Record)[key], - copies - ); + if ( + types.isSharedArrayBuffer(value) || + types.isRegExp(value) || + types.isDataView(value) || + types.isBoxedPrimitive(value) || + types.isNativeError(value) || + types.isPromise(value) || + types.isArgumentsObject(value) + ) { + return false; } - return copy; + return isPassivePlainObject(value, workflowGlobal, seen); } -export function prepareRetainedStepInput( +export function isRetainedSerializationPassive( value: unknown, workflowGlobal: Record -): RetainedStepInputPreparation { - if ( - !isHostDispatchPristine() || - !isPassivelyCloneable(value, workflowGlobal, new WeakSet()) - ) { - return { retainable: false }; - } - return { - retainable: true, - value: cloneIntoPristineRealm(value, new WeakMap()), - }; +): boolean { + return ( + verifySerializationPins(workflowGlobal) && + isPassive(value, workflowGlobal, new WeakSet()) + ); } diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index ad980e1ce7..936652c379 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,7 +31,7 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; -import { prepareRetainedStepInput } from './retained-step-input.js'; +import { isRetainedSerializationPassive } from './retained-step-input.js'; export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -497,30 +497,30 @@ export async function handleSuspension({ // racing with concurrent handlers on step execution. const createdStepCorrelationIds = new Set(); - // All-or-nothing: clones are only used when EVERY new step input in the - // batch is passive. A mixed batch would let an unsafe sibling's - // serialization (which can run getters) mutate objects a clone already - // snapshotted, changing what the other step durably receives relative to - // the ordinary in-order VM traversal. + // Serialization always runs through the one ordinary path below, so the + // durable bytes cannot depend on retention. What retention needs to know is + // whether that serialization will execute workflow code (getters, hooks, + // patched prototype members) — side effects a cold replay would not repeat. + // If any input in the batch is not provably passive, the caller demotes the + // session so the side effects land in a VM that is about to be discarded, + // exactly like the pre-retention runtime. let retainedStepInputsSafe = true; - const clonesByCorrelationId = new Map(); if (prepareForRetention) { for (const queueItem of stepItems) { if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - const prepared = prepareRetainedStepInput( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, - suspension.globalThis - ); - if (!prepared.retainable) { + if ( + !isRetainedSerializationPassive( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + thisVal: queueItem.thisVal, + }, + suspension.globalThis + ) + ) { retainedStepInputsSafe = false; - clonesByCorrelationId.clear(); break; } - clonesByCorrelationId.set(queueItem.correlationId, prepared.value); } } @@ -557,16 +557,15 @@ export async function handleSuspension({ if (stepsNeedingCreation.has(queueItem.correlationId)) { ops.push( (async () => { - const clone = clonesByCorrelationId.get(queueItem.correlationId); const dehydratedInput = await dehydrateStepArguments( - clone ?? { + { args: queueItem.args, closureVars: queueItem.closureVars, thisVal: queueItem.thisVal, }, runId, encryptionKey, - clone ? globalThis : suspension.globalThis, + suspension.globalThis, false, compression ); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index ef190f64e1..5a7f5b2288 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -3,6 +3,7 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; +import { registerSerializationPins } from './serialization-pins.js'; import { installUint8ArrayBase64 } from './uint8array-base64.js'; import { createRandomUUID } from './uuid.js'; @@ -148,12 +149,11 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { // Freeze VM-realm and session-local constructor objects — including // their prototype chains, which `Symbol.hasInstance` lookup walks (the // Date wrapper delegates statics to the unfrozen original VM Date via - // setPrototypeOf). A binding that references the shared host intrinsic - // stays unfrozen: both serialization paths dispatch through that same - // object, so a mutation cannot make them disagree (and freezing it - // would affect the whole process). Host Function.prototype's - // @@hasInstance is spec-immutable; host Object.prototype is verified - // separately (see isHostDispatchPristine). + // setPrototypeOf) — so reducer `instanceof` dispatch can never execute a + // workflow-planted hook while a retained VM's inputs serialize. A binding + // that references the shared host intrinsic stays unfrozen: freezing it + // would affect the whole process, and host-realm members run host code, + // which cannot touch retained VM state. if ( (typeof value === 'function' || (typeof value === 'object' && value !== null)) && @@ -333,6 +333,10 @@ export function createContext(options: CreateContextOptions) { g.exports = {}; (g as any).module = { exports: g.exports }; + // Capture the serialization-touched prototype members while the realm is + // provably pristine — no workflow code has run yet. + registerSerializationPins(g); + return { context, globalThis: g, diff --git a/packages/core/src/vm/serialization-pins.ts b/packages/core/src/vm/serialization-pins.ts new file mode 100644 index 0000000000..57bc4a3de2 --- /dev/null +++ b/packages/core/src/vm/serialization-pins.ts @@ -0,0 +1,115 @@ +/** + * Pinned copies of the workflow-realm prototype members that step-argument + * serialization executes, captured at context creation before any workflow + * code runs. + * + * The retained-VM fast path serializes step inputs through the ordinary + * serialization pipeline, which is only safe when that traversal provably + * executes no workflow code. For plain objects and arrays the traversal is + * own-property reads only; for the supported built-ins it also runs a small, + * measured set of prototype members (iteration protocols and getters — see + * the "touches only pinned members" test, which derives this list + * empirically). If workflow code replaced any of them, serializing that type + * would execute the replacement, so retention verifies each member is still + * the pinned original. Host-realm instances need no verification: their + * members run host code, which cannot reach the retained VM's state. + */ + +interface SerializationPins { + readonly mapPrototype: object; + readonly mapIterator: unknown; + readonly mapIteratorPrototype: object; + readonly mapIteratorNext: unknown; + readonly setPrototype: object; + readonly setIterator: unknown; + readonly setIteratorPrototype: object; + readonly setIteratorNext: unknown; + readonly datePrototype: object; + readonly dateGetDate: unknown; + readonly dateToISOString: unknown; + readonly typedArrayPrototype: object; + readonly typedArrayBuffer: unknown; + readonly typedArrayByteOffset: unknown; + readonly typedArrayByteLength: unknown; + readonly arrayBufferPrototype: object; + readonly arrayBufferByteLength: unknown; +} + +const registry = new WeakMap(); + +function ownValue(target: object, key: string | symbol): unknown { + const descriptor = Object.getOwnPropertyDescriptor(target, key); + return descriptor && 'value' in descriptor ? descriptor.value : undefined; +} + +function ownGetter(target: object, key: string | symbol): unknown { + return Object.getOwnPropertyDescriptor(target, key)?.get; +} + +export function registerSerializationPins(g: typeof globalThis): void { + const mapPrototype = g.Map.prototype; + const mapIteratorPrototype = Object.getPrototypeOf( + new g.Map()[Symbol.iterator]() + ) as object; + const setPrototype = g.Set.prototype; + const setIteratorPrototype = Object.getPrototypeOf( + new g.Set()[Symbol.iterator]() + ) as object; + // `g.Date` may already be the deterministic wrapper; its `prototype` + // data property is the realm's real Date.prototype either way. + const datePrototype = ownValue(g.Date, 'prototype') as object; + const typedArrayPrototype = Object.getPrototypeOf( + g.Uint8Array.prototype + ) as object; + const arrayBufferPrototype = g.ArrayBuffer.prototype; + + registry.set(g, { + mapPrototype, + mapIterator: ownValue(mapPrototype, Symbol.iterator), + mapIteratorPrototype, + mapIteratorNext: ownValue(mapIteratorPrototype, 'next'), + setPrototype, + setIterator: ownValue(setPrototype, Symbol.iterator), + setIteratorPrototype, + setIteratorNext: ownValue(setIteratorPrototype, 'next'), + datePrototype, + dateGetDate: ownValue(datePrototype, 'getDate'), + dateToISOString: ownValue(datePrototype, 'toISOString'), + typedArrayPrototype, + typedArrayBuffer: ownGetter(typedArrayPrototype, 'buffer'), + typedArrayByteOffset: ownGetter(typedArrayPrototype, 'byteOffset'), + typedArrayByteLength: ownGetter(typedArrayPrototype, 'byteLength'), + arrayBufferPrototype, + arrayBufferByteLength: ownGetter(arrayBufferPrototype, 'byteLength'), + }); +} + +export function getSerializationPins( + g: Record +): SerializationPins | undefined { + return registry.get(g); +} + +/** + * Whether every pinned member is still the original. All reads are + * own-descriptor reads — verification itself can never execute workflow code. + */ +export function verifySerializationPins(g: Record): boolean { + const pins = registry.get(g); + if (!pins) return false; + return ( + ownValue(pins.mapPrototype, Symbol.iterator) === pins.mapIterator && + ownValue(pins.mapIteratorPrototype, 'next') === pins.mapIteratorNext && + ownValue(pins.setPrototype, Symbol.iterator) === pins.setIterator && + ownValue(pins.setIteratorPrototype, 'next') === pins.setIteratorNext && + ownValue(pins.datePrototype, 'getDate') === pins.dateGetDate && + ownValue(pins.datePrototype, 'toISOString') === pins.dateToISOString && + ownGetter(pins.typedArrayPrototype, 'buffer') === pins.typedArrayBuffer && + ownGetter(pins.typedArrayPrototype, 'byteOffset') === + pins.typedArrayByteOffset && + ownGetter(pins.typedArrayPrototype, 'byteLength') === + pins.typedArrayByteLength && + ownGetter(pins.arrayBufferPrototype, 'byteLength') === + pins.arrayBufferByteLength + ); +} From 58514fdf176830a8cd9d72fafc7d430d15b606af Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:50:07 -0700 Subject: [PATCH 24/30] refactor(core): freeze built-in prototypes instead of pinning members (v3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the pin approach's structural hole: the class reducer READS value.constructor through Map.prototype — a data property when pristine (so member instrumentation never listed it), but executable the moment workflow code redefines it as a getter. Pinning what serialization executes misses what it reads. Freeze the accepted built-ins' prototypes wholesale (Map/Set/Date + iterator prototypes, %TypedArray% + subclass prototypes, ArrayBuffer): reads and executes are both immutable, and a patch attempt now throws loudly at the patch site instead of silently degrading. Deletes vm/serialization-pins.ts; the walker requires Object.isFrozen on the realm prototype (also covering realms where the freeze never ran). Also restores the host-dispatch pristineness check the v2 cut lost: workflow code can reach shared host constructors (exposed classes, structuredClone results) and plant workflow-realm Symbol.hasInstance hooks or WORKFLOW_SERIALIZE statics that reducers would execute during retained serialization. Host-realm built-in instances decline for the same reason; host-realm plain data (hydrated results) stays retainable. --- .changeset/retain-workflow-vm.md | 2 +- .../v5/api-reference/workflow-globals.mdx | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- .../src/runtime/retained-step-input.test.ts | 130 +++++++------- .../core/src/runtime/retained-step-input.ts | 165 +++++++++++++----- packages/core/src/vm/index.ts | 38 +++- packages/core/src/vm/serialization-pins.ts | 115 ------------ 7 files changed, 228 insertions(+), 226 deletions(-) delete mode 100644 packages/core/src/vm/serialization-pins.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index e1853096a9..9db15ed117 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function` and reducer-referenced global bindings) are frozen in the workflow sandbox. +Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function`, the prototypes of `Map`/`Set`/`Date`/typed arrays/`ArrayBuffer`, and reducer-referenced global bindings) are frozen in the workflow sandbox. diff --git a/docs/content/docs/v5/api-reference/workflow-globals.mdx b/docs/content/docs/v5/api-reference/workflow-globals.mdx index 59e8458b22..6b49709597 100644 --- a/docs/content/docs/v5/api-reference/workflow-globals.mdx +++ b/docs/content/docs/v5/api-reference/workflow-globals.mdx @@ -103,4 +103,4 @@ The following are **not available** in workflow functions. Move this logic to [s - **`WeakRef` and `FinalizationRegistry`**: garbage-collection timing is not deterministic, so observing it would make workflow code impossible to replay. (`WeakMap` and `WeakSet` remain available — they do not expose GC state.) - **`Atomics.waitAsync`**: a wall-clock timer, which cannot be replayed. Use [`sleep()`](/docs/api-reference/workflow/sleep) instead. - **Async `WebAssembly` compilation** (`compile`, `instantiate`, `compileStreaming`, `instantiateStreaming`): resolves on compile-thread timing. The synchronous `new WebAssembly.Module()` and `new WebAssembly.Instance()` constructors remain available. -- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), and the global bindings for serialization-relevant built-ins (`Map`, `Set`, `Date`, `RegExp`, typed arrays, `Headers`, `URL`, and similar) are frozen. Serialization dispatches on these, so redefining them would make replay unfaithful. +- **Mutating core intrinsics**: `Object`, `Array`, and `Function` (constructors and prototypes), the prototypes of `Map`, `Set`, `Date`, typed arrays, and `ArrayBuffer`, and the global bindings for serialization-relevant built-ins (`RegExp`, `Headers`, `URL`, and similar) are frozen. Serialization reads through these, so redefining them would make replay unfaithful. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 17322d7ca9..b1e881f0f2 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -80,7 +80,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary, as does any input if the prototype members serialization relies on have been modified. +- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary. - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 43c0ab46df..b84492deff 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -1,13 +1,13 @@ import vm from 'node:vm'; import { describe, expect, it } from 'vitest'; import { dehydrateStepArguments } from '../serialization.js'; -import { createContext } from '../vm/index.js'; +import { createContext, freezeSerializationIntrinsics } from '../vm/index.js'; import { isRetainedSerializationPassive } from './retained-step-input.js'; const seed = 'retained-step-input'; const fixedTimestamp = 1_700_000_000_000; -function makeContext() { +function makeContext({ freeze = true } = {}) { const { context, globalThis: workflowGlobal } = createContext({ seed, fixedTimestamp, @@ -27,6 +27,7 @@ function makeContext() { (workflowGlobal as any)[name] = (globalThis as any)[name]; } } + if (freeze) freezeSerializationIntrinsics(workflowGlobal); return { context, workflowGlobal }; } @@ -45,7 +46,7 @@ describe('isRetainedSerializationPassive', () => { expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); }); - it('accepts the supported built-ins with pristine pins', () => { + it('accepts the supported built-ins on a frozen realm', () => { const { context, workflowGlobal } = makeContext(); for (const expression of [ 'new Map([["k", { ok: true }]])', @@ -61,18 +62,23 @@ describe('isRetainedSerializationPassive', () => { } }); - it('accepts host-realm instances (hydrated step results)', () => { + it('accepts host-realm plain data but not host-realm built-ins', () => { const { workflowGlobal } = makeContext(); - const value = { - map: new Map([['k', 1]]), - date: new Date(1234), - bytes: new Uint8Array([9]), - }; - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + // Hydrated step results are host-realm plain objects/arrays. + expect( + isRetainedSerializationPassive({ nested: [{ ok: true }] }, workflowGlobal) + ).toBe(true); + // Host built-in prototypes cannot be frozen (process-shared) and are + // reachable from workflow code, so host-realm instances decline. + expect( + isRetainedSerializationPassive(new Map([['k', 1]]), workflowGlobal) + ).toBe(false); + expect(isRetainedSerializationPassive(new Date(0), workflowGlobal)).toBe( + false + ); }); - it('declines types whose serialization is not pinned', () => { + it('declines types whose serialization surface is not frozen', () => { const { context, workflowGlobal } = makeContext(); for (const expression of [ '/workflow/gi', @@ -179,66 +185,64 @@ describe('isRetainedSerializationPassive', () => { } }); - it('declines everything once a pinned member is patched', () => { + it('freezing makes prototype patches impossible, so retention persists', () => { const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + const map = vm.runInContext('new Map([["k", 1]])', context); + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - vm.runInContext( - `globalThis.__origIterator = Map.prototype[Symbol.iterator]; - Map.prototype[Symbol.iterator] = function () { return globalThis.__origIterator.call(this); };`, - context - ); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + // Redefining a member the serializer reads throws on the frozen prototype. + for (const attempt of [ + '"use strict"; Object.defineProperty(Map.prototype, "constructor", { get() { return 1; } })', + '"use strict"; Map.prototype[Symbol.iterator] = function () {};', + '"use strict"; Date.prototype.toISOString = function () { return "x"; };', + '"use strict"; Object.defineProperty(Float32Array.prototype, "buffer", { get() {} })', + ]) { + expect(() => vm.runInContext(attempt, context)).toThrow( + /not extensible|read only|Cannot redefine/ + ); + } + // Nothing changed, so the built-ins remain retainable. + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + }); - vm.runInContext( - 'Map.prototype[Symbol.iterator] = globalThis.__origIterator;', - context - ); + it('declines built-ins when the realm was never frozen', () => { + const { context, workflowGlobal } = makeContext({ freeze: false }); + const map = vm.runInContext('new Map()', context); + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(false); + // Plain data does not depend on the built-in prototypes. + const plain = vm.runInContext('({ ok: true })', context); expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); - it('declines per pinned member: iterator next, Date methods', () => { - for (const patch of [ - 'Object.getPrototypeOf((new Map())[Symbol.iterator]()).next = function () {};', - 'Object.getPrototypeOf((new Set())[Symbol.iterator]()).next = function () {};', - 'Date.prototype.toISOString = function () { return "spoofed"; };', - 'Date.prototype.getDate = function () { return 1; };', - ]) { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - vm.runInContext(patch, context); + it('declines retention while a host dispatch constructor is hooked', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + Object.defineProperty(Headers, Symbol.hasInstance, { + value: () => false, + configurable: true, + }); + try { expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } finally { + delete (Headers as any)[Symbol.hasInstance]; } - }); - - it('declines typed arrays whose subclass prototype shadows a pinned getter', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - Object.defineProperty(Float32Array.prototype, "buffer", { - get() { return new ArrayBuffer(0); }, configurable: true, - }); - return new Float32Array([1.5]); - })()`, - context - ); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - // Pins themselves are intact, so unrelated values stay retainable. - const plain = vm.runInContext('({ ok: true })', context); expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); }); }); -describe('serialization touches only pinned members', () => { - // THE coupling test: the pin list in vm/serialization-pins.ts is only sound - // if it covers every prototype member `dehydrateStepArguments` executes for - // the supported built-ins. Wrap every configurable member on the relevant - // prototypes with a recorder and assert the serializer hits nothing beyond - // the pinned set. If serde changes what it touches, this fails loudly — - // update the pin list (and this list) together. - const PINNED = new Set([ +describe('serialization touches only the frozen surface', () => { + // THE coupling test: retention is only sound if every prototype member + // `dehydrateStepArguments` executes for the supported built-ins lives on an + // object `freezeSerializationIntrinsics` freezes. Wrap every configurable + // member on the relevant prototypes (in an unfrozen realm) with a recorder + // and assert the serializer hits nothing beyond this measured set — every + // entry of which is on a frozen prototype in production. If serde starts + // touching something new, this fails loudly: extend the freeze (and this + // list) together. + const FROZEN_SURFACE = new Set([ 'Map.prototype.Symbol(Symbol.iterator)', '%MapIteratorPrototype%.next', 'Set.prototype.Symbol(Symbol.iterator)', @@ -252,7 +256,8 @@ describe('serialization touches only pinned members', () => { ]); it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { - const { context, workflowGlobal } = makeContext(); + // Unfrozen realm: the recorders themselves need to redefine members. + const { context, workflowGlobal } = makeContext({ freeze: false }); const g = workflowGlobal as any; const touched = new Set(); @@ -329,7 +334,10 @@ describe('serialization touches only pinned members', () => { false ); for (const name of touched) { - expect(PINNED, `unpinned member executed: ${name}`).toContain(name); + expect( + FROZEN_SURFACE, + `member outside the frozen surface executed: ${name}` + ).toContain(name); } } }); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 0e478cd8a2..9f49fe2954 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -1,8 +1,87 @@ import { types } from 'node:util'; -import { - getSerializationPins, - verifySerializationPins, -} from '../vm/serialization-pins.js'; +import { WORKFLOW_SERIALIZE } from '@workflow/serde'; + +// Host constructors that serialization dispatches on (`value instanceof +// global.X`), plus Object/Array, whose statics and prototypes the class +// reducer and devalue's tag lookup read for host-prototype values (hydrated +// step results are host-realm plain objects). The workflow VM's own +// intrinsics are frozen (see vm/index.ts), but host intrinsics are shared +// with the whole process and cannot be frozen — and workflow code can reach +// them (exposed host classes, `structuredClone` results) and plant +// workflow-realm hooks. Verified instead: any dirt declines retention, so a +// planted hook can never execute while a retained VM's inputs serialize. +const HOST_DISPATCH_CONSTRUCTORS = [ + 'Object', + 'Array', + 'Function', + 'Map', + 'Set', + 'Date', + 'RegExp', + 'ArrayBuffer', + 'SharedArrayBuffer', + 'DataView', + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + 'Headers', + 'URL', + 'URLSearchParams', + 'DOMException', + 'AbortController', + 'AbortSignal', + 'Request', + 'Response', + 'ReadableStream', + 'WritableStream', + 'TransformStream', +] as const; + +function hasOwn(target: object, key: string | symbol): boolean { + return Object.getOwnPropertyDescriptor(target, key) !== undefined; +} + +function isHostDispatchPristine(): boolean { + const hostGlobal = globalThis as unknown as Record; + for (const name of HOST_DISPATCH_CONSTRUCTORS) { + const constructor = hostGlobal[name]; + if (constructor === undefined) continue; + if (hasOwn(constructor as object, Symbol.hasInstance)) return false; + } + for (const constructor of [Object, Array]) { + if ( + hasOwn(constructor, WORKFLOW_SERIALIZE) || + hasOwn(constructor, 'classId') + ) { + return false; + } + } + for (const [prototype, constructor] of [ + [Object.prototype, Object], + [Array.prototype, Array], + ] as const) { + if ( + hasOwn(prototype, Symbol.toStringTag) || + ownDataProperty(prototype, 'constructor') !== constructor + ) { + return false; + } + } + // Constructor prototype chains end at host Object.prototype, where an + // added @@hasInstance would be found by dispatch lookup. (Host + // Function.prototype's @@hasInstance is spec non-configurable.) + if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; + return true; +} // Own data-property read that never performs a property Get — workflow code // can redefine its globals (or their `prototype` slots) with accessors, and @@ -32,6 +111,23 @@ function constructorPrototype( : undefined; } +// The workflow realm's prototype for a supported built-in, but only once the +// sandbox froze it: serialization both executes members on these prototypes +// (iterators, getters) and reads others (`constructor`), so a mutable +// prototype — including one in a realm where freezeSerializationIntrinsics +// never ran — is not provably passive. Host-realm instances of these +// built-ins decline for the same reason: host prototypes cannot be frozen +// and are reachable from workflow code (e.g. via structuredClone results). +function frozenRealmPrototype( + workflowGlobal: Record, + constructorName: string +): object | undefined { + const prototype = constructorPrototype(workflowGlobal, constructorName); + return prototype !== undefined && Object.isFrozen(prototype) + ? prototype + : undefined; +} + function hasAllowedPrototype( value: object, workflowGlobal: Record, @@ -105,11 +201,8 @@ function isPassiveMap( workflowGlobal: Record, seen: WeakSet ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype !== pins?.mapPrototype && prototype !== Map.prototype) { - return false; - } + if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; // Host forEach iterates via internal slots — no realm members execute. @@ -126,11 +219,8 @@ function isPassiveSet( workflowGlobal: Record, seen: WeakSet ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype !== pins?.setPrototype && prototype !== Set.prototype) { - return false; - } + if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; Set.prototype.forEach.call(value, (entryValue: unknown) => { @@ -143,10 +233,9 @@ function isPassiveDate( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); return ( - (prototype === pins?.datePrototype || prototype === Date.prototype) && + prototype === frozenRealmPrototype(workflowGlobal, 'Date') && hasNoOwnProperties(value) ); } @@ -155,21 +244,24 @@ function isPassiveTypedArray( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); - if (prototype === null || types.isProxy(prototype)) return false; - const parent = Object.getPrototypeOf(prototype); - const hostTypedArrayPrototype = Object.getPrototypeOf(Uint8Array.prototype); if ( - parent !== pins?.typedArrayPrototype && - parent !== hostTypedArrayPrototype + prototype === null || + types.isProxy(prototype) || + !Object.isFrozen(prototype) ) { return false; } - // The measured getters resolve on %TypedArray%.prototype; a shadow on the - // subclass prototype (e.g. Float32Array.prototype) would intercept them. - for (const name of ['buffer', 'byteOffset', 'byteLength']) { - if (Object.getOwnPropertyDescriptor(prototype, name)) return false; + // The subclass prototype must chain to the realm's frozen %TypedArray% + // parent, where the buffer/byteOffset/byteLength getters live. + const uint8Prototype = frozenRealmPrototype(workflowGlobal, 'Uint8Array'); + if (uint8Prototype === undefined) return false; + const typedArrayPrototype = Object.getPrototypeOf(uint8Prototype); + if ( + Object.getPrototypeOf(prototype) !== typedArrayPrototype || + !Object.isFrozen(typedArrayPrototype) + ) { + return false; } // Own keys on a typed array are exactly its canonical indices. if ( @@ -179,28 +271,18 @@ function isPassiveTypedArray( ) { return false; } - // The pinned/native getter is safe to invoke; reject SharedArrayBuffer - // backing (cross-thread mutation is unserializable either way). - const bufferGetter = ( - pins !== undefined && parent === pins.typedArrayPrototype - ? pins.typedArrayBuffer - : Object.getOwnPropertyDescriptor(hostTypedArrayPrototype, 'buffer')?.get - ) as (() => unknown) | undefined; - return ( - bufferGetter !== undefined && - !types.isSharedArrayBuffer(bufferGetter.call(value)) - ); + // Frozen chain ⇒ the getters are the originals; safe to invoke. Reject + // SharedArrayBuffer backing (cross-thread mutation is unserializable). + return !types.isSharedArrayBuffer((value as Uint8Array).buffer); } function isPassiveArrayBuffer( value: object, workflowGlobal: Record ): boolean { - const pins = getSerializationPins(workflowGlobal); const prototype = Object.getPrototypeOf(value); return ( - (prototype === pins?.arrayBufferPrototype || - prototype === ArrayBuffer.prototype) && + prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && hasNoOwnProperties(value) ); } @@ -250,8 +332,8 @@ function isPassivePlainObject( * * Passive values: primitives; plain objects and arrays (own enumerable * string-keyed data properties only — devalue traverses these purely via own - * reads); and Map/Set/Date/typed arrays/ArrayBuffer instances whose measured - * prototype members are verified pinned (see vm/serialization-pins.ts). + * reads); and workflow-realm Map/Set/Date/typed arrays/ArrayBuffer whose + * prototypes the sandbox froze (see freezeSerializationIntrinsics). * Everything else — proxies, accessors, functions, custom classes, RegExp, * hidden keys — declines, serializes exactly the same way, and the session * falls back to ordinary replay for that boundary. @@ -304,7 +386,6 @@ export function isRetainedSerializationPassive( workflowGlobal: Record ): boolean { return ( - verifySerializationPins(workflowGlobal) && - isPassive(value, workflowGlobal, new WeakSet()) + isHostDispatchPristine() && isPassive(value, workflowGlobal, new WeakSet()) ); } diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 5a7f5b2288..376fd01961 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -3,7 +3,6 @@ import { types } from 'node:util'; import { runInContext, createContext as vmCreateContext } from 'node:vm'; import { WorkflowRuntimeError } from '@workflow/errors'; import seedrandom from 'seedrandom'; -import { registerSerializationPins } from './serialization-pins.js'; import { installUint8ArrayBase64 } from './uint8array-base64.js'; import { createRandomUUID } from './uuid.js'; @@ -141,6 +140,39 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { // The typed-array constructors dispatch `instanceof` through their shared // %TypedArray% parent. Object.freeze(Object.getPrototypeOf(g.Uint8Array)); + // Prototypes of the built-ins the retained fast path accepts as step + // arguments (see runtime/retained-step-input.ts): serialization both + // executes members on them (iteration protocols, getters) and merely READS + // others (`constructor` via the class reducer), so pinning individual + // members is not enough — a replaced data property becomes executable the + // moment it is redefined as an accessor. Frozen wholesale, nothing on them + // can ever be workflow-planted. RegExp/DataView prototypes stay mutable; + // those types always take the ordinary replay path. + Object.freeze(g.Map.prototype); + Object.freeze(Object.getPrototypeOf(new g.Map()[Symbol.iterator]())); + Object.freeze(g.Set.prototype); + Object.freeze(Object.getPrototypeOf(new g.Set()[Symbol.iterator]())); + // biome-ignore lint/style/noNonNullAssertion: Date always has a prototype + Object.freeze(Object.getOwnPropertyDescriptor(g.Date, 'prototype')!.value); + Object.freeze(Object.getPrototypeOf(g.Uint8Array.prototype)); + for (const name of [ + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', + ]) { + const constructor = (g as unknown as Record)[name]; + if (constructor !== undefined) Object.freeze(constructor.prototype); + } + Object.freeze(g.ArrayBuffer.prototype); const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); @@ -333,10 +365,6 @@ export function createContext(options: CreateContextOptions) { g.exports = {}; (g as any).module = { exports: g.exports }; - // Capture the serialization-touched prototype members while the realm is - // provably pristine — no workflow code has run yet. - registerSerializationPins(g); - return { context, globalThis: g, diff --git a/packages/core/src/vm/serialization-pins.ts b/packages/core/src/vm/serialization-pins.ts deleted file mode 100644 index 57bc4a3de2..0000000000 --- a/packages/core/src/vm/serialization-pins.ts +++ /dev/null @@ -1,115 +0,0 @@ -/** - * Pinned copies of the workflow-realm prototype members that step-argument - * serialization executes, captured at context creation before any workflow - * code runs. - * - * The retained-VM fast path serializes step inputs through the ordinary - * serialization pipeline, which is only safe when that traversal provably - * executes no workflow code. For plain objects and arrays the traversal is - * own-property reads only; for the supported built-ins it also runs a small, - * measured set of prototype members (iteration protocols and getters — see - * the "touches only pinned members" test, which derives this list - * empirically). If workflow code replaced any of them, serializing that type - * would execute the replacement, so retention verifies each member is still - * the pinned original. Host-realm instances need no verification: their - * members run host code, which cannot reach the retained VM's state. - */ - -interface SerializationPins { - readonly mapPrototype: object; - readonly mapIterator: unknown; - readonly mapIteratorPrototype: object; - readonly mapIteratorNext: unknown; - readonly setPrototype: object; - readonly setIterator: unknown; - readonly setIteratorPrototype: object; - readonly setIteratorNext: unknown; - readonly datePrototype: object; - readonly dateGetDate: unknown; - readonly dateToISOString: unknown; - readonly typedArrayPrototype: object; - readonly typedArrayBuffer: unknown; - readonly typedArrayByteOffset: unknown; - readonly typedArrayByteLength: unknown; - readonly arrayBufferPrototype: object; - readonly arrayBufferByteLength: unknown; -} - -const registry = new WeakMap(); - -function ownValue(target: object, key: string | symbol): unknown { - const descriptor = Object.getOwnPropertyDescriptor(target, key); - return descriptor && 'value' in descriptor ? descriptor.value : undefined; -} - -function ownGetter(target: object, key: string | symbol): unknown { - return Object.getOwnPropertyDescriptor(target, key)?.get; -} - -export function registerSerializationPins(g: typeof globalThis): void { - const mapPrototype = g.Map.prototype; - const mapIteratorPrototype = Object.getPrototypeOf( - new g.Map()[Symbol.iterator]() - ) as object; - const setPrototype = g.Set.prototype; - const setIteratorPrototype = Object.getPrototypeOf( - new g.Set()[Symbol.iterator]() - ) as object; - // `g.Date` may already be the deterministic wrapper; its `prototype` - // data property is the realm's real Date.prototype either way. - const datePrototype = ownValue(g.Date, 'prototype') as object; - const typedArrayPrototype = Object.getPrototypeOf( - g.Uint8Array.prototype - ) as object; - const arrayBufferPrototype = g.ArrayBuffer.prototype; - - registry.set(g, { - mapPrototype, - mapIterator: ownValue(mapPrototype, Symbol.iterator), - mapIteratorPrototype, - mapIteratorNext: ownValue(mapIteratorPrototype, 'next'), - setPrototype, - setIterator: ownValue(setPrototype, Symbol.iterator), - setIteratorPrototype, - setIteratorNext: ownValue(setIteratorPrototype, 'next'), - datePrototype, - dateGetDate: ownValue(datePrototype, 'getDate'), - dateToISOString: ownValue(datePrototype, 'toISOString'), - typedArrayPrototype, - typedArrayBuffer: ownGetter(typedArrayPrototype, 'buffer'), - typedArrayByteOffset: ownGetter(typedArrayPrototype, 'byteOffset'), - typedArrayByteLength: ownGetter(typedArrayPrototype, 'byteLength'), - arrayBufferPrototype, - arrayBufferByteLength: ownGetter(arrayBufferPrototype, 'byteLength'), - }); -} - -export function getSerializationPins( - g: Record -): SerializationPins | undefined { - return registry.get(g); -} - -/** - * Whether every pinned member is still the original. All reads are - * own-descriptor reads — verification itself can never execute workflow code. - */ -export function verifySerializationPins(g: Record): boolean { - const pins = registry.get(g); - if (!pins) return false; - return ( - ownValue(pins.mapPrototype, Symbol.iterator) === pins.mapIterator && - ownValue(pins.mapIteratorPrototype, 'next') === pins.mapIteratorNext && - ownValue(pins.setPrototype, Symbol.iterator) === pins.setIterator && - ownValue(pins.setIteratorPrototype, 'next') === pins.setIteratorNext && - ownValue(pins.datePrototype, 'getDate') === pins.dateGetDate && - ownValue(pins.datePrototype, 'toISOString') === pins.dateToISOString && - ownGetter(pins.typedArrayPrototype, 'buffer') === pins.typedArrayBuffer && - ownGetter(pins.typedArrayPrototype, 'byteOffset') === - pins.typedArrayByteOffset && - ownGetter(pins.typedArrayPrototype, 'byteLength') === - pins.typedArrayByteLength && - ownGetter(pins.arrayBufferPrototype, 'byteLength') === - pins.arrayBufferByteLength - ); -} From 93aa3a5b88dce017341de90a7177550261e01b56 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:05:01 -0700 Subject: [PATCH 25/30] fix(core): harden the passivity checker's own execution surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Capture Map/Set forEach and the %TypedArray% buffer getter as module- load primordials: the checker previously invoked live host methods that workflow code can reach (structuredClone(new Map()).constructor) and replace with delegating workflow-realm closures. - Typed arrays must have one of the realm's real frozen subclass prototypes by identity — 'frozen and chains to %TypedArray%' admitted manufactured frozen hostile prototypes with delegating buffer getters. --- .../src/runtime/retained-step-input.test.ts | 49 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 61 +++++++++++++------ 2 files changed, 90 insertions(+), 20 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index b84492deff..58573edc22 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -342,3 +342,52 @@ describe('serialization touches only the frozen surface', () => { } }); }); + +describe('checker execution surface', () => { + it('never invokes live host collection methods', () => { + const { context, workflowGlobal } = makeContext(); + const map = vm.runInContext('new Map([["k", 1]])', context); + + let invoked = 0; + const original = Map.prototype.forEach; + // Simulate workflow code having replaced the reachable host method. + Map.prototype.forEach = function ( + this: Map, + ...args: [any] + ) { + invoked++; + return original.apply(this, args); + }; + try { + // The checker uses the module-captured primordial: same verdict, + // replaced method never runs. + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + expect(invoked).toBe(0); + } finally { + Map.prototype.forEach = original; + } + }); + + it('declines typed arrays re-prototyped onto a frozen hostile prototype', () => { + const { context, workflowGlobal } = makeContext(); + const value = vm.runInContext( + `(() => { + globalThis.__retainedTestCalls = 0; + const realGetter = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), "buffer").get; + const hostile = Object.create( + Object.getPrototypeOf(Uint8Array.prototype), + { buffer: { get() { globalThis.__retainedTestCalls++; return realGetter.call(this); } } } + ); + Object.freeze(hostile); + const ta = new Uint8Array([1, 2]); + Object.setPrototypeOf(ta, hostile); + return ta; + })()`, + context + ); + + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 9f49fe2954..0da2ed82e3 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -46,6 +46,33 @@ const HOST_DISPATCH_CONSTRUCTORS = [ 'TransformStream', ] as const; +// Host primordials captured at module load — before any workflow code can +// exist in the process — so the checker itself never invokes a live host +// method workflow code could have replaced (host constructors are reachable +// via e.g. `structuredClone(new Map()).constructor`). +const hostMapForEach = Map.prototype.forEach; +const hostSetForEach = Set.prototype.forEach; +// biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists +const hostTypedArrayBuffer = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), + 'buffer' +)!.get!; + +const TYPED_ARRAY_CONSTRUCTORS = [ + 'Int8Array', + 'Uint8Array', + 'Uint8ClampedArray', + 'Int16Array', + 'Uint16Array', + 'Int32Array', + 'Uint32Array', + 'Float16Array', + 'Float32Array', + 'Float64Array', + 'BigInt64Array', + 'BigUint64Array', +] as const; + function hasOwn(target: object, key: string | symbol): boolean { return Object.getOwnPropertyDescriptor(target, key) !== undefined; } @@ -205,8 +232,9 @@ function isPassiveMap( if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; - // Host forEach iterates via internal slots — no realm members execute. - Map.prototype.forEach.call(value, (entryValue: unknown, key: unknown) => { + // The captured host forEach iterates via internal slots — no realm + // members (and no live, replaceable host members) execute. + hostMapForEach.call(value, (entryValue: unknown, key: unknown) => { passive &&= isPassive(key, workflowGlobal, seen) && isPassive(entryValue, workflowGlobal, seen); @@ -223,7 +251,7 @@ function isPassiveSet( if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; - Set.prototype.forEach.call(value, (entryValue: unknown) => { + hostSetForEach.call(value, (entryValue: unknown) => { passive &&= isPassive(entryValue, workflowGlobal, seen); }); return passive; @@ -244,22 +272,15 @@ function isPassiveTypedArray( value: object, workflowGlobal: Record ): boolean { + // Identity against the finite set of the realm's real (frozen) typed-array + // prototypes — NOT "anything chaining to %TypedArray%": a workflow can + // manufacture a frozen hostile prototype with a delegating `buffer` getter + // and setPrototypeOf a real typed array onto it. const prototype = Object.getPrototypeOf(value); if ( - prototype === null || - types.isProxy(prototype) || - !Object.isFrozen(prototype) - ) { - return false; - } - // The subclass prototype must chain to the realm's frozen %TypedArray% - // parent, where the buffer/byteOffset/byteLength getters live. - const uint8Prototype = frozenRealmPrototype(workflowGlobal, 'Uint8Array'); - if (uint8Prototype === undefined) return false; - const typedArrayPrototype = Object.getPrototypeOf(uint8Prototype); - if ( - Object.getPrototypeOf(prototype) !== typedArrayPrototype || - !Object.isFrozen(typedArrayPrototype) + !TYPED_ARRAY_CONSTRUCTORS.some( + (name) => prototype === frozenRealmPrototype(workflowGlobal, name) + ) ) { return false; } @@ -271,9 +292,9 @@ function isPassiveTypedArray( ) { return false; } - // Frozen chain ⇒ the getters are the originals; safe to invoke. Reject - // SharedArrayBuffer backing (cross-thread mutation is unserializable). - return !types.isSharedArrayBuffer((value as Uint8Array).buffer); + // Read the backing buffer via the captured host getter (internal slots + // work cross-realm); reject SharedArrayBuffer backing. + return !types.isSharedArrayBuffer(hostTypedArrayBuffer.call(value)); } function isPassiveArrayBuffer( From fb78452b5e21520c9493dd88c103c3dc60d40a5a Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:22:58 -0700 Subject: [PATCH 26/30] fix(core): checker uses module-load primordials; verify inherited serializer statics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The walker resolved Object.getOwnPropertyDescriptor, Reflect.ownKeys, Array.isArray, Number/String helpers, and Object.getPrototypeOf/isFrozen from live host globals workflow code can reach and replace; all are now module-load captures, so the checker can never execute a planted delegate. - The class reducer reads cls[WORKFLOW_SERIALIZE]/cls.classId as inherited Gets, so isHostDispatchPristine now also verifies host Function.prototype and Object.prototype carry no serializer statics. Generic replacement of shared host statics (Object.keys, Array.from, …) via realm escape remains the documented host-reachability boundary, tracked by the realm-local intrinsics follow-up. --- .../src/runtime/retained-step-input.test.ts | 47 +++++++++++++++ .../core/src/runtime/retained-step-input.ts | 60 +++++++++++-------- 2 files changed, 81 insertions(+), 26 deletions(-) diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 58573edc22..482443844c 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -391,3 +391,50 @@ describe('checker execution surface', () => { expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); }); }); + +describe('walker primordials', () => { + it('uses captured primordials, not live host statics', () => { + const { context, workflowGlobal } = makeContext(); + const map = vm.runInContext('new Map([["k", { ok: true }]])', context); + + // If the walker consulted the live host statics, these would throw. + const originalDescriptor = Object.getOwnPropertyDescriptor; + const originalOwnKeys = Reflect.ownKeys; + (Object as any).getOwnPropertyDescriptor = () => { + throw new Error('live getOwnPropertyDescriptor used'); + }; + (Reflect as any).ownKeys = () => { + throw new Error('live ownKeys used'); + }; + try { + expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); + } finally { + (Object as any).getOwnPropertyDescriptor = originalDescriptor; + (Reflect as any).ownKeys = originalOwnKeys; + } + }); + + it('declines when host Function.prototype carries serializer statics', () => { + const { context, workflowGlobal } = makeContext(); + const plain = vm.runInContext('({ ok: true })', context); + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + + Object.defineProperty( + Function.prototype, + Symbol.for('workflow-serialize'), + { + get() { + return undefined; + }, + configurable: true, + } + ); + try { + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); + } finally { + delete (Function.prototype as any)[Symbol.for('workflow-serialize')]; + } + + expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); + }); +}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index 0da2ed82e3..e18a5db19e 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -48,8 +48,16 @@ const HOST_DISPATCH_CONSTRUCTORS = [ // Host primordials captured at module load — before any workflow code can // exist in the process — so the checker itself never invokes a live host -// method workflow code could have replaced (host constructors are reachable -// via e.g. `structuredClone(new Map()).constructor`). +// member workflow code could have replaced (host constructors are reachable +// via e.g. `structuredClone(new Map()).constructor` or exposed classes). +const hostGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; +const hostGetPrototypeOf = Object.getPrototypeOf; +const hostIsFrozen = Object.isFrozen; +const hostOwnKeys = Reflect.ownKeys; +const hostIsArray = Array.isArray; +const hostIsInteger = Number.isInteger; +const hostNumber = Number; +const hostString = String; const hostMapForEach = Map.prototype.forEach; const hostSetForEach = Set.prototype.forEach; // biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists @@ -74,7 +82,7 @@ const TYPED_ARRAY_CONSTRUCTORS = [ ] as const; function hasOwn(target: object, key: string | symbol): boolean { - return Object.getOwnPropertyDescriptor(target, key) !== undefined; + return hostGetOwnPropertyDescriptor(target, key) !== undefined; } function isHostDispatchPristine(): boolean { @@ -84,11 +92,11 @@ function isHostDispatchPristine(): boolean { if (constructor === undefined) continue; if (hasOwn(constructor as object, Symbol.hasInstance)) return false; } - for (const constructor of [Object, Array]) { - if ( - hasOwn(constructor, WORKFLOW_SERIALIZE) || - hasOwn(constructor, 'classId') - ) { + // The class reducer reads `cls[WORKFLOW_SERIALIZE]` / `cls.classId` as + // inherited Gets, so the constructors' whole prototype chains — host + // Function.prototype and Object.prototype — must be clean too. + for (const target of [Object, Array, Function.prototype, Object.prototype]) { + if (hasOwn(target, WORKFLOW_SERIALIZE) || hasOwn(target, 'classId')) { return false; } } @@ -114,7 +122,7 @@ function isHostDispatchPristine(): boolean { // can redefine its globals (or their `prototype` slots) with accessors, and // validation must not execute workflow-owned code. function ownDataProperty(target: object, key: string): unknown { - const descriptor = Object.getOwnPropertyDescriptor(target, key); + const descriptor = hostGetOwnPropertyDescriptor(target, key); return descriptor && 'value' in descriptor ? descriptor.value : undefined; } @@ -150,7 +158,7 @@ function frozenRealmPrototype( constructorName: string ): object | undefined { const prototype = constructorPrototype(workflowGlobal, constructorName); - return prototype !== undefined && Object.isFrozen(prototype) + return prototype !== undefined && hostIsFrozen(prototype) ? prototype : undefined; } @@ -160,7 +168,7 @@ function hasAllowedPrototype( workflowGlobal: Record, constructorName: string ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === constructorPrototype( @@ -171,12 +179,12 @@ function hasAllowedPrototype( } function isArrayIndex(key: string): boolean { - const index = Number(key); + const index = hostNumber(key); return ( - Number.isInteger(index) && + hostIsInteger(index) && index >= 0 && index < 2 ** 32 - 1 && - String(index) === key + hostString(index) === key ); } @@ -187,7 +195,7 @@ function isPassiveArrayProperty( seen: WeakSet ): boolean { if (key === 'length') return true; - const descriptor = Object.getOwnPropertyDescriptor(array, key); + const descriptor = hostGetOwnPropertyDescriptor(array, key); if (!descriptor) return false; // Only own enumerable data indices are passive. Anything hidden — symbol // tags, non-enumerable properties, accessors — can be observed by @@ -209,7 +217,7 @@ function isPassiveArray( ): boolean { return ( hasAllowedPrototype(value, workflowGlobal, 'Array') && - Reflect.ownKeys(value).every((key) => + hostOwnKeys(value).every((key) => isPassiveArrayProperty(value, key, workflowGlobal, seen) ) ); @@ -220,7 +228,7 @@ function isPassiveArray( // symbol could still be observed through other dispatch lookups, and clean // instances are the overwhelmingly common case anyway. function hasNoOwnProperties(value: object): boolean { - return Reflect.ownKeys(value).length === 0; + return hostOwnKeys(value).length === 0; } function isPassiveMap( @@ -228,7 +236,7 @@ function isPassiveMap( workflowGlobal: Record, seen: WeakSet ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; @@ -247,7 +255,7 @@ function isPassiveSet( workflowGlobal: Record, seen: WeakSet ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; if (!hasNoOwnProperties(value)) return false; let passive = true; @@ -261,7 +269,7 @@ function isPassiveDate( value: object, workflowGlobal: Record ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === frozenRealmPrototype(workflowGlobal, 'Date') && hasNoOwnProperties(value) @@ -276,7 +284,7 @@ function isPassiveTypedArray( // prototypes — NOT "anything chaining to %TypedArray%": a workflow can // manufacture a frozen hostile prototype with a delegating `buffer` getter // and setPrototypeOf a real typed array onto it. - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); if ( !TYPED_ARRAY_CONSTRUCTORS.some( (name) => prototype === frozenRealmPrototype(workflowGlobal, name) @@ -286,7 +294,7 @@ function isPassiveTypedArray( } // Own keys on a typed array are exactly its canonical indices. if ( - !Reflect.ownKeys(value).every( + !hostOwnKeys(value).every( (key) => typeof key === 'string' && isArrayIndex(key) ) ) { @@ -301,7 +309,7 @@ function isPassiveArrayBuffer( value: object, workflowGlobal: Record ): boolean { - const prototype = Object.getPrototypeOf(value); + const prototype = hostGetPrototypeOf(value); return ( prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && hasNoOwnProperties(value) @@ -314,7 +322,7 @@ function isPassiveObjectProperty( workflowGlobal: Record, seen: WeakSet ): boolean { - const descriptor = Object.getOwnPropertyDescriptor(object, key); + const descriptor = hostGetOwnPropertyDescriptor(object, key); if (!descriptor) return false; // Only own enumerable string-keyed data properties are passive. Anything // hidden — symbol tags, non-enumerable properties, accessors — can be @@ -335,7 +343,7 @@ function isPassivePlainObject( seen: WeakSet ): boolean { if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; - return Reflect.ownKeys(value).every((key) => + return hostOwnKeys(value).every((key) => isPassiveObjectProperty(value, key, workflowGlobal, seen) ); } @@ -378,7 +386,7 @@ function isPassive( if (seen.has(value)) return true; seen.add(value); - if (Array.isArray(value)) return isPassiveArray(value, workflowGlobal, seen); + if (hostIsArray(value)) return isPassiveArray(value, workflowGlobal, seen); if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); From 59e21d64596770f6487f0c656415b04636a9211b Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:41:00 -0700 Subject: [PATCH 27/30] fix(core): stale-suspension generation token; cover BigInt toString MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Suspension signals capture ctx.suspensionGeneration when scheduled and no-op if the session resumed past that boundary. The harmful interleaving was already unreachable (queue items are deleted on consume, completion writes state synchronously, nextTick precedes timers) — the token turns those ordering facts into an explicit invariant. - The BigInt reducer calls .toString() on primitives from host code, which resolves on host BigInt.prototype: its identity joins the host dispatch check, and the VM BigInt.prototype is frozen besides. --- packages/core/src/private.ts | 7 +++++++ .../src/runtime/retained-step-input.test.ts | 21 +++++++++++++++++++ .../core/src/runtime/retained-step-input.ts | 7 +++++++ packages/core/src/step.ts | 4 ++++ packages/core/src/vm/index.ts | 1 + packages/core/src/workflow.ts | 2 ++ 6 files changed, 42 insertions(+) diff --git a/packages/core/src/private.ts b/packages/core/src/private.ts index 36072b290c..96815fa4e5 100644 --- a/packages/core/src/private.ts +++ b/packages/core/src/private.ts @@ -133,6 +133,13 @@ export interface WorkflowOrchestratorContext { runId: string; encryptionKey: CryptoKey | undefined; globalThis: typeof globalThis; + /** + * Increments on every retained-session resume. Suspension signals capture + * it when scheduled and no-op if it moved — a timer queued at boundary N + * must not signal after the session resumed into boundary N+1, so the + * state machine never has to reason about cross-boundary timer ordering. + */ + suspensionGeneration: number; eventsConsumer: EventsConsumer; /** * Map of pending invocations keyed by correlationId. diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts index 482443844c..ce8eafa023 100644 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ b/packages/core/src/runtime/retained-step-input.test.ts @@ -392,6 +392,27 @@ describe('checker execution surface', () => { }); }); +describe('bigint serialization', () => { + it('declines retention while host BigInt.prototype.toString is replaced', () => { + const { context, workflowGlobal } = makeContext(); + const value = vm.runInContext('({ big: 42n })', context); + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + + const original = BigInt.prototype.toString; + // biome-ignore lint/suspicious/noGlobalAssign: simulating workflow-realm tampering + BigInt.prototype.toString = function (this: bigint, ...args: [number?]) { + return original.apply(this, args); + }; + try { + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); + } finally { + BigInt.prototype.toString = original; + } + + expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); + }); +}); + describe('walker primordials', () => { it('uses captured primordials, not live host statics', () => { const { context, workflowGlobal } = makeContext(); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts index e18a5db19e..7fec0156da 100644 --- a/packages/core/src/runtime/retained-step-input.ts +++ b/packages/core/src/runtime/retained-step-input.ts @@ -58,6 +58,7 @@ const hostIsArray = Array.isArray; const hostIsInteger = Number.isInteger; const hostNumber = Number; const hostString = String; +const hostBigIntToString = BigInt.prototype.toString; const hostMapForEach = Map.prototype.forEach; const hostSetForEach = Set.prototype.forEach; // biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists @@ -115,6 +116,12 @@ function isHostDispatchPristine(): boolean { // added @@hasInstance would be found by dispatch lookup. (Host // Function.prototype's @@hasInstance is spec non-configurable.) if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; + // The BigInt reducer calls `.toString()` on bigint primitives from host + // code, which resolves on the HOST BigInt.prototype (primitives are + // realm-less; method lookup uses the running code's realm). + if (ownDataProperty(BigInt.prototype, 'toString') !== hostBigIntToString) { + return false; + } return true; } diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index 1b14165b91..40d20bc914 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -56,7 +56,11 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { // Crucially, if we got here, then this step Promise does // not resolve so that the user workflow code does not proceed any further. // Notify the workflow handler that this step has not been run / has not completed yet. + const generation = ctx.suspensionGeneration; scheduleWhenIdle(ctx, () => { + // A retained session may have resumed past this boundary while + // the timer was queued; a stale signal must not fire. + if (generation !== ctx.suspensionGeneration) return; ctx.onWorkflowError( new WorkflowSuspension(ctx.invocationsQueue, ctx.globalThis) ); diff --git a/packages/core/src/vm/index.ts b/packages/core/src/vm/index.ts index 376fd01961..0d32faad6d 100644 --- a/packages/core/src/vm/index.ts +++ b/packages/core/src/vm/index.ts @@ -173,6 +173,7 @@ export function freezeSerializationIntrinsics(g: typeof globalThis): void { if (constructor !== undefined) Object.freeze(constructor.prototype); } Object.freeze(g.ArrayBuffer.prototype); + Object.freeze(g.BigInt.prototype); const hostGlobal = globalThis as unknown as Record; for (const name of SERIALIZATION_BINDINGS) { const descriptor = Object.getOwnPropertyDescriptor(g, name); diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 83a8154ca0..70beca73fd 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -452,6 +452,7 @@ function createWorkflowSession({ promiseQueueHolder.current = value; }, pendingDeliveries: 0, + suspensionGeneration: 0, pendingDeliveryBarriers: new Map(), replayPayloadCache, }; @@ -1155,6 +1156,7 @@ function createWorkflowSession({ } const interruption = withResolvers(); state = { type: 'running', interruption }; + workflowContext.suspensionGeneration++; eventsConsumer.append(nextEvents.slice(consumedEvents.length)); return { type: 'resumed', From bb2098ab347ce66a9bac129e0785f24bbdef8a36 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:47:00 -0700 Subject: [PATCH 28/30] feat(core): retain the workflow VM across inline steps (primitive args) Keeps the suspended workflow VM, its events consumer, and the paused async stack alive across inline step executions within one invocation. Each loop iteration appends only the newly written events instead of replaying the entire event log in a fresh VM, so step-to-step overhead stays flat as runs grow. - WorkflowSession wraps executeWorkflow: suspended sessions expose resume(events) which appends to the retained EventsConsumer and lets the parked run() continuation settle; any divergence (unexpected suspension shape, consumer error) demotes to full replay permanently - Retention is gated per boundary: only suspensions whose queued step inputs are all primitives (null/undefined/boolean/number/string) are retainable, because serializing primitives executes no workflow code; a follow-up widens this to plain data and standard built-ins - Suspensions with hooks, waits, or attributes always fall back - A suspension generation token invalidates stale timer callbacks from an abandoned suspension so they cannot advance a resumed VM - WORKFLOW_RETAINED_VM=0 kill switch; telemetry records workflow.execution.mode = replay | retained Part 2 of the retained-VM stack (#2990); requires the determinism hardening in part 1. --- .changeset/retain-workflow-vm.md | 2 +- .../docs/v5/configuration/runtime-tuning.mdx | 2 +- packages/core/src/retained-vm-loop.test.ts | 20 - .../src/runtime/retained-step-input.test.ts | 461 ----- .../core/src/runtime/retained-step-input.ts | 427 ----- .../core/src/runtime/suspension-handler.ts | 33 +- packages/core/src/workflow.test.ts | 62 + packages/core/src/workflow.ts | 1512 ++++++++--------- 8 files changed, 830 insertions(+), 1689 deletions(-) delete mode 100644 packages/core/src/runtime/retained-step-input.test.ts delete mode 100644 packages/core/src/runtime/retained-step-input.ts diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 9db15ed117..25d38b0d57 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -3,4 +3,4 @@ 'workflow': patch --- -Retain workflow execution across passive-data inline steps within one invocation; boundaries whose argument serialization could execute workflow code fall back to replay, and `WORKFLOW_RETAINED_VM=0` disables retention. `crypto.subtle.digest` in workflow functions now computes synchronously via `node:crypto`, so digest timing is deterministic under replay. `WeakRef`, `FinalizationRegistry`, `Atomics.waitAsync`, and async `WebAssembly` compilation are no longer exposed in workflow functions (GC observation and wall-clock timing cannot be replayed), and the serialization-relevant intrinsics (`Object`/`Array`/`Function`, the prototypes of `Map`/`Set`/`Date`/typed arrays/`ArrayBuffer`, and reducer-referenced global bindings) are frozen in the workflow sandbox. +Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. Boundaries whose step arguments are not primitive values fall back to ordinary replay. diff --git a/docs/content/docs/v5/configuration/runtime-tuning.mdx b/docs/content/docs/v5/configuration/runtime-tuning.mdx index 72a4679171..9007fb7c2a 100644 --- a/docs/content/docs/v5/configuration/runtime-tuning.mdx +++ b/docs/content/docs/v5/configuration/runtime-tuning.mdx @@ -81,7 +81,7 @@ For example, a workflow can run a 10-minute inline step even with `WORKFLOW_REPL - Default: enabled - Keeps the suspended workflow VM alive across inline steps within one invocation, so each iteration of the inline loop appends only the newly written events instead of replaying the whole event log in a fresh VM. - Suspensions involving hooks, waits, or attributes — and any replay divergence — always fall back to a full replay. -- Step inputs made of plain data (objects, arrays, primitives) and standard built-ins (`Map`, `Set`, `Date`, typed arrays, `ArrayBuffer`) remain retainable. Anything whose serialization could execute workflow code — accessors, proxies, functions, custom class serializers, `RegExp` — falls back to replay for that boundary. +- Step inputs of primitive values remain retainable; anything whose serialization could execute workflow code falls back to replay for that boundary. (Support for plain objects, arrays, and standard built-ins lands in a follow-up.) - Set `0` or `false` to replay from scratch in a fresh VM on every iteration. ### `WORKFLOW_INLINE_OWNERSHIP` diff --git a/packages/core/src/retained-vm-loop.test.ts b/packages/core/src/retained-vm-loop.test.ts index 2085b1edad..d24b59bcc2 100644 --- a/packages/core/src/retained-vm-loop.test.ts +++ b/packages/core/src/retained-vm-loop.test.ts @@ -100,17 +100,6 @@ const mixedBatchWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP" } globalThis.__private_workflows = new Map([["workflow", workflow]]);`; -// Map/Date/typed-array arguments serialize through pinned prototype members -// (see vm/serialization-pins.ts), so these boundaries stay retainable. -const builtinArgsWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); - const s2 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s2"); - async function workflow() { - const a = await s1({ index: new Map([["k", 1]]), when: new Date(1234) }); - const b = await s2(new Uint8Array([1, 2, 3])); - return a + b; - } - globalThis.__private_workflows = new Map([["workflow", workflow]]);`; - // `crypto.subtle.digest` computes synchronously via node:crypto, so a // digest-using VM stays quiescent at suspension and remains retainable. const digestWorkflow = `const s1 = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("r_s1"); @@ -330,15 +319,6 @@ describe('retained VM through the inline replay loop', () => { expect(vmBuilds).toBeGreaterThan(1); }); - it('retains boundaries whose args are supported built-ins', async () => { - const { vmBuilds, output } = await drive( - 'wrun_retained_builtins', - builtinArgsWorkflow - ); - expect(output).toBeInstanceOf(Uint8Array); - expect(vmBuilds).toBe(1); - }); - it('retains a VM that used the synchronous crypto.subtle.digest', async () => { const { vmBuilds, output } = await drive( 'wrun_retained_digest', diff --git a/packages/core/src/runtime/retained-step-input.test.ts b/packages/core/src/runtime/retained-step-input.test.ts deleted file mode 100644 index ce8eafa023..0000000000 --- a/packages/core/src/runtime/retained-step-input.test.ts +++ /dev/null @@ -1,461 +0,0 @@ -import vm from 'node:vm'; -import { describe, expect, it } from 'vitest'; -import { dehydrateStepArguments } from '../serialization.js'; -import { createContext, freezeSerializationIntrinsics } from '../vm/index.js'; -import { isRetainedSerializationPassive } from './retained-step-input.js'; - -const seed = 'retained-step-input'; -const fixedTimestamp = 1_700_000_000_000; - -function makeContext({ freeze = true } = {}) { - const { context, globalThis: workflowGlobal } = createContext({ - seed, - fixedTimestamp, - }); - // Mimic the globals workflow.ts installs before any serialization happens - // (the stream/request reducers dispatch on them unguarded). - for (const name of [ - 'ReadableStream', - 'WritableStream', - 'TransformStream', - 'Request', - 'Response', - 'AbortController', - 'AbortSignal', - ]) { - if ((workflowGlobal as any)[name] === undefined) { - (workflowGlobal as any)[name] = (globalThis as any)[name]; - } - } - if (freeze) freezeSerializationIntrinsics(workflowGlobal); - return { context, workflowGlobal }; -} - -describe('isRetainedSerializationPassive', () => { - it('accepts plain cross-realm data', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `({ - nested: [{ ok: true }, "text", 42n], - sparse: [1, , 3], - flag: false, - })`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - }); - - it('accepts the supported built-ins on a frozen realm', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - 'new Map([["k", { ok: true }]])', - 'new Set([1, "two"])', - 'new Date(1234)', - 'new Uint8Array([1, 2, 3])', - 'new Float32Array([1.5])', - 'new ArrayBuffer(8)', - '({ when: new Date(0), bytes: new Uint8Array(2), index: new Map() })', - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - } - }); - - it('accepts host-realm plain data but not host-realm built-ins', () => { - const { workflowGlobal } = makeContext(); - // Hydrated step results are host-realm plain objects/arrays. - expect( - isRetainedSerializationPassive({ nested: [{ ok: true }] }, workflowGlobal) - ).toBe(true); - // Host built-in prototypes cannot be frozen (process-shared) and are - // reachable from workflow code, so host-realm instances decline. - expect( - isRetainedSerializationPassive(new Map([['k', 1]]), workflowGlobal) - ).toBe(false); - expect(isRetainedSerializationPassive(new Date(0), workflowGlobal)).toBe( - false - ); - }); - - it('declines types whose serialization surface is not frozen', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - '/workflow/gi', - 'new DataView(new ArrayBuffer(8))', - 'new SharedArrayBuffer(8)', - 'new Uint8Array(new SharedArrayBuffer(4))', - 'new Error("boom")', - 'new (class Sub extends Map {})()', - 'Object.assign(new Map(), { expando: 1 })', - 'Object.create(null)', - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } - }); - - it('declines accessors without invoking them', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - return { get value() { globalThis.__retainedTestCalls++; return 1; } }; - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines proxies without invoking their traps', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - return new Proxy({ value: 1 }, { - ownKeys(target) { - globalThis.__retainedTestCalls++; - return Reflect.ownKeys(target); - } - }); - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines custom class serializers without invoking them', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - class Value { - static classId = "test/Value"; - static [Symbol.for("workflow-serialize")](instance) { - globalThis.__retainedTestCalls++; - return { value: instance.value }; - } - constructor(value) { this.value = value; } - } - return new Value(1); - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); - - it('declines hidden own keys (symbols, non-enumerables, constructor)', () => { - const { context, workflowGlobal } = makeContext(); - for (const expression of [ - `(() => { - const tagged = { plain: true }; - Object.defineProperty(tagged, Symbol.for("WORKFLOW_ABORT_STREAM_NAME"), { - value: "abort-stream", enumerable: false, - }); - return tagged; - })()`, - `(() => { - const hidden = { plain: true }; - Object.defineProperty(hidden, "signal", { - get() { return { aborted: false }; }, enumerable: false, - }); - return hidden; - })()`, - `(() => { - const arr = [1]; - Object.defineProperty(arr, "constructor", { - value: class Fake { static classId = "fake"; }, enumerable: false, - }); - return arr; - })()`, - `(() => { - const arr = [1, 2]; - Object.defineProperty(arr, "0", { value: 7, enumerable: false }); - return arr; - })()`, - ]) { - const value = vm.runInContext(expression, context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } - }); - - it('freezing makes prototype patches impossible, so retention persists', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", 1]])', context); - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - - // Redefining a member the serializer reads throws on the frozen prototype. - for (const attempt of [ - '"use strict"; Object.defineProperty(Map.prototype, "constructor", { get() { return 1; } })', - '"use strict"; Map.prototype[Symbol.iterator] = function () {};', - '"use strict"; Date.prototype.toISOString = function () { return "x"; };', - '"use strict"; Object.defineProperty(Float32Array.prototype, "buffer", { get() {} })', - ]) { - expect(() => vm.runInContext(attempt, context)).toThrow( - /not extensible|read only|Cannot redefine/ - ); - } - // Nothing changed, so the built-ins remain retainable. - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - }); - - it('declines built-ins when the realm was never frozen', () => { - const { context, workflowGlobal } = makeContext({ freeze: false }); - const map = vm.runInContext('new Map()', context); - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(false); - // Plain data does not depend on the built-in prototypes. - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); - - it('declines retention while a host dispatch constructor is hooked', () => { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - - Object.defineProperty(Headers, Symbol.hasInstance, { - value: () => false, - configurable: true, - }); - try { - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - } finally { - delete (Headers as any)[Symbol.hasInstance]; - } - - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); -}); - -describe('serialization touches only the frozen surface', () => { - // THE coupling test: retention is only sound if every prototype member - // `dehydrateStepArguments` executes for the supported built-ins lives on an - // object `freezeSerializationIntrinsics` freezes. Wrap every configurable - // member on the relevant prototypes (in an unfrozen realm) with a recorder - // and assert the serializer hits nothing beyond this measured set — every - // entry of which is on a frozen prototype in production. If serde starts - // touching something new, this fails loudly: extend the freeze (and this - // list) together. - const FROZEN_SURFACE = new Set([ - 'Map.prototype.Symbol(Symbol.iterator)', - '%MapIteratorPrototype%.next', - 'Set.prototype.Symbol(Symbol.iterator)', - '%SetIteratorPrototype%.next', - 'Date.prototype.getDate', - 'Date.prototype.toISOString', - '%TypedArray%.prototype.buffer', - '%TypedArray%.prototype.byteOffset', - '%TypedArray%.prototype.byteLength', - 'ArrayBuffer.prototype.byteLength', - ]); - - it('for Map, Set, Date, typed arrays, and ArrayBuffer', async () => { - // Unfrozen realm: the recorders themselves need to redefine members. - const { context, workflowGlobal } = makeContext({ freeze: false }); - const g = workflowGlobal as any; - const touched = new Set(); - - const wrapPrototype = (prototype: object, label: string) => { - for (const key of Reflect.ownKeys(prototype)) { - if (key === 'constructor') continue; - const descriptor = Object.getOwnPropertyDescriptor(prototype, key); - if (!descriptor || !descriptor.configurable) continue; - const name = `${label}.${String(key)}`; - if (typeof descriptor.value === 'function') { - const original = descriptor.value; - Object.defineProperty(prototype, key, { - ...descriptor, - value: function (this: unknown, ...args: unknown[]) { - touched.add(name); - return original.apply(this, args); - }, - }); - } else if (descriptor.get) { - const originalGet = descriptor.get; - Object.defineProperty(prototype, key, { - ...descriptor, - get() { - touched.add(name); - return originalGet.call(this); - }, - set: descriptor.set, - }); - } - } - }; - - const values = vm.runInContext( - `({ - map: new Map([["k", 1]]), - set: new Set([1, 2]), - date: new Date(1234), - f32: new Float32Array([1.5, 2.5]), - u8: new Uint8Array([1, 2, 3]), - ab: new ArrayBuffer(8), - })`, - context - ); - - wrapPrototype(g.Map.prototype, 'Map.prototype'); - wrapPrototype( - Object.getPrototypeOf(new g.Map()[Symbol.iterator]()), - '%MapIteratorPrototype%' - ); - wrapPrototype(g.Set.prototype, 'Set.prototype'); - wrapPrototype( - Object.getPrototypeOf(new g.Set()[Symbol.iterator]()), - '%SetIteratorPrototype%' - ); - const datePrototype = Object.getOwnPropertyDescriptor(g.Date, 'prototype')! - .value as object; - wrapPrototype(datePrototype, 'Date.prototype'); - wrapPrototype( - Object.getPrototypeOf(g.Uint8Array.prototype), - '%TypedArray%.prototype' - ); - wrapPrototype(g.Uint8Array.prototype, 'Uint8Array.prototype'); - wrapPrototype(g.Float32Array.prototype, 'Float32Array.prototype'); - wrapPrototype(g.ArrayBuffer.prototype, 'ArrayBuffer.prototype'); - - for (const value of Object.values(values as Record)) { - touched.clear(); - await dehydrateStepArguments( - { args: [value], closureVars: undefined, thisVal: undefined }, - 'wrun_pin_coverage', - undefined, - g, - false, - false - ); - for (const name of touched) { - expect( - FROZEN_SURFACE, - `member outside the frozen surface executed: ${name}` - ).toContain(name); - } - } - }); -}); - -describe('checker execution surface', () => { - it('never invokes live host collection methods', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", 1]])', context); - - let invoked = 0; - const original = Map.prototype.forEach; - // Simulate workflow code having replaced the reachable host method. - Map.prototype.forEach = function ( - this: Map, - ...args: [any] - ) { - invoked++; - return original.apply(this, args); - }; - try { - // The checker uses the module-captured primordial: same verdict, - // replaced method never runs. - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - expect(invoked).toBe(0); - } finally { - Map.prototype.forEach = original; - } - }); - - it('declines typed arrays re-prototyped onto a frozen hostile prototype', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext( - `(() => { - globalThis.__retainedTestCalls = 0; - const realGetter = Object.getOwnPropertyDescriptor( - Object.getPrototypeOf(Uint8Array.prototype), "buffer").get; - const hostile = Object.create( - Object.getPrototypeOf(Uint8Array.prototype), - { buffer: { get() { globalThis.__retainedTestCalls++; return realGetter.call(this); } } } - ); - Object.freeze(hostile); - const ta = new Uint8Array([1, 2]); - Object.setPrototypeOf(ta, hostile); - return ta; - })()`, - context - ); - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - expect(vm.runInContext('globalThis.__retainedTestCalls', context)).toBe(0); - }); -}); - -describe('bigint serialization', () => { - it('declines retention while host BigInt.prototype.toString is replaced', () => { - const { context, workflowGlobal } = makeContext(); - const value = vm.runInContext('({ big: 42n })', context); - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - - const original = BigInt.prototype.toString; - // biome-ignore lint/suspicious/noGlobalAssign: simulating workflow-realm tampering - BigInt.prototype.toString = function (this: bigint, ...args: [number?]) { - return original.apply(this, args); - }; - try { - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(false); - } finally { - BigInt.prototype.toString = original; - } - - expect(isRetainedSerializationPassive(value, workflowGlobal)).toBe(true); - }); -}); - -describe('walker primordials', () => { - it('uses captured primordials, not live host statics', () => { - const { context, workflowGlobal } = makeContext(); - const map = vm.runInContext('new Map([["k", { ok: true }]])', context); - - // If the walker consulted the live host statics, these would throw. - const originalDescriptor = Object.getOwnPropertyDescriptor; - const originalOwnKeys = Reflect.ownKeys; - (Object as any).getOwnPropertyDescriptor = () => { - throw new Error('live getOwnPropertyDescriptor used'); - }; - (Reflect as any).ownKeys = () => { - throw new Error('live ownKeys used'); - }; - try { - expect(isRetainedSerializationPassive(map, workflowGlobal)).toBe(true); - } finally { - (Object as any).getOwnPropertyDescriptor = originalDescriptor; - (Reflect as any).ownKeys = originalOwnKeys; - } - }); - - it('declines when host Function.prototype carries serializer statics', () => { - const { context, workflowGlobal } = makeContext(); - const plain = vm.runInContext('({ ok: true })', context); - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - - Object.defineProperty( - Function.prototype, - Symbol.for('workflow-serialize'), - { - get() { - return undefined; - }, - configurable: true, - } - ); - try { - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(false); - } finally { - delete (Function.prototype as any)[Symbol.for('workflow-serialize')]; - } - - expect(isRetainedSerializationPassive(plain, workflowGlobal)).toBe(true); - }); -}); diff --git a/packages/core/src/runtime/retained-step-input.ts b/packages/core/src/runtime/retained-step-input.ts deleted file mode 100644 index 7fec0156da..0000000000 --- a/packages/core/src/runtime/retained-step-input.ts +++ /dev/null @@ -1,427 +0,0 @@ -import { types } from 'node:util'; -import { WORKFLOW_SERIALIZE } from '@workflow/serde'; - -// Host constructors that serialization dispatches on (`value instanceof -// global.X`), plus Object/Array, whose statics and prototypes the class -// reducer and devalue's tag lookup read for host-prototype values (hydrated -// step results are host-realm plain objects). The workflow VM's own -// intrinsics are frozen (see vm/index.ts), but host intrinsics are shared -// with the whole process and cannot be frozen — and workflow code can reach -// them (exposed host classes, `structuredClone` results) and plant -// workflow-realm hooks. Verified instead: any dirt declines retention, so a -// planted hook can never execute while a retained VM's inputs serialize. -const HOST_DISPATCH_CONSTRUCTORS = [ - 'Object', - 'Array', - 'Function', - 'Map', - 'Set', - 'Date', - 'RegExp', - 'ArrayBuffer', - 'SharedArrayBuffer', - 'DataView', - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', - 'Headers', - 'URL', - 'URLSearchParams', - 'DOMException', - 'AbortController', - 'AbortSignal', - 'Request', - 'Response', - 'ReadableStream', - 'WritableStream', - 'TransformStream', -] as const; - -// Host primordials captured at module load — before any workflow code can -// exist in the process — so the checker itself never invokes a live host -// member workflow code could have replaced (host constructors are reachable -// via e.g. `structuredClone(new Map()).constructor` or exposed classes). -const hostGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor; -const hostGetPrototypeOf = Object.getPrototypeOf; -const hostIsFrozen = Object.isFrozen; -const hostOwnKeys = Reflect.ownKeys; -const hostIsArray = Array.isArray; -const hostIsInteger = Number.isInteger; -const hostNumber = Number; -const hostString = String; -const hostBigIntToString = BigInt.prototype.toString; -const hostMapForEach = Map.prototype.forEach; -const hostSetForEach = Set.prototype.forEach; -// biome-ignore lint/style/noNonNullAssertion: the %TypedArray% buffer getter always exists -const hostTypedArrayBuffer = Object.getOwnPropertyDescriptor( - Object.getPrototypeOf(Uint8Array.prototype), - 'buffer' -)!.get!; - -const TYPED_ARRAY_CONSTRUCTORS = [ - 'Int8Array', - 'Uint8Array', - 'Uint8ClampedArray', - 'Int16Array', - 'Uint16Array', - 'Int32Array', - 'Uint32Array', - 'Float16Array', - 'Float32Array', - 'Float64Array', - 'BigInt64Array', - 'BigUint64Array', -] as const; - -function hasOwn(target: object, key: string | symbol): boolean { - return hostGetOwnPropertyDescriptor(target, key) !== undefined; -} - -function isHostDispatchPristine(): boolean { - const hostGlobal = globalThis as unknown as Record; - for (const name of HOST_DISPATCH_CONSTRUCTORS) { - const constructor = hostGlobal[name]; - if (constructor === undefined) continue; - if (hasOwn(constructor as object, Symbol.hasInstance)) return false; - } - // The class reducer reads `cls[WORKFLOW_SERIALIZE]` / `cls.classId` as - // inherited Gets, so the constructors' whole prototype chains — host - // Function.prototype and Object.prototype — must be clean too. - for (const target of [Object, Array, Function.prototype, Object.prototype]) { - if (hasOwn(target, WORKFLOW_SERIALIZE) || hasOwn(target, 'classId')) { - return false; - } - } - for (const [prototype, constructor] of [ - [Object.prototype, Object], - [Array.prototype, Array], - ] as const) { - if ( - hasOwn(prototype, Symbol.toStringTag) || - ownDataProperty(prototype, 'constructor') !== constructor - ) { - return false; - } - } - // Constructor prototype chains end at host Object.prototype, where an - // added @@hasInstance would be found by dispatch lookup. (Host - // Function.prototype's @@hasInstance is spec non-configurable.) - if (hasOwn(Object.prototype, Symbol.hasInstance)) return false; - // The BigInt reducer calls `.toString()` on bigint primitives from host - // code, which resolves on the HOST BigInt.prototype (primitives are - // realm-less; method lookup uses the running code's realm). - if (ownDataProperty(BigInt.prototype, 'toString') !== hostBigIntToString) { - return false; - } - return true; -} - -// Own data-property read that never performs a property Get — workflow code -// can redefine its globals (or their `prototype` slots) with accessors, and -// validation must not execute workflow-owned code. -function ownDataProperty(target: object, key: string): unknown { - const descriptor = hostGetOwnPropertyDescriptor(target, key); - return descriptor && 'value' in descriptor ? descriptor.value : undefined; -} - -function constructorPrototype( - realmGlobal: Record, - constructorName: string -): object | undefined { - const constructor = ownDataProperty(realmGlobal, constructorName); - if ( - (typeof constructor !== 'function' && typeof constructor !== 'object') || - constructor === null || - types.isProxy(constructor) - ) { - return undefined; - } - const prototype = ownDataProperty(constructor, 'prototype'); - return typeof prototype === 'object' && - prototype !== null && - !types.isProxy(prototype) - ? prototype - : undefined; -} - -// The workflow realm's prototype for a supported built-in, but only once the -// sandbox froze it: serialization both executes members on these prototypes -// (iterators, getters) and reads others (`constructor`), so a mutable -// prototype — including one in a realm where freezeSerializationIntrinsics -// never ran — is not provably passive. Host-realm instances of these -// built-ins decline for the same reason: host prototypes cannot be frozen -// and are reachable from workflow code (e.g. via structuredClone results). -function frozenRealmPrototype( - workflowGlobal: Record, - constructorName: string -): object | undefined { - const prototype = constructorPrototype(workflowGlobal, constructorName); - return prototype !== undefined && hostIsFrozen(prototype) - ? prototype - : undefined; -} - -function hasAllowedPrototype( - value: object, - workflowGlobal: Record, - constructorName: string -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === - constructorPrototype( - globalThis as unknown as Record, - constructorName - ) || prototype === constructorPrototype(workflowGlobal, constructorName) - ); -} - -function isArrayIndex(key: string): boolean { - const index = hostNumber(key); - return ( - hostIsInteger(index) && - index >= 0 && - index < 2 ** 32 - 1 && - hostString(index) === key - ); -} - -function isPassiveArrayProperty( - array: unknown[], - key: string | symbol, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (key === 'length') return true; - const descriptor = hostGetOwnPropertyDescriptor(array, key); - if (!descriptor) return false; - // Only own enumerable data indices are passive. Anything hidden — symbol - // tags, non-enumerable properties, accessors — can be observed by - // serialization dispatch (reducer probes, thenable checks, the class - // reducer) and can execute workflow code when read. - return ( - typeof key === 'string' && - isArrayIndex(key) && - descriptor.enumerable === true && - 'value' in descriptor && - isPassive(descriptor.value, workflowGlobal, seen) - ); -} - -function isPassiveArray( - value: unknown[], - workflowGlobal: Record, - seen: WeakSet -): boolean { - return ( - hasAllowedPrototype(value, workflowGlobal, 'Array') && - hostOwnKeys(value).every((key) => - isPassiveArrayProperty(value, key, workflowGlobal, seen) - ) - ); -} - -// Instances of the supported built-ins must carry no own properties at all: -// serialization never reads own properties on them, but an own accessor or -// symbol could still be observed through other dispatch lookups, and clean -// instances are the overwhelmingly common case anyway. -function hasNoOwnProperties(value: object): boolean { - return hostOwnKeys(value).length === 0; -} - -function isPassiveMap( - value: Map, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const prototype = hostGetPrototypeOf(value); - if (prototype !== frozenRealmPrototype(workflowGlobal, 'Map')) return false; - if (!hasNoOwnProperties(value)) return false; - let passive = true; - // The captured host forEach iterates via internal slots — no realm - // members (and no live, replaceable host members) execute. - hostMapForEach.call(value, (entryValue: unknown, key: unknown) => { - passive &&= - isPassive(key, workflowGlobal, seen) && - isPassive(entryValue, workflowGlobal, seen); - }); - return passive; -} - -function isPassiveSet( - value: Set, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const prototype = hostGetPrototypeOf(value); - if (prototype !== frozenRealmPrototype(workflowGlobal, 'Set')) return false; - if (!hasNoOwnProperties(value)) return false; - let passive = true; - hostSetForEach.call(value, (entryValue: unknown) => { - passive &&= isPassive(entryValue, workflowGlobal, seen); - }); - return passive; -} - -function isPassiveDate( - value: object, - workflowGlobal: Record -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === frozenRealmPrototype(workflowGlobal, 'Date') && - hasNoOwnProperties(value) - ); -} - -function isPassiveTypedArray( - value: object, - workflowGlobal: Record -): boolean { - // Identity against the finite set of the realm's real (frozen) typed-array - // prototypes — NOT "anything chaining to %TypedArray%": a workflow can - // manufacture a frozen hostile prototype with a delegating `buffer` getter - // and setPrototypeOf a real typed array onto it. - const prototype = hostGetPrototypeOf(value); - if ( - !TYPED_ARRAY_CONSTRUCTORS.some( - (name) => prototype === frozenRealmPrototype(workflowGlobal, name) - ) - ) { - return false; - } - // Own keys on a typed array are exactly its canonical indices. - if ( - !hostOwnKeys(value).every( - (key) => typeof key === 'string' && isArrayIndex(key) - ) - ) { - return false; - } - // Read the backing buffer via the captured host getter (internal slots - // work cross-realm); reject SharedArrayBuffer backing. - return !types.isSharedArrayBuffer(hostTypedArrayBuffer.call(value)); -} - -function isPassiveArrayBuffer( - value: object, - workflowGlobal: Record -): boolean { - const prototype = hostGetPrototypeOf(value); - return ( - prototype === frozenRealmPrototype(workflowGlobal, 'ArrayBuffer') && - hasNoOwnProperties(value) - ); -} - -function isPassiveObjectProperty( - object: object, - key: string | symbol, - workflowGlobal: Record, - seen: WeakSet -): boolean { - const descriptor = hostGetOwnPropertyDescriptor(object, key); - if (!descriptor) return false; - // Only own enumerable string-keyed data properties are passive. Anything - // hidden — symbol tags, non-enumerable properties, accessors — can be - // observed by serialization dispatch (reducer probes like `.signal`, - // thenable checks, the class reducer) and can execute workflow code. - return ( - typeof key === 'string' && - key !== '__proto__' && - descriptor.enumerable === true && - 'value' in descriptor && - isPassive(descriptor.value, workflowGlobal, seen) - ); -} - -function isPassivePlainObject( - value: object, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if (!hasAllowedPrototype(value, workflowGlobal, 'Object')) return false; - return hostOwnKeys(value).every((key) => - isPassiveObjectProperty(value, key, workflowGlobal, seen) - ); -} - -/** - * Whether serializing `value` through the ordinary pipeline provably executes - * no workflow code and draws no workflow-realm randomness. - * - * Retained sessions keep running after suspension, so serialization side - * effects there would desync the live VM from what a cold replay - * reconstructs (replay never re-serializes an already-created step). The - * bytes themselves cannot differ by mode — serialization happens once and - * every mode reads the same `step_created` event — so passivity is the only - * property retention needs. - * - * Passive values: primitives; plain objects and arrays (own enumerable - * string-keyed data properties only — devalue traverses these purely via own - * reads); and workflow-realm Map/Set/Date/typed arrays/ArrayBuffer whose - * prototypes the sandbox froze (see freezeSerializationIntrinsics). - * Everything else — proxies, accessors, functions, custom classes, RegExp, - * hidden keys — declines, serializes exactly the same way, and the session - * falls back to ordinary replay for that boundary. - */ -function isPassive( - value: unknown, - workflowGlobal: Record, - seen: WeakSet -): boolean { - if ( - value === null || - value === undefined || - typeof value === 'boolean' || - typeof value === 'bigint' || - typeof value === 'number' || - typeof value === 'string' - ) { - return true; - } - if (typeof value !== 'object' || types.isProxy(value)) return false; - if (seen.has(value)) return true; - seen.add(value); - - if (hostIsArray(value)) return isPassiveArray(value, workflowGlobal, seen); - if (types.isMap(value)) return isPassiveMap(value, workflowGlobal, seen); - if (types.isSet(value)) return isPassiveSet(value, workflowGlobal, seen); - if (types.isDate(value)) return isPassiveDate(value, workflowGlobal); - if (types.isTypedArray(value)) { - return isPassiveTypedArray(value, workflowGlobal); - } - if (types.isArrayBuffer(value)) { - return isPassiveArrayBuffer(value, workflowGlobal); - } - if ( - types.isSharedArrayBuffer(value) || - types.isRegExp(value) || - types.isDataView(value) || - types.isBoxedPrimitive(value) || - types.isNativeError(value) || - types.isPromise(value) || - types.isArgumentsObject(value) - ) { - return false; - } - return isPassivePlainObject(value, workflowGlobal, seen); -} - -export function isRetainedSerializationPassive( - value: unknown, - workflowGlobal: Record -): boolean { - return ( - isHostDispatchPristine() && isPassive(value, workflowGlobal, new WeakSet()) - ); -} diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 64c362c566..c23ab2e878 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -31,7 +31,19 @@ import * as Attribute from '../telemetry/semantic-conventions.js'; import { getAbortStreamIdFromToken } from '../util.js'; import { getMaxInlineSteps } from './constants.js'; import { type MutableEventLog, withPreconditionRetry } from './helpers.js'; -import { isRetainedSerializationPassive } from './retained-step-input.js'; + +// Serializing a primitive executes no code of any kind. BigInt is excluded: +// its encoding calls a prototype method. Widened to plain data and standard +// built-ins by the retained-input walker in a follow-up. +function isPrimitiveStepArgument(value: unknown): boolean { + return ( + value === null || + value === undefined || + typeof value === 'boolean' || + typeof value === 'number' || + typeof value === 'string' + ); +} export interface SuspensionHandlerParams { suspension: WorkflowSuspension; @@ -502,22 +514,19 @@ export async function handleSuspension({ // durable bytes cannot depend on retention. What retention needs to know is // whether that serialization will execute workflow code (getters, hooks, // patched prototype members) — side effects a cold replay would not repeat. - // If any input in the batch is not provably passive, the caller demotes the - // session so the side effects land in a VM that is about to be discarded, - // exactly like the pre-retention runtime. + // For now only primitive arguments are provably passive (serializing them + // executes no code at all); a follow-up widens this to plain data and the + // standard built-ins. If any input in the batch is not provably passive, + // the caller demotes the session so the side effects land in a VM that is + // about to be discarded, exactly like the pre-retention runtime. let retainedStepInputsSafe = true; if (prepareForRetention) { for (const queueItem of stepItems) { if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; if ( - !isRetainedSerializationPassive( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - thisVal: queueItem.thisVal, - }, - suspension.globalThis - ) + queueItem.thisVal !== undefined || + queueItem.closureVars !== undefined || + !queueItem.args.every(isPrimitiveStepArgument) ) { retainedStepInputsSafe = false; break; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 688bc2774d..6830603819 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -405,6 +405,68 @@ describe('runWorkflow', () => { ).toEqual({ type: 'replay' }); }); + it('falls back to replay (not failure) after mid-execution divergence', async () => { + const ops: Promise[] = []; + const workflowRun: WorkflowRun = { + runId: 'wrun_retained_mid_divergence', + workflowName: 'workflow', + status: 'running', + input: await dehydrateWorkflowArguments( + [], + 'wrun_retained_mid_divergence', + noEncryptionKey, + ops + ), + createdAt: new Date('2024-01-01T00:00:00.000Z'), + updatedAt: new Date('2024-01-01T00:00:00.000Z'), + startedAt: new Date('2024-01-01T00:00:00.000Z'), + deploymentId: 'test-deployment', + }; + const workflowCode = ` + const step = globalThis[Symbol.for("WORKFLOW_USE_STEP")]("step"); + async function workflow() { await step(); } + ${getWorkflowTransformCode('workflow')}`; + const events: Event[] = []; + + const suspended = await executeWorkflow({ + type: 'replay', + workflowCode, + workflowRun, + events, + encryptionKey: noEncryptionKey, + replayPayloadCache: new ReplayPayloadCache(noEncryptionKey), + }); + assert(suspended.type === 'suspended'); + + // A strict extension whose appended suffix the VM cannot consume: the + // resume itself starts, then diverges mid-execution. + await expect( + executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [ + { + eventId: 'event-alien', + runId: workflowRun.runId, + eventType: 'hook_received', + correlationId: 'hook_unknown', + eventData: {}, + createdAt: new Date('2024-01-01T00:00:01.000Z'), + } as Event, + ], + }) + ).rejects.toThrow(/could not consume event/); + + // The session must demote to replay — not throw "cannot resume". + expect( + await executeWorkflow({ + type: 'resume', + session: suspended.session, + events: [], + }) + ).toEqual({ type: 'replay' }); + }); + it('regenerates step correlation IDs independent of startedAt (turbo replay-stability)', async () => { // Turbo's first delivery synthesizes `startedAt` from the local clock, // while later (non-turbo) deliveries load the server-canonical `startedAt`. diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 539b2e199a..57f75435fe 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -151,31 +151,6 @@ type WorkflowSessionState = | { readonly type: 'replay' } | { readonly type: 'completed' }; -// The suspension counts are all derived from `steps` in the WorkflowSuspension -// constructor, so identical items mean an identical boundary. -function isSameSuspensionBoundary( - previous: WorkflowSuspension, - next: WorkflowSuspension -): boolean { - return ( - previous.steps.length === next.steps.length && - previous.steps.every((item, index) => item === next.steps[index]) - ); -} - -// A suspended VM is quiescent, but each parked step consumer signals its own -// (identical) suspension via scheduleWhenIdle — absorb those duplicates and -// treat anything else as unretainable. -function updateSuspendedSession( - suspension: WorkflowSuspension, - error: Error -): WorkflowSessionState { - return WorkflowSuspension.is(error) && - isSameSuspensionBoundary(suspension, error) - ? { type: 'suspended', suspension } - : { type: 'replay' }; -} - export interface WorkflowSession { readonly workflowRun: WorkflowRun; readonly argumentCount: number; @@ -322,7 +297,7 @@ export async function runWorkflow( return result.output; } -function createWorkflowSession({ +async function createWorkflowSession({ workflowCode, workflowRun, events, @@ -334,311 +309,303 @@ function createWorkflowSession({ session: WorkflowSession; execution: Promise; }> { - return (async () => { - const startedAt = workflowRun.startedAt; - if (!startedAt) { - throw new WorkflowRuntimeError( - `Workflow run "${workflowRun.runId}" has no "startedAt" timestamp (should not happen)` - ); - } - - // The deterministic RNG seed is derived from identifiers that are all - // known the instant the queue message arrives — `runId`, `workflowName`, - // and `deploymentId` — with no timestamp component. `runId` alone already - // makes the seed unique-per-run and replay-stable; `workflowName` and - // `deploymentId` are included for extra entropy. Dropping the timestamp - // means the seed no longer depends on `startedAt`/`createdAt`, so it (and - // the VM context) can be computed before any server round-trip. - // - // The VM's initial fixed clock is derived from the run's creation time, - // recovered from the ULID embedded in `runId` (also available immediately), - // falling back to the run snapshot's `createdAt` for non-ULID ids. This - // initial `fixedTimestamp` only governs `Date.now()` / `new Date()` in the - // window before the first event is consumed; thereafter `updateTimestamp` - // advances the VM clock to each consumed event's `createdAt` (see the - // EventsConsumer below), starting with `run_created`. - const fixedTimestamp = - runIdCreatedAt(workflowRun.runId) ?? +workflowRun.createdAt; - - // Get the port before creating VM context to avoid async operations - // affecting the deterministic timestamp - const isVercel = process.env.VERCEL_URL !== undefined; - // Load getPort lazily to prevent Turbopack from tracing get-port's - // fs ops (readdir, readFile) into the flow route bundle. The resolved - // port is cached per process (see get-port-lazy.ts), so this is cheap - // on replays after the first. - const workflowBaseUrl = createWorkflowBaseUrl( - isVercel - ? `https://${process.env.VERCEL_URL}` - : `http://localhost:${(await getPortLazy()) ?? 3000}` + const startedAt = workflowRun.startedAt; + if (!startedAt) { + throw new WorkflowRuntimeError( + `Workflow run "${workflowRun.runId}" has no "startedAt" timestamp (should not happen)` ); + } - const { - context, - globalThis: vmGlobalThis, - updateTimestamp, - } = createContext({ - seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, - fixedTimestamp, - }); + // The deterministic RNG seed is derived from identifiers that are all + // known the instant the queue message arrives — `runId`, `workflowName`, + // and `deploymentId` — with no timestamp component. `runId` alone already + // makes the seed unique-per-run and replay-stable; `workflowName` and + // `deploymentId` are included for extra entropy. Dropping the timestamp + // means the seed no longer depends on `startedAt`/`createdAt`, so it (and + // the VM context) can be computed before any server round-trip. + // + // The VM's initial fixed clock is derived from the run's creation time, + // recovered from the ULID embedded in `runId` (also available immediately), + // falling back to the run snapshot's `createdAt` for non-ULID ids. This + // initial `fixedTimestamp` only governs `Date.now()` / `new Date()` in the + // window before the first event is consumed; thereafter `updateTimestamp` + // advances the VM clock to each consumed event's `createdAt` (see the + // EventsConsumer below), starting with `run_created`. + const fixedTimestamp = + runIdCreatedAt(workflowRun.runId) ?? +workflowRun.createdAt; + + // Get the port before creating VM context to avoid async operations + // affecting the deterministic timestamp + const isVercel = process.env.VERCEL_URL !== undefined; + // Load getPort lazily to prevent Turbopack from tracing get-port's + // fs ops (readdir, readFile) into the flow route bundle. The resolved + // port is cached per process (see get-port-lazy.ts), so this is cheap + // on replays after the first. + const workflowBaseUrl = createWorkflowBaseUrl( + isVercel + ? `https://${process.env.VERCEL_URL}` + : `http://localhost:${(await getPortLazy()) ?? 3000}` + ); - const initialInterruption = withResolvers(); - let state: WorkflowSessionState = { - type: 'running', - interruption: initialInterruption, - }; + const { + context, + globalThis: vmGlobalThis, + updateTimestamp, + } = createContext({ + seed: `${workflowRun.runId}:${workflowRun.workflowName}:${workflowRun.deploymentId}`, + fixedTimestamp, + }); - const onWorkflowError = (error: Error): void => { - switch (state.type) { - case 'running': { - const { interruption } = state; - state = WorkflowSuspension.is(error) - ? { type: 'suspended', suspension: error } - : { type: 'replay' }; - interruption.reject(error); - return; - } - case 'suspended': - state = updateSuspendedSession(state.suspension, error); - return; - case 'replay': - case 'completed': - return; + const initialInterruption = withResolvers(); + let state: WorkflowSessionState = { + type: 'running', + interruption: initialInterruption, + }; + + const onWorkflowError = (error: Error): void => { + switch (state.type) { + case 'running': { + const { interruption } = state; + state = WorkflowSuspension.is(error) + ? { type: 'suspended', suspension: error } + : { type: 'replay' }; + interruption.reject(error); + return; } - state satisfies never; - }; - - const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); - const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => - new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) - ); - - // Create a mutable holder for the promise queue so the EventsConsumer - // can access the current queue state via a getter. The queue is mutated - // by step/hook/sleep callbacks as events are processed. - const promiseQueueHolder = { current: Promise.resolve() }; - - const eventsConsumer = new EventsConsumer(events, { - onConsumedEvent: (event) => { - updateTimestamp(+event.createdAt); - }, - onUnconsumedEvent: (event) => { - onWorkflowError( - new ReplayDivergenceError( - `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, - { eventId: event.eventId } - ) - ); - }, - getPromiseQueue: () => promiseQueueHolder.current, - }); - - const workflowContext: WorkflowOrchestratorContext = { - runId: workflowRun.runId, - encryptionKey, - worldCapabilities, - globalThis: vmGlobalThis, - onWorkflowError, - eventsConsumer, - // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so - // the time prefix fed to `ulid()` MUST be replay-stable across every - // delivery — otherwise a redelivery regenerates different correlation IDs - // and replay throws ReplayDivergenceError. `startedAt` is NOT safe here: - // under turbo the first delivery synthesizes `startedAt` from the local - // clock, but later (non-turbo) deliveries load the server-canonical - // `startedAt`, which differs by >=1ms. Use the same replay-stable value - // that already seeds the RNG and the VM clock (`fixedTimestamp`, recovered - // from the run ID's ULID and known the instant the message arrives). - generateUlid: () => ulid(fixedTimestamp), - generateNanoid, - invocationsQueue: new Map(), - // Use getter/setter so the EventsConsumer's getPromiseQueue() always - // sees the latest queue state as it's mutated by step/hook/sleep callbacks. - get promiseQueue() { - return promiseQueueHolder.current; - }, - set promiseQueue(value: Promise) { - promiseQueueHolder.current = value; - }, - pendingDeliveries: 0, - suspensionGeneration: 0, - pendingDeliveryBarriers: new Map(), - replayPayloadCache, - }; - - // Consume run lifecycle events - these are structural events that don't - // need special handling in the workflow, but must be consumed to advance - // past them in the event log - workflowContext.eventsConsumer.subscribe((event) => { - if (!event) { - return EventConsumerResult.NotConsumed; + case 'suspended': { + // A suspended VM is quiescent, but each parked step consumer + // signals its own (identical) suspension via scheduleWhenIdle — + // absorb those duplicates, treat anything else as unretainable. + // The suspension counts are all derived from `steps`, so identical + // items mean an identical boundary. + const { steps } = state.suspension; + const isDuplicateSignal = + WorkflowSuspension.is(error) && + error.steps.length === steps.length && + error.steps.every((item, index) => item === steps[index]); + if (!isDuplicateSignal) state = { type: 'replay' }; + return; } + case 'replay': + case 'completed': + return; + } + state satisfies never; + }; - // Consume run_created - every run has exactly one - if (event.eventType === 'run_created') { - return EventConsumerResult.Consumed; - } + const ulid = monotonicFactory(() => vmGlobalThis.Math.random()); + const generateNanoid = nanoid.customRandom(nanoid.urlAlphabet, 21, (size) => + new Uint8Array(size).map(() => 256 * vmGlobalThis.Math.random()) + ); - // Consume run_started - every run has exactly one - if (event.eventType === 'run_started') { - return EventConsumerResult.Consumed; - } + // Create a mutable holder for the promise queue so the EventsConsumer + // can access the current queue state via a getter. The queue is mutated + // by step/hook/sleep callbacks as events are processed. + const promiseQueueHolder = { current: Promise.resolve() }; + + const eventsConsumer = new EventsConsumer(events, { + onConsumedEvent: (event) => { + updateTimestamp(+event.createdAt); + }, + onUnconsumedEvent: (event) => { + onWorkflowError( + new ReplayDivergenceError( + `Replay could not consume event: eventType=${event.eventType}, correlationId=${event.correlationId}, eventId=${event.eventId}.`, + { eventId: event.eventId } + ) + ); + }, + getPromiseQueue: () => promiseQueueHolder.current, + }); - // Attribute writes performed from a step have no workflow-body call to - // consume them during replay; they are already reflected in the run - // snapshot and remain structural until a read API is introduced. - if ( - event.eventType === 'attr_set' && - event.eventData.writer.type === 'step' - ) { - return EventConsumerResult.Consumed; - } + const workflowContext: WorkflowOrchestratorContext = { + runId: workflowRun.runId, + encryptionKey, + worldCapabilities, + globalThis: vmGlobalThis, + onWorkflowError, + eventsConsumer, + // Correlation IDs (step_/wait_/hook_) are derived from `generateUlid`, so + // the time prefix fed to `ulid()` MUST be replay-stable across every + // delivery — otherwise a redelivery regenerates different correlation IDs + // and replay throws ReplayDivergenceError. `startedAt` is NOT safe here: + // under turbo the first delivery synthesizes `startedAt` from the local + // clock, but later (non-turbo) deliveries load the server-canonical + // `startedAt`, which differs by >=1ms. Use the same replay-stable value + // that already seeds the RNG and the VM clock (`fixedTimestamp`, recovered + // from the run ID's ULID and known the instant the message arrives). + generateUlid: () => ulid(fixedTimestamp), + generateNanoid, + invocationsQueue: new Map(), + // Use getter/setter so the EventsConsumer's getPromiseQueue() always + // sees the latest queue state as it's mutated by step/hook/sleep callbacks. + get promiseQueue() { + return promiseQueueHolder.current; + }, + set promiseQueue(value: Promise) { + promiseQueueHolder.current = value; + }, + pendingDeliveries: 0, + suspensionGeneration: 0, + pendingDeliveryBarriers: new Map(), + replayPayloadCache, + }; + // Consume run lifecycle events - these are structural events that don't + // need special handling in the workflow, but must be consumed to advance + // past them in the event log + workflowContext.eventsConsumer.subscribe((event) => { + if (!event) { return EventConsumerResult.NotConsumed; - }); + } - const useStep = createUseStep(workflowContext); - const createHook = createCreateHook(workflowContext); - const sleep = createSleep(workflowContext); - const setAttributes = createSetAttributes(workflowContext); - - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_USE_STEP] = useStep; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_SET_ATTRIBUTES] = setAttributes; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_CREATE_HOOK] = createHook; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_SLEEP] = sleep; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_GET_STREAM_ID] = (namespace?: string) => - getWorkflowRunStreamId(workflowRun.runId, namespace); - - // For the workflow VM, we store the context in a symbol on the `globalThis` object - const ctx: WorkflowMetadata = { - workflowName: workflowRun.workflowName, - workflowRunId: workflowRun.runId, - workflowStartedAt: new vmGlobalThis.Date(+startedAt), - url: workflowBaseUrl, - features: { encryption: !!encryptionKey }, - }; + // Consume run_created - every run has exactly one + if (event.eventType === 'run_created') { + return EventConsumerResult.Consumed; + } - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[STABLE_ULID] = ulid; + // Consume run_started - every run has exactly one + if (event.eventType === 'run_started') { + return EventConsumerResult.Consumed; + } - // NOTE: Will have a config override to use the custom fetch step. - // For now `fetch` must be explicitly imported from `workflow`. - vmGlobalThis.fetch = () => { - throw new vmGlobalThis.Error( - `Global "fetch" is unavailable in workflow functions. Use the "fetch" step function from "workflow" to make HTTP requests.\n\nLearn more: https://workflow-sdk.dev/err/${ERROR_SLUGS.FETCH_IN_WORKFLOW_FUNCTION}` - ); - }; + // Attribute writes performed from a step have no workflow-body call to + // consume them during replay; they are already reflected in the run + // snapshot and remain structural until a read API is introduced. + if ( + event.eventType === 'attr_set' && + event.eventData.writer.type === 'step' + ) { + return EventConsumerResult.Consumed; + } - // Override timeout/interval functions to throw helpful errors - // These are not supported in workflow functions because they rely on - // asynchronous scheduling which breaks deterministic replay - const timeoutErrorMessage = - 'Timeout functions like "setTimeout" and "setInterval" are not supported in workflow functions. Use the "sleep" function from "workflow" for time-based delays.'; + return EventConsumerResult.NotConsumed; + }); - (vmGlobalThis as any).setTimeout = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).setInterval = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearTimeout = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearInterval = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).setImmediate = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; - (vmGlobalThis as any).clearImmediate = () => { - throw new WorkflowRuntimeError(timeoutErrorMessage, { - slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, - }); - }; + const useStep = createUseStep(workflowContext); + const createHook = createCreateHook(workflowContext); + const sleep = createSleep(workflowContext); + const setAttributes = createSetAttributes(workflowContext); + + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_USE_STEP] = useStep; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_SET_ATTRIBUTES] = setAttributes; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_CREATE_HOOK] = createHook; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_SLEEP] = sleep; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_GET_STREAM_ID] = (namespace?: string) => + getWorkflowRunStreamId(workflowRun.runId, namespace); + + // For the workflow VM, we store the context in a symbol on the `globalThis` object + const ctx: WorkflowMetadata = { + workflowName: workflowRun.workflowName, + workflowRunId: workflowRun.runId, + workflowStartedAt: new vmGlobalThis.Date(+startedAt), + url: workflowBaseUrl, + features: { encryption: !!encryptionKey }, + }; + + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[WORKFLOW_CONTEXT_SYMBOL] = ctx; + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[STABLE_ULID] = ulid; + + // NOTE: Will have a config override to use the custom fetch step. + // For now `fetch` must be explicitly imported from `workflow`. + vmGlobalThis.fetch = () => { + throw new vmGlobalThis.Error( + `Global "fetch" is unavailable in workflow functions. Use the "fetch" step function from "workflow" to make HTTP requests.\n\nLearn more: https://workflow-sdk.dev/err/${ERROR_SLUGS.FETCH_IN_WORKFLOW_FUNCTION}` + ); + }; - // `AbortController` and `AbortSignal` in the workflow VM are hook-backed - // for deterministic replay. The controller's abort() queues a hook resumption, - // and signal.aborted is updated when the hook event is processed during replay. - (vmGlobalThis as any).AbortController = - createCreateAbortController(workflowContext); - const abortSignalStatics = createAbortSignalStatics(); - (vmGlobalThis as any).AbortSignal = { - abort: abortSignalStatics.abort, - any: abortSignalStatics.any, - timeout: abortSignalStatics.timeout, - }; + // Override timeout/interval functions to throw helpful errors + // These are not supported in workflow functions because they rely on + // asynchronous scheduling which breaks deterministic replay + const timeoutErrorMessage = + 'Timeout functions like "setTimeout" and "setInterval" are not supported in workflow functions. Use the "sleep" function from "workflow" for time-based delays.'; - // `Request` and `Response` are special built-in classes that invoke steps - // for the `json()`, `text()` and `arrayBuffer()` instance methods - class Request implements globalThis.Request { - cache!: globalThis.Request['cache']; - credentials!: globalThis.Request['credentials']; - destination!: globalThis.Request['destination']; - headers!: Headers; - integrity!: string; - method!: string; - mode!: globalThis.Request['mode']; - redirect!: globalThis.Request['redirect']; - referrer!: string; - referrerPolicy!: globalThis.Request['referrerPolicy']; - url!: string; - keepalive!: boolean; - signal!: AbortSignal; - duplex!: 'half'; - body!: ReadableStream | null; - - constructor(input: any, init?: RequestInit) { - // Handle URL input - if (typeof input === 'string' || input instanceof vmGlobalThis.URL) { - const urlString = String(input); - // Validate URL format - try { - new vmGlobalThis.URL(urlString); - this.url = urlString; - } catch (cause) { - throw new TypeError(`Failed to parse URL from ${urlString}`, { - cause, - }); - } - } else { - // Input is a Request object - clone its properties - this.url = input.url; - if (!init) { - this.method = input.method; - this.headers = new vmGlobalThis.Headers(input.headers); - this.body = input.body; - this.mode = input.mode; - this.credentials = input.credentials; - this.cache = input.cache; - this.redirect = input.redirect; - this.referrer = input.referrer; - this.referrerPolicy = input.referrerPolicy; - this.integrity = input.integrity; - this.keepalive = input.keepalive; - this.signal = input.signal; - this.duplex = input.duplex; - this.destination = input.destination; - return; - } - // If init is provided, merge: use source properties, then override with init - // Copy all properties from the source Request first + (vmGlobalThis as any).setTimeout = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).setInterval = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearTimeout = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearInterval = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).setImmediate = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + (vmGlobalThis as any).clearImmediate = () => { + throw new WorkflowRuntimeError(timeoutErrorMessage, { + slug: ERROR_SLUGS.TIMEOUT_FUNCTIONS_IN_WORKFLOW, + }); + }; + + // `AbortController` and `AbortSignal` in the workflow VM are hook-backed + // for deterministic replay. The controller's abort() queues a hook resumption, + // and signal.aborted is updated when the hook event is processed during replay. + (vmGlobalThis as any).AbortController = + createCreateAbortController(workflowContext); + const abortSignalStatics = createAbortSignalStatics(); + (vmGlobalThis as any).AbortSignal = { + abort: abortSignalStatics.abort, + any: abortSignalStatics.any, + timeout: abortSignalStatics.timeout, + }; + + // `Request` and `Response` are special built-in classes that invoke steps + // for the `json()`, `text()` and `arrayBuffer()` instance methods + class Request implements globalThis.Request { + cache!: globalThis.Request['cache']; + credentials!: globalThis.Request['credentials']; + destination!: globalThis.Request['destination']; + headers!: Headers; + integrity!: string; + method!: string; + mode!: globalThis.Request['mode']; + redirect!: globalThis.Request['redirect']; + referrer!: string; + referrerPolicy!: globalThis.Request['referrerPolicy']; + url!: string; + keepalive!: boolean; + signal!: AbortSignal; + duplex!: 'half'; + body!: ReadableStream | null; + + constructor(input: any, init?: RequestInit) { + // Handle URL input + if (typeof input === 'string' || input instanceof vmGlobalThis.URL) { + const urlString = String(input); + // Validate URL format + try { + new vmGlobalThis.URL(urlString); + this.url = urlString; + } catch (cause) { + throw new TypeError(`Failed to parse URL from ${urlString}`, { + cause, + }); + } + } else { + // Input is a Request object - clone its properties + this.url = input.url; + if (!init) { this.method = input.method; this.headers = new vmGlobalThis.Headers(input.headers); this.body = input.body; @@ -653,540 +620,551 @@ function createWorkflowSession({ this.signal = input.signal; this.duplex = input.duplex; this.destination = input.destination; + return; } + // If init is provided, merge: use source properties, then override with init + // Copy all properties from the source Request first + this.method = input.method; + this.headers = new vmGlobalThis.Headers(input.headers); + this.body = input.body; + this.mode = input.mode; + this.credentials = input.credentials; + this.cache = input.cache; + this.redirect = input.redirect; + this.referrer = input.referrer; + this.referrerPolicy = input.referrerPolicy; + this.integrity = input.integrity; + this.keepalive = input.keepalive; + this.signal = input.signal; + this.duplex = input.duplex; + this.destination = input.destination; + } - // Override with init options if provided - // Set method - if (init?.method) { - this.method = init.method.toUpperCase(); - } else if (typeof this.method !== 'string') { - // Fallback to default for string input case - this.method = 'GET'; - } - - // Set headers - if (init?.headers) { - this.headers = new vmGlobalThis.Headers(init.headers); - } else if ( - typeof input === 'string' || - input instanceof vmGlobalThis.URL - ) { - // For string/URL input, create empty headers - this.headers = new vmGlobalThis.Headers(); - } + // Override with init options if provided + // Set method + if (init?.method) { + this.method = init.method.toUpperCase(); + } else if (typeof this.method !== 'string') { + // Fallback to default for string input case + this.method = 'GET'; + } - // Set other properties with init values or defaults - if (init?.mode !== undefined) { - this.mode = init.mode; - } else if (typeof this.mode !== 'string') { - this.mode = 'cors'; - } + // Set headers + if (init?.headers) { + this.headers = new vmGlobalThis.Headers(init.headers); + } else if ( + typeof input === 'string' || + input instanceof vmGlobalThis.URL + ) { + // For string/URL input, create empty headers + this.headers = new vmGlobalThis.Headers(); + } - if (init?.credentials !== undefined) { - this.credentials = init.credentials; - } else if (typeof this.credentials !== 'string') { - this.credentials = 'same-origin'; - } + // Set other properties with init values or defaults + if (init?.mode !== undefined) { + this.mode = init.mode; + } else if (typeof this.mode !== 'string') { + this.mode = 'cors'; + } - // `any` cast here because @types/node v22 does not yet have `cache` - if ((init as any)?.cache !== undefined) { - this.cache = (init as any).cache; - } else if (typeof this.cache !== 'string') { - this.cache = 'default'; - } + if (init?.credentials !== undefined) { + this.credentials = init.credentials; + } else if (typeof this.credentials !== 'string') { + this.credentials = 'same-origin'; + } - if (init?.redirect !== undefined) { - this.redirect = init.redirect; - } else if (typeof this.redirect !== 'string') { - this.redirect = 'follow'; - } + // `any` cast here because @types/node v22 does not yet have `cache` + if ((init as any)?.cache !== undefined) { + this.cache = (init as any).cache; + } else if (typeof this.cache !== 'string') { + this.cache = 'default'; + } - if (init?.referrer !== undefined) { - this.referrer = init.referrer; - } else if (typeof this.referrer !== 'string') { - this.referrer = 'about:client'; - } + if (init?.redirect !== undefined) { + this.redirect = init.redirect; + } else if (typeof this.redirect !== 'string') { + this.redirect = 'follow'; + } - if (init?.referrerPolicy !== undefined) { - this.referrerPolicy = init.referrerPolicy; - } else if (typeof this.referrerPolicy !== 'string') { - this.referrerPolicy = ''; - } + if (init?.referrer !== undefined) { + this.referrer = init.referrer; + } else if (typeof this.referrer !== 'string') { + this.referrer = 'about:client'; + } - if (init?.integrity !== undefined) { - this.integrity = init.integrity; - } else if (typeof this.integrity !== 'string') { - this.integrity = ''; - } + if (init?.referrerPolicy !== undefined) { + this.referrerPolicy = init.referrerPolicy; + } else if (typeof this.referrerPolicy !== 'string') { + this.referrerPolicy = ''; + } - if (init?.keepalive !== undefined) { - this.keepalive = init.keepalive; - } else if (typeof this.keepalive !== 'boolean') { - this.keepalive = false; - } + if (init?.integrity !== undefined) { + this.integrity = init.integrity; + } else if (typeof this.integrity !== 'string') { + this.integrity = ''; + } - if (init?.signal !== undefined) { - // @ts-expect-error - AbortSignal stub - this.signal = init.signal; - } else if (!this.signal) { - // @ts-expect-error - AbortSignal stub - this.signal = { aborted: false }; - } + if (init?.keepalive !== undefined) { + this.keepalive = init.keepalive; + } else if (typeof this.keepalive !== 'boolean') { + this.keepalive = false; + } - if (!this.duplex) { - this.duplex = 'half'; - } + if (init?.signal !== undefined) { + // @ts-expect-error - AbortSignal stub + this.signal = init.signal; + } else if (!this.signal) { + // @ts-expect-error - AbortSignal stub + this.signal = { aborted: false }; + } - if (!this.destination) { - this.destination = 'document'; - } + if (!this.duplex) { + this.duplex = 'half'; + } - const body = init?.body; + if (!this.destination) { + this.destination = 'document'; + } - // Validate that GET/HEAD methods don't have a body - if ( - body !== null && - body !== undefined && - (this.method === 'GET' || this.method === 'HEAD') - ) { - throw new TypeError(`Request with GET/HEAD method cannot have body.`); - } + const body = init?.body; - // Store the original BodyInit for serialization - if (body !== null && body !== undefined) { - // Create a "fake" ReadableStream that stores the original body - // This avoids doing async work during workflow replay - this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { - [BODY_INIT_SYMBOL]: { - value: body, - writable: false, - }, - }); - } else { - this.body = null; - } + // Validate that GET/HEAD methods don't have a body + if ( + body !== null && + body !== undefined && + (this.method === 'GET' || this.method === 'HEAD') + ) { + throw new TypeError(`Request with GET/HEAD method cannot have body.`); } - clone(): Request { - ENOTSUP(); + // Store the original BodyInit for serialization + if (body !== null && body !== undefined) { + // Create a "fake" ReadableStream that stores the original body + // This avoids doing async work during workflow replay + this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { + [BODY_INIT_SYMBOL]: { + value: body, + writable: false, + }, + }); + } else { + this.body = null; } + } - get bodyUsed() { - return false; - } + clone(): Request { + ENOTSUP(); + } - // TODO: implement these - blob!: () => Promise; - formData!: () => Promise; + get bodyUsed() { + return false; + } - arrayBuffer!: () => Promise; - json!: () => Promise; - text!: () => Promise; + // TODO: implement these + blob!: () => Promise; + formData!: () => Promise; - async bytes() { - return new Uint8Array(await this.arrayBuffer()); - } + arrayBuffer!: () => Promise; + json!: () => Promise; + text!: () => Promise; + + async bytes() { + return new Uint8Array(await this.arrayBuffer()); } - vmGlobalThis.Request = Request; - - Object.defineProperties(Request.prototype, { - arrayBuffer: { - value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), - writable: true, - configurable: true, - }, - json: { - value: useStep<[], any>('__builtin_response_json'), - writable: true, - configurable: true, - }, - text: { - value: useStep<[], string>('__builtin_response_text'), - writable: true, - configurable: true, - }, - }); + } + vmGlobalThis.Request = Request; + + Object.defineProperties(Request.prototype, { + arrayBuffer: { + value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), + writable: true, + configurable: true, + }, + json: { + value: useStep<[], any>('__builtin_response_json'), + writable: true, + configurable: true, + }, + text: { + value: useStep<[], string>('__builtin_response_text'), + writable: true, + configurable: true, + }, + }); - class Response implements globalThis.Response { - type!: globalThis.Response['type']; - url!: string; - status!: number; - statusText!: string; - body!: ReadableStream | null; - headers!: Headers; - redirected!: boolean; - - constructor(body?: any, init?: ResponseInit) { - this.status = init?.status ?? 200; - this.statusText = init?.statusText ?? ''; - this.headers = new vmGlobalThis.Headers(init?.headers); - this.type = 'default'; - this.url = ''; - this.redirected = false; - - // Validate that null-body status codes don't have a body - // Per HTTP spec: 204 (No Content), 205 (Reset Content), and 304 (Not Modified) - if ( - body !== null && - body !== undefined && - (this.status === 204 || this.status === 205 || this.status === 304) - ) { - throw new TypeError( - `Response constructor: Invalid response status code ${this.status}` - ); - } + class Response implements globalThis.Response { + type!: globalThis.Response['type']; + url!: string; + status!: number; + statusText!: string; + body!: ReadableStream | null; + headers!: Headers; + redirected!: boolean; + + constructor(body?: any, init?: ResponseInit) { + this.status = init?.status ?? 200; + this.statusText = init?.statusText ?? ''; + this.headers = new vmGlobalThis.Headers(init?.headers); + this.type = 'default'; + this.url = ''; + this.redirected = false; + + // Validate that null-body status codes don't have a body + // Per HTTP spec: 204 (No Content), 205 (Reset Content), and 304 (Not Modified) + if ( + body !== null && + body !== undefined && + (this.status === 204 || this.status === 205 || this.status === 304) + ) { + throw new TypeError( + `Response constructor: Invalid response status code ${this.status}` + ); + } - // Store the original BodyInit for serialization - if (body !== null && body !== undefined) { - // Create a "fake" ReadableStream that stores the original body - // This avoids doing async work during workflow replay - this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { - [BODY_INIT_SYMBOL]: { - value: body, - writable: false, - }, - }); - } else { - this.body = null; - } + // Store the original BodyInit for serialization + if (body !== null && body !== undefined) { + // Create a "fake" ReadableStream that stores the original body + // This avoids doing async work during workflow replay + this.body = Object.create(vmGlobalThis.ReadableStream.prototype, { + [BODY_INIT_SYMBOL]: { + value: body, + writable: false, + }, + }); + } else { + this.body = null; } + } - // TODO: implement these - clone!: () => Response; - blob!: () => Promise; - formData!: () => Promise; + // TODO: implement these + clone!: () => Response; + blob!: () => Promise; + formData!: () => Promise; - get ok() { - return this.status >= 200 && this.status < 300; - } + get ok() { + return this.status >= 200 && this.status < 300; + } - get bodyUsed() { - return false; - } + get bodyUsed() { + return false; + } - arrayBuffer!: () => Promise; - json!: () => Promise; - text!: () => Promise; + arrayBuffer!: () => Promise; + json!: () => Promise; + text!: () => Promise; - async bytes() { - return new Uint8Array(await this.arrayBuffer()); - } + async bytes() { + return new Uint8Array(await this.arrayBuffer()); + } - static json(data: any, init?: ResponseInit): Response { - const body = JSON.stringify(data); - const headers = new vmGlobalThis.Headers(init?.headers); - if (!headers.has('content-type')) { - headers.set('content-type', 'application/json'); - } - return new Response(body, { ...init, headers }); + static json(data: any, init?: ResponseInit): Response { + const body = JSON.stringify(data); + const headers = new vmGlobalThis.Headers(init?.headers); + if (!headers.has('content-type')) { + headers.set('content-type', 'application/json'); } + return new Response(body, { ...init, headers }); + } - static error(): Response { - ENOTSUP(); - } + static error(): Response { + ENOTSUP(); + } - static redirect(url: string | URL, status: number = 302): Response { - // Validate status code - only specific redirect codes are allowed - if (![301, 302, 303, 307, 308].includes(status)) { - throw new RangeError( - `Invalid redirect status code: ${status}. Must be one of: 301, 302, 303, 307, 308` - ); - } + static redirect(url: string | URL, status: number = 302): Response { + // Validate status code - only specific redirect codes are allowed + if (![301, 302, 303, 307, 308].includes(status)) { + throw new RangeError( + `Invalid redirect status code: ${status}. Must be one of: 301, 302, 303, 307, 308` + ); + } - // Create response with Location header - const headers = new vmGlobalThis.Headers(); - headers.set('Location', String(url)); + // Create response with Location header + const headers = new vmGlobalThis.Headers(); + headers.set('Location', String(url)); - const response = Object.create(Response.prototype); - response.status = status; - response.statusText = ''; - response.headers = headers; - response.body = null; - response.type = 'default'; - response.url = ''; - response.redirected = false; + const response = Object.create(Response.prototype); + response.status = status; + response.statusText = ''; + response.headers = headers; + response.body = null; + response.type = 'default'; + response.url = ''; + response.redirected = false; - return response; - } + return response; } - vmGlobalThis.Response = Response; - - Object.defineProperties(Response.prototype, { - arrayBuffer: { - value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), - writable: true, - configurable: true, - }, - json: { - value: useStep<[], any>('__builtin_response_json'), - writable: true, - configurable: true, - }, - text: { - value: useStep<[], string>('__builtin_response_text'), - writable: true, - configurable: true, - }, - }); + } + vmGlobalThis.Response = Response; + + Object.defineProperties(Response.prototype, { + arrayBuffer: { + value: useStep<[], ArrayBuffer>('__builtin_response_array_buffer'), + writable: true, + configurable: true, + }, + json: { + value: useStep<[], any>('__builtin_response_json'), + writable: true, + configurable: true, + }, + text: { + value: useStep<[], string>('__builtin_response_text'), + writable: true, + configurable: true, + }, + }); - class ReadableStream implements globalThis.ReadableStream { - constructor() { - ENOTSUP(); - } + class ReadableStream implements globalThis.ReadableStream { + constructor() { + ENOTSUP(); + } - get locked() { - return false; - } + get locked() { + return false; + } - cancel(): any { - ENOTSUP(); - } + cancel(): any { + ENOTSUP(); + } - getReader(): any { - ENOTSUP(); - } + getReader(): any { + ENOTSUP(); + } - pipeThrough(): any { - ENOTSUP(); - } + pipeThrough(): any { + ENOTSUP(); + } - pipeTo(): any { - ENOTSUP(); - } + pipeTo(): any { + ENOTSUP(); + } - tee(): any { - ENOTSUP(); - } + tee(): any { + ENOTSUP(); + } - values(): any { - ENOTSUP(); - } + values(): any { + ENOTSUP(); + } - static from(): any { - ENOTSUP(); - } + static from(): any { + ENOTSUP(); + } - [Symbol.asyncIterator](): any { - ENOTSUP(); - } + [Symbol.asyncIterator](): any { + ENOTSUP(); } - vmGlobalThis.ReadableStream = ReadableStream; + } + vmGlobalThis.ReadableStream = ReadableStream; - class WritableStream implements globalThis.WritableStream { - constructor() { - ENOTSUP(); - } + class WritableStream implements globalThis.WritableStream { + constructor() { + ENOTSUP(); + } - get locked() { - return false; - } + get locked() { + return false; + } - abort(): any { - ENOTSUP(); - } + abort(): any { + ENOTSUP(); + } - close(): any { - ENOTSUP(); - } + close(): any { + ENOTSUP(); + } - getWriter(): any { - ENOTSUP(); - } + getWriter(): any { + ENOTSUP(); } - vmGlobalThis.WritableStream = WritableStream; + } + vmGlobalThis.WritableStream = WritableStream; - class TransformStream implements globalThis.TransformStream { - readable: globalThis.ReadableStream; - writable: globalThis.WritableStream; + class TransformStream implements globalThis.TransformStream { + readable: globalThis.ReadableStream; + writable: globalThis.WritableStream; - constructor() { - ENOTSUP(); - } + constructor() { + ENOTSUP(); } - vmGlobalThis.TransformStream = TransformStream; - - // Eventually we'll probably want to provide our own `console` object, - // but for now we'll just expose the global one. - vmGlobalThis.console = globalThis.console; - - // HACK: propagate symbol needed for AI gateway usage - const SYMBOL_FOR_REQ_CONTEXT = Symbol.for('@vercel/request-context'); - // @ts-expect-error - `@types/node` says symbol is not valid, but it does work - vmGlobalThis[SYMBOL_FOR_REQ_CONTEXT] = (globalThis as any)[ - SYMBOL_FOR_REQ_CONTEXT - ]; - - // Get a reference to the user-defined workflow function. - // The filename parameter ensures stack traces show a meaningful name - // (e.g., "example/workflows/99_e2e.ts") instead of "evalmachine.". - const parsedName = parseWorkflowName(workflowRun.workflowName); - const filename = parsedName?.moduleSpecifier || workflowRun.workflowName; - - // Evaluate the workflow bundle against the fresh context using a - // process-wide cache of the compiled `vm.Script`. The bundle is the same - // string for every replay and every invocation in this process, and - // compilation is a pure function of `(code, filename)`, so reusing the - // compiled Script across replays is determinism-safe: it produces the same - // workflow function and the same `filename` source attribution as - // re-parsing the bundle every time, but skips the (expensive) re-parse. - // Evaluating the bundle registers every workflow on - // `globalThis.__private_workflows`; the trailing lookup expression then - // retrieves the requested workflow function. The lookup is evaluated as a - // separate cached Script under the same `filename`, so error stack frames - // still attribute to the workflow's source file (`remapErrorStack` keys on - // `filename`). The one behavioural difference from the previous - // single-combined-string approach is the *line number* of an error thrown - // by the lookup expression itself: it now reports line 1 of the lookup - // Script rather than the line just past the end of the bundle. That path - // is rare (it requires the lookup `?.get(...)` expression to throw) and - // does not affect the workflow function or replay determinism. - // All SDK globals are installed; pin the serialization-consulted - // intrinsics before any workflow code can run. - freezeSerializationIntrinsics(vmGlobalThis); - - runCachedWorkflowScript(workflowCode, filename, context); - const workflowFn = runCachedWorkflowScript( - `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, - filename, - context + } + vmGlobalThis.TransformStream = TransformStream; + + // Eventually we'll probably want to provide our own `console` object, + // but for now we'll just expose the global one. + vmGlobalThis.console = globalThis.console; + + // HACK: propagate symbol needed for AI gateway usage + const SYMBOL_FOR_REQ_CONTEXT = Symbol.for('@vercel/request-context'); + // @ts-expect-error - `@types/node` says symbol is not valid, but it does work + vmGlobalThis[SYMBOL_FOR_REQ_CONTEXT] = (globalThis as any)[ + SYMBOL_FOR_REQ_CONTEXT + ]; + + // Get a reference to the user-defined workflow function. + // The filename parameter ensures stack traces show a meaningful name + // (e.g., "example/workflows/99_e2e.ts") instead of "evalmachine.". + const parsedName = parseWorkflowName(workflowRun.workflowName); + const filename = parsedName?.moduleSpecifier || workflowRun.workflowName; + + // Evaluate the workflow bundle against the fresh context using a + // process-wide cache of the compiled `vm.Script`. The bundle is the same + // string for every replay and every invocation in this process, and + // compilation is a pure function of `(code, filename)`, so reusing the + // compiled Script across replays is determinism-safe: it produces the same + // workflow function and the same `filename` source attribution as + // re-parsing the bundle every time, but skips the (expensive) re-parse. + // Evaluating the bundle registers every workflow on + // `globalThis.__private_workflows`; the trailing lookup expression then + // retrieves the requested workflow function. The lookup is evaluated as a + // separate cached Script under the same `filename`, so error stack frames + // still attribute to the workflow's source file (`remapErrorStack` keys on + // `filename`). The one behavioural difference from the previous + // single-combined-string approach is the *line number* of an error thrown + // by the lookup expression itself: it now reports line 1 of the lookup + // Script rather than the line just past the end of the bundle. That path + // is rare (it requires the lookup `?.get(...)` expression to throw) and + // does not affect the workflow function or replay determinism. + // All SDK globals are installed; pin the serialization-consulted + // intrinsics before any workflow code can run. + freezeSerializationIntrinsics(vmGlobalThis); + + runCachedWorkflowScript(workflowCode, filename, context); + const workflowFn = runCachedWorkflowScript( + `globalThis.__private_workflows?.get(${JSON.stringify(workflowRun.workflowName)})`, + filename, + context + ); + + if (typeof workflowFn !== 'function') { + throw new WorkflowNotRegisteredError(workflowRun.workflowName); + } + + // Chain workflow argument hydration onto the promiseQueue so that the + // unconsumed event check (which waits for the queue to drain) doesn't + // fire during the async gap between run_started consumption and the + // workflow function subscribing its first step callbacks. + let args: unknown[] = []; + workflowContext.promiseQueue = workflowContext.promiseQueue.then(async () => { + const prepared = await replayPayloadCache.prepareWorkflowInput(workflowRun); + args = await hydrateWorkflowArguments( + workflowRun.input, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + {}, + prepared ); + }); + await workflowContext.promiseQueue; + + const workflowExecution = (async (): Promise => { + return await workflowFn(...args); + })(); - if (typeof workflowFn !== 'function') { - throw new WorkflowNotRegisteredError(workflowRun.workflowName); + const failWorkflow = async (error: unknown): Promise => { + // Control-flow signals are handled by the runtime and do not mean the + // workflow has terminally failed. `onWorkflowError` already moved the + // state machine (e.g. to `replay` on divergence, so a later resume falls + // back instead of throwing) — leave it alone. + if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + throw error; } + state = { type: 'completed' }; - // Chain workflow argument hydration onto the promiseQueue so that the - // unconsumed event check (which waits for the queue to drain) doesn't - // fire during the async gap between run_started consumption and the - // workflow function subscribing its first step callbacks. - let args: unknown[] = []; - workflowContext.promiseQueue = workflowContext.promiseQueue.then( - async () => { - const prepared = - await replayPayloadCache.prepareWorkflowInput(workflowRun); - args = await hydrateWorkflowArguments( - workflowRun.input, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - {}, - prepared - ); - } + await drainPendingQueueItems( + workflowRun.runId, + workflowContext.invocationsQueue, + vmGlobalThis, + workflowRun, + 'failed', + runReadyBarrier ); - await workflowContext.promiseQueue; - const workflowExecution = (async (): Promise => { - return await workflowFn(...args); - })(); + throw error; + }; - const failWorkflow = async (error: unknown): Promise => { - state = { type: 'completed' }; - // Control-flow signals are handled by the runtime and do not mean the - // workflow has terminally failed. - if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + const waitForExecution = async ( + interruption: PromiseWithResolvers + ): Promise => { + let result: unknown; + try { + result = await Promise.race([workflowExecution, interruption.promise]); + } catch (error) { + if (state.type === 'suspended' && error === state.suspension) { throw error; } + return failWorkflow(error); + } + + state = { type: 'completed' }; + try { + const output = await dehydrateWorkflowReturnValue( + result, + workflowRun.runId, + encryptionKey, + vmGlobalThis, + false, + // Gate payload compression on the run's specVersion: only runs + // marked as possibly containing compressed payloads (spec >= 5) + // get gzip data. + (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION + ); await drainPendingQueueItems( workflowRun.runId, workflowContext.invocationsQueue, vmGlobalThis, workflowRun, - 'failed', + 'completed', runReadyBarrier ); - throw error; - }; - - const completeWorkflow = async ( - result: unknown - ): Promise => { - state = { type: 'completed' }; - try { - const output = await dehydrateWorkflowReturnValue( - result, - workflowRun.runId, - encryptionKey, - vmGlobalThis, - false, - // Gate payload compression on the run's specVersion: only runs - // marked as possibly containing compressed payloads (spec >= 5) - // get gzip data. - (workflowRun.specVersion ?? 0) >= SPEC_VERSION_SUPPORTS_COMPRESSION - ); - - await drainPendingQueueItems( - workflowRun.runId, - workflowContext.invocationsQueue, - vmGlobalThis, - workflowRun, - 'completed', - runReadyBarrier - ); - - return { output, resultType: typeof result }; - } catch (error) { - return failWorkflow(error); - } - }; - - const waitForExecution = async ( - interruption: PromiseWithResolvers - ): Promise => { - let result: unknown; - try { - result = await Promise.race([workflowExecution, interruption.promise]); - } catch (error) { - if (state.type === 'suspended' && error === state.suspension) { - throw error; - } - return failWorkflow(error); - } - return completeWorkflow(result); - }; + return { output, resultType: typeof result }; + } catch (error) { + return failWorkflow(error); + } + }; - const session: WorkflowSession = { - workflowRun, - argumentCount: args.length, - resume(nextEvents) { - switch (state.type) { - case 'suspended': { - const consumedEvents = eventsConsumer.events; - const isStrictExtension = - nextEvents.length > consumedEvents.length && - consumedEvents.every( - (event, index) => event.eventId === nextEvents[index]?.eventId - ); - if (!isStrictExtension) { - state = { type: 'replay' }; - return { type: 'replay' }; - } - const interruption = withResolvers(); - state = { type: 'running', interruption }; - workflowContext.suspensionGeneration++; - eventsConsumer.append(nextEvents.slice(consumedEvents.length)); - return { - type: 'resumed', - execution: waitForExecution(interruption), - }; - } - case 'replay': - return { type: 'replay' }; - case 'completed': - case 'running': - throw new WorkflowRuntimeError( - `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + const session: WorkflowSession = { + workflowRun, + argumentCount: args.length, + resume(nextEvents) { + switch (state.type) { + case 'suspended': { + const consumedEvents = eventsConsumer.events; + const isStrictExtension = + nextEvents.length > consumedEvents.length && + consumedEvents.every( + (event, index) => event.eventId === nextEvents[index]?.eventId ); + if (!isStrictExtension) { + state = { type: 'replay' }; + return { type: 'replay' }; + } + const interruption = withResolvers(); + state = { type: 'running', interruption }; + workflowContext.suspensionGeneration++; + eventsConsumer.append(nextEvents.slice(consumedEvents.length)); + return { + type: 'resumed', + execution: waitForExecution(interruption), + }; } - state satisfies never; - }, - }; + case 'replay': + return { type: 'replay' }; + case 'completed': + case 'running': + throw new WorkflowRuntimeError( + `Cannot resume ${state.type} workflow "${workflowRun.runId}"` + ); + } + state satisfies never; + }, + }; - return { - session, - execution: waitForExecution(initialInterruption), - }; - })(); + return { + session, + execution: waitForExecution(initialInterruption), + }; } From c61f9888190e9e003e1ab4e7f173a6f44c4577b8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:22:00 -0700 Subject: [PATCH 29/30] chore: retrigger vercel deployments From 3cd4d0747605f4c92731990d2490710a58782fb8 Mon Sep 17 00:00:00 2001 From: Nathan Colosimo <110621881+NathanColosimo@users.noreply.github.com> Date: Fri, 24 Jul 2026 14:23:16 -0700 Subject: [PATCH 30/30] simplify retention: single decision site in suspension catch, steps-only allow-list gate, drop prepareForRetention param --- .changeset/retain-workflow-vm.md | 4 +- packages/core/src/runtime.ts | 88 +++++++++---------- .../core/src/runtime/suspension-handler.ts | 27 ++---- packages/core/src/workflow.ts | 9 +- 4 files changed, 59 insertions(+), 69 deletions(-) diff --git a/.changeset/retain-workflow-vm.md b/.changeset/retain-workflow-vm.md index 25d38b0d57..718282d550 100644 --- a/.changeset/retain-workflow-vm.md +++ b/.changeset/retain-workflow-vm.md @@ -1,6 +1,6 @@ --- -'@workflow/core': patch -'workflow': patch +'@workflow/core': minor +'workflow': minor --- Retain workflow execution across inline steps within one invocation; `WORKFLOW_RETAINED_VM=0` disables retention. Boundaries whose step arguments are not primitive values fall back to ordinary replay. diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index 3c6ae0d375..b30a9568d0 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -361,9 +361,11 @@ function openHookAndWaitState(events: Event[]): { } /** - * Retain only pure step boundaries with no out-of-band continuation source. - * Attributes require replay; hooks and waits can wake another invocation. - * `WORKFLOW_RETAINED_VM=0` disables retention entirely. + * Retain only pure step boundaries (every suspension item is a step — any + * other item type, present or future, is unretainable by default) with no + * out-of-band continuation source: attributes require replay; hooks and + * waits can wake another invocation. `WORKFLOW_RETAINED_VM=0` disables + * retention entirely. * * Quiescence assumes workflow code stays inside the sandbox's determinism * contract. Escaping to the host realm (e.g. recovering the host `Function` @@ -373,21 +375,15 @@ function openHookAndWaitState(events: Event[]): { */ function canRetainWorkflowSession( suspension: WorkflowSuspension, - events: Event[] + openHookWaitState: { openHook: boolean; openWait: boolean } ): boolean { - if ( - !isVmRetentionEnabled() || - suspension.stepCount === 0 || - suspension.hookCount > 0 || - suspension.waitCount > 0 || - suspension.attributeCount > 0 || - suspension.hookDisposedCount > 0 || - suspension.abortCount > 0 - ) { - return false; - } - const { openHook, openWait } = openHookAndWaitState(events); - return !openHook && !openWait; + return ( + isVmRetentionEnabled() && + suspension.steps.length > 0 && + suspension.steps.every((item) => item.type === 'step') && + !openHookWaitState.openHook && + !openHookWaitState.openWait + ); } /** @@ -1603,12 +1599,11 @@ export function workflowEntrypoint( // point and the inline executeStep mutates eventsCursor. preInlineWriteCursor = eventsCursor; - let executionMode = workflowExecution.type; runtimeLogger.debug('Starting workflow execution', { workflowRunId: runId, loopIteration, eventCount: events.length, - executionMode, + executionMode: workflowExecution.type, }); replayStart = Date.now(); let workflowResult: WorkflowExecutionResult = @@ -1621,7 +1616,6 @@ export function workflowEntrypoint( : { type: 'replay' }; if (workflowResult.type === 'replay') { - executionMode = 'replay'; workflowExecution = { type: 'replay' }; // Start every missing decrypt/decompress operation // before VM setup. Web Crypto work can overlap bundle @@ -1648,15 +1642,13 @@ export function workflowEntrypoint( } if (workflowResult.type === 'suspended') { - workflowExecution = canRetainWorkflowSession( - workflowResult.suspension, - events - ) - ? { - type: 'retained', - session: workflowResult.session, - } - : { type: 'replay' }; + // Park the live session; the suspension catch below + // makes the one retention decision — keep it for the + // next iteration or discard it for a fresh replay. + workflowExecution = { + type: 'retained', + session: workflowResult.session, + }; throw workflowResult.suspension; } @@ -1665,7 +1657,7 @@ export function workflowEntrypoint( workflowRunId: runId, loopIteration, replayMs: Date.now() - replayStart, - executionMode, + executionMode: workflowExecution.type, }); // Workflow completed. Send the snapshot but do NOT @@ -1790,15 +1782,7 @@ export function workflowEntrypoint( requestId, eventLog: suspensionLog, runReadyBarrier, - prepareForRetention: - workflowExecution.type === 'retained', }); - if ( - workflowExecution.type === 'retained' && - !suspensionResult.retainedStepInputsSafe - ) { - workflowExecution = { type: 'replay' }; - } } catch (suspensionError) { // A suspension create whose stale (412) rejection // survived the in-guard reload retries: schedule an @@ -1889,6 +1873,28 @@ export function workflowEntrypoint( return; } eventsCursor = suspensionLog.cursor; + + // Open hooks/waits in the cumulative log (this + // suspension's writes are already merged in), computed + // once for the retention decision here and the + // delta/turbo gates below. + const openHookWaitState = + openHookAndWaitState(cachedEvents); + + // The single retention decision: keep the parked + // session only across a pure step boundary with no + // out-of-band continuation source and provably + // passive step inputs. + if ( + workflowExecution.type === 'retained' && + !( + canRetainWorkflowSession(err, openHookWaitState) && + suspensionResult.retainedStepInputsSafe + ) + ) { + workflowExecution = { type: 'replay' }; + } + preStepBlockingMs += suspensionResult.hookCreationMs; if ( suspensionResult.hasAttributeEvents && @@ -2209,12 +2215,6 @@ export function workflowEntrypoint( // this the replay-budget check at the top of the // next loop iteration would (incorrectly) charge // the step body against the budget. - // Open hooks/waits in the cumulative log, computed - // once for the two gates below. - const openHookWaitState = openHookAndWaitState( - cachedEvents ?? [] - ); - // Inline-delta fast path gate. We request the delta — // and on the next iteration consume it in place of the // events.list — only when ALL hold: diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index c23ab2e878..72f5411f3c 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -69,11 +69,6 @@ export interface SuspensionHandlerParams { * where `run_started` was already awaited up front. */ runReadyBarrier?: Promise; - /** - * Prepare new step inputs without traversing workflow-owned objects during - * serialization. Enabled only while the caller is holding a retained VM. - */ - prepareForRetention?: boolean; } /** @@ -225,7 +220,6 @@ export async function handleSuspension({ requestId, eventLog, runReadyBarrier, - prepareForRetention = false, }: SuspensionHandlerParams): Promise { const runId = run.runId; @@ -519,20 +513,13 @@ export async function handleSuspension({ // standard built-ins. If any input in the batch is not provably passive, // the caller demotes the session so the side effects land in a VM that is // about to be discarded, exactly like the pre-retention runtime. - let retainedStepInputsSafe = true; - if (prepareForRetention) { - for (const queueItem of stepItems) { - if (!stepsNeedingCreation.has(queueItem.correlationId)) continue; - if ( - queueItem.thisVal !== undefined || - queueItem.closureVars !== undefined || - !queueItem.args.every(isPrimitiveStepArgument) - ) { - retainedStepInputsSafe = false; - break; - } - } - } + const retainedStepInputsSafe = stepItems.every( + (item) => + !stepsNeedingCreation.has(item.correlationId) || + (item.thisVal === undefined && + item.closureVars === undefined && + item.args.every(isPrimitiveStepArgument)) + ); // Lazy inline start: defer the step_created write for up to // `getMaxInlineSteps()` steps the caller will run inline (in parallel). Each diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 4088a0888f..f56580e0d0 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -1060,10 +1060,13 @@ async function createWorkflowSession({ const failWorkflow = async (error: unknown): Promise => { // Control-flow signals are handled by the runtime and do not mean the - // workflow has terminally failed. `onWorkflowError` already moved the - // state machine (e.g. to `replay` on divergence, so a later resume falls - // back instead of throwing) — leave it alone. + // workflow has terminally failed. `onWorkflowError` usually already moved + // the state machine, but a divergence can also arrive via a step + // promise's direct rejection (bypassing `onWorkflowError`) — demote so + // every control-flow path converges on `replay` and a later resume falls + // back instead of throwing. if (WorkflowSuspension.is(error) || ReplayDivergenceError.is(error)) { + if (state.type === 'running') state = { type: 'replay' }; throw error; } state = { type: 'completed' };