From 296207f3b457bcd727a66ee62bc8814f3c9492aa Mon Sep 17 00:00:00 2001 From: gcharang <21151592+gcharang@users.noreply.github.com> Date: Sat, 15 Aug 2026 01:32:29 +0400 Subject: [PATCH] fix: align run lifecycle service and release notes --- .../flowsafe-lifecycle-release-quality.md | 7 ++ docs/approval-system.md | 4 +- docs/flowsafe-architecture.md | 6 +- docs/observability-and-quality.md | 2 +- packages/fleet-control/CHANGELOG.md | 6 ++ packages/flowsafe/CHANGELOG.md | 8 ++ .../src/do-runner/durable-object.test.ts | 20 ++++ .../flowsafe/src/do-runner/durable-object.ts | 10 +- .../flowsafe/src/host-kit/flowsafe-worker.ts | 95 ++++++++++--------- .../flowsafe/src/host-kit/stream-router.ts | 2 +- 10 files changed, 106 insertions(+), 54 deletions(-) create mode 100644 .changeset/flowsafe-lifecycle-release-quality.md diff --git a/.changeset/flowsafe-lifecycle-release-quality.md b/.changeset/flowsafe-lifecycle-release-quality.md new file mode 100644 index 0000000..f9e93e8 --- /dev/null +++ b/.changeset/flowsafe-lifecycle-release-quality.md @@ -0,0 +1,7 @@ +--- +"@proofoftech/flowsafe": patch +--- + +Use one approval-service assembly for public approval requests and terminal cleanup so SLA, resume, audit, notification, streaming, and separation-of-duties policy cannot drift between those paths. + +Make the Durable Object principal diagnostic lifecycle-neutral and document termination, deadline expiry, approval abandonment, and cleanup ordering across the architecture and observability guides. diff --git a/docs/approval-system.md b/docs/approval-system.md index a2e8c93..1075389 100644 --- a/docs/approval-system.md +++ b/docs/approval-system.md @@ -95,7 +95,9 @@ Roles are enforced server-side: - `reviewer` and `admin` can perform review operations according to the service rules. - Other roles may list or inspect only where the host's configured policy permits it. -- Per-workflow start and resume authorization belongs to the run router, not the approval record. +- Per-workflow start, resume, and terminate authorization belongs to the run router, not the approval record. + +Termination and deadline expiration never decide or resume an approval. They atomically fence new decisions against the terminal intent, then abandon every open record through system-attributed stale supersession. SLA escalation remains independent and never cancels a run. Authentication and actor-to-role mapping remain host responsibilities. diff --git a/docs/flowsafe-architecture.md b/docs/flowsafe-architecture.md index c425b06..a26860c 100644 --- a/docs/flowsafe-architecture.md +++ b/docs/flowsafe-architecture.md @@ -58,7 +58,7 @@ See [API reference map](api-reference.md) for exact import paths. ## Runner and storage -`init(env, options)` creates a `RunnerRuntime` and import-swapped `createWorkflow`/`createStep` factories. Definitions retain Mastra's builder shape; start and resume go through the runtime. +`init(env, options)` creates a `RunnerRuntime` and import-swapped `createWorkflow`/`createStep` factories. Definitions retain Mastra's builder shape. Start, resume, cancellation, and deadline expiration go through the runtime. The runtime: @@ -66,11 +66,13 @@ The runtime: - requires a caller-minted run id; - derives workflow scope and connector execution identity; - obtains per-leg request context before `createRun()`; -- serializes start and resume per run; +- serializes execution and lifecycle mutations per run; - persists snapshots through the D1 Mastra store; - publishes authoritative `RunSummary` values at lifecycle boundaries; - persists requester identity, attempt tokens, and monotonic resume ordinals in the authoritative workflow snapshot. +Cancellation and deadline expiration persist an exact terminal intent before interrupting active work. Terminal cleanup then abandons open approvals, discards a live agent-schedule receipt, releases run ownership, and marks cleanup complete. Retries resume that ordering from the persisted lifecycle state. + `DurableObjectRunner` binds one object to `workflowId:runId`. Every request must name the same values as the object's `id.name`. The runtime does not mint `breakwater.isolationScope`. One deployment serves one organization, so connector idempotency and rate limits are deployment-wide. The key remains reserved, and provider-supplied values are dropped. diff --git a/docs/observability-and-quality.md b/docs/observability-and-quality.md index e548504..b9e8d54 100644 --- a/docs/observability-and-quality.md +++ b/docs/observability-and-quality.md @@ -203,7 +203,7 @@ Do not log caught errors again through a generic serializer without reapplying t ### Durable execution -- start/resume/status latency and errors; +- start, resume, status, terminate, and deadline-sweep latency and errors; - runs by current status and age; - suspended runs by workflow and gate age; - restart recovery; diff --git a/packages/fleet-control/CHANGELOG.md b/packages/fleet-control/CHANGELOG.md index 655bdfc..96c2fd9 100644 --- a/packages/fleet-control/CHANGELOG.md +++ b/packages/fleet-control/CHANGELOG.md @@ -6,6 +6,12 @@ - 1f6a13a: Add a fenced, replay-safe `forceDecommissionDeployment()` escape hatch for ordinary deployments whose retained specification inputs are unavailable. + The operation always runs under `withDeploymentLease()` and persists the normal teardown phases. It removes control secrets, disables and verifies public ingress, detaches the custom domain, and deletes the exact persisted D1 database ID after asserting its fleet-owned name. Provider `404` responses converge as already absent, so retries resume after any completed mutation. Success deletes the fleet ledger row and emits the normal audit surface with `forced: true`. + + Force decommission never fetches an artifact, rebuilds a specification digest, or enters the `FLEET_SPEC_DIGEST` or version-ownership attestation path. It does not delete the ordinary Worker script, application R2 buckets, or control-plane retention data. After success, the host remains responsible for deleting its separate retention row and revoking its gateway key. + + The built-in `WranglerLoopBackend` implements the required spec-free route-API operations. Custom ordinary-Worker backends must implement `forceDecommissionStep()` with equivalent fenced checks. Workers for Platforms deployments fail closed because their dispatch and trusted-resource topology cannot use the ordinary-Worker route API. A `database-create-authorized` record also fails closed because its durable row cannot prove the exact database ID after a lost create response. + ### Patch Changes - Updated dependencies [1f6a13a] diff --git a/packages/flowsafe/CHANGELOG.md b/packages/flowsafe/CHANGELOG.md index 488ad52..102a063 100644 --- a/packages/flowsafe/CHANGELOG.md +++ b/packages/flowsafe/CHANGELOG.md @@ -6,6 +6,14 @@ - 1f6a13a: Add idempotent run termination and deadline maintenance to the Durable Object host kit. + `createRunRouter()` now exposes authenticated workflow termination, and the agent host exposes the equivalent agent-run route. Cancellation is legal from every nonterminal wait, retry, suspension, and running state. It persists a structured `CANCELLED` envelope before cleanup, abandons open approvals without minting or resuming, discards an executing agent-schedule receipt, and releases ownership only after the terminal snapshot commits. Retries and Durable Object eviction re-drive the same persisted intent. A disputed economic settlement returns a structured `409` before active work is interrupted. + + Starts and resumes accept `deadlineMs`. `RunSummary` reports the resulting deadline, and the maintenance Durable Object runs a bounded, cursor-resumable deadline duty. Each overdue run uses an owner-Durable-Object compare-and-swap transition to `timed_out` with a `TIMED_OUT` envelope. Deadline expiry abandons open approvals, while the existing SLA sweep remains escalation-only. Maintenance health now reports deadline scheduling and completion timestamps. + + Hosts composed with `createFlowsafeWorker()` receive the workflow route and deadline duty. Hosts that expose lifecycle termination through a custom `DurableObjectRunner` must override `runLifecycle()` with approval-abandonment hooks and implement `release()` on their `runOwnership()` store. Agent hosts with begun schedule-dispatch leases must also provide the exact-run discard hook. Ordinary starts and resumes remain lifecycle-metadata-free unless they use a deadline or another trusted lifecycle projection. + + The release adds no required D1 table and keeps the existing deployment-identity header, trusted execution-principal header, separation-of-duties policy, snapshot authority, and 39-character storage-prefix limit. + ## 0.15.0 ### Minor Changes diff --git a/packages/flowsafe/src/do-runner/durable-object.test.ts b/packages/flowsafe/src/do-runner/durable-object.test.ts index 08b140a..e91b2ea 100644 --- a/packages/flowsafe/src/do-runner/durable-object.test.ts +++ b/packages/flowsafe/src/do-runner/durable-object.test.ts @@ -274,11 +274,31 @@ describe('DurableObjectRunner.fetch', () => { ); expect(response.status).toBe(403); + await expect(response.json()).resolves.toEqual({ + error: 'run request carries no valid trusted execution principal', + }); expect(reserve).not.toHaveBeenCalled(); expect(runtime.status).not.toHaveBeenCalled(); expect(runtime.start).not.toHaveBeenCalled(); }); + it('uses a lifecycle-neutral missing-principal diagnostic on terminal routes', async () => { + const runner = new TestRunner(undefined, makeProductionEnv()); + for (const path of [ + '/runs/gated/run-missing-principal/terminate', + '/runs/gated/run-missing-principal/deadline', + ]) { + const response = await runner.fetch( + deploymentIdentityRequest(`http://do${path}`, { method: 'POST' }), + ); + + expect(response.status).toBe(403); + await expect(response.json()).resolves.toEqual({ + error: 'run request carries no valid trusted execution principal', + }); + } + }); + it('rejects a new resume leg without requester kind before runtime work', async () => { const runtime = { resume: vi.fn(), diff --git a/packages/flowsafe/src/do-runner/durable-object.ts b/packages/flowsafe/src/do-runner/durable-object.ts index c81f827..549269e 100644 --- a/packages/flowsafe/src/do-runner/durable-object.ts +++ b/packages/flowsafe/src/do-runner/durable-object.ts @@ -201,12 +201,12 @@ export abstract class DurableObjectRunner { return value; } - #startPrincipal(request: Request): ExecutionPrincipal { + #trustedExecutionPrincipal(request: Request): ExecutionPrincipal { const encoded = request.headers.get(EXECUTION_PRINCIPAL_HEADER); const principal = encoded ? decodeExecutionPrincipal(encoded) : undefined; if (!principal) { throw new DurableObjectRunIdentityError( - 'run start carries no valid trusted execution principal', + 'run request carries no valid trusted execution principal', ); } return principal; @@ -455,7 +455,7 @@ export abstract class DurableObjectRunner { if (request.method === 'POST' && segments.length === 1) { return this.#withOperationLock(async () => { - const principal = this.#startPrincipal(request); + const principal = this.#trustedExecutionPrincipal(request); const body = await readJson(request); if (!body || typeof body.workflowId !== 'string') { return json({ error: 'workflowId is required' }, 400); @@ -671,7 +671,7 @@ export abstract class DurableObjectRunner { runId ) { this.#assertRunIdentity(workflowId, runId); - const principal = this.#startPrincipal(request); + const principal = this.#trustedExecutionPrincipal(request); const runtime = this.#ensureRuntime(); const preflightOwner = await this.runOwnership(this.env).owner( 'run', @@ -719,7 +719,7 @@ export abstract class DurableObjectRunner { runId ) { this.#assertRunIdentity(workflowId, runId); - const principal = this.#startPrincipal(request); + const principal = this.#trustedExecutionPrincipal(request); const body = (await readJson(request)) ?? {}; const cas: RunLifecycleCas = { expectedRevision: body.expectedRevision as number, diff --git a/packages/flowsafe/src/host-kit/flowsafe-worker.ts b/packages/flowsafe/src/host-kit/flowsafe-worker.ts index 99349ab..4131ee9 100644 --- a/packages/flowsafe/src/host-kit/flowsafe-worker.ts +++ b/packages/flowsafe/src/host-kit/flowsafe-worker.ts @@ -16,6 +16,8 @@ import type { ApprovalAuditEvent, ApprovalDatabase, ApprovalNotificationSink, + ApprovalService, + ApprovalStore, ApprovalStreamSink, SelfDecisionPolicy, } from '../approval-api/index.js'; @@ -453,6 +455,39 @@ export interface FlowsafeRunnerLifecycleOptions { discardScheduleDispatch?: DurableObjectRunLifecycleHooks['discardScheduleDispatch']; } +interface ConfiguredApprovalServiceOptions { + store: ApprovalStore; + waitUntil?: (promise: Promise) => void; + notify?: ApprovalNotificationSink; + allowSelfDecision: SelfDecisionPolicy; + stream?: ApprovalStreamSink; +} + +function buildConfiguredApprovalService( + config: FlowsafeRunnerLifecycleConfig, + env: Env, + topology: Pick, + options: ConfiguredApprovalServiceOptions, +): ApprovalService { + return buildHostApprovalService(options.store, { + deploymentTag: env.DEPLOYMENT_TENANT, + systemPrincipalId: config.systemPrincipalId, + defaultSlaSeconds: numberVar( + env.APPROVAL_SLA_SECONDS, + 4 * 60 * 60, + 'APPROVAL_SLA_SECONDS', + ), + resumeRun: config.buildResumeRun + ? config.buildResumeRun(topology.resumeRecord, env) + : topology.resumeRecord, + queue: auditQueueFor(env), + waitUntil: options.waitUntil, + notify: options.notify, + allowSelfDecision: options.allowSelfDecision, + stream: options.stream, + }); +} + /** * Builds the Runner DO's terminal-cleanup hooks from the same approval-service * configuration as createFlowsafeWorker. Hosts supply only the optional DO @@ -479,32 +514,19 @@ export function createFlowsafeRunnerLifecycle( const hubTopology = env.HUB ? createHubTopology(env.HUB, env.DEPLOYMENT_IDENTITY_SECRET) : undefined; - const service = buildHostApprovalService( - approvalStoreFactoryFor(env.DB, storageTablePrefix).store(), - { - deploymentTag: env.DEPLOYMENT_TENANT, - systemPrincipalId: config.systemPrincipalId, - defaultSlaSeconds: numberVar( - env.APPROVAL_SLA_SECONDS, - 4 * 60 * 60, - 'APPROVAL_SLA_SECONDS', - ), - resumeRun: config.buildResumeRun - ? config.buildResumeRun(topology.resumeRecord, env) - : topology.resumeRecord, - queue: auditQueueFor(env), - waitUntil: options.waitUntil, - notify: config.notify?.(env), - allowSelfDecision, - stream: hubTopology - ? (event) => { - const send = hubTopology.publish(event); - options.waitUntil?.(send); - return send; - } - : undefined, - }, - ); + const service = buildConfiguredApprovalService(config, env, topology, { + store: approvalStoreFactoryFor(env.DB, storageTablePrefix).store(), + waitUntil: options.waitUntil, + notify: config.notify?.(env), + allowSelfDecision, + stream: hubTopology + ? (event) => { + const send = hubTopology.publish(event); + options.waitUntil?.(send); + return send; + } + : undefined, + }); return { abandonApprovals: (workflowId, runId, status) => abandonApprovalsForRun( @@ -761,30 +783,15 @@ export function createFlowsafeWorker( storeFactory: approvalStoreFactoryFor(env.DB, storageTablePrefix), deploymentTag: env.DEPLOYMENT_TENANT, buildService: (store) => - buildHostApprovalService(store, { - deploymentTag: env.DEPLOYMENT_TENANT, - systemPrincipalId: config.systemPrincipalId, - defaultSlaSeconds: numberVar( - env.APPROVAL_SLA_SECONDS, - 4 * 60 * 60, - 'APPROVAL_SLA_SECONDS', - ), - // The one topology-specific piece: decisions resume the run - // through its DO stub. - resumeRun: config.buildResumeRun - ? config.buildResumeRun(topology.resumeRecord, env) - : topology.resumeRecord, - queue: auditQueueFor(env), + buildConfiguredApprovalService(config, env, topology, { + store, waitUntil, notify, - // Fetch-scope hub fan-out: every request-path mutation reaches the - // deployment hub, kept alive by ctx.waitUntil (DL-020). Undefined when no - // HUB is bound, so a non-streaming host is byte-identical to before. stream, allowSelfDecision: selfDecision, }), // The resolver's canSelfDecide display hint reads the SAME policy the - // service enforces (passed to buildHostApprovalService above), so the + // service enforces (passed to the shared service builder above), so the // /workflows echo can never contradict decide(). allowSelfDecision: selfDecision, }); diff --git a/packages/flowsafe/src/host-kit/stream-router.ts b/packages/flowsafe/src/host-kit/stream-router.ts index 364c61f..07422a5 100644 --- a/packages/flowsafe/src/host-kit/stream-router.ts +++ b/packages/flowsafe/src/host-kit/stream-router.ts @@ -55,7 +55,7 @@ export type StreamRouter = (request: Request) => Promise; /** * A DO stub that forwards a raw WS-upgrade Request and returns the 101 Response * unmodified. The runner's structural RunnerStubLike (do-run-topology.ts) models - * only the JSON `fetch(url, init)` overload used for start/status/resume; the + * only the JSON `fetch(url, init)` overload used for run lifecycle routes; the * REAL DurableObjectStub also forwards a raw `Request` -> `Response`, which the * WebSocket upgrade needs. This narrow local shape names exactly that overload * so the forward is typed without widening the shared RunnerStubLike seam. The