diff --git a/docs/CODEMAP.md b/docs/CODEMAP.md index 2a7114cd..b1d543a4 100644 --- a/docs/CODEMAP.md +++ b/docs/CODEMAP.md @@ -10,7 +10,7 @@ _99 entries — tsup.config `entry`. Regenerate with `agent-docs`._ | [`./app-auth`](api/app-auth.md) | 11 | `platform` | | [`./assets`](api/assets.md) | 44 | — | | [`./assistant`](api/assistant.md) | 56 | `runtime`, `web-react` | -| [`./billing`](api/billing.md) | 32 | — | +| [`./billing`](api/billing.md) | 34 | — | | [`./brand`](api/brand.md) | 5 | — | | [`./brand-extraction`](api/brand-extraction.md) | 19 | — | | [`./catalog`](api/catalog.md) | 10 | `runtime` | @@ -146,9 +146,9 @@ Depends on: `runtime`, `web-react` ## `./billing` -Source: `src/billing/index.ts` · 32 exports +Source: `src/billing/index.ts` · 34 exports -`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` +`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyConditionalWrites`, `DurableWorkspaceKeyCreateInput`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` [Full API →](api/billing.md) diff --git a/docs/api/billing.md b/docs/api/billing.md index a141ce2b..100a74b8 100644 --- a/docs/api/billing.md +++ b/docs/api/billing.md @@ -4,7 +4,7 @@ Source: `src/billing/index.ts` -32 exports. +34 exports. ### `createIdentityBoundWorkspaceKeyManager` @@ -38,6 +38,22 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyConditionalWrites` + +`interface` — Optional compare-and-set lifecycle writes for stores that support fencing. + +```ts +interface DurableWorkspaceKeyConditionalWrites +``` + +### `DurableWorkspaceKeyCreateInput` + +`interface` — The exact request identity that must be reused when recovering a create. + +```ts +interface DurableWorkspaceKeyCreateInput +``` + ### `DurableWorkspaceKeyManager` `interface` — Identity-bound durable key manager API. diff --git a/docs/codemap.json b/docs/codemap.json index 515e50b9..bd6be588 100644 --- a/docs/codemap.json +++ b/docs/codemap.json @@ -785,6 +785,18 @@ "signature": "(opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager", "doc": "Create a workspace key manager that handles key provisioning and budget tracking" }, + { + "name": "DurableWorkspaceKeyConditionalWrites", + "kind": "interface", + "signature": "interface DurableWorkspaceKeyConditionalWrites", + "doc": "Optional compare-and-set lifecycle writes for stores that support fencing." + }, + { + "name": "DurableWorkspaceKeyCreateInput", + "kind": "interface", + "signature": "interface DurableWorkspaceKeyCreateInput", + "doc": "The exact request identity that must be reused when recovering a create." + }, { "name": "DurableWorkspaceKeyManager", "kind": "interface", diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 4a8c8e30..d7b96d7c 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1012,6 +1012,22 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyConditionalWrites` + +`interface` — Optional compare-and-set lifecycle writes for stores that support fencing. + +```ts +interface DurableWorkspaceKeyConditionalWrites +``` + +### `DurableWorkspaceKeyCreateInput` + +`interface` — The exact request identity that must be reused when recovering a create. + +```ts +interface DurableWorkspaceKeyCreateInput +``` + ### `DurableWorkspaceKeyManager` `interface` — Identity-bound durable key manager API. diff --git a/docs/llms.txt b/docs/llms.txt index 53149778..4cd57f75 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -10,7 +10,7 @@ _Generated by agent-docs from tsup.config `entry`; 99 entries. Regenerate with ` - [`./app-auth`](api/app-auth.md): 11 exports — AppAuth, AppAuthConfig, AppAuthEmailClient, AppAuthEmailConfig, AppAuthInstance, AppAuthSchema, AppAuthSession, AppAuthSocialConfig, … - [`./assets`](api/assets.md): 44 exports — ApprovalEvent, ApprovalEventSchema, AssetContentMap, assetCreateJsonSchema, AssetFormat, AssetSpec, AssetStatus, AssetVariant, … - [`./assistant`](api/assistant.md): 56 exports — adaptTranscript, AssistantChat, AssistantClient, AssistantClientConfig, AssistantClientInputError, AssistantClientProvider, AssistantDeliveryMode, AssistantDock, … -- [`./billing`](api/billing.md): 32 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, DurableWorkspaceKeyProvisioner, DurableWorkspaceKeyProvisioningRecord, … +- [`./billing`](api/billing.md): 34 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyConditionalWrites, DurableWorkspaceKeyCreateInput, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, … - [`./brand`](api/brand.md): 5 exports — BrandHeader, BrandHeaderProps, Logo, LogoProps, TangleKnot - [`./brand-extraction`](api/brand-extraction.md): 19 exports — BrandColor, BrandExtractionResult, BrandFont, BrandImage, BrandKit, BrandLogoCandidate, decideBrandKit, DecidedBrandKit, … - [`./catalog`](api/catalog.md): 10 exports — buildCatalog, CatalogModel, catalogModelForId, fetchModelCatalog, isChatCapableModel, ModelCatalog, normalizeModelId, resolveCatalogModelId, … diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 3d87817a..396ceb84 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -41,6 +41,8 @@ export interface DurableWorkspaceKeyRecord { sourceKeyFingerprint: string /** The persisted name used to recover a remote create after a crash. Null only for pre-name rows. */ name: string | null + /** The persisted retry identity for the remote create. Absent on legacy rows without a persisted create identity. */ + idempotencyKey?: string | null keyId: string keyEncrypted: string budgetUsd: number @@ -63,9 +65,22 @@ type DurableWorkspaceKeyCreateResult = Awaited[0]): Promise + /** True only when replay returns the original child for the persisted request identity. */ + supportsIdempotentCreate?: true + /** Create one child key. Non-idempotent providers recover through stable-name discovery only. */ + createKey(input: DurableWorkspaceKeyCreateInput): Promise getKey(id: string): Promise<{ budgetUsd?: number | null budgetSpent?: number @@ -80,6 +95,20 @@ export interface DurableWorkspaceKeyProvisioner { }): Promise> } +/** Optional compare-and-set lifecycle writes for stores that support fencing. */ +export interface DurableWorkspaceKeyConditionalWrites { + /** Return false when the row was no longer provisioning. */ + markProvisioningRemote(input: { id: string; keyId: string }): Promise + /** Return false when the row was no longer provisioning. */ + markActive(input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }): Promise +} + /** Persistence operations for identity-bound key lifecycle state. */ export interface DurableWorkspaceKeyStore { /** Return the active row for this exact product scope. */ @@ -98,7 +127,12 @@ export interface DurableWorkspaceKeyStore { expiresAt: Date budgetUsd: number }): Promise - /** Keep a failed cleanup visible and schedule a later retry. */ + /** + * Optional fenced writes. The manager uses false to compensate a stale + * creator. Stores without this field retain the pre-fence behavior. + */ + conditionalWrites?: DurableWorkspaceKeyConditionalWrites + /** Keep a failed cleanup visible and schedule a later retry. Terminal rows are immutable. */ markRevocationPending(input: { id: string error?: string | null @@ -151,14 +185,26 @@ export interface DurableWorkspaceKeyManager { ensureKey(identity: WorkspaceKeyIdentity, options?: { budgetUsd?: number }): Promise /** Read usage without returning the child secret. */ getUsage(identity: WorkspaceKeyIdentity): Promise - /** Retry due cleanup rows. A supplied owner/workspace limits the retry to its scope. */ - retryPendingRevocations(scope?: Pick): Promise + /** + * Retry due cleanup rows. A supplied owner/workspace limits the retry to its + * scope. Empty provisioning rows also require the full identity so a retry + * can use the original source binding safely. + */ + retryPendingRevocations( + scope?: Pick, + identity?: WorkspaceKeyIdentity, + ): Promise } /** Configuration for {@link createIdentityBoundWorkspaceKeyManager}. */ export interface DurableWorkspaceKeyManagerOptions { store: DurableWorkspaceKeyStore provisioner: DurableWorkspaceKeyProvisioner + /** + * Control-plane client for historical get/revoke/list operations. Use this + * when the source credential can rotate or disappear. It never mints keys. + */ + recoveryProvisioner?: Pick crypto: KeyCrypto /** Product partition. Products must use separate values. */ product: WorkspaceKeyProduct @@ -275,6 +321,30 @@ function isProvisioningId(value: string): boolean { return value.startsWith('provisioning:') } +async function markProvisioningRemote( + store: DurableWorkspaceKeyStore, + input: { id: string; keyId: string }, +): Promise { + if (store.conditionalWrites) return store.conditionalWrites.markProvisioningRemote(input) + await store.markProvisioningRemote(input) + return true +} + +async function markActive( + store: DurableWorkspaceKeyStore, + input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }, +): Promise { + if (store.conditionalWrites) return store.conditionalWrites.markActive(input) + await store.markActive(input) + return true +} + function usageFromRemote( row: DurableWorkspaceKeyRecord, remote: Awaited>, @@ -337,6 +407,7 @@ export function createIdentityBoundWorkspaceKeyManager( const now = options.now ?? (() => new Date()) const isRemoteMissing = options.isRemoteMissing ?? defaultRemoteMissing + const recoveryProvisioner = options.recoveryProvisioner ?? options.provisioner const localLocks = new Map>() async function withLocalLock(scope: string, work: () => Promise): Promise { @@ -452,7 +523,7 @@ export function createIdentityBoundWorkspaceKeyManager( return true } try { - await options.provisioner.revokeKey(row.keyId) + await recoveryProvisioner.revokeKey(row.keyId) await options.store.markRevoked(row.id, now()) return true } catch (error) { @@ -470,26 +541,71 @@ export function createIdentityBoundWorkspaceKeyManager( } } - async function cleanupProvisioning(row: DurableWorkspaceKeyRecord, identity?: WorkspaceKeyIdentity): Promise { - let candidates: Array<{ id: string }> + async function provisioningCandidates(row: DurableWorkspaceKeyRecord): Promise<{ + name: string + candidates: Array<{ id: string }> + } | null> { if (row.keyId && !isProvisioningId(row.keyId)) { - candidates = [{ id: row.keyId }] - } else { - const name = row.name?.trim() || options.legacyNameForRecord?.(row)?.trim() - if (!name) { - await options.store.markRevocationPending({ - id: row.id, - error: 'provisioning row has no recoverable remote name; migrate the row or configure legacyNameForRecord', - nextAttemptAt: now(), - incrementAttempts: false, - }) - return false - } - candidates = await options.provisioner.findCreatedKeys({ + return { name: row.name?.trim() ?? '', candidates: [{ id: row.keyId }] } + } + const name = row.name?.trim() || options.legacyNameForRecord?.(row)?.trim() + if (!name) return null + return { + name, + candidates: await recoveryProvisioner.findCreatedKeys({ name, product: row.product, sourceKeyId: row.sourceKeyId, + }), + } + } + + async function probeProvisioning( + row: DurableWorkspaceKeyRecord, + identity: WorkspaceKeyIdentity | undefined, + name: string, + ): Promise | null> { + const idempotencyKey = row.idempotencyKey?.trim() + if (!identity || !sameIdentity(row, identity, product) + || !options.provisioner.supportsIdempotentCreate || !idempotencyKey) return [] + try { + const created = await options.provisioner.createKey({ + name, + product: row.product, + budgetUsd: row.budgetUsd, + expiresAt: row.expiresAt.toISOString(), + idempotencyKey, + }) + const remoteId = created.id?.trim() + if (remoteId) return [{ id: remoteId }] + } catch (error) { + await options.store.markRevocationPending({ + id: row.id, + error: errorMessage(error, 'remote provisioning recovery failed'), + nextAttemptAt: new Date(now().getTime() + retryDelay(row.revocationAttempts + 1)), + incrementAttempts: true, }) + return null + } + return (await provisioningCandidates({ ...row, name, keyId: `provisioning:${idempotencyKey}` }))?.candidates ?? [] + } + + async function cleanupProvisioning(row: DurableWorkspaceKeyRecord, identity?: WorkspaceKeyIdentity): Promise { + const resolved = await provisioningCandidates(row) + if (!resolved) { + await options.store.markRevocationPending({ + id: row.id, + error: 'provisioning row has no recoverable remote name; migrate the row or configure legacyNameForRecord', + nextAttemptAt: now(), + incrementAttempts: false, + }) + return false + } + let { name, candidates } = resolved + if (candidates.length === 0) { + const probed = await probeProvisioning(row, identity, name) + if (probed === null) return false + candidates = probed } const scope = { workspaceId: row.workspaceId, ownerUserId: row.ownerUserId, product: row.product } @@ -502,7 +618,7 @@ export function createIdentityBoundWorkspaceKeyManager( for (const keyId of toRevoke) { try { - await options.provisioner.revokeKey(keyId) + await recoveryProvisioner.revokeKey(keyId) } catch (error) { if (isRemoteMissing(error)) continue await options.store.markRevocationPending({ @@ -515,10 +631,16 @@ export function createIdentityBoundWorkspaceKeyManager( } } - if (toRevoke.length > 0 || candidates.length === 0 || activeKeyId !== null) { + if (candidates.length > 0) { await options.store.markRevoked(row.id, now()) } else { - await options.store.markOrphaned(row.id, 'no unowned remote child matched the durable provisioning record') + await options.store.markRevocationPending({ + id: row.id, + error: 'no remote child matched the durable provisioning record; retrying crash recovery', + nextAttemptAt: new Date(now().getTime() + retryDelay(row.revocationAttempts + 1)), + incrementAttempts: true, + }) + return false } return true } @@ -571,14 +693,28 @@ export function createIdentityBoundWorkspaceKeyManager( } } - async function retryPendingRevocations(scopeInput?: Pick): Promise { + async function retryPendingRevocations( + scopeInput?: Pick, + identityInput?: WorkspaceKeyIdentity, + ): Promise { + const identity = identityInput ? normalizeIdentity(identityInput) : undefined + if (identity && scopeInput + && (identity.workspaceId !== scopeInput.workspaceId || identity.ownerUserId !== scopeInput.ownerUserId)) { + throw new Error('workspace child key retry identity does not match its scope') + } if (scopeInput) { const workspaceId = scopeInput.workspaceId.trim() const ownerUserId = scopeInput.ownerUserId.trim() if (!workspaceId || !ownerUserId) throw new Error('workspace child key retry scope is incomplete') const scope = scopeKey({ workspaceId, ownerUserId, product }) return withLocalLock(scope, () => withDurableLease(scope, async () => ( - await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId })).completed)) + await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId }, identity)).completed)) + } + + if (identity) { + const scope = scopeKey(scopeFor(identity, product)) + return withLocalLock(scope, () => withDurableLease(scope, async () => ( + await retryPendingRevocationsUnlocked({ workspaceId: identity.workspaceId, ownerUserId: identity.ownerUserId }, identity)).completed)) } const rows = await options.store.listPendingRevocations({ product, now: now(), limit: PENDING_REVOCATION_BATCH_SIZE }) @@ -612,7 +748,7 @@ export function createIdentityBoundWorkspaceKeyManager( let remote: Awaited> try { - remote = await options.provisioner.getKey(row.keyId) + remote = await recoveryProvisioner.getKey(row.keyId) } catch (error) { if (isRemoteMissing(error)) { await options.store.markRevoked(row.id, now()) @@ -645,6 +781,7 @@ export function createIdentityBoundWorkspaceKeyManager( const name = (options.nameForIdentity?.(identity, mintOperationId) ?? defaultKeyName(identity, product, mintOperationId)).trim() if (!name) throw new Error('workspace child key remote name is required') const rowId = operationId() + const idempotencyKey = `workspace-key:${rowId}` const provisioningRow: DurableWorkspaceKeyProvisioningRecord = { id: rowId, workspaceId: identity.workspaceId, @@ -654,6 +791,7 @@ export function createIdentityBoundWorkspaceKeyManager( sourceKeyId: sourceKeyId(identity), sourceKeyFingerprint: identity.sourceKeyFingerprint, name, + idempotencyKey, keyId: `provisioning:${mintOperationId}`, keyEncrypted: '', budgetUsd, @@ -673,6 +811,7 @@ export function createIdentityBoundWorkspaceKeyManager( product, budgetUsd, expiresAt: expiresAt.toISOString(), + idempotencyKey, }) } catch (error) { // Keep cleanup running even when the state write fails. The remote @@ -695,7 +834,7 @@ export function createIdentityBoundWorkspaceKeyManager( if (!remoteId || !secret) { if (remoteId) { try { - await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) + await markProvisioningRemote(options.store, { id: rowId, keyId: remoteId }) } catch { // revokePending uses the returned id even when this state write fails. } @@ -710,31 +849,53 @@ export function createIdentityBoundWorkspaceKeyManager( throw new Error('remote create returned no usable child key') } - onRemoteKey?.(provisioningRow, remoteId) + let cleanupRow = provisioningRow + onRemoteKey?.(cleanupRow, remoteId) + + async function retainRetiredCreatorCleanup(): Promise { + // A retired row cannot hold another pending transition. Persist the + // returned child separately before revocation so a failed revoke survives restart. + const retainedRow = { ...provisioningRow, id: operationId(), keyId: remoteId } + try { + await options.store.insertProvisioning(retainedRow) + } catch (cause) { + throw new Error('returned child cleanup record could not be persisted', { cause }) + } + cleanupRow = retainedRow + onRemoteKey?.(cleanupRow, remoteId) + } try { - await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) + const recorded = await markProvisioningRemote(options.store, { id: rowId, keyId: remoteId }) + if (!recorded) { + await retainRetiredCreatorCleanup() + throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') + } const keyEncrypted = await options.crypto.encrypt(secret) const remoteExpiresAt = created.expiresAt?.trim() const activeExpiresAt = remoteExpiresAt ? new Date(remoteExpiresAt) : expiresAt if (!Number.isFinite(activeExpiresAt.getTime())) throw new Error('remote child key expiry is malformed') const activeBudgetUsd = created.budgetUsd ?? budgetUsd if (!Number.isFinite(activeBudgetUsd) || activeBudgetUsd < 0) throw new Error('remote child key budget is malformed') - await options.store.markActive({ + const activated = await markActive(options.store, { id: rowId, keyId: remoteId, keyEncrypted, expiresAt: activeExpiresAt, budgetUsd: activeBudgetUsd, }) + if (!activated) { + await retainRetiredCreatorCleanup() + throw new Error('workspace child-key provisioning row was retired before activation') + } } catch (error) { // A persistence failure must not strand the already-created remote key. try { - await markPending(rowId, error, false) + await markPending(cleanupRow.id, error, false) } catch { // revokePending below still has the remote id and attempts cleanup. } - await revokePending({ ...provisioningRow, keyId: remoteId, status: 'revocation_pending' }, identity) + await revokePending({ ...cleanupRow, keyId: remoteId, status: 'revocation_pending' }, identity) throw error } diff --git a/src/billing/index.ts b/src/billing/index.ts index c7b91765..b603f5e3 100644 --- a/src/billing/index.ts +++ b/src/billing/index.ts @@ -403,6 +403,8 @@ export { createIdentityBoundWorkspaceKeyManager, type DurableWorkspaceKeyManager, type DurableWorkspaceKeyManagerOptions, + type DurableWorkspaceKeyCreateInput, + type DurableWorkspaceKeyConditionalWrites, type DurableWorkspaceKeyProvisioner, type DurableWorkspaceKeyProvisioningRecord, type DurableWorkspaceKeyRecord, diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index f9b2974a..6077cb50 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { createIdentityBoundWorkspaceKeyManager, + type DurableWorkspaceKeyConditionalWrites, type DurableWorkspaceKeyManager, type DurableWorkspaceKeyProvisioner, type DurableWorkspaceKeyRecord, @@ -46,7 +47,37 @@ function makeHarness() { let renewalDelayMs = 0 let failRenewal = false let failMarkPending = false - let lastCreateInput: { name: string; product: string; budgetUsd: number; expiresAt: string } | null = null + let createStartedResolve: (() => void) | null = null + const createInputs: Array<{ name: string; product: string; budgetUsd: number; expiresAt: string; idempotencyKey: string }> = [] + + const markProvisioningRemote = async (input: { id: string; keyId: string }): Promise => { + const row = rows.get(input.id) + if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return false + row.keyId = input.keyId + return true + } + const markActive = async (input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }): Promise => { + const row = rows.get(input.id) + if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return false + row.keyId = input.keyId + row.keyEncrypted = input.keyEncrypted + row.expiresAt = input.expiresAt + row.budgetUsd = input.budgetUsd + row.status = 'active' + return true + } + const conditionalWrites: DurableWorkspaceKeyConditionalWrites = { + markProvisioningRemote, + markActive, + } const store: DurableWorkspaceKeyStore = { async getActive(scope) { @@ -68,23 +99,17 @@ function makeHarness() { rows.set(record.id, { ...record }) }, async markProvisioningRemote(input) { - const row = rows.get(input.id) - if (!row) throw new Error(`unknown row ${input.id}`) - row.keyId = input.keyId + await markProvisioningRemote(input) }, async markActive(input) { - const row = rows.get(input.id) - if (!row) throw new Error(`unknown row ${input.id}`) - row.keyId = input.keyId - row.keyEncrypted = input.keyEncrypted - row.expiresAt = input.expiresAt - row.budgetUsd = input.budgetUsd - row.status = 'active' + await markActive(input) }, + conditionalWrites, async markRevocationPending(input) { if (failMarkPending) throw new Error('state store unavailable') const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status === 'revoked' || row.status === 'orphaned') return row.status = 'revocation_pending' row.nextRevocationAt = input.nextAttemptAt row.lastRevocationError = input.error ?? null @@ -132,9 +157,15 @@ function makeHarness() { }, } + const createdByIdentity = new Map() const provisioner: DurableWorkspaceKeyProvisioner = { + supportsIdempotentCreate: true, async createKey(input) { - lastCreateInput = input + createInputs.push(input) + const previous = createdByIdentity.get(input.idempotencyKey) + if (previous) return { id: previous.id, key: previous.key, budgetUsd: previous.budgetUsd, expiresAt: previous.expiresAt } + createStartedResolve?.() + createStartedResolve = null if (createDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, createDelayMs)) keyNumber += 1 const id = returnMissingId ? '' : `remote-${keyNumber}` @@ -151,6 +182,7 @@ function makeHarness() { expiresAt: input.expiresAt, revoked: false, }) + createdByIdentity.set(input.idempotencyKey, remote.get(remoteId)!) const failure = createFailure createFailure = null returnMissingId = false @@ -219,6 +251,7 @@ function makeHarness() { sourceKeyId, sourceKeyFingerprint: sourceKeyId ? `fingerprint-${sourceKeyId}` : 'fingerprint-none', name: `router:workspace-1:crashed-attempt`, + idempotencyKey: `workspace-key:row-${rowNumber}`, keyId: 'provisioning:crashed-attempt', keyEncrypted: '', budgetUsd: 25, @@ -253,7 +286,9 @@ function makeHarness() { setReturnMissingId(value: boolean) { returnMissingId = value }, failRevocationFor(value: string | null) { failNextRevocationFor = value }, setRevocationFailure(value: Error) { revocationFailure = value }, - getLastCreateInput() { return lastCreateInput }, + getLastCreateInput() { return createInputs.at(-1) ?? null }, + getCreateInputs() { return createInputs }, + waitForCreateStart() { return new Promise((resolve) => { createStartedResolve = resolve }) }, getRenewalCount() { return renewalCount }, } } @@ -273,6 +308,37 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect([...h.rows.values()].filter((row) => row.status === 'active')).toHaveLength(2) }) + it('keeps legacy void lifecycle stores compatible', async () => { + const h = makeHarness() + const { conditionalWrites: _conditionalWrites, ...legacyMethods } = h.store + const legacyStore: DurableWorkspaceKeyStore = { + ...legacyMethods, + async markProvisioningRemote(input) { + await h.store.markProvisioningRemote(input) + }, + async markActive(input) { + await h.store.markActive(input) + }, + } + const manager = createIdentityBoundWorkspaceKeyManager({ + store: legacyStore, + provisioner: h.provisioner, + crypto: { + async encrypt(value) { return `encrypted:${value}` }, + async decrypt(value) { return value.slice('encrypted:'.length) }, + }, + product: 'router', + defaultBudgetUsd: 25, + now: () => new Date(START), + }) + + const result = await manager.ensureKey(h.identity()) + + expect(result.usage.keyId).toBe('remote-1') + expect([...h.rows.values()].find((row) => row.keyId === 'remote-1')?.status).toBe('active') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + }) + it('uses the durable lease across manager instances', async () => { const h = makeHarness() h.setCreateDelay(25) @@ -335,6 +401,131 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(h.rows.get(row.id)?.status).toBe('revoked') }) + it('keeps an undiscovered committed child pending when creation is not idempotent', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + h.remote.set('hidden-original', { + id: 'hidden-original', key: 'original-secret', name: row.name!, product: row.product, + sourceKeyId: row.sourceKeyId, budgetUsd: 25, budgetSpent: 0, + expiresAt: row.expiresAt.toISOString(), revoked: false, + }) + const manager = h.manager('router', { + provisioner: { ...h.provisioner, supportsIdempotentCreate: undefined, findCreatedKeys: async () => [] }, + }) + await expect(manager.ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(0) + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') + expect(h.remote.get('hidden-original')?.revoked).toBe(false) + h.advance(60_000) + await h.manager('router').ensureKey(h.identity()) + expect(h.remote.get('hidden-original')?.revoked).toBe(true) + }) + + it('probes an empty provisioning row with its persisted create identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + + const result = await h.manager('router').ensureKey(h.identity()) + const inputs = h.getCreateInputs() + + expect(inputs[0]).toMatchObject({ + name: row.name, + idempotencyKey: row.idempotencyKey, + budgetUsd: row.budgetUsd, + expiresAt: row.expiresAt.toISOString(), + }) + expect(inputs[1]?.idempotencyKey).not.toBe(row.idempotencyKey) + expect(h.remote.get('remote-1')?.revoked).toBe(true) + expect(row.status).toBe('revoked') + expect(result.usage.keyId).toBe('remote-2') + }) + + it('replays an idempotent create to revoke the original child hidden from discovery', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + const original = await h.provisioner.createKey({ name: row.name!, product: row.product, + budgetUsd: row.budgetUsd, expiresAt: row.expiresAt.toISOString(), idempotencyKey: row.idempotencyKey! }) + const manager = h.manager('router', { + provisioner: { ...h.provisioner, findCreatedKeys: async () => [] }, + }) + const result = await manager.ensureKey(h.identity()) + if (!original.id) throw new Error('Fixture create did not return its original id') + expect(h.remote.get(original.id)?.revoked).toBe(true) + expect(h.remote.size).toBe(2) + expect(result.usage.keyId).not.toBe(original.id) + expect(h.getCreateInputs()[1]?.idempotencyKey).toBe(row.idempotencyKey) + }) + + it('keeps a legacy empty row pending instead of inventing a new create identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + delete row.idempotencyKey + + await expect(h.manager('router').ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(0) + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') + }) + + it('retries an empty pending row when the caller supplies its full identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + h.rows.set(row.id, { ...row, status: 'revocation_pending', nextRevocationAt: new Date(START) }) + + expect(await h.manager('router').retryPendingRevocations({ + workspaceId: 'workspace-1', + ownerUserId: 'owner-1', + }, h.identity())).toBe(1) + expect(h.rows.get(row.id)?.status).toBe('revoked') + expect(h.remote.get('remote-1')?.revoked).toBe(true) + }) + + it('keeps a source-mismatched empty provisioning row pending instead of orphaning it', async () => { + const h = makeHarness() + const row = h.insertProvisioning({ sourceKeyId: 'old-source', sourceKeyFingerprint: 'old-fingerprint' }) + + await expect(h.manager('router').ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') + expect(h.rows.get(row.id)?.status).not.toBe('orphaned') + }) + + it('uses the recovery provisioner for historical reads and revokes after source rotation', async () => { + const h = makeHarness() + const first = await h.manager('router').ensureKey(h.identity()) + const recoveryRevocations: string[] = [] + const recoveryProvisioner: DurableWorkspaceKeyProvisioner = { + ...h.provisioner, + async revokeKey(id) { + recoveryRevocations.push(id) + return h.provisioner.revokeKey(id) + }, + } + const sourceProvisioner: DurableWorkspaceKeyProvisioner = { + ...h.provisioner, + async getKey() { throw new Error('rotated source is no longer authorized') }, + async revokeKey() { throw new Error('rotated source cannot revoke historical child') }, + async findCreatedKeys() { throw new Error('rotated source cannot list historical children') }, + } + const manager = createIdentityBoundWorkspaceKeyManager({ + store: h.store, + provisioner: sourceProvisioner, + recoveryProvisioner, + crypto: { + async encrypt(value) { return `encrypted:${value}` }, + async decrypt(value) { return value.slice('encrypted:'.length) }, + }, + product: 'router', + defaultBudgetUsd: 25, + now: () => new Date(START), + }) + + expect((await manager.getUsage(h.identity()))?.keyId).toBe(first.usage.keyId) + h.remote.get(first.usage.keyId)!.expiresAt = new Date(START - 1).toISOString() + const replacement = await manager.ensureKey(h.identity()) + + expect(recoveryRevocations).toContain(first.usage.keyId) + expect(replacement.usage.keyId).not.toBe(first.usage.keyId) + }) + it('requires an explicit legacy-name resolver for pre-name provisioning rows', async () => { const h = makeHarness() const row = h.insertProvisioning({ name: null }) @@ -558,6 +749,57 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect([...h.remote.values()][0]?.revoked).toBe(true) }) + it('fences a paused creator when another worker retires its provisioning row', async () => { + const h = makeHarness() + h.setCreateDelay(20) + const inFlight = h.manager('router').ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find((candidate) => candidate.status === 'provisioning') + expect(row).toBeDefined() + row!.status = 'revoked' + + await expect(inFlight).rejects.toThrow('retired before the remote id was recorded') + expect([...h.rows.values()].some((candidate) => candidate.status === 'active')).toBe(false) + expect([...h.remote.values()][0]?.revoked).toBe(true) + }) + + it.each(['record-remote', 'activate'] as const)('retains failed cleanup after a retired creator loses %s fencing', async stage => { + const h = makeHarness() + h.setCreateDelay(20) + const manager = h.manager('router') + const inFlight = manager.ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find(candidate => candidate.status === 'provisioning')! + if (stage === 'record-remote') row.status = 'revoked' + else h.store.conditionalWrites!.markActive = async () => { row.status = 'revoked'; return false } + h.failRevocationFor('remote-1') + await expect(inFlight).rejects.toThrow('retired before') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + expect([...h.rows.values()].some(candidate => candidate.status === 'revocation_pending' + && candidate.keyId === 'remote-1')).toBe(true) + expect(row.status).toBe('revoked') + const restarted = h.manager('router') + await expect(restarted.ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(1) + h.advance(60_000) + expect(await restarted.retryPendingRevocations()).toBe(1) + expect(h.remote.get('remote-1')?.revoked).toBe(true) + }) + + it('reports a failed compensation write when both durable storage and revocation are unavailable', async () => { + const h = makeHarness() + h.setCreateDelay(20) + const inFlight = h.manager('router').ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find(candidate => candidate.status === 'provisioning')! + row.status = 'revoked' + h.store.insertProvisioning = async () => { throw new Error('storage unavailable') } + h.failRevocationFor('remote-1') + await expect(inFlight).rejects.toThrow('cleanup record could not be persisted') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + expect([...h.rows.values()].some(candidate => candidate.status === 'revocation_pending')).toBe(false) + }) + it('fails before remote spend when encryption is unavailable', async () => { const h = makeHarness() let creates = 0