From 17f79afb032b006cece51edfc21575372ee24559 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 20:09:52 -0600 Subject: [PATCH 01/10] fix(billing): fence durable key recovery --- docs/CODEMAP.md | 6 +- docs/api/billing.md | 10 ++- docs/codemap.json | 6 ++ docs/llms-full.txt | 8 ++ docs/llms.txt | 2 +- src/billing/identity-bound.ts | 127 ++++++++++++++++++++++----- src/billing/index.ts | 1 + tests/billing-identity-bound.test.ts | 95 +++++++++++++++++++- 8 files changed, 223 insertions(+), 32 deletions(-) diff --git a/docs/CODEMAP.md b/docs/CODEMAP.md index aed848df..52d2392f 100644 --- a/docs/CODEMAP.md +++ b/docs/CODEMAP.md @@ -10,7 +10,7 @@ _97 entries — tsup.config `entry`. Regenerate with `agent-docs`._ | [`./app-auth`](api/app-auth.md) | 11 | `platform` | | [`./assets`](api/assets.md) | 44 | — | | [`./assistant`](api/assistant.md) | 56 | `runtime`, `web-react` | -| [`./billing`](api/billing.md) | 32 | — | +| [`./billing`](api/billing.md) | 33 | — | | [`./brand`](api/brand.md) | 5 | — | | [`./brand-extraction`](api/brand-extraction.md) | 19 | — | | [`./catalog`](api/catalog.md) | 10 | `runtime` | @@ -144,9 +144,9 @@ Depends on: `runtime`, `web-react` ## `./billing` -Source: `src/billing/index.ts` · 32 exports +Source: `src/billing/index.ts` · 33 exports -`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` +`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyCreateInput`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` [Full API →](api/billing.md) diff --git a/docs/api/billing.md b/docs/api/billing.md index a141ce2b..36c2b096 100644 --- a/docs/api/billing.md +++ b/docs/api/billing.md @@ -4,7 +4,7 @@ Source: `src/billing/index.ts` -32 exports. +33 exports. ### `createIdentityBoundWorkspaceKeyManager` @@ -38,6 +38,14 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyCreateInput` + +`interface` — The exact request identity that must be reused when recovering a create. + +```ts +interface DurableWorkspaceKeyCreateInput +``` + ### `DurableWorkspaceKeyManager` `interface` — Identity-bound durable key manager API. diff --git a/docs/codemap.json b/docs/codemap.json index 6fb5b379..57c3e466 100644 --- a/docs/codemap.json +++ b/docs/codemap.json @@ -785,6 +785,12 @@ "signature": "(opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager", "doc": "Create a workspace key manager that handles key provisioning and budget tracking" }, + { + "name": "DurableWorkspaceKeyCreateInput", + "kind": "interface", + "signature": "interface DurableWorkspaceKeyCreateInput", + "doc": "The exact request identity that must be reused when recovering a create." + }, { "name": "DurableWorkspaceKeyManager", "kind": "interface", diff --git a/docs/llms-full.txt b/docs/llms-full.txt index 9273dd1a..cc1eeef9 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1012,6 +1012,14 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyCreateInput` + +`interface` — The exact request identity that must be reused when recovering a create. + +```ts +interface DurableWorkspaceKeyCreateInput +``` + ### `DurableWorkspaceKeyManager` `interface` — Identity-bound durable key manager API. diff --git a/docs/llms.txt b/docs/llms.txt index d62a18b4..9ca21c27 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -10,7 +10,7 @@ _Generated by agent-docs from tsup.config `entry`; 97 entries. Regenerate with ` - [`./app-auth`](api/app-auth.md): 11 exports — AppAuth, AppAuthConfig, AppAuthEmailClient, AppAuthEmailConfig, AppAuthInstance, AppAuthSchema, AppAuthSession, AppAuthSocialConfig, … - [`./assets`](api/assets.md): 44 exports — ApprovalEvent, ApprovalEventSchema, AssetContentMap, assetCreateJsonSchema, AssetFormat, AssetSpec, AssetStatus, AssetVariant, … - [`./assistant`](api/assistant.md): 56 exports — adaptTranscript, AssistantChat, AssistantClient, AssistantClientConfig, AssistantClientInputError, AssistantClientProvider, AssistantDeliveryMode, AssistantDock, … -- [`./billing`](api/billing.md): 32 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, DurableWorkspaceKeyProvisioner, DurableWorkspaceKeyProvisioningRecord, … +- [`./billing`](api/billing.md): 33 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyCreateInput, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, DurableWorkspaceKeyProvisioner, … - [`./brand`](api/brand.md): 5 exports — BrandHeader, BrandHeaderProps, Logo, LogoProps, TangleKnot - [`./brand-extraction`](api/brand-extraction.md): 19 exports — BrandColor, BrandExtractionResult, BrandFont, BrandImage, BrandKit, BrandLogoCandidate, decideBrandKit, DecidedBrandKit, … - [`./catalog`](api/catalog.md): 10 exports — buildCatalog, CatalogModel, catalogModelForId, fetchModelCatalog, isChatCapableModel, ModelCatalog, normalizeModelId, resolveCatalogModelId, … diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 3d87817a..fd410f43 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -41,6 +41,8 @@ export interface DurableWorkspaceKeyRecord { sourceKeyFingerprint: string /** The persisted name used to recover a remote create after a crash. Null only for pre-name rows. */ name: string | null + /** The persisted retry identity for the remote create. Null only for pre-identity rows. */ + idempotencyKey: string | null keyId: string keyEncrypted: string budgetUsd: number @@ -63,9 +65,24 @@ type DurableWorkspaceKeyCreateResult = Awaited[0]): Promise + /** + * Create one child key. Reusing `idempotencyKey` must be safe for the same + * request body. Providers without native idempotency still get crash-safe + * cleanup because the manager persists and searches the stable name. + */ + createKey(input: DurableWorkspaceKeyCreateInput): Promise getKey(id: string): Promise<{ budgetUsd?: number | null budgetSpent?: number @@ -88,9 +105,9 @@ export interface DurableWorkspaceKeyStore { listProvisioning(scope: DurableWorkspaceKeyScope): Promise /** Insert before the remote create so a crashed create can be recovered. */ insertProvisioning(record: DurableWorkspaceKeyProvisioningRecord): Promise - /** Save the remote id while the row remains in provisioning state. */ + /** Save the remote id only while the row remains in provisioning state. */ markProvisioningRemote(input: { id: string; keyId: string }): Promise - /** Promote a fully encrypted row to active state. */ + /** Promote a fully encrypted row only from provisioning state. */ markActive(input: { id: string keyId: string @@ -98,7 +115,7 @@ export interface DurableWorkspaceKeyStore { expiresAt: Date budgetUsd: number }): Promise - /** Keep a failed cleanup visible and schedule a later retry. */ + /** Keep a failed cleanup visible and schedule a later retry. Terminal rows are immutable. */ markRevocationPending(input: { id: string error?: string | null @@ -159,6 +176,11 @@ export interface DurableWorkspaceKeyManager { export interface DurableWorkspaceKeyManagerOptions { store: DurableWorkspaceKeyStore provisioner: DurableWorkspaceKeyProvisioner + /** + * Control-plane client for historical get/revoke/list operations. Use this + * when the source credential can rotate or disappear. It never mints keys. + */ + recoveryProvisioner?: Pick crypto: KeyCrypto /** Product partition. Products must use separate values. */ product: WorkspaceKeyProduct @@ -275,6 +297,11 @@ function isProvisioningId(value: string): boolean { return value.startsWith('provisioning:') } +function idempotencyKeyForRecord(row: Pick): string { + const value = row.idempotencyKey?.trim() + return value || `workspace-key:${row.id}` +} + function usageFromRemote( row: DurableWorkspaceKeyRecord, remote: Awaited>, @@ -337,6 +364,7 @@ export function createIdentityBoundWorkspaceKeyManager( const now = options.now ?? (() => new Date()) const isRemoteMissing = options.isRemoteMissing ?? defaultRemoteMissing + const recoveryProvisioner = options.recoveryProvisioner ?? options.provisioner const localLocks = new Map>() async function withLocalLock(scope: string, work: () => Promise): Promise { @@ -452,7 +480,7 @@ export function createIdentityBoundWorkspaceKeyManager( return true } try { - await options.provisioner.revokeKey(row.keyId) + await recoveryProvisioner.revokeKey(row.keyId) await options.store.markRevoked(row.id, now()) return true } catch (error) { @@ -470,26 +498,69 @@ export function createIdentityBoundWorkspaceKeyManager( } } - async function cleanupProvisioning(row: DurableWorkspaceKeyRecord, identity?: WorkspaceKeyIdentity): Promise { - let candidates: Array<{ id: string }> + async function provisioningCandidates(row: DurableWorkspaceKeyRecord): Promise<{ + name: string + candidates: Array<{ id: string }> + } | null> { if (row.keyId && !isProvisioningId(row.keyId)) { - candidates = [{ id: row.keyId }] - } else { - const name = row.name?.trim() || options.legacyNameForRecord?.(row)?.trim() - if (!name) { - await options.store.markRevocationPending({ - id: row.id, - error: 'provisioning row has no recoverable remote name; migrate the row or configure legacyNameForRecord', - nextAttemptAt: now(), - incrementAttempts: false, - }) - return false - } - candidates = await options.provisioner.findCreatedKeys({ + return { name: row.name?.trim() ?? '', candidates: [{ id: row.keyId }] } + } + const name = row.name?.trim() || options.legacyNameForRecord?.(row)?.trim() + if (!name) return null + return { + name, + candidates: await recoveryProvisioner.findCreatedKeys({ name, product: row.product, sourceKeyId: row.sourceKeyId, + }), + } + } + + async function probeProvisioning( + row: DurableWorkspaceKeyRecord, + identity: WorkspaceKeyIdentity | undefined, + name: string, + ): Promise | null> { + if (!identity || !sameIdentity(row, identity, product)) return [] + try { + const created = await options.provisioner.createKey({ + name, + product: row.product, + budgetUsd: row.budgetUsd, + expiresAt: row.expiresAt.toISOString(), + idempotencyKey: idempotencyKeyForRecord(row), }) + const remoteId = created.id?.trim() + if (remoteId) return [{ id: remoteId }] + } catch (error) { + await options.store.markRevocationPending({ + id: row.id, + error: errorMessage(error, 'remote provisioning recovery failed'), + nextAttemptAt: new Date(now().getTime() + retryDelay(row.revocationAttempts + 1)), + incrementAttempts: true, + }) + return null + } + return (await provisioningCandidates({ ...row, name, keyId: `provisioning:${idempotencyKeyForRecord(row)}` }))?.candidates ?? [] + } + + async function cleanupProvisioning(row: DurableWorkspaceKeyRecord, identity?: WorkspaceKeyIdentity): Promise { + const resolved = await provisioningCandidates(row) + if (!resolved) { + await options.store.markRevocationPending({ + id: row.id, + error: 'provisioning row has no recoverable remote name; migrate the row or configure legacyNameForRecord', + nextAttemptAt: now(), + incrementAttempts: false, + }) + return false + } + let { name, candidates } = resolved + if (candidates.length === 0) { + const probed = await probeProvisioning(row, identity, name) + if (probed === null) return false + candidates = probed } const scope = { workspaceId: row.workspaceId, ownerUserId: row.ownerUserId, product: row.product } @@ -502,7 +573,7 @@ export function createIdentityBoundWorkspaceKeyManager( for (const keyId of toRevoke) { try { - await options.provisioner.revokeKey(keyId) + await recoveryProvisioner.revokeKey(keyId) } catch (error) { if (isRemoteMissing(error)) continue await options.store.markRevocationPending({ @@ -515,10 +586,16 @@ export function createIdentityBoundWorkspaceKeyManager( } } - if (toRevoke.length > 0 || candidates.length === 0 || activeKeyId !== null) { + if (candidates.length > 0) { await options.store.markRevoked(row.id, now()) } else { - await options.store.markOrphaned(row.id, 'no unowned remote child matched the durable provisioning record') + await options.store.markRevocationPending({ + id: row.id, + error: 'no remote child matched the durable provisioning record; retrying crash recovery', + nextAttemptAt: new Date(now().getTime() + retryDelay(row.revocationAttempts + 1)), + incrementAttempts: true, + }) + return false } return true } @@ -612,7 +689,7 @@ export function createIdentityBoundWorkspaceKeyManager( let remote: Awaited> try { - remote = await options.provisioner.getKey(row.keyId) + remote = await recoveryProvisioner.getKey(row.keyId) } catch (error) { if (isRemoteMissing(error)) { await options.store.markRevoked(row.id, now()) @@ -654,6 +731,7 @@ export function createIdentityBoundWorkspaceKeyManager( sourceKeyId: sourceKeyId(identity), sourceKeyFingerprint: identity.sourceKeyFingerprint, name, + idempotencyKey: `workspace-key:${rowId}`, keyId: `provisioning:${mintOperationId}`, keyEncrypted: '', budgetUsd, @@ -673,6 +751,7 @@ export function createIdentityBoundWorkspaceKeyManager( product, budgetUsd, expiresAt: expiresAt.toISOString(), + idempotencyKey: idempotencyKeyForRecord(provisioningRow), }) } catch (error) { // Keep cleanup running even when the state write fails. The remote diff --git a/src/billing/index.ts b/src/billing/index.ts index c7b91765..98cd90df 100644 --- a/src/billing/index.ts +++ b/src/billing/index.ts @@ -403,6 +403,7 @@ export { createIdentityBoundWorkspaceKeyManager, type DurableWorkspaceKeyManager, type DurableWorkspaceKeyManagerOptions, + type DurableWorkspaceKeyCreateInput, type DurableWorkspaceKeyProvisioner, type DurableWorkspaceKeyProvisioningRecord, type DurableWorkspaceKeyRecord, diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index f9b2974a..c796d871 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -46,7 +46,8 @@ function makeHarness() { let renewalDelayMs = 0 let failRenewal = false let failMarkPending = false - let lastCreateInput: { name: string; product: string; budgetUsd: number; expiresAt: string } | null = null + let createStartedResolve: (() => void) | null = null + const createInputs: Array<{ name: string; product: string; budgetUsd: number; expiresAt: string; idempotencyKey: string }> = [] const store: DurableWorkspaceKeyStore = { async getActive(scope) { @@ -70,11 +71,13 @@ function makeHarness() { async markProvisioningRemote(input) { const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return row.keyId = input.keyId }, async markActive(input) { const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return row.keyId = input.keyId row.keyEncrypted = input.keyEncrypted row.expiresAt = input.expiresAt @@ -85,6 +88,7 @@ function makeHarness() { if (failMarkPending) throw new Error('state store unavailable') const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status === 'revoked' || row.status === 'orphaned') return row.status = 'revocation_pending' row.nextRevocationAt = input.nextAttemptAt row.lastRevocationError = input.error ?? null @@ -134,7 +138,9 @@ function makeHarness() { const provisioner: DurableWorkspaceKeyProvisioner = { async createKey(input) { - lastCreateInput = input + createInputs.push(input) + createStartedResolve?.() + createStartedResolve = null if (createDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, createDelayMs)) keyNumber += 1 const id = returnMissingId ? '' : `remote-${keyNumber}` @@ -219,6 +225,7 @@ function makeHarness() { sourceKeyId, sourceKeyFingerprint: sourceKeyId ? `fingerprint-${sourceKeyId}` : 'fingerprint-none', name: `router:workspace-1:crashed-attempt`, + idempotencyKey: `workspace-key:row-${rowNumber}`, keyId: 'provisioning:crashed-attempt', keyEncrypted: '', budgetUsd: 25, @@ -253,7 +260,9 @@ function makeHarness() { setReturnMissingId(value: boolean) { returnMissingId = value }, failRevocationFor(value: string | null) { failNextRevocationFor = value }, setRevocationFailure(value: Error) { revocationFailure = value }, - getLastCreateInput() { return lastCreateInput }, + getLastCreateInput() { return createInputs.at(-1) ?? null }, + getCreateInputs() { return createInputs }, + waitForCreateStart() { return new Promise((resolve) => { createStartedResolve = resolve }) }, getRenewalCount() { return renewalCount }, } } @@ -335,6 +344,72 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(h.rows.get(row.id)?.status).toBe('revoked') }) + it('probes an empty provisioning row with its persisted create identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + + const result = await h.manager('router').ensureKey(h.identity()) + const inputs = h.getCreateInputs() + + expect(inputs[0]).toMatchObject({ + name: row.name, + idempotencyKey: row.idempotencyKey, + budgetUsd: row.budgetUsd, + expiresAt: row.expiresAt.toISOString(), + }) + expect(inputs[1]?.idempotencyKey).not.toBe(row.idempotencyKey) + expect(h.remote.get('remote-1')?.revoked).toBe(true) + expect(row.status).toBe('revoked') + expect(result.usage.keyId).toBe('remote-2') + }) + + it('keeps a source-mismatched empty provisioning row pending instead of orphaning it', async () => { + const h = makeHarness() + const row = h.insertProvisioning({ sourceKeyId: 'old-source', sourceKeyFingerprint: 'old-fingerprint' }) + + await expect(h.manager('router').ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') + expect(h.rows.get(row.id)?.status).not.toBe('orphaned') + }) + + it('uses the recovery provisioner for historical reads and revokes after source rotation', async () => { + const h = makeHarness() + const first = await h.manager('router').ensureKey(h.identity()) + const recoveryRevocations: string[] = [] + const recoveryProvisioner: DurableWorkspaceKeyProvisioner = { + ...h.provisioner, + async revokeKey(id) { + recoveryRevocations.push(id) + return h.provisioner.revokeKey(id) + }, + } + const sourceProvisioner: DurableWorkspaceKeyProvisioner = { + ...h.provisioner, + async getKey() { throw new Error('rotated source is no longer authorized') }, + async revokeKey() { throw new Error('rotated source cannot revoke historical child') }, + async findCreatedKeys() { throw new Error('rotated source cannot list historical children') }, + } + const manager = createIdentityBoundWorkspaceKeyManager({ + store: h.store, + provisioner: sourceProvisioner, + recoveryProvisioner, + crypto: { + async encrypt(value) { return `encrypted:${value}` }, + async decrypt(value) { return value.slice('encrypted:'.length) }, + }, + product: 'router', + defaultBudgetUsd: 25, + now: () => new Date(START), + }) + + expect((await manager.getUsage(h.identity()))?.keyId).toBe(first.usage.keyId) + h.remote.get(first.usage.keyId)!.expiresAt = new Date(START - 1).toISOString() + const replacement = await manager.ensureKey(h.identity()) + + expect(recoveryRevocations).toContain(first.usage.keyId) + expect(replacement.usage.keyId).not.toBe(first.usage.keyId) + }) + it('requires an explicit legacy-name resolver for pre-name provisioning rows', async () => { const h = makeHarness() const row = h.insertProvisioning({ name: null }) @@ -558,6 +633,20 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect([...h.remote.values()][0]?.revoked).toBe(true) }) + it('fences a paused creator when another worker retires its provisioning row', async () => { + const h = makeHarness() + h.setCreateDelay(20) + const inFlight = h.manager('router').ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find((candidate) => candidate.status === 'provisioning') + expect(row).toBeDefined() + row!.status = 'revoked' + + await expect(inFlight).rejects.toThrow('not persisted as active') + expect([...h.rows.values()].some((candidate) => candidate.status === 'active')).toBe(false) + expect([...h.remote.values()][0]?.revoked).toBe(true) + }) + it('fails before remote spend when encryption is unavailable', async () => { const h = makeHarness() let creates = 0 From de9c5cce339c827b630d8e3a2e5f86605c5da8e9 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 20:38:02 -0600 Subject: [PATCH 02/10] fix(billing): enforce recovery fences --- src/billing/identity-bound.ts | 40 ++++++++++++++++++++++------ tests/billing-identity-bound.test.ts | 21 ++++++++++++--- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index fd410f43..af4ed422 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -106,7 +106,7 @@ export interface DurableWorkspaceKeyStore { /** Insert before the remote create so a crashed create can be recovered. */ insertProvisioning(record: DurableWorkspaceKeyProvisioningRecord): Promise /** Save the remote id only while the row remains in provisioning state. */ - markProvisioningRemote(input: { id: string; keyId: string }): Promise + markProvisioningRemote(input: { id: string; keyId: string }): Promise /** Promote a fully encrypted row only from provisioning state. */ markActive(input: { id: string @@ -114,7 +114,7 @@ export interface DurableWorkspaceKeyStore { keyEncrypted: string expiresAt: Date budgetUsd: number - }): Promise + }): Promise /** Keep a failed cleanup visible and schedule a later retry. Terminal rows are immutable. */ markRevocationPending(input: { id: string @@ -168,8 +168,15 @@ export interface DurableWorkspaceKeyManager { ensureKey(identity: WorkspaceKeyIdentity, options?: { budgetUsd?: number }): Promise /** Read usage without returning the child secret. */ getUsage(identity: WorkspaceKeyIdentity): Promise - /** Retry due cleanup rows. A supplied owner/workspace limits the retry to its scope. */ - retryPendingRevocations(scope?: Pick): Promise + /** + * Retry due cleanup rows. A supplied owner/workspace limits the retry to its + * scope. Empty provisioning rows also require the full identity so a retry + * can use the original source binding safely. + */ + retryPendingRevocations( + scope?: Pick, + identity?: WorkspaceKeyIdentity, + ): Promise } /** Configuration for {@link createIdentityBoundWorkspaceKeyManager}. */ @@ -648,14 +655,29 @@ export function createIdentityBoundWorkspaceKeyManager( } } - async function retryPendingRevocations(scopeInput?: Pick): Promise { + async function retryPendingRevocations( + scopeInput?: Pick, + identityInput?: WorkspaceKeyIdentity, + ): Promise { + const identity = identityInput ? normalizeIdentity(identityInput) : undefined + if (identity && scopeInput + && (identity.workspaceId !== scopeInput.workspaceId || identity.ownerUserId !== scopeInput.ownerUserId)) { + throw new Error('workspace child key retry identity does not match its scope') + } + const activeIdentity = identity if (scopeInput) { const workspaceId = scopeInput.workspaceId.trim() const ownerUserId = scopeInput.ownerUserId.trim() if (!workspaceId || !ownerUserId) throw new Error('workspace child key retry scope is incomplete') const scope = scopeKey({ workspaceId, ownerUserId, product }) return withLocalLock(scope, () => withDurableLease(scope, async () => ( - await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId })).completed)) + await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId }, activeIdentity)).completed)) + } + + if (identity) { + const scope = scopeKey(scopeFor(identity, product)) + return withLocalLock(scope, () => withDurableLease(scope, async () => ( + await retryPendingRevocationsUnlocked({ workspaceId: identity.workspaceId, ownerUserId: identity.ownerUserId }, identity)).completed)) } const rows = await options.store.listPendingRevocations({ product, now: now(), limit: PENDING_REVOCATION_BATCH_SIZE }) @@ -792,20 +814,22 @@ export function createIdentityBoundWorkspaceKeyManager( onRemoteKey?.(provisioningRow, remoteId) try { - await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) + const recorded = await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) + if (!recorded) throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') const keyEncrypted = await options.crypto.encrypt(secret) const remoteExpiresAt = created.expiresAt?.trim() const activeExpiresAt = remoteExpiresAt ? new Date(remoteExpiresAt) : expiresAt if (!Number.isFinite(activeExpiresAt.getTime())) throw new Error('remote child key expiry is malformed') const activeBudgetUsd = created.budgetUsd ?? budgetUsd if (!Number.isFinite(activeBudgetUsd) || activeBudgetUsd < 0) throw new Error('remote child key budget is malformed') - await options.store.markActive({ + const activated = await options.store.markActive({ id: rowId, keyId: remoteId, keyEncrypted, expiresAt: activeExpiresAt, budgetUsd: activeBudgetUsd, }) + if (!activated) throw new Error('workspace child-key provisioning row was retired before activation') } catch (error) { // A persistence failure must not strand the already-created remote key. try { diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index c796d871..232f47de 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -71,18 +71,20 @@ function makeHarness() { async markProvisioningRemote(input) { const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) - if (row.status !== 'provisioning') return + if (row.status !== 'provisioning') return false row.keyId = input.keyId + return true }, async markActive(input) { const row = rows.get(input.id) if (!row) throw new Error(`unknown row ${input.id}`) - if (row.status !== 'provisioning') return + if (row.status !== 'provisioning') return false row.keyId = input.keyId row.keyEncrypted = input.keyEncrypted row.expiresAt = input.expiresAt row.budgetUsd = input.budgetUsd row.status = 'active' + return true }, async markRevocationPending(input) { if (failMarkPending) throw new Error('state store unavailable') @@ -363,6 +365,19 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(result.usage.keyId).toBe('remote-2') }) + it('retries an empty pending row when the caller supplies its full identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + h.rows.set(row.id, { ...row, status: 'revocation_pending', nextRevocationAt: new Date(START) }) + + expect(await h.manager('router').retryPendingRevocations({ + workspaceId: 'workspace-1', + ownerUserId: 'owner-1', + }, h.identity())).toBe(1) + expect(h.rows.get(row.id)?.status).toBe('revoked') + expect(h.remote.get('remote-1')?.revoked).toBe(true) + }) + it('keeps a source-mismatched empty provisioning row pending instead of orphaning it', async () => { const h = makeHarness() const row = h.insertProvisioning({ sourceKeyId: 'old-source', sourceKeyFingerprint: 'old-fingerprint' }) @@ -642,7 +657,7 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(row).toBeDefined() row!.status = 'revoked' - await expect(inFlight).rejects.toThrow('not persisted as active') + await expect(inFlight).rejects.toThrow('retired before the remote id was recorded') expect([...h.rows.values()].some((candidate) => candidate.status === 'active')).toBe(false) expect([...h.remote.values()][0]?.revoked).toBe(true) }) From 30f7b7348140701c607171a7c56056c3fc6e7ef5 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 20:43:20 -0600 Subject: [PATCH 03/10] fix(billing): preserve lifecycle store compatibility --- src/billing/identity-bound.ts | 28 ++++++++++++++++++++------ tests/billing-identity-bound.test.ts | 30 ++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index af4ed422..1e41c3bb 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -105,16 +105,24 @@ export interface DurableWorkspaceKeyStore { listProvisioning(scope: DurableWorkspaceKeyScope): Promise /** Insert before the remote create so a crashed create can be recovered. */ insertProvisioning(record: DurableWorkspaceKeyProvisioningRecord): Promise - /** Save the remote id only while the row remains in provisioning state. */ - markProvisioningRemote(input: { id: string; keyId: string }): Promise - /** Promote a fully encrypted row only from provisioning state. */ + /** + * Save the remote id only while the row remains in provisioning state. + * Return false when the conditional update matched no provisioning row. + * Legacy stores may return undefined after a successful write. + */ + markProvisioningRemote(input: { id: string; keyId: string }): Promise + /** + * Promote a fully encrypted row only from provisioning state. + * Return false when the conditional update matched no provisioning row. + * Legacy stores may return undefined after a successful write. + */ markActive(input: { id: string keyId: string keyEncrypted: string expiresAt: Date budgetUsd: number - }): Promise + }): Promise /** Keep a failed cleanup visible and schedule a later retry. Terminal rows are immutable. */ markRevocationPending(input: { id: string @@ -309,6 +317,10 @@ function idempotencyKeyForRecord(row: Pick>, @@ -815,7 +827,9 @@ export function createIdentityBoundWorkspaceKeyManager( try { const recorded = await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) - if (!recorded) throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') + if (!lifecycleWriteSucceeded(recorded)) { + throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') + } const keyEncrypted = await options.crypto.encrypt(secret) const remoteExpiresAt = created.expiresAt?.trim() const activeExpiresAt = remoteExpiresAt ? new Date(remoteExpiresAt) : expiresAt @@ -829,7 +843,9 @@ export function createIdentityBoundWorkspaceKeyManager( expiresAt: activeExpiresAt, budgetUsd: activeBudgetUsd, }) - if (!activated) throw new Error('workspace child-key provisioning row was retired before activation') + if (!lifecycleWriteSucceeded(activated)) { + throw new Error('workspace child-key provisioning row was retired before activation') + } } catch (error) { // A persistence failure must not strand the already-created remote key. try { diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 232f47de..09f58072 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -284,6 +284,36 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect([...h.rows.values()].filter((row) => row.status === 'active')).toHaveLength(2) }) + it('keeps legacy void lifecycle stores compatible', async () => { + const h = makeHarness() + const legacyStore: DurableWorkspaceKeyStore = { + ...h.store, + async markProvisioningRemote(input) { + await h.store.markProvisioningRemote(input) + }, + async markActive(input) { + await h.store.markActive(input) + }, + } + const manager = createIdentityBoundWorkspaceKeyManager({ + store: legacyStore, + provisioner: h.provisioner, + crypto: { + async encrypt(value) { return `encrypted:${value}` }, + async decrypt(value) { return value.slice('encrypted:'.length) }, + }, + product: 'router', + defaultBudgetUsd: 25, + now: () => new Date(START), + }) + + const result = await manager.ensureKey(h.identity()) + + expect(result.usage.keyId).toBe('remote-1') + expect([...h.rows.values()].find((row) => row.keyId === 'remote-1')?.status).toBe('active') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + }) + it('uses the durable lease across manager instances', async () => { const h = makeHarness() h.setCreateDelay(25) From e0de1c353f1338ef2ece37f85bc3d12c71101a50 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 20:49:36 -0600 Subject: [PATCH 04/10] fix(billing): preserve lifecycle store api --- docs/CODEMAP.md | 6 +-- docs/api/billing.md | 10 +++- docs/codemap.json | 6 +++ docs/llms-full.txt | 8 ++++ docs/llms.txt | 2 +- src/billing/identity-bound.ts | 69 ++++++++++++++++++++-------- src/billing/index.ts | 1 + tests/billing-identity-bound.test.ts | 50 ++++++++++++++------ 8 files changed, 113 insertions(+), 39 deletions(-) diff --git a/docs/CODEMAP.md b/docs/CODEMAP.md index 52d2392f..9333ed75 100644 --- a/docs/CODEMAP.md +++ b/docs/CODEMAP.md @@ -10,7 +10,7 @@ _97 entries — tsup.config `entry`. Regenerate with `agent-docs`._ | [`./app-auth`](api/app-auth.md) | 11 | `platform` | | [`./assets`](api/assets.md) | 44 | — | | [`./assistant`](api/assistant.md) | 56 | `runtime`, `web-react` | -| [`./billing`](api/billing.md) | 33 | — | +| [`./billing`](api/billing.md) | 34 | — | | [`./brand`](api/brand.md) | 5 | — | | [`./brand-extraction`](api/brand-extraction.md) | 19 | — | | [`./catalog`](api/catalog.md) | 10 | `runtime` | @@ -144,9 +144,9 @@ Depends on: `runtime`, `web-react` ## `./billing` -Source: `src/billing/index.ts` · 33 exports +Source: `src/billing/index.ts` · 34 exports -`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyCreateInput`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` +`createIdentityBoundWorkspaceKeyManager`, `createPlatformBalanceManager`, `createTcloudKeyProvisioner`, `createWorkspaceKeyManager`, `DurableWorkspaceKeyConditionalWrites`, `DurableWorkspaceKeyCreateInput`, `DurableWorkspaceKeyManager`, `DurableWorkspaceKeyManagerOptions`, `DurableWorkspaceKeyProvisioner`, `DurableWorkspaceKeyProvisioningRecord`, `DurableWorkspaceKeyRecord`, `DurableWorkspaceKeyScope`, `DurableWorkspaceKeyStore`, `KeyCrypto`, `KeyProvisioner`, `PlanLimit`, `PlatformBalanceInfo`, `PlatformBalanceManager`, `PlatformBalanceManagerOptions`, `PlatformBillingClient`, `PlatformIdentity`, `PlatformProductUsage`, `SharedBillingState`, `TcloudKeyClient`, `WorkspaceKeyIdentity`, `WorkspaceKeyManager`, `WorkspaceKeyManagerOptions`, `WorkspaceKeyProduct`, `WorkspaceKeyRecord`, `WorkspaceKeyStatus`, `WorkspaceKeyStore`, `WorkspaceKeyUsage`, `WorkspaceModelKeyUsage`, `WorkspaceRuntimeKey` [Full API →](api/billing.md) diff --git a/docs/api/billing.md b/docs/api/billing.md index 36c2b096..100a74b8 100644 --- a/docs/api/billing.md +++ b/docs/api/billing.md @@ -4,7 +4,7 @@ Source: `src/billing/index.ts` -33 exports. +34 exports. ### `createIdentityBoundWorkspaceKeyManager` @@ -38,6 +38,14 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyConditionalWrites` + +`interface` — Optional compare-and-set lifecycle writes for stores that support fencing. + +```ts +interface DurableWorkspaceKeyConditionalWrites +``` + ### `DurableWorkspaceKeyCreateInput` `interface` — The exact request identity that must be reused when recovering a create. diff --git a/docs/codemap.json b/docs/codemap.json index 57c3e466..3d465564 100644 --- a/docs/codemap.json +++ b/docs/codemap.json @@ -785,6 +785,12 @@ "signature": "(opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager", "doc": "Create a workspace key manager that handles key provisioning and budget tracking" }, + { + "name": "DurableWorkspaceKeyConditionalWrites", + "kind": "interface", + "signature": "interface DurableWorkspaceKeyConditionalWrites", + "doc": "Optional compare-and-set lifecycle writes for stores that support fencing." + }, { "name": "DurableWorkspaceKeyCreateInput", "kind": "interface", diff --git a/docs/llms-full.txt b/docs/llms-full.txt index cc1eeef9..6a83da54 100644 --- a/docs/llms-full.txt +++ b/docs/llms-full.txt @@ -1012,6 +1012,14 @@ Source: `src/billing/index.ts` (opts: WorkspaceKeyManagerOptions) => WorkspaceKeyManager ``` +### `DurableWorkspaceKeyConditionalWrites` + +`interface` — Optional compare-and-set lifecycle writes for stores that support fencing. + +```ts +interface DurableWorkspaceKeyConditionalWrites +``` + ### `DurableWorkspaceKeyCreateInput` `interface` — The exact request identity that must be reused when recovering a create. diff --git a/docs/llms.txt b/docs/llms.txt index 9ca21c27..cd94c3a8 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -10,7 +10,7 @@ _Generated by agent-docs from tsup.config `entry`; 97 entries. Regenerate with ` - [`./app-auth`](api/app-auth.md): 11 exports — AppAuth, AppAuthConfig, AppAuthEmailClient, AppAuthEmailConfig, AppAuthInstance, AppAuthSchema, AppAuthSession, AppAuthSocialConfig, … - [`./assets`](api/assets.md): 44 exports — ApprovalEvent, ApprovalEventSchema, AssetContentMap, assetCreateJsonSchema, AssetFormat, AssetSpec, AssetStatus, AssetVariant, … - [`./assistant`](api/assistant.md): 56 exports — adaptTranscript, AssistantChat, AssistantClient, AssistantClientConfig, AssistantClientInputError, AssistantClientProvider, AssistantDeliveryMode, AssistantDock, … -- [`./billing`](api/billing.md): 33 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyCreateInput, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, DurableWorkspaceKeyProvisioner, … +- [`./billing`](api/billing.md): 34 exports — createIdentityBoundWorkspaceKeyManager, createPlatformBalanceManager, createTcloudKeyProvisioner, createWorkspaceKeyManager, DurableWorkspaceKeyConditionalWrites, DurableWorkspaceKeyCreateInput, DurableWorkspaceKeyManager, DurableWorkspaceKeyManagerOptions, … - [`./brand`](api/brand.md): 5 exports — BrandHeader, BrandHeaderProps, Logo, LogoProps, TangleKnot - [`./brand-extraction`](api/brand-extraction.md): 19 exports — BrandColor, BrandExtractionResult, BrandFont, BrandImage, BrandKit, BrandLogoCandidate, decideBrandKit, DecidedBrandKit, … - [`./catalog`](api/catalog.md): 10 exports — buildCatalog, CatalogModel, catalogModelForId, fetchModelCatalog, isChatCapableModel, ModelCatalog, normalizeModelId, resolveCatalogModelId, … diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 1e41c3bb..15c98b4d 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -97,6 +97,20 @@ export interface DurableWorkspaceKeyProvisioner { }): Promise> } +/** Optional compare-and-set lifecycle writes for stores that support fencing. */ +export interface DurableWorkspaceKeyConditionalWrites { + /** Return false when the row was no longer provisioning. */ + markProvisioningRemote(input: { id: string; keyId: string }): Promise + /** Return false when the row was no longer provisioning. */ + markActive(input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }): Promise +} + /** Persistence operations for identity-bound key lifecycle state. */ export interface DurableWorkspaceKeyStore { /** Return the active row for this exact product scope. */ @@ -105,24 +119,21 @@ export interface DurableWorkspaceKeyStore { listProvisioning(scope: DurableWorkspaceKeyScope): Promise /** Insert before the remote create so a crashed create can be recovered. */ insertProvisioning(record: DurableWorkspaceKeyProvisioningRecord): Promise - /** - * Save the remote id only while the row remains in provisioning state. - * Return false when the conditional update matched no provisioning row. - * Legacy stores may return undefined after a successful write. - */ - markProvisioningRemote(input: { id: string; keyId: string }): Promise - /** - * Promote a fully encrypted row only from provisioning state. - * Return false when the conditional update matched no provisioning row. - * Legacy stores may return undefined after a successful write. - */ + /** Save the remote id while the row remains in provisioning state. */ + markProvisioningRemote(input: { id: string; keyId: string }): Promise + /** Promote a fully encrypted row to active state. */ markActive(input: { id: string keyId: string keyEncrypted: string expiresAt: Date budgetUsd: number - }): Promise + }): Promise + /** + * Optional fenced writes. The manager uses false to compensate a stale + * creator. Stores without this field retain the pre-fence behavior. + */ + conditionalWrites?: DurableWorkspaceKeyConditionalWrites /** Keep a failed cleanup visible and schedule a later retry. Terminal rows are immutable. */ markRevocationPending(input: { id: string @@ -317,8 +328,28 @@ function idempotencyKeyForRecord(row: Pick { + if (store.conditionalWrites) return store.conditionalWrites.markProvisioningRemote(input) + await store.markProvisioningRemote(input) + return true +} + +async function markActive( + store: DurableWorkspaceKeyStore, + input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }, +): Promise { + if (store.conditionalWrites) return store.conditionalWrites.markActive(input) + await store.markActive(input) + return true } function usageFromRemote( @@ -808,7 +839,7 @@ export function createIdentityBoundWorkspaceKeyManager( if (!remoteId || !secret) { if (remoteId) { try { - await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) + await markProvisioningRemote(options.store, { id: rowId, keyId: remoteId }) } catch { // revokePending uses the returned id even when this state write fails. } @@ -826,8 +857,8 @@ export function createIdentityBoundWorkspaceKeyManager( onRemoteKey?.(provisioningRow, remoteId) try { - const recorded = await options.store.markProvisioningRemote({ id: rowId, keyId: remoteId }) - if (!lifecycleWriteSucceeded(recorded)) { + const recorded = await markProvisioningRemote(options.store, { id: rowId, keyId: remoteId }) + if (!recorded) { throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') } const keyEncrypted = await options.crypto.encrypt(secret) @@ -836,14 +867,14 @@ export function createIdentityBoundWorkspaceKeyManager( if (!Number.isFinite(activeExpiresAt.getTime())) throw new Error('remote child key expiry is malformed') const activeBudgetUsd = created.budgetUsd ?? budgetUsd if (!Number.isFinite(activeBudgetUsd) || activeBudgetUsd < 0) throw new Error('remote child key budget is malformed') - const activated = await options.store.markActive({ + const activated = await markActive(options.store, { id: rowId, keyId: remoteId, keyEncrypted, expiresAt: activeExpiresAt, budgetUsd: activeBudgetUsd, }) - if (!lifecycleWriteSucceeded(activated)) { + if (!activated) { throw new Error('workspace child-key provisioning row was retired before activation') } } catch (error) { diff --git a/src/billing/index.ts b/src/billing/index.ts index 98cd90df..b603f5e3 100644 --- a/src/billing/index.ts +++ b/src/billing/index.ts @@ -404,6 +404,7 @@ export { type DurableWorkspaceKeyManager, type DurableWorkspaceKeyManagerOptions, type DurableWorkspaceKeyCreateInput, + type DurableWorkspaceKeyConditionalWrites, type DurableWorkspaceKeyProvisioner, type DurableWorkspaceKeyProvisioningRecord, type DurableWorkspaceKeyRecord, diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 09f58072..fa15cf37 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { createIdentityBoundWorkspaceKeyManager, + type DurableWorkspaceKeyConditionalWrites, type DurableWorkspaceKeyManager, type DurableWorkspaceKeyProvisioner, type DurableWorkspaceKeyRecord, @@ -49,6 +50,35 @@ function makeHarness() { let createStartedResolve: (() => void) | null = null const createInputs: Array<{ name: string; product: string; budgetUsd: number; expiresAt: string; idempotencyKey: string }> = [] + const markProvisioningRemote = async (input: { id: string; keyId: string }): Promise => { + const row = rows.get(input.id) + if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return false + row.keyId = input.keyId + return true + } + const markActive = async (input: { + id: string + keyId: string + keyEncrypted: string + expiresAt: Date + budgetUsd: number + }): Promise => { + const row = rows.get(input.id) + if (!row) throw new Error(`unknown row ${input.id}`) + if (row.status !== 'provisioning') return false + row.keyId = input.keyId + row.keyEncrypted = input.keyEncrypted + row.expiresAt = input.expiresAt + row.budgetUsd = input.budgetUsd + row.status = 'active' + return true + } + const conditionalWrites: DurableWorkspaceKeyConditionalWrites = { + markProvisioningRemote, + markActive, + } + const store: DurableWorkspaceKeyStore = { async getActive(scope) { return [...rows.values()] @@ -69,23 +99,12 @@ function makeHarness() { rows.set(record.id, { ...record }) }, async markProvisioningRemote(input) { - const row = rows.get(input.id) - if (!row) throw new Error(`unknown row ${input.id}`) - if (row.status !== 'provisioning') return false - row.keyId = input.keyId - return true + await markProvisioningRemote(input) }, async markActive(input) { - const row = rows.get(input.id) - if (!row) throw new Error(`unknown row ${input.id}`) - if (row.status !== 'provisioning') return false - row.keyId = input.keyId - row.keyEncrypted = input.keyEncrypted - row.expiresAt = input.expiresAt - row.budgetUsd = input.budgetUsd - row.status = 'active' - return true + await markActive(input) }, + conditionalWrites, async markRevocationPending(input) { if (failMarkPending) throw new Error('state store unavailable') const row = rows.get(input.id) @@ -286,8 +305,9 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { it('keeps legacy void lifecycle stores compatible', async () => { const h = makeHarness() + const { conditionalWrites: _conditionalWrites, ...legacyMethods } = h.store const legacyStore: DurableWorkspaceKeyStore = { - ...h.store, + ...legacyMethods, async markProvisioningRemote(input) { await h.store.markProvisioningRemote(input) }, From 82a46c8f391c3d8bad819d0c56f36bfd11b20848 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 21:00:39 -0600 Subject: [PATCH 05/10] test(billing): cover legacy recovery probes --- tests/billing-identity-bound.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index fa15cf37..bbaf2189 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -415,6 +415,23 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(result.usage.keyId).toBe('remote-2') }) + it('recovers a legacy empty row when the probe returns no remote id', async () => { + const h = makeHarness() + const row = h.insertProvisioning({ idempotencyKey: null }) + h.setReturnMissingId(true) + + const result = await h.manager('router').ensureKey(h.identity()) + const inputs = h.getCreateInputs() + + expect(inputs[0]).toMatchObject({ + name: row.name, + idempotencyKey: `workspace-key:${row.id}`, + }) + expect(h.remote.get('orphan-1')?.revoked).toBe(true) + expect(h.rows.get(row.id)?.status).toBe('revoked') + expect(result.usage.keyId).toBe('remote-2') + }) + it('retries an empty pending row when the caller supplies its full identity', async () => { const h = makeHarness() const row = h.insertProvisioning() From 6364bf5be479e488ddceefef52e7ec198aa6d9bf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 21:03:11 -0600 Subject: [PATCH 06/10] test(billing): assert probe identity separation --- tests/billing-identity-bound.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index bbaf2189..4def664b 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -427,6 +427,7 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { name: row.name, idempotencyKey: `workspace-key:${row.id}`, }) + expect(inputs[1]?.idempotencyKey).not.toBe(inputs[0]?.idempotencyKey) expect(h.remote.get('orphan-1')?.revoked).toBe(true) expect(h.rows.get(row.id)?.status).toBe('revoked') expect(result.usage.keyId).toBe('remote-2') From f8c368000d16eeeebb3a181ba0aefaf5bc055b24 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Wed, 2 Sep 2026 21:29:25 -0600 Subject: [PATCH 07/10] fix(billing): preserve released record shape --- src/billing/identity-bound.ts | 4 ++-- tests/billing-identity-bound.test.ts | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 15c98b4d..5f30c8db 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -41,8 +41,8 @@ export interface DurableWorkspaceKeyRecord { sourceKeyFingerprint: string /** The persisted name used to recover a remote create after a crash. Null only for pre-name rows. */ name: string | null - /** The persisted retry identity for the remote create. Null only for pre-identity rows. */ - idempotencyKey: string | null + /** The persisted retry identity for the remote create. Missing on rows written before 0.46.55. */ + idempotencyKey?: string | null keyId: string keyEncrypted: string budgetUsd: number diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 4def664b..5cd513eb 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -415,9 +415,10 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(result.usage.keyId).toBe('remote-2') }) - it('recovers a legacy empty row when the probe returns no remote id', async () => { + it('recovers a legacy empty row without a persisted retry identity', async () => { const h = makeHarness() - const row = h.insertProvisioning({ idempotencyKey: null }) + const row = h.insertProvisioning() + delete row.idempotencyKey h.setReturnMissingId(true) const result = await h.manager('router').ensureKey(h.identity()) From afe3142b27f715025b373a533618b066e4e01db3 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 8 Sep 2026 18:07:38 -0700 Subject: [PATCH 08/10] fix(billing): require proven idempotency before recovery replay --- src/billing/identity-bound.ts | 29 ++++++------- tests/billing-identity-bound.test.ts | 61 +++++++++++++++++++++------- 2 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 5f30c8db..66dee3ba 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -77,11 +77,9 @@ export interface DurableWorkspaceKeyCreateInput { /** The remote operations required by the durable manager. */ export interface DurableWorkspaceKeyProvisioner { - /** - * Create one child key. Reusing `idempotencyKey` must be safe for the same - * request body. Providers without native idempotency still get crash-safe - * cleanup because the manager persists and searches the stable name. - */ + /** True only when replay returns the original child for the persisted request identity. */ + supportsIdempotentCreate?: true + /** Create one child key. Non-idempotent providers recover through stable-name discovery only. */ createKey(input: DurableWorkspaceKeyCreateInput): Promise getKey(id: string): Promise<{ budgetUsd?: number | null @@ -323,11 +321,6 @@ function isProvisioningId(value: string): boolean { return value.startsWith('provisioning:') } -function idempotencyKeyForRecord(row: Pick): string { - const value = row.idempotencyKey?.trim() - return value || `workspace-key:${row.id}` -} - async function markProvisioningRemote( store: DurableWorkspaceKeyStore, input: { id: string; keyId: string }, @@ -572,14 +565,16 @@ export function createIdentityBoundWorkspaceKeyManager( identity: WorkspaceKeyIdentity | undefined, name: string, ): Promise | null> { - if (!identity || !sameIdentity(row, identity, product)) return [] + const idempotencyKey = row.idempotencyKey?.trim() + if (!identity || !sameIdentity(row, identity, product) + || !options.provisioner.supportsIdempotentCreate || !idempotencyKey) return [] try { const created = await options.provisioner.createKey({ name, product: row.product, budgetUsd: row.budgetUsd, expiresAt: row.expiresAt.toISOString(), - idempotencyKey: idempotencyKeyForRecord(row), + idempotencyKey, }) const remoteId = created.id?.trim() if (remoteId) return [{ id: remoteId }] @@ -592,7 +587,7 @@ export function createIdentityBoundWorkspaceKeyManager( }) return null } - return (await provisioningCandidates({ ...row, name, keyId: `provisioning:${idempotencyKeyForRecord(row)}` }))?.candidates ?? [] + return (await provisioningCandidates({ ...row, name, keyId: `provisioning:${idempotencyKey}` }))?.candidates ?? [] } async function cleanupProvisioning(row: DurableWorkspaceKeyRecord, identity?: WorkspaceKeyIdentity): Promise { @@ -707,14 +702,13 @@ export function createIdentityBoundWorkspaceKeyManager( && (identity.workspaceId !== scopeInput.workspaceId || identity.ownerUserId !== scopeInput.ownerUserId)) { throw new Error('workspace child key retry identity does not match its scope') } - const activeIdentity = identity if (scopeInput) { const workspaceId = scopeInput.workspaceId.trim() const ownerUserId = scopeInput.ownerUserId.trim() if (!workspaceId || !ownerUserId) throw new Error('workspace child key retry scope is incomplete') const scope = scopeKey({ workspaceId, ownerUserId, product }) return withLocalLock(scope, () => withDurableLease(scope, async () => ( - await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId }, activeIdentity)).completed)) + await retryPendingRevocationsUnlocked({ workspaceId, ownerUserId }, identity)).completed)) } if (identity) { @@ -787,6 +781,7 @@ export function createIdentityBoundWorkspaceKeyManager( const name = (options.nameForIdentity?.(identity, mintOperationId) ?? defaultKeyName(identity, product, mintOperationId)).trim() if (!name) throw new Error('workspace child key remote name is required') const rowId = operationId() + const idempotencyKey = `workspace-key:${rowId}` const provisioningRow: DurableWorkspaceKeyProvisioningRecord = { id: rowId, workspaceId: identity.workspaceId, @@ -796,7 +791,7 @@ export function createIdentityBoundWorkspaceKeyManager( sourceKeyId: sourceKeyId(identity), sourceKeyFingerprint: identity.sourceKeyFingerprint, name, - idempotencyKey: `workspace-key:${rowId}`, + idempotencyKey, keyId: `provisioning:${mintOperationId}`, keyEncrypted: '', budgetUsd, @@ -816,7 +811,7 @@ export function createIdentityBoundWorkspaceKeyManager( product, budgetUsd, expiresAt: expiresAt.toISOString(), - idempotencyKey: idempotencyKeyForRecord(provisioningRow), + idempotencyKey, }) } catch (error) { // Keep cleanup running even when the state write fails. The remote diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 5cd513eb..85628ed8 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -157,9 +157,13 @@ function makeHarness() { }, } + const createdByIdentity = new Map() const provisioner: DurableWorkspaceKeyProvisioner = { + supportsIdempotentCreate: true, async createKey(input) { createInputs.push(input) + const previous = createdByIdentity.get(input.idempotencyKey) + if (previous) return { id: previous.id, key: previous.key, budgetUsd: previous.budgetUsd, expiresAt: previous.expiresAt } createStartedResolve?.() createStartedResolve = null if (createDelayMs > 0) await new Promise((resolve) => setTimeout(resolve, createDelayMs)) @@ -178,6 +182,7 @@ function makeHarness() { expiresAt: input.expiresAt, revoked: false, }) + createdByIdentity.set(input.idempotencyKey, remote.get(remoteId)!) const failure = createFailure createFailure = null returnMissingId = false @@ -396,6 +401,26 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(h.rows.get(row.id)?.status).toBe('revoked') }) + it('keeps an undiscovered committed child pending when creation is not idempotent', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + h.remote.set('hidden-original', { + id: 'hidden-original', key: 'original-secret', name: row.name!, product: row.product, + sourceKeyId: row.sourceKeyId, budgetUsd: 25, budgetSpent: 0, + expiresAt: row.expiresAt.toISOString(), revoked: false, + }) + const manager = h.manager('router', { + provisioner: { ...h.provisioner, supportsIdempotentCreate: undefined, findCreatedKeys: async () => [] }, + }) + await expect(manager.ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(0) + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') + expect(h.remote.get('hidden-original')?.revoked).toBe(false) + h.advance(60_000) + await h.manager('router').ensureKey(h.identity()) + expect(h.remote.get('hidden-original')?.revoked).toBe(true) + }) + it('probes an empty provisioning row with its persisted create identity', async () => { const h = makeHarness() const row = h.insertProvisioning() @@ -415,23 +440,29 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect(result.usage.keyId).toBe('remote-2') }) - it('recovers a legacy empty row without a persisted retry identity', async () => { + it('replays an idempotent create to revoke the original child hidden from discovery', async () => { const h = makeHarness() const row = h.insertProvisioning() - delete row.idempotencyKey - h.setReturnMissingId(true) + const original = await h.provisioner.createKey({ name: row.name!, product: row.product, + budgetUsd: row.budgetUsd, expiresAt: row.expiresAt.toISOString(), idempotencyKey: row.idempotencyKey! }) + const manager = h.manager('router', { + provisioner: { ...h.provisioner, findCreatedKeys: async () => [] }, + }) + const result = await manager.ensureKey(h.identity()) + expect(h.remote.get(original.id)?.revoked).toBe(true) + expect(h.remote.size).toBe(2) + expect(result.usage.keyId).not.toBe(original.id) + expect(h.getCreateInputs()[1]?.idempotencyKey).toBe(row.idempotencyKey) + }) - const result = await h.manager('router').ensureKey(h.identity()) - const inputs = h.getCreateInputs() + it('keeps a legacy empty row pending instead of inventing a new create identity', async () => { + const h = makeHarness() + const row = h.insertProvisioning() + delete row.idempotencyKey - expect(inputs[0]).toMatchObject({ - name: row.name, - idempotencyKey: `workspace-key:${row.id}`, - }) - expect(inputs[1]?.idempotencyKey).not.toBe(inputs[0]?.idempotencyKey) - expect(h.remote.get('orphan-1')?.revoked).toBe(true) - expect(h.rows.get(row.id)?.status).toBe('revoked') - expect(result.usage.keyId).toBe('remote-2') + await expect(h.manager('router').ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(0) + expect(h.rows.get(row.id)?.status).toBe('revocation_pending') }) it('retries an empty pending row when the caller supplies its full identity', async () => { @@ -734,7 +765,9 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { it('fails before remote spend when encryption is unavailable', async () => { const h = makeHarness() let creates = 0 - const provisioner: DurableWorkspaceKeyProvisioner = { + const createdByIdentity = new Map() + const provisioner: DurableWorkspaceKeyProvisioner = { + supportsIdempotentCreate: true, ...h.provisioner, async createKey(input) { creates += 1 From dae7dac0ed272fd67503a481272ad6c7f2e6fd6c Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 8 Sep 2026 18:19:43 -0700 Subject: [PATCH 09/10] test(billing): validate recovery fixture identity --- src/billing/identity-bound.ts | 2 +- tests/billing-identity-bound.test.ts | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index 66dee3ba..ed05ff8b 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -41,7 +41,7 @@ export interface DurableWorkspaceKeyRecord { sourceKeyFingerprint: string /** The persisted name used to recover a remote create after a crash. Null only for pre-name rows. */ name: string | null - /** The persisted retry identity for the remote create. Missing on rows written before 0.46.55. */ + /** The persisted retry identity for the remote create. Absent on legacy rows without a persisted create identity. */ idempotencyKey?: string | null keyId: string keyEncrypted: string diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 85628ed8..47482534 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -449,6 +449,7 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { provisioner: { ...h.provisioner, findCreatedKeys: async () => [] }, }) const result = await manager.ensureKey(h.identity()) + if (!original.id) throw new Error('Fixture create did not return its original id') expect(h.remote.get(original.id)?.revoked).toBe(true) expect(h.remote.size).toBe(2) expect(result.usage.keyId).not.toBe(original.id) @@ -765,9 +766,7 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { it('fails before remote spend when encryption is unavailable', async () => { const h = makeHarness() let creates = 0 - const createdByIdentity = new Map() - const provisioner: DurableWorkspaceKeyProvisioner = { - supportsIdempotentCreate: true, + const provisioner: DurableWorkspaceKeyProvisioner = { ...h.provisioner, async createKey(input) { creates += 1 From f393726bf22cba34da43dc7e57e18c73570b3226 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Tue, 8 Sep 2026 18:24:27 -0700 Subject: [PATCH 10/10] fix(billing): retain stale creator compensation for retry --- src/billing/identity-bound.ts | 22 ++++++++++++++--- tests/billing-identity-bound.test.ts | 37 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/billing/identity-bound.ts b/src/billing/identity-bound.ts index ed05ff8b..396ceb84 100644 --- a/src/billing/identity-bound.ts +++ b/src/billing/identity-bound.ts @@ -849,11 +849,26 @@ export function createIdentityBoundWorkspaceKeyManager( throw new Error('remote create returned no usable child key') } - onRemoteKey?.(provisioningRow, remoteId) + let cleanupRow = provisioningRow + onRemoteKey?.(cleanupRow, remoteId) + + async function retainRetiredCreatorCleanup(): Promise { + // A retired row cannot hold another pending transition. Persist the + // returned child separately before revocation so a failed revoke survives restart. + const retainedRow = { ...provisioningRow, id: operationId(), keyId: remoteId } + try { + await options.store.insertProvisioning(retainedRow) + } catch (cause) { + throw new Error('returned child cleanup record could not be persisted', { cause }) + } + cleanupRow = retainedRow + onRemoteKey?.(cleanupRow, remoteId) + } try { const recorded = await markProvisioningRemote(options.store, { id: rowId, keyId: remoteId }) if (!recorded) { + await retainRetiredCreatorCleanup() throw new Error('workspace child-key provisioning row was retired before the remote id was recorded') } const keyEncrypted = await options.crypto.encrypt(secret) @@ -870,16 +885,17 @@ export function createIdentityBoundWorkspaceKeyManager( budgetUsd: activeBudgetUsd, }) if (!activated) { + await retainRetiredCreatorCleanup() throw new Error('workspace child-key provisioning row was retired before activation') } } catch (error) { // A persistence failure must not strand the already-created remote key. try { - await markPending(rowId, error, false) + await markPending(cleanupRow.id, error, false) } catch { // revokePending below still has the remote id and attempts cleanup. } - await revokePending({ ...provisioningRow, keyId: remoteId, status: 'revocation_pending' }, identity) + await revokePending({ ...cleanupRow, keyId: remoteId, status: 'revocation_pending' }, identity) throw error } diff --git a/tests/billing-identity-bound.test.ts b/tests/billing-identity-bound.test.ts index 47482534..6077cb50 100644 --- a/tests/billing-identity-bound.test.ts +++ b/tests/billing-identity-bound.test.ts @@ -763,6 +763,43 @@ describe('createIdentityBoundWorkspaceKeyManager', () => { expect([...h.remote.values()][0]?.revoked).toBe(true) }) + it.each(['record-remote', 'activate'] as const)('retains failed cleanup after a retired creator loses %s fencing', async stage => { + const h = makeHarness() + h.setCreateDelay(20) + const manager = h.manager('router') + const inFlight = manager.ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find(candidate => candidate.status === 'provisioning')! + if (stage === 'record-remote') row.status = 'revoked' + else h.store.conditionalWrites!.markActive = async () => { row.status = 'revoked'; return false } + h.failRevocationFor('remote-1') + await expect(inFlight).rejects.toThrow('retired before') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + expect([...h.rows.values()].some(candidate => candidate.status === 'revocation_pending' + && candidate.keyId === 'remote-1')).toBe(true) + expect(row.status).toBe('revoked') + const restarted = h.manager('router') + await expect(restarted.ensureKey(h.identity())).rejects.toThrow('cleanup is pending') + expect(h.getCreateInputs()).toHaveLength(1) + h.advance(60_000) + expect(await restarted.retryPendingRevocations()).toBe(1) + expect(h.remote.get('remote-1')?.revoked).toBe(true) + }) + + it('reports a failed compensation write when both durable storage and revocation are unavailable', async () => { + const h = makeHarness() + h.setCreateDelay(20) + const inFlight = h.manager('router').ensureKey(h.identity()) + await h.waitForCreateStart() + const row = [...h.rows.values()].find(candidate => candidate.status === 'provisioning')! + row.status = 'revoked' + h.store.insertProvisioning = async () => { throw new Error('storage unavailable') } + h.failRevocationFor('remote-1') + await expect(inFlight).rejects.toThrow('cleanup record could not be persisted') + expect(h.remote.get('remote-1')?.revoked).toBe(false) + expect([...h.rows.values()].some(candidate => candidate.status === 'revocation_pending')).toBe(false) + }) + it('fails before remote spend when encryption is unavailable', async () => { const h = makeHarness() let creates = 0