From 683b30e41febe1e2d9b9eeccc64d7f7185faaf7d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 13:16:59 -0600 Subject: [PATCH 001/327] fix(powersync): serialize tracking startup --- .../powersync-db-collection/src/powersync.ts | 48 +++++++- .../tests/on-demand-sync.test.ts | 103 ++++++++++++++++++ 2 files changed, 148 insertions(+), 3 deletions(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 76b8dedd3..af01a25a7 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -323,6 +323,7 @@ function createPowerSyncCollectionConfig< let disposeTracking: | ((options?: { context?: LockContext }) => Promise) | null = null + let trackingSetup: Promise | null = null if (syncMode === `eager`) { return runEagerSync() @@ -337,6 +338,13 @@ function createPowerSyncCollectionConfig< async function safelyDisposeTracking( context?: LockContext, ): Promise { + // Cleanup can race trigger creation. Wait until the disposer has been + // published so an abort cannot strand a freshly-created trigger. + const setup = trackingSetup + if (setup) { + await setup.catch(() => undefined) + } + const dispose = disposeTracking if (!dispose) { return @@ -346,6 +354,25 @@ function createPowerSyncCollectionConfig< await dispose(context ? { context } : undefined) } + async function establishTracking( + options: Parameters[0], + appliedReceipts: Array, + ): Promise { + const setup = (async () => { + const dispose = await createDiffTrigger(options, appliedReceipts) + disposeTracking = dispose + })() + trackingSetup = setup + + try { + await setup + } finally { + if (trackingSetup === setup) { + trackingSetup = null + } + } + } + async function createDiffTrigger( options: { setupContext?: LockContext @@ -398,6 +425,17 @@ function createPowerSyncCollectionConfig< } async function flushDiffRecords(): Promise { + // PowerSync can notify after creating the tracking table but before its + // create call returns. Preserve that notification until the disposer, + // which proves the trigger is usable, has been published. + const setup = trackingSetup + if (setup) { + await setup.catch(() => undefined) + } + if (!disposeTracking) { + return + } + const ignoredReceipts: Array = [] await database .writeTransaction(async (context) => { @@ -515,7 +553,7 @@ function createPowerSyncCollectionConfig< onUnload = await restConfig.onLoad?.() const appliedReceipts: Array = [] - disposeTracking = await createDiffTrigger( + await establishTracking( { // Initial eager hydration must make the source usable before // PowerSync can persist a mutation queued during startup. @@ -567,7 +605,8 @@ function createPowerSyncCollectionConfig< let stopped = false const hasStopped = () => stopped - start().catch((error) => + const startup = start() + void startup.catch((error) => database.logger.error( `Could not start syncing process for ${viewName} into ${trackedTableName}`, error, @@ -581,6 +620,9 @@ function createPowerSyncCollectionConfig< const loadSubset = async ( options?: LoadSubsetOptions, ): Promise => { + if (hasStopped()) return + // Never create a trigger that has no observer to drain its diff table. + await startup if (hasStopped()) return const appliedReceipts: Array = [] @@ -642,7 +684,7 @@ function createPowerSyncCollectionConfig< await flushDiffRecordsWithContext(ctx, appliedReceipts) await safelyDisposeTracking(ctx) - disposeTracking = await createDiffTrigger( + await establishTracking( { setupContext: ctx, when: { diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 8d1dc3412..bfc798c1d 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -12,6 +12,7 @@ import { lt, or, } from '@tanstack/db' +import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' @@ -2229,6 +2230,108 @@ describe(`On-Demand Sync Mode`, () => { }) } + it(`flushes eager changes that arrive before the tracking handle is published`, async () => { + const db = await createDatabase() + await createTestProducts(db) + + let flushTrackingChanges: + | ((event: { changedTables: Array }) => Promise | void) + | undefined + vi.spyOn(db, `onChangeWithCallback`).mockImplementation((handler) => { + flushTrackingChanges = handler?.onChange + return () => {} + }) + + const triggerCreated = pDefer() + const publishTrackingHandle = pDefer() + const createDiffTrigger = db.triggers.createDiffTrigger.bind(db.triggers) + vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation( + async (options) => { + const dispose = await createDiffTrigger(options) + triggerCreated.resolve() + await publishTrackingHandle.promise + return dispose + }, + ) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + }), + ) + onTestFinished(() => collection.cleanup()) + + await triggerCreated.promise + await db.execute(` + INSERT INTO products (id, name, price, category) + VALUES ('during-startup', 'During startup', 300, 'electronics') + `) + + expect(flushTrackingChanges).toBeDefined() + const flush = Promise.resolve( + flushTrackingChanges!({ + changedTables: [collection.utils.getMeta().trackedTableName], + }), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + publishTrackingHandle.resolve() + await Promise.all([flush, collection.stateWhenReady()]) + + expect(collection.get(`during-startup`)?.name).toBe(`During startup`) + }) + + it(`does not create tracking when change observation fails to start`, async () => { + const db = await createDatabase() + const startupError = new Error(`change observation failed`) + vi.spyOn(db.logger, `error`).mockImplementation(() => {}) + vi.spyOn(console, `error`).mockImplementation(() => {}) + vi.spyOn(db, `onChangeWithCallback`).mockImplementation(() => { + throw startupError + }) + const createDiffTrigger = vi.spyOn(db.triggers, `createDiffTrigger`) + + const collection = makeCollection(db) + onTestFinished(() => collection.cleanup()) + await collection.stateWhenReady() + + const query = categoryQuery(collection, `electronics`) + onTestFinished(() => query.cleanup()) + + await expect(query.preload()).rejects.toBe(startupError) + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + + it(`disposes tracking that finishes starting during collection cleanup`, async () => { + const db = await createDatabase() + vi.spyOn(db, `onChangeWithCallback`).mockImplementation(() => () => {}) + + const triggerStarted = pDefer() + const finishTrigger = pDefer() + const dispose = vi.fn(async () => {}) + vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation(async () => { + triggerStarted.resolve() + await finishTrigger.promise + return dispose + }) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + }), + ) + + await triggerStarted.promise + collection.cleanup() + finishTrigger.resolve() + + await vi.waitFor(() => { + expect(dispose).toHaveBeenCalledTimes(1) + }) + }) + it(`should start tracking again when a subset is loaded after every subset was unloaded`, async () => { const db = await createDatabase() await createTestProducts(db) From 3ce3e5499472555e4cbc360b4997f83dcb4dff16 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 13:32:38 -0600 Subject: [PATCH 002/327] fix(powersync): cancel suspended startup work --- .../powersync-db-collection/src/powersync.ts | 13 +++++- .../tests/load-hooks.test.ts | 29 ++++++++++++ .../tests/on-demand-sync.test.ts | 45 +++++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index af01a25a7..55e1184b5 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -550,7 +550,12 @@ function createPowerSyncCollectionConfig< let onUnload: CleanupFn | void | null = null start(async () => { - onUnload = await restConfig.onLoad?.() + const cleanup = await restConfig.onLoad?.() + if (abortController.signal.aborted) { + cleanup?.() + return + } + onUnload = cleanup const appliedReceipts: Array = [] await establishTracking( @@ -624,6 +629,12 @@ function createPowerSyncCollectionConfig< // Never create a trigger that has no observer to drain its diff table. await startup if (hasStopped()) return + if ( + options && + (releasedSubsets.has(options) || options.signal?.aborted) + ) { + return + } const appliedReceipts: Array = [] if (options) { diff --git a/packages/powersync-db-collection/tests/load-hooks.test.ts b/packages/powersync-db-collection/tests/load-hooks.test.ts index cc428816e..b5094f615 100644 --- a/packages/powersync-db-collection/tests/load-hooks.test.ts +++ b/packages/powersync-db-collection/tests/load-hooks.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' import { createCollection, createLiveQueryCollection, eq } from '@tanstack/db' +import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' @@ -91,6 +92,34 @@ describe(`Sync Streams`, () => { expect(collection.status).toBe(`error`) }) + it(`eager mode: releases a load hook that resolves after cleanup`, async () => { + const db = await createDatabase() + const releaseLoad = pDefer() + const loadStarted = pDefer() + const cleanupLoad = vi.fn() + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(async () => {}) + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + onLoad: async () => { + loadStarted.resolve() + await releaseLoad.promise + return cleanupLoad + }, + }), + ) + + await loadStarted.promise + collection.cleanup() + releaseLoad.resolve() + + await vi.waitFor(() => expect(cleanupLoad).toHaveBeenCalledOnce()) + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + it(`on-demand mode: should call onLoadSubset/onUnloadSubset for each live query`, async () => { const db = await createDatabase() await createTestProducts(db) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index bfc798c1d..dc5aa20a0 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2230,6 +2230,51 @@ describe(`On-Demand Sync Mode`, () => { }) } + it(`does not acquire a subset released while tracking startup is suspended`, async () => { + const db = await createDatabase() + const onLoadSubset = vi.fn() + const createDiffTrigger = vi.spyOn(db.triggers, `createDiffTrigger`) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const abortController = new AbortController() + const request = { + where: eq(`category`, `electronics`), + signal: abortController.signal, + } + const load = sync.loadSubset(request) + + // Release the request before start() crosses its first async boundary. + abortController.abort() + sync.unloadSubset?.(request) + + try { + await load + + expect(onLoadSubset).not.toHaveBeenCalled() + expect(createDiffTrigger).not.toHaveBeenCalled() + } finally { + sync.cleanup?.() + } + }) + it(`flushes eager changes that arrive before the tracking handle is published`, async () => { const db = await createDatabase() await createTestProducts(db) From 040ebc8e8013c5222d905f9d7f6eb6666eb64899 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 14:02:25 -0600 Subject: [PATCH 003/327] docs: add PowerSync startup changeset --- .changeset/fix-powersync-tracking-startup.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-powersync-tracking-startup.md diff --git a/.changeset/fix-powersync-tracking-startup.md b/.changeset/fix-powersync-tracking-startup.md new file mode 100644 index 000000000..eeb52f8a5 --- /dev/null +++ b/.changeset/fix-powersync-tracking-startup.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': patch +--- + +Serialize PowerSync tracking startup so changes and cleanup cannot race an unpublished trigger, and cancel subset or load-hook work released while startup is suspended. From 4fb485a8c18b737ec60db260c07c1207974115e7 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:05:01 +0000 Subject: [PATCH 004/327] ci: apply automated fixes --- .../tests/on-demand-sync.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index dc5aa20a0..5e5b8f7c1 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2355,11 +2355,13 @@ describe(`On-Demand Sync Mode`, () => { const triggerStarted = pDefer() const finishTrigger = pDefer() const dispose = vi.fn(async () => {}) - vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation(async () => { - triggerStarted.resolve() - await finishTrigger.promise - return dispose - }) + vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation( + async () => { + triggerStarted.resolve() + await finishTrigger.promise + return dispose + }, + ) const collection = createCollection( powerSyncCollectionOptions({ From 1ac77eb372026dff74c09bb0fa707d5cd012071d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 14:56:02 -0600 Subject: [PATCH 005/327] fix(electric): cancel bounded refresh waits --- .../electric-db-collection/src/electric.ts | 30 ++++- .../tests/electric.test.ts | 106 ++++++++++++++++++ 2 files changed, 134 insertions(+), 2 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 39e6540a6..1066be217 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -625,10 +625,34 @@ function createLoadSubsetDedupe>({ // long-poll requests promptly. Bound the wait so on-demand live queries don't // remain loading until the long-poll naturally times out. // If the refresh fails or times out, we fall through to requestSnapshot which - // still works. + // still works. Cleanup or request cancellation ends the wait without starting + // a snapshot that no current demand can use. if (stream.isUpToDate) { let timeoutId: ReturnType | undefined + let removeAbortListeners = () => {} try { + const abortSignals = new Set( + [signal, opts.signal].filter( + (candidate): candidate is AbortSignal => candidate !== undefined, + ), + ) + const aborted = new Promise((resolve) => { + const onAbort = () => resolve() + if (Array.from(abortSignals).some((candidate) => candidate.aborted)) { + resolve() + return + } + + abortSignals.forEach((candidate) => + candidate.addEventListener(`abort`, onAbort, { once: true }), + ) + removeAbortListeners = () => { + abortSignals.forEach((candidate) => + candidate.removeEventListener(`abort`, onAbort), + ) + } + }) + await Promise.race([ stream.forceDisconnectAndRefresh(), new Promise((resolve) => { @@ -637,6 +661,7 @@ function createLoadSubsetDedupe>({ FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS, ) }), + aborted, ]) } catch (error) { if (handleSnapshotError(error, `forceDisconnectAndRefresh`)) { @@ -647,11 +672,12 @@ function createLoadSubsetDedupe>({ error, ) } finally { + removeAbortListeners() clearTimeout(timeoutId) } } - if (opts.signal?.aborted) return + if (signal.aborted || opts.signal?.aborted) return // Upstream limitation: ShapeStream.requestSnapshot() publishes its rows // through the stream callback before its Promise resolves. It accepts no diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index c913f8d97..6afe302d0 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2896,6 +2896,112 @@ describe(`Electric Integration`, () => { } }) + it(`should cancel a pending refresh wait when the collection is cleaned up`, async () => { + vi.useFakeTimers() + let resolveRefresh: () => void = () => {} + const refresh = new Promise((resolve) => { + resolveRefresh = resolve + }) + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-cleanup-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + let loadSettled = false + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ).then(() => { + loadSettled = true + }) + + await Promise.resolve() + await testCollection.cleanup() + await vi.advanceTimersByTimeAsync(0) + + expect(loadSettled).toBe(true) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + + resolveRefresh() + await refresh + await load + expect(mockRequestSnapshot).not.toHaveBeenCalled() + } finally { + resolveRefresh() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + + it(`should retry a refresh wait after the requesting demand is aborted`, async () => { + vi.useFakeTimers() + let resolveRefresh: () => void = () => {} + const refresh = new Promise((resolve) => { + resolveRefresh = resolve + }) + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-abort-retry-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const abortController = new AbortController() + let abortedLoadSettled = false + const abortedLoad = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ).then(() => { + abortedLoadSettled = true + }) + + await Promise.resolve() + abortController.abort() + await vi.advanceTimersByTimeAsync(0) + + expect(abortedLoadSettled).toBe(true) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + await testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(2) + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + await testCollection.cleanup() + await abortedLoad + } finally { + resolveRefresh() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + it(`should clear the refresh timeout when refresh settles early`, async () => { vi.useFakeTimers() try { From a14459bd47fc24c45a03dfb8469aba6a554bd31f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 15:02:40 -0600 Subject: [PATCH 006/327] test(electric): simplify refresh lifecycle setup --- .../tests/electric.test.ts | 62 ++++++++----------- 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 6afe302d0..3abd7ccbf 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2659,6 +2659,20 @@ describe(`Electric Integration`, () => { // Tests for syncMode configuration describe(`syncMode configuration`, () => { + const createOnDemandCollection = (id: string) => + createCollection( + electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + it(`should not request snapshots during subscription in eager mode`, () => { vi.clearAllMocks() @@ -2898,26 +2912,14 @@ describe(`Electric Integration`, () => { it(`should cancel a pending refresh wait when the collection is cleaned up`, async () => { vi.useFakeTimers() - let resolveRefresh: () => void = () => {} - const refresh = new Promise((resolve) => { - resolveRefresh = resolve - }) + const refresh = createDeferred() try { mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) - const testCollection = createCollection( - electricCollectionOptions({ - id: `on-demand-refresh-cleanup-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - }, - syncMode: `on-demand`, - getKey: (item: Row) => item.id as number, - startSync: true, - }), + const testCollection = createOnDemandCollection( + `on-demand-refresh-cleanup-test`, ) let loadSettled = false @@ -2935,12 +2937,12 @@ describe(`Electric Integration`, () => { expect(mockRequestSnapshot).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) - resolveRefresh() - await refresh + refresh.resolve() + await refresh.promise await load expect(mockRequestSnapshot).not.toHaveBeenCalled() } finally { - resolveRefresh() + refresh.resolve() await vi.runOnlyPendingTimersAsync() vi.useRealTimers() } @@ -2948,26 +2950,14 @@ describe(`Electric Integration`, () => { it(`should retry a refresh wait after the requesting demand is aborted`, async () => { vi.useFakeTimers() - let resolveRefresh: () => void = () => {} - const refresh = new Promise((resolve) => { - resolveRefresh = resolve - }) + const refresh = createDeferred() try { mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) - const testCollection = createCollection( - electricCollectionOptions({ - id: `on-demand-refresh-abort-retry-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - }, - syncMode: `on-demand`, - getKey: (item: Row) => item.id as number, - startSync: true, - }), + const testCollection = createOnDemandCollection( + `on-demand-refresh-abort-retry-test`, ) const abortController = new AbortController() let abortedLoadSettled = false @@ -2996,7 +2986,7 @@ describe(`Electric Integration`, () => { await testCollection.cleanup() await abortedLoad } finally { - resolveRefresh() + refresh.resolve() await vi.runOnlyPendingTimersAsync() vi.useRealTimers() } From 37e69fb1eb9858956c00ce1c737786cd33cf2921 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 25 Aug 2026 15:03:16 -0600 Subject: [PATCH 007/327] chore: add Electric refresh cancellation changeset --- .changeset/cancel-electric-refresh-wait.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/cancel-electric-refresh-wait.md diff --git a/.changeset/cancel-electric-refresh-wait.md b/.changeset/cancel-electric-refresh-wait.md new file mode 100644 index 000000000..311ce222a --- /dev/null +++ b/.changeset/cancel-electric-refresh-wait.md @@ -0,0 +1,5 @@ +--- +'@tanstack/electric-db-collection': patch +--- + +Cancel an on-demand refresh wait when its request or collection is cleaned up, preventing snapshots from starting after teardown. From 01f187b4d3372450063a9ae4a1894cfae96718b4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 07:32:02 -0600 Subject: [PATCH 008/327] fix(electric): skip aborted refresh startup --- .../electric-db-collection/src/electric.ts | 6 ++-- .../tests/electric.test.ts | 35 ++++++++++++++++++- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 1066be217..e0686a8b0 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -579,7 +579,9 @@ function createLoadSubsetDedupe>({ const loadSubset = async (opts: LoadSubsetOptions) => { const commitCursor = getCommitCursor() - if (opts.signal?.aborted) return + const isAborted = (): boolean => + signal.aborted || opts.signal?.aborted === true + if (isAborted()) return if (isBufferingInitialSync()) { const snapshotParams = compileSQL(opts, compileOptions) @@ -677,7 +679,7 @@ function createLoadSubsetDedupe>({ } } - if (signal.aborted || opts.signal?.aborted) return + if (isAborted()) return // Upstream limitation: ShapeStream.requestSnapshot() publishes its rows // through the stream callback before its Promise resolves. It accepts no diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 3abd7ccbf..c804bcdf1 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2659,7 +2659,15 @@ describe(`Electric Integration`, () => { // Tests for syncMode configuration describe(`syncMode configuration`, () => { - const createOnDemandCollection = (id: string) => + const createOnDemandCollection = ( + id: string, + ): Collection< + Row, + number, + ElectricCollectionUtils, + StandardSchemaV1, + Row + > => createCollection( electricCollectionOptions({ id, @@ -2948,6 +2956,31 @@ describe(`Electric Integration`, () => { } }) + it(`does not start a refresh when the collection signal is already aborted`, async () => { + mockStream.isUpToDate = true + const abortController = new AbortController() + abortController.abort() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-already-aborted-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: abortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + await testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() + expect(mockRequestSnapshot).not.toHaveBeenCalled() + await testCollection.cleanup() + }) + it(`should retry a refresh wait after the requesting demand is aborted`, async () => { vi.useFakeTimers() const refresh = createDeferred() From eac96d1abd84f3a99fda24a0811ea1ef41978d7e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 07:41:19 -0600 Subject: [PATCH 009/327] test(electric): preserve inferred collection key type --- packages/electric-db-collection/tests/electric.test.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index c804bcdf1..96f549fa0 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2659,15 +2659,7 @@ describe(`Electric Integration`, () => { // Tests for syncMode configuration describe(`syncMode configuration`, () => { - const createOnDemandCollection = ( - id: string, - ): Collection< - Row, - number, - ElectricCollectionUtils, - StandardSchemaV1, - Row - > => + const createOnDemandCollection = (id: string) => createCollection( electricCollectionOptions({ id, From 86e2cece5e7ef11b4f2292c847a4ed8e6efc0831 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 09:22:35 -0600 Subject: [PATCH 010/327] fix(powersync): make subset lifecycles atomic --- .../powersync-db-collection/src/powersync.ts | 321 +++++++++++------- .../tests/on-demand-sync.test.ts | 273 +++++++++++++++ 2 files changed, 480 insertions(+), 114 deletions(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 55e1184b5..c5bac6fcd 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -605,9 +605,24 @@ function createPowerSyncCollectionConfig< // On-demand mode. // Registers a diff trigger for the active WHERE expressions. function runOnDemandSync() { - const unloadSubsetCallbacks = new Map() + type DemandRecord = { + options: LoadSubsetOptions + state: `provisional` | `active` | `released` | `failed` + cleanup?: CleanupFn + } + type PendingRelease = { + options: LoadSubsetOptions + failures: number + } + + const demands = new Map() const releasedSubsets = new WeakSet() + const pendingReleases: Array = [] let stopped = false + let lifecycleGeneration = 0 + let trackingRevision = 0 + let drainingReleases = false + let releaseRetryTimer: ReturnType | undefined const hasStopped = () => stopped const startup = start() @@ -618,82 +633,45 @@ function createPowerSyncCollectionConfig< ), ) - // Tracks all active WHERE expressions for on-demand sync filtering. - // Each loadSubset call pushes its predicate; unloadSubset removes it. - const activeWhereExpressions: Array = [] - - const loadSubset = async ( - options?: LoadSubsetOptions, - ): Promise => { - if (hasStopped()) return - // Never create a trigger that has no observer to drain its diff table. - await startup - if (hasStopped()) return - if ( - options && - (releasedSubsets.has(options) || options.signal?.aborted) - ) { - return - } + const activeWhereExpressions = () => + Array.from(demands.values()) + .filter((demand) => demand.state === `active`) + .map((demand) => demand.options.where) + + const rebuildTracking = async (): Promise => { + const generation = lifecycleGeneration + const revision = trackingRevision + const isCurrent = () => + !hasStopped() && + lifecycleGeneration === generation && + trackingRevision === revision const appliedReceipts: Array = [] - if (options) { - activeWhereExpressions.push(options.where) - const cleanup = await restConfig.onLoadSubset?.(options) - if (hasStopped()) { - cleanup?.() - return - } - if (cleanup) { - if (releasedSubsets.has(options) || options.signal?.aborted) { - cleanup() - } else { - unloadSubsetCallbacks.set(options, cleanup) - } - } - } - - // No predicates remain, so stop tracking entirely. Both calls are no-ops - // when no tracking table is currently active. - if (activeWhereExpressions.length === 0) { - await database.writeLock(async (ctx) => { - await flushDiffRecordsWithContext(ctx, appliedReceipts) - await safelyDisposeTracking(ctx) - }) - await Promise.all(appliedReceipts) - return - } - - const combinedWhere = - activeWhereExpressions.length === 1 - ? activeWhereExpressions[0] - : or( - activeWhereExpressions[0], - activeWhereExpressions[1], - ...activeWhereExpressions.slice(2), - ) - - const compiledNewData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, - ) - - const compiledOldData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, - ) - - const compiledView = compileSQLite({ where: combinedWhere }) - - const newDataWhenClause = toInlinedWhereClause(compiledNewData) - const oldDataWhenClause = toInlinedWhereClause(compiledOldData) - const viewWhereClause = toInlinedWhereClause(compiledView) - await database.writeLock(async (ctx) => { - // Replace any active tracking with one covering the new set of - // predicates. + if (!isCurrent()) return await flushDiffRecordsWithContext(ctx, appliedReceipts) + if (!isCurrent()) return await safelyDisposeTracking(ctx) + if (!isCurrent()) return + + const active = activeWhereExpressions() + if (active.length === 0) return + const combinedWhere = + active.length === 1 + ? active[0] + : or(active[0], active[1], ...active.slice(2)) + const compiledNewData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'NEW.data' }, + ) + const compiledOldData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'OLD.data' }, + ) + const compiledView = compileSQLite({ where: combinedWhere }) + const newDataWhenClause = toInlinedWhereClause(compiledNewData) + const oldDataWhenClause = toInlinedWhereClause(compiledOldData) + const viewWhereClause = toInlinedWhereClause(compiledView) await establishTracking( { @@ -717,10 +695,56 @@ function createPowerSyncCollectionConfig< }, appliedReceipts, ) + if (!isCurrent()) await safelyDisposeTracking(ctx) }) await Promise.all(appliedReceipts) } + const loadSubset = async ( + options: LoadSubsetOptions, + ): Promise => { + if (hasStopped()) return + // Never create a trigger that has no observer to drain its diff table. + await startup + if ( + hasStopped() || + releasedSubsets.has(options) || + options.signal?.aborted + ) { + return + } + + const demand: DemandRecord = { options, state: `provisional` } + demands.set(options, demand) + trackingRevision++ + try { + const cleanup = await restConfig.onLoadSubset?.(options) + if (cleanup) demand.cleanup = cleanup + } catch (error) { + demand.state = `failed` + demands.delete(options) + trackingRevision++ + throw error + } + + if ( + hasStopped() || + releasedSubsets.has(options) || + options.signal?.aborted || + demands.get(options) !== demand + ) { + demand.state = `released` + demands.delete(options) + trackingRevision++ + demand.cleanup?.() + return + } + + demand.state = `active` + trackingRevision++ + await rebuildTracking() + } + const toInlinedWhereClause = (compiled: { where?: string params: Array @@ -733,56 +757,111 @@ function createPowerSyncCollectionConfig< ) } - const unloadSubset = async (options: LoadSubsetOptions) => { - releasedSubsets.add(options) - unloadSubsetCallbacks.get(options)?.() - unloadSubsetCallbacks.delete(options) - - const idx = activeWhereExpressions.indexOf(options.where) - if (idx !== -1) { - activeWhereExpressions.splice(idx, 1) - } - - // Evict rows that were exclusively loaded by the departing predicate. - // These are rows matching the departing WHERE that are no longer covered - // by any remaining active predicate. + const performPhysicalRelease = async ( + options: LoadSubsetOptions, + ): Promise => { const compiledDeparting = compileSQLite({ where: options.where }) const departingWhereSQL = toInlinedWhereClause(compiledDeparting) + let rowsToEvict: Array<{ id: string }> + for (;;) { + if (hasStopped()) return + const revision = trackingRevision + const active = activeWhereExpressions() + let evictionSQL: string + if (active.length === 0) { + evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` + } else { + const combinedRemaining = + active.length === 1 + ? active[0]! + : or(active[0], active[1], ...active.slice(2)) + const compiledRemaining = compileSQLite({ + where: combinedRemaining, + }) + const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) + evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + } - let evictionSQL: string - if (activeWhereExpressions.length === 0) { - evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` - } else { - const combinedRemaining = - activeWhereExpressions.length === 1 - ? activeWhereExpressions[0]! - : or( - activeWhereExpressions[0], - activeWhereExpressions[1], - ...activeWhereExpressions.slice(2), - ) - const compiledRemaining = compileSQLite({ - where: combinedRemaining, - }) - const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) - evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) + if (hasStopped()) return + if (trackingRevision === revision) break } - - const rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) if (rowsToEvict.length > 0) { begin() for (const { id } of rowsToEvict) { write({ type: `delete`, key: id }) } - // Eviction does not establish new subset coverage. Keep trigger - // replacement in the same unload turn even when this delete waits - // behind a persisting mutation; the later load tracks its own - // establishing receipts. void commit() } + await rebuildTracking() + } + + function scheduleReleaseDrain(delay = 0): void { + if (hasStopped() || drainingReleases || releaseRetryTimer) return + if (delay > 0) { + releaseRetryTimer = setTimeout(() => { + releaseRetryTimer = undefined + void drainReleases() + }, delay) + return + } + void drainReleases() + } + + async function drainReleases(): Promise { + if (hasStopped() || drainingReleases) return + drainingReleases = true + let retryDelay = 0 + try { + while (!hasStopped() && pendingReleases.length > 0) { + const pending = pendingReleases[0]! + try { + await performPhysicalRelease(pending.options) + pendingReleases.shift() + } catch (error) { + pending.failures++ + retryDelay = Math.min(1000 * 2 ** (pending.failures - 1), 30000) + database.logger.error( + `Could not release subset tracking for ${viewName}; retrying`, + error, + ) + break + } + } + } finally { + drainingReleases = false + } + if (pendingReleases.length > 0) scheduleReleaseDrain(retryDelay) + } + + const unloadSubset = (options: LoadSubsetOptions): void => { + releasedSubsets.add(options) + const demand = demands.get(options) + if ( + !demand || + demand.state === `released` || + demand.state === `failed` + ) { + return + } - // Recreate the diff trigger for the remaining active WHERE expressions. - await loadSubset() + const wasActive = demand.state === `active` + demand.state = `released` + demands.delete(options) + trackingRevision++ + try { + demand.cleanup?.() + } catch (error) { + database.logger.error( + `Could not clean up subset hook for ${viewName}`, + error, + ) + } + + if (wasActive) { + pendingReleases.push({ options, failures: 0 }) + scheduleReleaseDrain() + } } markReady() @@ -790,16 +869,30 @@ function createPowerSyncCollectionConfig< return { cleanup: () => { stopped = true + lifecycleGeneration++ + trackingRevision++ + clearTimeout(releaseRetryTimer) + releaseRetryTimer = undefined database.logger.info( `Sync has been stopped for ${viewName} into ${trackedTableName}`, ) abortController.abort() - for (const cleanup of unloadSubsetCallbacks.values()) cleanup() - unloadSubsetCallbacks.clear() - activeWhereExpressions.length = 0 + for (const demand of demands.values()) { + try { + demand.cleanup?.() + } catch (error) { + database.logger.error( + `Could not clean up subset hook for ${viewName}`, + error, + ) + } + demand.state = `released` + } + demands.clear() + pendingReleases.length = 0 }, loadSubset: (options: LoadSubsetOptions) => loadSubset(options), - unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options), + unloadSubset, } } }, diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 5e5b8f7c1..ea54dbe1c 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2275,6 +2275,279 @@ describe(`On-Demand Sync Mode`, () => { } }) + it(`does not start queued tracking after collection cleanup`, async () => { + const db = await createDatabase() + const queued = pDefer() + let runQueuedWriteLock!: () => Promise + vi.spyOn(db, `writeLock`).mockImplementation( + (callback) => + new Promise((resolve, reject) => { + runQueuedWriteLock = async () => { + try { + await callback({} as never) + resolve(undefined as never) + } catch (error) { + reject(error) + } + } + queued.resolve() + }) as never, + ) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const load = sync.loadSubset({ where: eq(`category`, `electronics`) }) + await queued.promise + sync.cleanup?.() + await runQueuedWriteLock() + await load + + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + + it(`does not retain a predicate whose load hook rejects`, async () => { + const db = await createDatabase() + const hookFailure = new Error(`subset hook failed`) + const onLoadSubset = vi + .fn() + .mockRejectedValueOnce(hookFailure) + .mockResolvedValueOnce(undefined) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + try { + await expect( + sync.loadSubset({ where: eq(`category`, `electronics`) }), + ).rejects.toBe(hookFailure) + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + + const when = createDiffTrigger.mock.calls.at(-1)?.[0].when + expect(when?.INSERT).toContain(`clothing`) + expect(when?.INSERT).not.toContain(`electronics`) + } finally { + sync.cleanup?.() + } + }) + + it(`does not publish a provisional hook through another active demand`, async () => { + const db = await createDatabase() + const firstHook = pDefer() + const onLoadSubset = vi + .fn() + .mockReturnValueOnce(firstHook.promise) + .mockResolvedValueOnce(undefined) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const provisional = sync.loadSubset({ + where: eq(`category`, `electronics`), + }) + await vi.waitFor(() => expect(onLoadSubset).toHaveBeenCalledTimes(1)) + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + + const when = createDiffTrigger.mock.calls.at(-1)?.[0].when + expect(when?.INSERT).toContain(`clothing`) + expect(when?.INSERT).not.toContain(`electronics`) + + firstHook.resolve() + await provisional + sync.cleanup?.() + }) + + it(`hands subset release to the adapter without returning a promise`, async () => { + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const request = { where: eq(`category`, `electronics`) } + + await sync.loadSubset(request) + const release = ( + sync.unloadSubset as (options: typeof request) => unknown + )(request) + try { + expect(release).toBeUndefined() + } finally { + await Promise.resolve(release) + sync.cleanup?.() + } + }) + + it(`retries physical subset release after asynchronous adapter failure`, async () => { + vi.useFakeTimers() + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + const getAll = vi + .spyOn(db, `getAll`) + .mockRejectedValueOnce(new Error(`transient eviction failure`)) + .mockResolvedValueOnce([]) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const request = { where: eq(`category`, `electronics`) } + + try { + await sync.loadSubset(request) + expect(sync.unloadSubset(request)).toBeUndefined() + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(1)) + + await vi.advanceTimersByTimeAsync(1000) + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(2)) + } finally { + sync.cleanup?.() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + + it(`recomputes eviction when another demand activates during release`, async () => { + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + const firstEviction = pDefer>() + const getAll = vi + .spyOn(db, `getAll`) + .mockReturnValueOnce(firstEviction.promise) + .mockResolvedValueOnce([]) + const write = vi.fn() + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write, + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const departing = { where: eq(`category`, `electronics`) } + + try { + await sync.loadSubset(departing) + sync.unloadSubset(departing) + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(1)) + + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + firstEviction.resolve([{ id: `row-now-owned-by-clothing` }]) + + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(2)) + expect(write).not.toHaveBeenCalledWith({ + type: `delete`, + key: `row-now-owned-by-clothing`, + }) + } finally { + firstEviction.resolve([]) + sync.cleanup?.() + } + }) + it(`flushes eager changes that arrive before the tracking handle is published`, async () => { const db = await createDatabase() await createTestProducts(db) From 1bfba4e1cd10f1caae2cca2f34c0d235c8cec3cd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 09:22:46 -0600 Subject: [PATCH 011/327] fix(electric): stop snapshots after cleanup --- .../electric-db-collection/src/electric.ts | 5 +- .../tests/electric.test.ts | 53 +++++++++++++++++++ 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index e0686a8b0..c478fe59e 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -587,12 +587,13 @@ function createLoadSubsetDedupe>({ const snapshotParams = compileSQL(opts, compileOptions) try { const { data: rows } = await stream.fetchSnapshot(snapshotParams) - if (opts.signal?.aborted || !isBufferingInitialSync()) { + if (isAborted() || !isBufferingInitialSync()) { debug(`${logPrefix}Ignoring snapshot - sync completed while fetching`) return } if (rows.length > 0) { + if (isAborted()) return begin() for (const row of rows) { write({ @@ -605,7 +606,7 @@ function createLoadSubsetDedupe>({ debug(`${logPrefix}Applied snapshot with ${rows.length} rows`) } } catch (error) { - if (opts.signal?.aborted) return + if (isAborted()) return if (handleSnapshotError(error, `fetchSnapshot`)) { return } diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 96f549fa0..6d3955943 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2948,6 +2948,59 @@ describe(`Electric Integration`, () => { } }) + it(`does not start buffered snapshot publication after adapter cleanup`, async () => { + const snapshot = createDeferred<{ + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + mockFetchSnapshot.mockReturnValueOnce(snapshot.promise) + const options = electricCollectionOptions({ + id: `progressive-snapshot-cleanup-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const begin = vi.fn() + const write = vi.fn() + const commit = vi.fn(() => true as const) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin, + write, + commit, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!controls || typeof controls === `function` || !controls.loadSubset) { + throw new Error(`Expected progressive sync controls`) + } + + const load = controls.loadSubset({ limit: 10 }) + controls.cleanup?.() + snapshot.resolve({ + data: [ + { + key: `1`, + value: { id: 1, name: `Late snapshot user` }, + headers: { operation: `insert` }, + }, + ], + }) + if (load !== true) await load + + expect(begin).not.toHaveBeenCalled() + expect(write).not.toHaveBeenCalled() + expect(commit).not.toHaveBeenCalled() + }) + it(`does not start a refresh when the collection signal is already aborted`, async () => { mockStream.isUpToDate = true const abortController = new AbortController() From 0d0a613dfe1737f701afe70e970c55a32125427e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 10:52:32 -0600 Subject: [PATCH 012/327] test(db): model applied subset settlement --- .../db/tests/load-subset-full-flow-model.ts | 195 ++++++++++++++++++ ...saction-refinement-oracle.property.test.ts | 134 ++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 59ab9fc74..90a974872 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -1,6 +1,8 @@ export type FullFlowOwnerId = string export type FullFlowSessionId = string export type FullFlowDemandId = string +export type FullFlowSourceId = string +export type FullFlowTransactionId = string export type LoadSubsetFullFlowEvent = | { @@ -48,6 +50,34 @@ export type LoadSubsetFullFlowEvent = type: `runContinuation` taskId: string } + | { + type: `stageSyncTransaction` + transactionId: FullFlowTransactionId + sourceId: FullFlowSourceId + rowKeys: ReadonlyArray + } + | { + type: `commitSyncTransaction` + transactionId: FullFlowTransactionId + parked: boolean + signalAborted: boolean + } + | { + type: `enterSyncApplication` + transactionId: FullFlowTransactionId + } + | { + type: `abortSyncTransaction` + transactionId: FullFlowTransactionId + } + | { + type: `publishSyncTransaction` + transactionId: FullFlowTransactionId + } + | { + type: `settleSyncReceipt` + transactionId: FullFlowTransactionId + } export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` @@ -113,6 +143,12 @@ export function projectTransportLoads( case `advanceWindowRevision`: case `scheduleContinuation`: case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: break } } @@ -176,6 +212,12 @@ export function projectAuthorizedContinuationStarts( } case `applyAuthoritativeRows`: case `releaseDemand`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: break } } @@ -200,3 +242,156 @@ export function projectRetainedRowKeys( return [...retainedRows].sort() } + +export type ExpectedSyncReceiptState = `pending` | `resolved` | `rejected` + +export type ExpectedPublicRow = { + sourceId: FullFlowSourceId + rowKey: string +} + +export type ExpectedSyncTransactionObservation = { + visibleRows: Array + publishedBatches: Array> + callbackReads: Array> + receipts: Array<{ + transactionId: FullFlowTransactionId + state: ExpectedSyncReceiptState + }> +} + +type SyncTransactionState = + | `staged` + | `committed` + | `parked` + | `applying` + | `published` + | `resolved` + | `rejected` + +type ProjectedSyncTransaction = { + sourceId: FullFlowSourceId + rowKeys: ReadonlyArray + state: SyncTransactionState +} + +function sortPublicRows( + rows: Iterable, +): Array { + return [...rows].sort((left, right) => + left.sourceId === right.sourceId + ? left.rowKey.localeCompare(right.rowKey) + : left.sourceId.localeCompare(right.sourceId), + ) +} + +/** + * Projects the sync transaction's public contract without consulting the + * collection queue. Abort can still win while work is staged, committed, or + * parked. Once application starts, publication is irrevocable. A receipt does + * not resolve until the published batch and callback-time reads are visible. + */ +export function projectSyncTransactions( + history: ReadonlyArray, +): ExpectedSyncTransactionObservation { + const transactions = new Map< + FullFlowTransactionId, + ProjectedSyncTransaction + >() + const visibleRows = new Map() + const publishedBatches: Array> = [] + const callbackReads: Array> = [] + + for (const event of history) { + switch (event.type) { + case `stageSyncTransaction`: + transactions.set(event.transactionId, { + sourceId: event.sourceId, + rowKeys: event.rowKeys, + state: `staged`, + }) + break + case `commitSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if (!transaction || transaction.state !== `staged`) break + transaction.state = event.signalAborted + ? `rejected` + : event.parked + ? `parked` + : `committed` + break + } + case `enterSyncApplication`: { + const transaction = transactions.get(event.transactionId) + if ( + transaction?.state === `committed` || + transaction?.state === `parked` + ) { + transaction.state = `applying` + } + break + } + case `abortSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if ( + transaction?.state === `staged` || + transaction?.state === `committed` || + transaction?.state === `parked` + ) { + transaction.state = `rejected` + } + break + } + case `publishSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if (transaction?.state !== `applying`) break + const batch = transaction.rowKeys.map((rowKey) => ({ + sourceId: transaction.sourceId, + rowKey, + })) + for (const row of batch) { + visibleRows.set(`${row.sourceId}\u0000${row.rowKey}`, row) + } + transaction.state = `published` + publishedBatches.push(sortPublicRows(batch)) + callbackReads.push(sortPublicRows(visibleRows.values())) + break + } + case `settleSyncReceipt`: { + const transaction = transactions.get(event.transactionId) + if (transaction?.state === `published`) { + transaction.state = `resolved` + } + break + } + case `requestDemand`: + case `applyAuthoritativeRows`: + case `releaseDemand`: + case `restartSession`: + case `cleanupSession`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + break + } + } + + return { + visibleRows: sortPublicRows(visibleRows.values()), + publishedBatches, + callbackReads, + receipts: [...transactions] + .flatMap(([transactionId, transaction]) => { + const state = + transaction.state === `resolved` + ? `resolved` + : transaction.state === `rejected` + ? `rejected` + : `pending` + return [{ transactionId, state } as const] + }) + .sort((left, right) => + left.transactionId.localeCompare(right.transactionId), + ), + } +} diff --git a/packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts new file mode 100644 index 000000000..424c447b9 --- /dev/null +++ b/packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { createTransaction } from '../../src/transactions.js' +import { projectSyncTransactions } from '../load-subset-full-flow-model.js' +import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' + +type Row = { id: string; group: string } + +describe(`loadSubset transaction refinement`, () => { + it.each([`at-commit`, `while-parked`, `after-publication-starts`] as const)( + `matches the independent receipt and publication model when aborting %s`, + async (abortPhase) => { + const sourceId = `transaction-refinement-${abortPhase}` + const transactionId = `subset-transaction` + const remoteRow: Row = { id: `remote`, group: `requested` } + const history: Array = [ + { + type: `stageSyncTransaction`, + transactionId, + sourceId, + rowKeys: [remoteRow.id], + }, + { + type: `commitSyncTransaction`, + transactionId, + parked: true, + signalAborted: abortPhase === `at-commit`, + }, + ] + const controller = new AbortController() + const persistence = createDeferred() + const publishedBatches: Array> = [] + const callbackReads: Array> = [] + const source = createCollection({ + id: sourceId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: ({ signal }) => { + begin() + write({ type: `insert`, value: remoteRow }) + if (abortPhase === `at-commit`) controller.abort() + return commit(signal) + }, + } + }, + }, + }) + source.startSyncImmediate() + const blocker = createTransaction({ + mutationFn: () => persistence.promise, + }) + blocker.mutate(() => + source.insert({ id: `local`, group: `outside-request` }), + ) + const subscription = source.subscribeChanges( + (changes) => { + const remoteKeys = changes + .filter((change) => change.key === remoteRow.id) + .map((change) => String(change.key)) + if (remoteKeys.length === 0) return + publishedBatches.push(remoteKeys) + callbackReads.push(source.has(remoteRow.id) ? [remoteRow.id] : []) + if (abortPhase === `after-publication-starts`) { + controller.abort() + } + }, + { includeInitialState: false }, + ) + const load = source._sync.loadSubset({ signal: controller.signal }) + expect(load).toBeInstanceOf(Promise) + + try { + if (abortPhase === `while-parked`) { + controller.abort() + history.push({ type: `abortSyncTransaction`, transactionId }) + } else if (abortPhase === `after-publication-starts`) { + history.push( + { type: `enterSyncApplication`, transactionId }, + { type: `publishSyncTransaction`, transactionId }, + { type: `abortSyncTransaction`, transactionId }, + { type: `settleSyncReceipt`, transactionId }, + ) + } + + persistence.resolve() + await blocker.isPersisted.promise + + if (abortPhase !== `after-publication-starts`) { + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } else { + await expect(load).resolves.toEqual( + expect.objectContaining({ collectionId: sourceId }), + ) + } + + const expected = projectSyncTransactions(history) + const visibleRows = source.has(remoteRow.id) + ? [{ sourceId, rowKey: remoteRow.id }] + : [] + + expect(visibleRows).toEqual(expected.visibleRows) + expect(publishedBatches).toEqual( + expected.publishedBatches.map((batch) => + batch.map(({ rowKey }) => rowKey), + ), + ) + expect(callbackReads).toEqual( + expected.callbackReads.map((rows) => + rows.map(({ rowKey }) => rowKey), + ), + ) + expect(expected.receipts).toEqual([ + { + transactionId, + state: + abortPhase === `after-publication-starts` + ? `resolved` + : `rejected`, + }, + ]) + } finally { + persistence.resolve() + await blocker.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await source.cleanup() + } + }, + ) +}) From 0fa43ba6a4db96606d76fab7fee9e8505e85f025 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 10:57:19 -0600 Subject: [PATCH 013/327] test(db): model replay publication --- .../db/tests/load-subset-full-flow-model.ts | 217 +++++++++++++++++ ...-replay-refinement-oracle.property.test.ts | 223 ++++++++++++++++++ 2 files changed, 440 insertions(+) create mode 100644 packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 90a974872..2c52f28a6 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -3,6 +3,11 @@ export type FullFlowSessionId = string export type FullFlowDemandId = string export type FullFlowSourceId = string export type FullFlowTransactionId = string +export type FullFlowVersionedRow = { + sourceId: FullFlowSourceId + rowKey: string + version: number +} export type LoadSubsetFullFlowEvent = | { @@ -78,6 +83,27 @@ export type LoadSubsetFullFlowEvent = type: `settleSyncReceipt` transactionId: FullFlowTransactionId } + | { + type: `establishPublication` + sourceId: FullFlowSourceId + rows: ReadonlyArray + } + | { + type: `startReplay` + attemptId: string + sourceId: FullFlowSourceId + } + | { + type: `writeReplayRows` + attemptId: string + rows: ReadonlyArray + acceptedByCore: boolean + } + | { + type: `settleReplay` + attemptId: string + outcome: `resolve` | `reject` + } export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` @@ -149,6 +175,10 @@ export function projectTransportLoads( case `abortSyncTransaction`: case `publishSyncTransaction`: case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: break } } @@ -218,6 +248,10 @@ export function projectAuthorizedContinuationStarts( case `abortSyncTransaction`: case `publishSyncTransaction`: case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: break } } @@ -372,6 +406,10 @@ export function projectSyncTransactions( case `advanceWindowRevision`: case `scheduleContinuation`: case `runContinuation`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: break } } @@ -395,3 +433,182 @@ export function projectSyncTransactions( ), } } + +export type ExpectedVersionedChange = { + type: `insert` | `update` | `delete` + row: FullFlowVersionedRow + previousVersion?: number +} + +export type ExpectedReplayObservation = { + coreRows: Array + visibleRows: Array + publishedBatches: Array> +} + +type ProjectedReplayAttempt = { + outcome?: `resolve` | `reject` +} + +type ProjectedReplaySession = { + sourceId: FullFlowSourceId + currentAttemptId: string + attempts: Map + baseline: Map +} + +function versionedRowIdentity(row: FullFlowVersionedRow): string { + return `${row.sourceId}\u0000${row.rowKey}` +} + +function sortVersionedRows( + rows: Iterable, +): Array { + return [...rows].sort((left, right) => + versionedRowIdentity(left).localeCompare(versionedRowIdentity(right)), + ) +} + +function versionedPublicationDiff( + baseline: ReadonlyMap, + replacement: ReadonlyMap, +): Array { + const changes: Array = [] + for (const [identity, previous] of baseline) { + const next = replacement.get(identity) + if (!next) { + changes.push({ type: `delete`, row: previous }) + } else if (next.version !== previous.version) { + changes.push({ + type: `update`, + row: next, + previousVersion: previous.version, + }) + } + } + for (const [identity, row] of replacement) { + if (!baseline.has(identity)) changes.push({ type: `insert`, row }) + } + return changes.sort((left, right) => + versionedRowIdentity(left.row).localeCompare( + versionedRowIdentity(right.row), + ), + ) +} + +/** + * Projects truncate replay as a replacement protocol. Core rows and last-good + * publication are independent domains: truncate clears core immediately, but + * public rows change only after every overlapping attempt settles and the + * newest attempt succeeds. + */ +export function projectReplayPublication( + history: ReadonlyArray, +): ExpectedReplayObservation { + const coreRows = new Map() + const visibleRows = new Map() + const publishedBatches: Array> = [] + const sessions = new Map() + const attemptSessions = new Map() + + for (const event of history) { + switch (event.type) { + case `establishPublication`: { + const batch: Array = [] + for (const row of event.rows) { + const identity = versionedRowIdentity(row) + coreRows.set(identity, row) + visibleRows.set(identity, row) + batch.push({ type: `insert`, row }) + } + if (batch.length > 0) publishedBatches.push(batch) + break + } + case `startReplay`: { + let session = sessions.get(event.sourceId) + if (!session) { + session = { + sourceId: event.sourceId, + currentAttemptId: event.attemptId, + attempts: new Map(), + baseline: new Map( + [...visibleRows].filter( + ([, row]) => row.sourceId === event.sourceId, + ), + ), + } + sessions.set(event.sourceId, session) + } + session.currentAttemptId = event.attemptId + session.attempts.set(event.attemptId, {}) + attemptSessions.set(event.attemptId, session) + for (const [identity, row] of coreRows) { + if (row.sourceId === event.sourceId) coreRows.delete(identity) + } + break + } + case `writeReplayRows`: + if (event.acceptedByCore) { + for (const row of event.rows) { + coreRows.set(versionedRowIdentity(row), row) + } + } + break + case `settleReplay`: { + const session = attemptSessions.get(event.attemptId) + const attempt = session?.attempts.get(event.attemptId) + if (!session || !attempt) break + attempt.outcome = event.outcome + if ([...session.attempts.values()].some(({ outcome }) => !outcome)) { + break + } + + const current = session.attempts.get(session.currentAttemptId) + if (current?.outcome === `resolve`) { + const replacement = new Map( + [...coreRows].filter( + ([, row]) => row.sourceId === session.sourceId, + ), + ) + const changes = versionedPublicationDiff( + session.baseline, + replacement, + ) + for (const [identity, row] of visibleRows) { + if (row.sourceId === session.sourceId) visibleRows.delete(identity) + } + for (const [identity, row] of replacement) { + visibleRows.set(identity, row) + } + if (changes.length > 0) publishedBatches.push(changes) + } + sessions.delete(session.sourceId) + for (const attemptId of session.attempts.keys()) { + attemptSessions.delete(attemptId) + } + break + } + case `requestDemand`: + case `applyAuthoritativeRows`: + case `releaseDemand`: + case `restartSession`: + case `cleanupSession`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + break + } + } + + return { + coreRows: sortVersionedRows(coreRows.values()), + visibleRows: sortVersionedRows(visibleRows.values()), + publishedBatches, + } +} diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts new file mode 100644 index 000000000..a000b3c36 --- /dev/null +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { projectReplayPublication } from '../load-subset-full-flow-model.js' +import { flushPromises } from '../utils.js' +import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { + ChangeMessage, + ChangeMessageOrDeleteKeyMessage, + LoadSubsetOptions, +} from '../../src/types.js' + +type Row = { id: string; version: number } + +describe(`loadSubset replay refinement`, () => { + function createHarness(sourceId: string) { + let begin!: () => void + let write!: (message: ChangeMessageOrDeleteKeyMessage) => void + let commit!: () => void + let truncate!: () => void + let loadCount = 0 + const pending: Array<{ + options: LoadSubsetOptions + deferred: ReturnType> + }> = [] + const batches: Array< + Array<{ + type: `insert` | `update` | `delete` + row: { sourceId: string; rowKey: string; version: number } + previousVersion?: number + }> + > = [] + const visible = new Map() + const source = createCollection({ + id: sourceId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: (options) => { + loadCount++ + if (loadCount === 1) { + begin() + write({ type: `insert`, value: { id: `row`, version: 1 } }) + commit() + return true + } + const deferred = createDeferred() + pending.push({ options, deferred }) + return deferred.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const subscription = source.subscribeChanges( + (changes: Array>) => { + const batch = changes.map((change) => { + if (change.type === `delete`) visible.delete(String(change.key)) + else visible.set(String(change.key), { ...change.value }) + return { + type: change.type, + row: { + sourceId, + rowKey: String(change.key), + version: change.value.version, + }, + ...(change.previousValue === undefined + ? {} + : { previousVersion: change.previousValue.version }), + } + }) + if (batch.length > 0) batches.push(batch) + }, + ) + + const replaceCore = (version: number) => { + begin() + write({ type: `insert`, value: { id: `row`, version } }) + commit() + } + const startReplay = async () => { + begin() + truncate() + commit() + await flushPromises() + } + const coreRows = () => + source.toArray.map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })) + const visibleRows = () => + [...visible.values()].map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })) + + return { + source, + subscription, + pending, + batches, + replaceCore, + startReplay, + coreRows, + visibleRows, + } + } + + it(`retains the last complete publication when replay fails after writing`, async () => { + const sourceId = `replay-refinement-failure` + const row = (version: number) => ({ + sourceId, + rowKey: `row`, + version, + }) + const history: Array = [ + { type: `establishPublication`, sourceId, rows: [row(1)] }, + ] + const harness = createHarness(sourceId) + + try { + harness.subscription.requestSnapshot({ optimizedOnly: false }) + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) + + harness.replaceCore(2) + history.push({ + type: `writeReplayRows`, + attemptId: `replay-1`, + rows: [row(2)], + acceptedByCore: true, + }) + harness.pending[0]?.deferred.reject(new Error(`replay failed`)) + history.push({ + type: `settleReplay`, + attemptId: `replay-1`, + outcome: `reject`, + }) + await flushPromises() + + const expected = projectReplayPublication(history) + expect(harness.coreRows()).toEqual(expected.coreRows) + expect(harness.visibleRows()).toEqual(expected.visibleRows) + expect(harness.batches).toEqual(expected.publishedBatches) + } finally { + harness.subscription.unsubscribe() + await harness.source.cleanup() + } + }) + + it(`waits for every overlapping replay before publishing the newest success`, async () => { + const sourceId = `replay-refinement-overlap` + const row = (version: number) => ({ + sourceId, + rowKey: `row`, + version, + }) + const history: Array = [ + { type: `establishPublication`, sourceId, rows: [row(1)] }, + ] + const harness = createHarness(sourceId) + + try { + harness.subscription.requestSnapshot({ optimizedOnly: false }) + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-2`, sourceId }) + + expect(harness.pending[0]?.options.signal?.aborted).toBe(true) + harness.replaceCore(3) + history.push({ + type: `writeReplayRows`, + attemptId: `replay-2`, + rows: [row(3)], + acceptedByCore: true, + }) + harness.pending[1]?.deferred.resolve() + history.push({ + type: `settleReplay`, + attemptId: `replay-2`, + outcome: `resolve`, + }) + await flushPromises() + + const beforeObsoleteSettlement = projectReplayPublication(history) + expect(harness.visibleRows()).toEqual( + beforeObsoleteSettlement.visibleRows, + ) + expect(harness.batches).toEqual(beforeObsoleteSettlement.publishedBatches) + + harness.pending[0]?.deferred.reject( + new DOMException(`obsolete`, `AbortError`), + ) + history.push({ + type: `settleReplay`, + attemptId: `replay-1`, + outcome: `reject`, + }) + await flushPromises() + + const expected = projectReplayPublication(history) + expect(harness.coreRows()).toEqual(expected.coreRows) + expect(harness.visibleRows()).toEqual(expected.visibleRows) + expect(harness.batches).toEqual(expected.publishedBatches) + } finally { + for (const replay of harness.pending) replay.deferred.resolve() + harness.subscription.unsubscribe() + await harness.source.cleanup() + } + }) +}) From 12024581e3b6c5d4eefc0dc8340081c0b5446472 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 11:00:40 -0600 Subject: [PATCH 014/327] test(db): model cross-source readiness --- .../db/tests/load-subset-full-flow-model.ts | 123 ++++++++++++++ ...adiness-refinement-oracle.property.test.ts | 156 ++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 2c52f28a6..d020d4a72 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -104,6 +104,19 @@ export type LoadSubsetFullFlowEvent = attemptId: string outcome: `resolve` | `reject` } + | { + type: `registerSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + } + | { + type: `settleSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + outcome: `resolve` | `reject` + } export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` @@ -179,6 +192,8 @@ export function projectTransportLoads( case `startReplay`: case `writeReplayRows`: case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: break } } @@ -252,6 +267,8 @@ export function projectAuthorizedContinuationStarts( case `startReplay`: case `writeReplayRows`: case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: break } } @@ -410,6 +427,8 @@ export function projectSyncTransactions( case `startReplay`: case `writeReplayRows`: case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: break } } @@ -602,6 +621,8 @@ export function projectReplayPublication( case `abortSyncTransaction`: case `publishSyncTransaction`: case `settleSyncReceipt`: + case `registerSourceDemand`: + case `settleSourceDemand`: break } } @@ -612,3 +633,105 @@ export function projectReplayPublication( publishedBatches, } } + +export type ExpectedSourceReadiness = { + status: `loading` | `ready` | `error` | `cleaned-up` + pendingSources: Array + failedSources: Array +} + +/** Projects initial live-query readiness across every reachable source. */ +export function projectSourceReadiness( + history: ReadonlyArray, +): ExpectedSourceReadiness { + const demands = new Map< + string, + { + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + state: `pending` | `resolved` | `rejected` + } + >() + let currentSession: FullFlowSessionId | undefined + let cleanedUp = false + + for (const event of history) { + switch (event.type) { + case `registerSourceDemand`: + currentSession ??= event.sessionId + if (event.sessionId !== currentSession) break + cleanedUp = false + demands.set(`${event.sourceId}\u0000${event.demandId}`, { + sessionId: event.sessionId, + sourceId: event.sourceId, + state: `pending`, + }) + break + case `settleSourceDemand`: { + if (event.sessionId !== currentSession) break + const demand = demands.get(`${event.sourceId}\u0000${event.demandId}`) + if (demand) + demand.state = event.outcome === `resolve` ? `resolved` : `rejected` + break + } + case `cleanupSession`: + if (event.sessionId === currentSession) { + cleanedUp = true + demands.clear() + } + break + case `restartSession`: + currentSession = event.nextSessionId + cleanedUp = false + demands.clear() + break + case `requestDemand`: + case `applyAuthoritativeRows`: + case `releaseDemand`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + break + } + } + + const currentDemands = [...demands.values()].filter( + ({ sessionId }) => sessionId === currentSession, + ) + const pendingSources = [ + ...new Set( + currentDemands + .filter(({ state }) => state === `pending`) + .map(({ sourceId }) => sourceId), + ), + ].sort() + const failedSources = [ + ...new Set( + currentDemands + .filter(({ state }) => state === `rejected`) + .map(({ sourceId }) => sourceId), + ), + ].sort() + + return { + status: cleanedUp + ? `cleaned-up` + : failedSources.length > 0 + ? `error` + : pendingSources.length > 0 || currentDemands.length === 0 + ? `loading` + : `ready`, + pendingSources, + failedSources, + } +} diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts new file mode 100644 index 000000000..375f0ae24 --- /dev/null +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts @@ -0,0 +1,156 @@ +import { expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { BTreeIndex } from '../../src/index.js' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { projectSourceReadiness } from '../load-subset-full-flow-model.js' +import { flushPromises } from '../utils.js' +import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' + +type Row = { id: string; group: string } + +it.each([`resolve`, `reject`, `cleanup`] as const)( + `matches cross-source initial readiness through %s`, + async (secondOutcome) => { + const sessionId = `session-1` + const leftId = `readiness-left-${secondOutcome}` + const rightId = `readiness-right-${secondOutcome}` + const leftDelivery = createDeferred() + const rightDelivery = createDeferred() + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId, + sourceId: leftId, + demandId: `all`, + }, + { + type: `registerSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + }, + ] + const createSource = ( + id: string, + row: Row, + delivery: ReturnType>, + ) => + createCollection({ + id, + getKey: (value) => value.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: () => + delivery.promise.then(async () => { + begin() + write({ type: `insert`, value: row }) + const applied = commit() + if (applied !== true) await applied + return { hasMore: false, appliedRowKeys: [row.id] } + }), + unloadSubset: () => {}, + } + }, + }, + }) + const left = createSource( + leftId, + { id: `left`, group: `shared` }, + leftDelivery, + ) + const right = createSource( + rightId, + { id: `right`, group: `shared` }, + rightDelivery, + ) + const live = createLiveQueryCollection({ + id: `readiness-live-${secondOutcome}`, + query: (q) => + q + .from({ left }) + .innerJoin({ right }, ({ left: leftRow, right: rightRow }) => + eq(leftRow.group, rightRow.group), + ) + .select(({ left: leftRow, right: rightRow }) => ({ + leftId: leftRow.id, + rightId: rightRow.id, + })), + startSync: true, + }) + const preload = live.preload() + void preload.catch(() => undefined) + + try { + expect(live.status).toBe(projectSourceReadiness(history).status) + + leftDelivery.resolve() + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: leftId, + demandId: `all`, + outcome: `resolve`, + }) + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(live.toArray).toEqual([]) + + if (secondOutcome === `cleanup`) { + await live.cleanup() + history.push({ type: `cleanupSession`, sessionId }) + expect(live.status).toBe(projectSourceReadiness(history).status) + + rightDelivery.resolve() + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + outcome: `resolve`, + }) + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(live.toArray).toEqual([]) + return + } else if (secondOutcome === `resolve`) { + rightDelivery.resolve() + } else { + rightDelivery.reject(new Error(`right source failed`)) + } + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + outcome: secondOutcome, + }) + await flushPromises() + + const expected = projectSourceReadiness(history) + expect(live.status).toBe(expected.status) + if (secondOutcome === `resolve`) { + await expect(preload).resolves.toBeUndefined() + expect(live.toArray).toEqual([ + expect.objectContaining({ leftId: `left`, rightId: `right` }), + ]) + } else { + await expect(preload).rejects.toThrow(`right source failed`) + expect(expected.failedSources).toEqual([rightId]) + } + } finally { + leftDelivery.resolve() + rightDelivery.resolve() + await live.cleanup() + await Promise.all([left.cleanup(), right.cleanup()]) + } + }, +) From 5b2a7bbacd0dbefa08e1b9d6b17a22bc7ac0eede Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 11:02:34 -0600 Subject: [PATCH 015/327] test(db): add refinement metamorphic laws --- ...d-subset-refinement-model.property.test.ts | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 packages/db/tests/query/load-subset-refinement-model.property.test.ts diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts new file mode 100644 index 000000000..86dfe9e5d --- /dev/null +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -0,0 +1,190 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect } from 'vitest' +import { + projectReplayPublication, + projectSyncTransactions, +} from '../load-subset-full-flow-model.js' +import { oraclePropertyOptions } from '../oracle-config.js' +import type { + FullFlowVersionedRow, + LoadSubsetFullFlowEvent, +} from '../load-subset-full-flow-model.js' + +function successfulTransaction( + transactionId: string, + sourceId: string, + rowKey: string, +): Array { + return [ + { + type: `stageSyncTransaction`, + transactionId, + sourceId, + rowKeys: [rowKey], + }, + { + type: `commitSyncTransaction`, + transactionId, + parked: false, + signalAborted: false, + }, + { type: `enterSyncApplication`, transactionId }, + { type: `publishSyncTransaction`, transactionId }, + { type: `settleSyncReceipt`, transactionId }, + ] +} + +fcTest.prop( + [ + fc.string({ minLength: 1, maxLength: 4 }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + oraclePropertyOptions(50), +)( + `commuting independent transactions preserves final public state and receipts`, + (leftKey, rightKey) => { + const left = successfulTransaction(`left-tx`, `left-source`, leftKey) + const right = successfulTransaction(`right-tx`, `right-source`, rightKey) + const leftThenRight = projectSyncTransactions([...left, ...right]) + const rightThenLeft = projectSyncTransactions([...right, ...left]) + + // Event-batch order is intentionally observable and may differ. The + // metamorphic law concerns the final independent state and receipts. + expect(leftThenRight.visibleRows).toEqual(rightThenLeft.visibleRows) + expect(leftThenRight.receipts).toEqual(rightThenLeft.receipts) + }, +) + +function overlappingReplayHistory( + baseline: FullFlowVersionedRow, + replacement: FullFlowVersionedRow, + oldAttemptId: string, + newAttemptId: string, + settlementOrder: `old-first` | `new-first`, +): Array { + const settlements: Array = + settlementOrder === `old-first` + ? [ + { + type: `settleReplay`, + attemptId: oldAttemptId, + outcome: `reject`, + }, + { + type: `settleReplay`, + attemptId: newAttemptId, + outcome: `resolve`, + }, + ] + : [ + { + type: `settleReplay`, + attemptId: newAttemptId, + outcome: `resolve`, + }, + { + type: `settleReplay`, + attemptId: oldAttemptId, + outcome: `reject`, + }, + ] + return [ + { + type: `establishPublication`, + sourceId: baseline.sourceId, + rows: [baseline], + }, + { + type: `startReplay`, + attemptId: oldAttemptId, + sourceId: baseline.sourceId, + }, + { + type: `startReplay`, + attemptId: newAttemptId, + sourceId: baseline.sourceId, + }, + { + type: `writeReplayRows`, + attemptId: newAttemptId, + rows: [replacement], + acceptedByCore: true, + }, + ...settlements, + ] +} + +fcTest.prop( + [fc.integer({ min: -10, max: 10 }), fc.integer({ min: -10, max: 10 })], + oraclePropertyOptions(50), +)( + `overlapping replay settlement order does not change the newest complete replacement`, + (baselineVersion, replacementVersion) => { + const baseline = { + sourceId: `source`, + rowKey: `row`, + version: baselineVersion, + } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } + + expect( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `old-first`, + ), + ), + ).toEqual( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `new-first`, + ), + ), + ) + }, +) + +fcTest.prop([fc.integer({ min: -10, max: 10 })], oraclePropertyOptions(50))( + `replay attempt names are observationally erased`, + (replacementVersion) => { + const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } + + expect( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ), + ), + ).toEqual( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `renamed-old`, + `renamed-new`, + `new-first`, + ), + ), + ) + }, +) From eaf1ac7d412f30e1b32d4e09a573515a029cab93 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 13:21:16 -0600 Subject: [PATCH 016/327] test(db): complete subset refinement laws --- .../db/tests/load-subset-full-flow-model.ts | 123 ++++- ...d-subset-refinement-model.property.test.ts | 441 +++++++++++++++++- 2 files changed, 561 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index d020d4a72..8de894376 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -3,6 +3,7 @@ export type FullFlowSessionId = string export type FullFlowDemandId = string export type FullFlowSourceId = string export type FullFlowTransactionId = string +export type FullFlowAcquisitionId = string export type FullFlowVersionedRow = { sourceId: FullFlowSourceId rowKey: string @@ -117,6 +118,23 @@ export type LoadSubsetFullFlowEvent = demandId: FullFlowDemandId outcome: `resolve` | `reject` } + | { + type: `startAcquisition` + acquisitionId: FullFlowAcquisitionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + } + | { + type: `attachAcquisitionOwner` + acquisitionId: FullFlowAcquisitionId + ownerId: FullFlowOwnerId + } + | { + type: `settleAcquisition` + acquisitionId: FullFlowAcquisitionId + outcome: `resolve` | `reject` + rowKeys: ReadonlyArray + } export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` @@ -194,6 +212,9 @@ export function projectTransportLoads( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: break } } @@ -269,6 +290,9 @@ export function projectAuthorizedContinuationStarts( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: break } } @@ -429,6 +453,9 @@ export function projectSyncTransactions( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: break } } @@ -463,6 +490,7 @@ export type ExpectedReplayObservation = { coreRows: Array visibleRows: Array publishedBatches: Array> + callbackReads: Array> } type ProjectedReplayAttempt = { @@ -527,6 +555,7 @@ export function projectReplayPublication( const coreRows = new Map() const visibleRows = new Map() const publishedBatches: Array> = [] + const callbackReads: Array> = [] const sessions = new Map() const attemptSessions = new Map() @@ -540,7 +569,10 @@ export function projectReplayPublication( visibleRows.set(identity, row) batch.push({ type: `insert`, row }) } - if (batch.length > 0) publishedBatches.push(batch) + if (batch.length > 0) { + publishedBatches.push(batch) + callbackReads.push(sortVersionedRows(visibleRows.values())) + } break } case `startReplay`: { @@ -599,7 +631,10 @@ export function projectReplayPublication( for (const [identity, row] of replacement) { visibleRows.set(identity, row) } - if (changes.length > 0) publishedBatches.push(changes) + if (changes.length > 0) { + publishedBatches.push(changes) + callbackReads.push(sortVersionedRows(visibleRows.values())) + } } sessions.delete(session.sourceId) for (const attemptId of session.attempts.keys()) { @@ -623,6 +658,9 @@ export function projectReplayPublication( case `settleSyncReceipt`: case `registerSourceDemand`: case `settleSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: break } } @@ -631,6 +669,7 @@ export function projectReplayPublication( coreRows: sortVersionedRows(coreRows.values()), visibleRows: sortVersionedRows(visibleRows.values()), publishedBatches, + callbackReads, } } @@ -701,6 +740,9 @@ export function projectSourceReadiness( case `startReplay`: case `writeReplayRows`: case `settleReplay`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: break } } @@ -735,3 +777,80 @@ export function projectSourceReadiness( failedSources, } } + +export type ExpectedAcquisitionObservation = { + physicalStarts: Array + owners: Array<{ + ownerId: FullFlowOwnerId + state: `pending` | `resolved` | `rejected` + rowKeys: Array + }> + visibleRowKeys: Array +} + +/** + * Projects the semantic result of physical acquisition sharing. + * + * A physical acquisition may serve one or many logical owners. Sharing may + * reduce transport starts, but it cannot change any owner's settlement or the + * rows made visible by successful work. + */ +export function projectAcquisitionSettlement( + history: ReadonlyArray, +): ExpectedAcquisitionObservation { + const acquisitions = new Map< + FullFlowAcquisitionId, + { + owners: Set + state: `pending` | `resolved` | `rejected` + rowKeys: Array + } + >() + const physicalStarts: Array = [] + const visibleRowKeys = new Set() + + for (const event of history) { + switch (event.type) { + case `startAcquisition`: + if (!acquisitions.has(event.acquisitionId)) { + acquisitions.set(event.acquisitionId, { + owners: new Set(), + state: `pending`, + rowKeys: [], + }) + physicalStarts.push(event.acquisitionId) + } + break + case `attachAcquisitionOwner`: + acquisitions.get(event.acquisitionId)?.owners.add(event.ownerId) + break + case `settleAcquisition`: { + const acquisition = acquisitions.get(event.acquisitionId) + if (!acquisition || acquisition.state !== `pending`) break + acquisition.state = + event.outcome === `resolve` ? `resolved` : `rejected` + acquisition.rowKeys = [...new Set(event.rowKeys)].sort() + if (acquisition.state === `resolved`) { + acquisition.rowKeys.forEach((rowKey) => visibleRowKeys.add(rowKey)) + } + break + } + default: + break + } + } + + return { + physicalStarts, + owners: [...acquisitions.values()] + .flatMap((acquisition) => + [...acquisition.owners].map((ownerId) => ({ + ownerId, + state: acquisition.state, + rowKeys: acquisition.state === `resolved` ? acquisition.rowKeys : [], + })), + ) + .sort((left, right) => left.ownerId.localeCompare(right.ownerId)), + visibleRowKeys: [...visibleRowKeys].sort(), + } +} diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 86dfe9e5d..94b37db7b 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1,14 +1,22 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { expect } from 'vitest' +import { expect, it } from 'vitest' +import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { + projectAcquisitionSettlement, + projectAdapterLifecycle, + projectAuthorizedContinuationStarts, projectReplayPublication, + projectRetainedRowKeys, + projectSourceReadiness, projectSyncTransactions, + projectTransportLoads, } from '../load-subset-full-flow-model.js' import { oraclePropertyOptions } from '../oracle-config.js' import type { FullFlowVersionedRow, LoadSubsetFullFlowEvent, } from '../load-subset-full-flow-model.js' +import type { LoadSubsetOptions, LoadSubsetResult } from '../../src/types.js' function successfulTransaction( transactionId: string, @@ -55,6 +63,304 @@ fcTest.prop( }, ) +function enumerateDemandLifecycles(): Array> { + const histories: Array> = [] + const visit = ( + history: Array, + unseenOwners: ReadonlyArray, + activeOwners: ReadonlyArray, + ) => { + histories.push(history) + if (history.length === 4) return + + for (const ownerId of unseenOwners) { + for (const alreadyAborted of [false, true]) { + visit( + [ + ...history, + { + type: `requestDemand`, + ownerId, + sessionId: `session`, + demandId: `demand`, + alreadyAborted, + }, + ], + unseenOwners.filter((owner) => owner !== ownerId), + alreadyAborted ? activeOwners : [...activeOwners, ownerId], + ) + } + } + for (const ownerId of activeOwners) { + visit( + [ + ...history, + { + type: `releaseDemand`, + ownerId, + demandId: `demand`, + rowKeys: [], + finalRowOwner: false, + invalidatesAdapterEvidence: false, + }, + ], + unseenOwners, + activeOwners.filter((owner) => owner !== ownerId), + ) + } + } + + visit([], [`owner-a`, `owner-b`], []) + return histories +} + +it(`exhaustively conserves adapter starts and releases for two owners`, () => { + for (const history of enumerateDemandLifecycles()) { + const lifecycle = projectAdapterLifecycle(history) + const activeOwners = new Set() + + for (const event of lifecycle) { + if (event.type === `invoke`) { + expect(activeOwners.has(event.ownerId), JSON.stringify(history)).toBe( + false, + ) + activeOwners.add(event.ownerId) + } else { + expect( + activeOwners.delete(event.ownerId), + JSON.stringify(history), + ).toBe(true) + } + } + } +}) + +function renameHistoryIds( + history: ReadonlyArray, + suffix: string, +): Array { + return history.map((event) => { + switch (event.type) { + case `requestDemand`: + return { + ...event, + ownerId: `${event.ownerId}-${suffix}`, + sessionId: `${event.sessionId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `applyAuthoritativeRows`: + case `releaseDemand`: + return { + ...event, + ownerId: `${event.ownerId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `registerSourceDemand`: + case `settleSourceDemand`: + return { + ...event, + sessionId: `${event.sessionId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `cleanupSession`: + return { ...event, sessionId: `${event.sessionId}-${suffix}` } + case `restartSession`: + return { + ...event, + previousSessionId: `${event.previousSessionId}-${suffix}`, + nextSessionId: `${event.nextSessionId}-${suffix}`, + } + case `advanceWindowRevision`: + return { ...event, sessionId: `${event.sessionId}-${suffix}` } + case `scheduleContinuation`: + return { + ...event, + taskId: `${event.taskId}-${suffix}`, + sessionId: `${event.sessionId}-${suffix}`, + } + case `runContinuation`: + return { ...event, taskId: `${event.taskId}-${suffix}` } + case `stageSyncTransaction`: + return { + ...event, + transactionId: `${event.transactionId}-${suffix}`, + } + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + return { + ...event, + transactionId: `${event.transactionId}-${suffix}`, + } + case `startAcquisition`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `attachAcquisitionOwner`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + ownerId: `${event.ownerId}-${suffix}`, + } + case `settleAcquisition`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + } + default: + return event + } + }) +} + +fcTest.prop( + [fc.string({ minLength: 1, maxLength: 4 })], + oraclePropertyOptions(50), +)(`source demand names are observationally erased`, (suffix) => { + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `demand-b`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + outcome: `resolve`, + }, + ] + + expect(projectSourceReadiness(renameHistoryIds(history, suffix))).toEqual( + projectSourceReadiness(history), + ) +}) + +fcTest.prop( + [fc.string({ minLength: 1, maxLength: 4 })], + oraclePropertyOptions(50), +)( + `demand, owner, session, and task names preserve projected laws`, + (suffix) => { + const demandHistory: Array = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + ownerId: `owner`, + demandId: `demand`, + rowKeys: [`row`], + }, + { + type: `releaseDemand`, + ownerId: `owner`, + demandId: `demand`, + rowKeys: [`row`], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + ] + const continuationHistory: Array = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + alreadyAborted: false, + }, + { + type: `scheduleContinuation`, + taskId: `task`, + sessionId: `session`, + windowRevision: 0, + }, + { type: `runContinuation`, taskId: `task` }, + ] + + const renamedDemand = renameHistoryIds(demandHistory, suffix) + expect(projectTransportLoads(renamedDemand)).toBe( + projectTransportLoads(demandHistory), + ) + expect(projectRetainedRowKeys(renamedDemand)).toEqual( + projectRetainedRowKeys(demandHistory), + ) + expect( + projectAdapterLifecycle(renamedDemand).map(({ type }) => type), + ).toEqual(projectAdapterLifecycle(demandHistory).map(({ type }) => type)) + expect( + projectAuthorizedContinuationStarts( + renameHistoryIds(continuationHistory, suffix), + ), + ).toBe(projectAuthorizedContinuationStarts(continuationHistory)) + }, +) + +fcTest.prop( + [fc.string({ minLength: 1, maxLength: 4 })], + oraclePropertyOptions(50), +)(`transaction names do not change publication semantics`, (suffix) => { + const history = successfulTransaction(`transaction`, `source`, `row`) + const original = projectSyncTransactions(history) + const renamed = projectSyncTransactions(renameHistoryIds(history, suffix)) + + expect({ + visibleRows: renamed.visibleRows, + publishedBatches: renamed.publishedBatches, + callbackReads: renamed.callbackReads, + receiptStates: renamed.receipts.map(({ state }) => state), + }).toEqual({ + visibleRows: original.visibleRows, + publishedBatches: original.publishedBatches, + callbackReads: original.callbackReads, + receiptStates: original.receipts.map(({ state }) => state), + }) +}) + +fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + oraclePropertyOptions(50), +)(`acquisition and owner names are semantically erased`, (rowKeys, suffix) => { + const history = acquisitionHistory(`shared`, rowKeys) + const renamed = renameHistoryIds(history, suffix) + + const normalizeOwners = ( + observation: ReturnType, + ) => ({ + owners: observation.owners.map(({ state, rowKeys: keys }) => ({ + state, + rowKeys: keys, + })), + visibleRowKeys: observation.visibleRowKeys, + }) + + expect(normalizeOwners(projectAcquisitionSettlement(renamed))).toEqual( + normalizeOwners(projectAcquisitionSettlement(history)), + ) +}) + function overlappingReplayHistory( baseline: FullFlowVersionedRow, replacement: FullFlowVersionedRow, @@ -188,3 +494,136 @@ fcTest.prop([fc.integer({ min: -10, max: 10 })], oraclePropertyOptions(50))( ) }, ) + +type AcquisitionTopology = `shared` | `separate` + +function acquisitionHistory( + topology: AcquisitionTopology, + rowKeys: ReadonlyArray, +): Array { + const start = (acquisitionId: string): LoadSubsetFullFlowEvent => ({ + type: `startAcquisition`, + acquisitionId, + sourceId: `source`, + demandId: `exact-demand`, + }) + const attach = ( + acquisitionId: string, + ownerId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `attachAcquisitionOwner`, + acquisitionId, + ownerId, + }) + const settle = (acquisitionId: string): LoadSubsetFullFlowEvent => ({ + type: `settleAcquisition`, + acquisitionId, + outcome: `resolve`, + rowKeys, + }) + + return topology === `shared` + ? [ + start(`shared-acquisition`), + attach(`shared-acquisition`, `owner-a`), + attach(`shared-acquisition`, `owner-b`), + settle(`shared-acquisition`), + ] + : [ + start(`acquisition-a`), + attach(`acquisition-a`, `owner-a`), + settle(`acquisition-a`), + start(`acquisition-b`), + attach(`acquisition-b`, `owner-b`), + settle(`acquisition-b`), + ] +} + +function semanticAcquisitionResult( + history: ReadonlyArray, +) { + const { owners, visibleRowKeys } = projectAcquisitionSettlement(history) + return { owners, visibleRowKeys } +} + +async function runAcquisitionTopology( + topology: AcquisitionTopology, + rowKeys: ReadonlyArray, +) { + let physicalStarts = 0 + const createDeduplicator = () => + new DeduplicatedLoadSubset({ + loadSubset: () => { + physicalStarts++ + return Promise.resolve({ + hasMore: false, + appliedRowKeys: rowKeys, + } satisfies LoadSubsetResult) + }, + }) + const shared = createDeduplicator() + const ownerDeduplicators = + topology === `shared` ? [shared, shared] : [shared, createDeduplicator()] + const options: LoadSubsetOptions = { limit: rowKeys.length } + const results = await Promise.all( + ownerDeduplicators.map((deduplicator) => deduplicator.loadSubset(options)), + ) + + return { + physicalStarts, + owners: results.map((result, index) => ({ + ownerId: index === 0 ? `owner-a` : `owner-b`, + state: `resolved` as const, + rowKeys: + result === true || result === undefined + ? [] + : [...(result.appliedRowKeys ?? [])].map(String).sort(), + })), + visibleRowKeys: [ + ...new Set( + results.flatMap((result) => + result === true || result === undefined + ? [] + : [...(result.appliedRowKeys ?? [])].map(String), + ), + ), + ].sort(), + } +} + +fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + ], + oraclePropertyOptions(50), +)( + `sharing an exact physical acquisition changes work, not logical results`, + async (rowKeys) => { + const sharedHistory = acquisitionHistory(`shared`, rowKeys) + const separateHistory = acquisitionHistory(`separate`, rowKeys) + const sharedExpected = projectAcquisitionSettlement(sharedHistory) + const separateExpected = projectAcquisitionSettlement(separateHistory) + + expect(semanticAcquisitionResult(sharedHistory)).toEqual( + semanticAcquisitionResult(separateHistory), + ) + expect(sharedExpected.physicalStarts).toHaveLength(1) + expect(separateExpected.physicalStarts).toHaveLength(2) + + const sharedActual = await runAcquisitionTopology(`shared`, rowKeys) + const separateActual = await runAcquisitionTopology(`separate`, rowKeys) + expect({ + owners: sharedActual.owners, + visibleRowKeys: sharedActual.visibleRowKeys, + }).toEqual(semanticAcquisitionResult(sharedHistory)) + expect({ + owners: separateActual.owners, + visibleRowKeys: separateActual.visibleRowKeys, + }).toEqual(semanticAcquisitionResult(separateHistory)) + expect(sharedActual.physicalStarts).toBe(1) + expect(separateActual.physicalStarts).toBe(2) + }, +) From 3059bc821a156b3818632e9c2bbd500f3992f2ef Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 13:21:24 -0600 Subject: [PATCH 017/327] test(db): verify replay publication coherence --- ...-replay-refinement-oracle.property.test.ts | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts index a000b3c36..d26472bef 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' +import { createLiveQueryCollection } from '../../src/query/index.js' import { projectReplayPublication } from '../load-subset-full-flow-model.js' import { flushPromises } from '../utils.js' -import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { + FullFlowVersionedRow, + LoadSubsetFullFlowEvent, +} from '../load-subset-full-flow-model.js' import type { ChangeMessage, ChangeMessageOrDeleteKeyMessage, @@ -30,7 +34,6 @@ describe(`loadSubset replay refinement`, () => { previousVersion?: number }> > = [] - const visible = new Map() const source = createCollection({ id: sourceId, getKey: (row) => row.id, @@ -60,11 +63,19 @@ describe(`loadSubset replay refinement`, () => { }, }, }) - const subscription = source.subscribeChanges( + const downstream = createLiveQueryCollection({ + id: `${sourceId}-downstream`, + query: (q) => + q.from({ row: source }).select(({ row }) => ({ + id: row.id, + version: row.version, + })), + startSync: true, + }) + const callbackReads: Array> = [] + const subscription = downstream.subscribeChanges( (changes: Array>) => { const batch = changes.map((change) => { - if (change.type === `delete`) visible.delete(String(change.key)) - else visible.set(String(change.key), { ...change.value }) return { type: change.type, row: { @@ -77,8 +88,18 @@ describe(`loadSubset replay refinement`, () => { : { previousVersion: change.previousValue.version }), } }) - if (batch.length > 0) batches.push(batch) + if (batch.length > 0) { + batches.push(batch) + callbackReads.push( + downstream.toArray.map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })), + ) + } }, + { includeInitialState: true }, ) const replaceCore = (version: number) => { @@ -99,7 +120,7 @@ describe(`loadSubset replay refinement`, () => { version, })) const visibleRows = () => - [...visible.values()].map(({ id, version }) => ({ + downstream.toArray.map(({ id, version }) => ({ sourceId, rowKey: id, version, @@ -107,9 +128,11 @@ describe(`loadSubset replay refinement`, () => { return { source, + downstream, subscription, pending, batches, + callbackReads, replaceCore, startReplay, coreRows, @@ -130,7 +153,7 @@ describe(`loadSubset replay refinement`, () => { const harness = createHarness(sourceId) try { - harness.subscription.requestSnapshot({ optimizedOnly: false }) + await harness.downstream.preload() await harness.startReplay() history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) @@ -153,9 +176,13 @@ describe(`loadSubset replay refinement`, () => { expect(harness.coreRows()).toEqual(expected.coreRows) expect(harness.visibleRows()).toEqual(expected.visibleRows) expect(harness.batches).toEqual(expected.publishedBatches) + expect(harness.callbackReads).toEqual(expected.callbackReads) } finally { harness.subscription.unsubscribe() - await harness.source.cleanup() + await Promise.all([ + harness.downstream.cleanup(), + harness.source.cleanup(), + ]) } }) @@ -172,7 +199,7 @@ describe(`loadSubset replay refinement`, () => { const harness = createHarness(sourceId) try { - harness.subscription.requestSnapshot({ optimizedOnly: false }) + await harness.downstream.preload() await harness.startReplay() history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) await harness.startReplay() @@ -199,6 +226,9 @@ describe(`loadSubset replay refinement`, () => { beforeObsoleteSettlement.visibleRows, ) expect(harness.batches).toEqual(beforeObsoleteSettlement.publishedBatches) + expect(harness.callbackReads).toEqual( + beforeObsoleteSettlement.callbackReads, + ) harness.pending[0]?.deferred.reject( new DOMException(`obsolete`, `AbortError`), @@ -214,10 +244,14 @@ describe(`loadSubset replay refinement`, () => { expect(harness.coreRows()).toEqual(expected.coreRows) expect(harness.visibleRows()).toEqual(expected.visibleRows) expect(harness.batches).toEqual(expected.publishedBatches) + expect(harness.callbackReads).toEqual(expected.callbackReads) } finally { for (const replay of harness.pending) replay.deferred.resolve() harness.subscription.unsubscribe() - await harness.source.cleanup() + await Promise.all([ + harness.downstream.cleanup(), + harness.source.cleanup(), + ]) } }) }) From 2dd6678e95885e92bb452d633d75bfa9da7146d9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 13:21:33 -0600 Subject: [PATCH 018/327] test(adapters): replay subset ownership model --- .../tests/electric-live-query.test.ts | 99 +++++++++++++++++++ .../tests/on-demand-sync.test.ts | 95 ++++++++++++++++++ 2 files changed, 194 insertions(+) diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index 572fb90be..1bceff471 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -8,10 +8,15 @@ import { lt, } from '@tanstack/db' import { electricCollectionOptions } from '../src/electric' +import { + projectRetainedRowKeys, + projectTransportLoads, +} from '../../db/tests/load-subset-full-flow-model' import type { ElectricCollectionUtils } from '../src/electric' import type { Collection } from '@tanstack/db' import type { Message } from '@electric-sql/client' import type { StandardSchemaV1 } from '@standard-schema/spec' +import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' // Sample user type for tests type User = { @@ -1316,4 +1321,98 @@ describe(`Electric Collection - loadSubset deduplication`, () => { // Still 2 calls - third was covered by the union of first two expect(mockRequestSnapshot).toHaveBeenCalledTimes(2) }) + + it(`matches the shared remount history after final-owner release`, async () => { + const electricCollection = createElectricCollectionWithSyncMode(`on-demand`) + const row = sampleUsers[0]! + const history: Array = [ + { + type: `requestDemand`, + ownerId: `owner-1`, + sessionId: `session-1`, + demandId: `active-users`, + alreadyAborted: false, + }, + ] + const createLive = (id: string) => + createLiveQueryCollection({ + id, + startSync: true, + query: (q) => + q + .from({ user: electricCollection }) + .where(({ user }) => eq(user.active, true)), + }) + simulateInitialSync([]) + mockRequestSnapshot.mockResolvedValue({ + data: [ + { + headers: { operation: `insert` }, + key: row.id, + value: row, + }, + ], + }) + const first = createLive(`electric-conformance-first`) + let second: ReturnType | undefined + + try { + await first.preload() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-1`, + demandId: `active-users`, + rowKeys: [String(row.id)], + }) + expect(first.toArray.map(({ id }) => String(id))).toEqual([ + String(row.id), + ]) + + await first.cleanup() + history.push( + { + type: `releaseDemand`, + ownerId: `owner-1`, + demandId: `active-users`, + rowKeys: [String(row.id)], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + { + type: `restartSession`, + previousSessionId: `session-1`, + nextSessionId: `session-2`, + }, + { + type: `requestDemand`, + ownerId: `owner-2`, + sessionId: `session-2`, + demandId: `active-users`, + alreadyAborted: false, + }, + ) + + second = createLive(`electric-conformance-second`) + await second.preload() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-2`, + demandId: `active-users`, + rowKeys: [String(row.id)], + }) + + expect(mockRequestSnapshot).toHaveBeenCalledTimes( + projectTransportLoads(history), + ) + expect(second.toArray.map(({ id }) => String(id))).toEqual( + projectRetainedRowKeys(history), + ) + } finally { + await Promise.all([ + first.cleanup(), + second?.cleanup() ?? Promise.resolve(), + electricCollection.cleanup(), + ]) + } + }) }) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 8d1dc3412..12dfcf1cc 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -14,6 +14,11 @@ import { } from '@tanstack/db' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' +import { + projectRetainedRowKeys, + projectTransportLoads, +} from '../../db/tests/load-subset-full-flow-model' +import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' const APP_SCHEMA = new Schema({ products: new Table({ @@ -1792,6 +1797,96 @@ describe(`On-Demand Sync Mode`, () => { { timeout: 2000 }, ) }) + + it(`matches the shared remount history after final-owner release`, async () => { + const db = await createDatabase() + await createTestProducts(db) + let transportLoads = 0 + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset: () => { + transportLoads++ + }, + }), + ) + await collection.stateWhenReady() + const createLive = () => + createLiveQueryCollection({ + query: (q) => + q + .from({ product: collection }) + .where(({ product }) => eq(product.category, `electronics`)), + }) + const first = createLive() + let second: ReturnType | undefined + const history: Array = [ + { + type: `requestDemand`, + ownerId: `owner-1`, + sessionId: `session-1`, + demandId: `electronics`, + alreadyAborted: false, + }, + ] + + try { + await first.preload() + const rowKeys = first.toArray.map(({ id }) => String(id)).sort() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-1`, + demandId: `electronics`, + rowKeys, + }) + + await first.cleanup() + history.push( + { + type: `releaseDemand`, + ownerId: `owner-1`, + demandId: `electronics`, + rowKeys, + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + { + type: `restartSession`, + previousSessionId: `session-1`, + nextSessionId: `session-2`, + }, + { + type: `requestDemand`, + ownerId: `owner-2`, + sessionId: `session-2`, + demandId: `electronics`, + alreadyAborted: false, + }, + ) + await vi.waitFor(() => expect(collection.size).toBe(0)) + + second = createLive() + await second.preload() + const reloadedKeys = second.toArray.map(({ id }) => String(id)).sort() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-2`, + demandId: `electronics`, + rowKeys: reloadedKeys, + }) + + expect(transportLoads).toBe(projectTransportLoads(history)) + expect(reloadedKeys).toEqual(projectRetainedRowKeys(history)) + } finally { + await Promise.all([ + first.cleanup(), + second?.cleanup() ?? Promise.resolve(), + collection.cleanup(), + ]) + } + }) }) describe(`Overlapping data across queries`, () => { From 1121a59466cc14f3b9c4d637a84ab119b95e7121 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 13:43:41 -0600 Subject: [PATCH 019/327] test(db): harden subset refinement oracles --- packages/db/package.json | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 5 + .../db/tests/load-subset-full-flow-model.ts | 47 +- ...d-subset-refinement-model.property.test.ts | 771 +++++++++++------- ...-replay-refinement-oracle.property.test.ts | 24 +- .../tests/electric-live-query.test.ts | 2 +- .../tests/on-demand-sync.test.ts | 20 +- 7 files changed, 540 insertions(+), 331 deletions(-) diff --git a/packages/db/package.json b/packages/db/package.json index 382d2ee90..764db63b6 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts" + "test:oracles": "vitest --run tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.property.test.ts tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts tests/query/load-subset-transaction-refinement-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 7d32a5b8f..e5d83ca23 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -735,6 +735,11 @@ create recursive Collection machinery. | Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | | Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | | Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | +| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | +| End-to-end demand, continuation, and outcome-free boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | +| Subset acquisition, readiness, receipt, and replay refinement laws | `packages/db/tests/query/load-subset-refinement-*.property.test.ts` | +| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.property.test.ts` | +| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 8de894376..3d0ba53fa 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -1,3 +1,12 @@ +/** + * A shared event vocabulary for small, independent refinement projections. + * + * This is deliberately not a second implementation of the Collection state + * machine. Each projector owns one law and ignores unrelated events. The + * lifecycle command model generates legal acquisition/release histories; + * boundary suites compare these projections with public Collection + * observations at the points where planes meet. + */ export type FullFlowOwnerId = string export type FullFlowSessionId = string export type FullFlowDemandId = string @@ -24,6 +33,10 @@ export type LoadSubsetFullFlowEvent = demandId: FullFlowDemandId rowKeys: ReadonlyArray } + | { + type: `settleDemandWithoutEvidence` + demandId: FullFlowDemandId + } | { type: `releaseDemand` ownerId: FullFlowOwnerId @@ -170,29 +183,41 @@ export function projectAdapterLifecycle( /** * Projects physical transport work from adapter evidence lifetime. * - * Request settlement alone is not evidence. Only an applied authoritative row - * publication makes the exact demand reusable, and an unload that invalidates - * that evidence forces the next owner to fetch again. + * Concurrent owners attach to one in-flight exact demand. Settlement alone is + * not reusable evidence: only an applied authoritative row publication makes + * the demand reusable, and an unload that invalidates that evidence forces the + * next owner to fetch again. */ export function projectTransportLoads( history: ReadonlyArray, ): number { const reusableDemands = new Set() + const inFlightDemands = new Set() let loads = 0 for (const event of history) { switch (event.type) { case `requestDemand`: - if (!event.alreadyAborted && !reusableDemands.has(event.demandId)) { + if ( + !event.alreadyAborted && + !reusableDemands.has(event.demandId) && + !inFlightDemands.has(event.demandId) + ) { loads++ + inFlightDemands.add(event.demandId) } break case `applyAuthoritativeRows`: + inFlightDemands.delete(event.demandId) reusableDemands.add(event.demandId) break + case `settleDemandWithoutEvidence`: + inFlightDemands.delete(event.demandId) + break case `releaseDemand`: if (event.invalidatesAdapterEvidence) { reusableDemands.delete(event.demandId) + inFlightDemands.delete(event.demandId) } break case `restartSession`: @@ -277,6 +302,7 @@ export function projectAuthorizedContinuationStarts( break } case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: case `releaseDemand`: case `stageSyncTransaction`: case `commitSyncTransaction`: @@ -441,6 +467,7 @@ export function projectSyncTransactions( } case `requestDemand`: case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: case `releaseDemand`: case `restartSession`: case `cleanupSession`: @@ -465,14 +492,14 @@ export function projectSyncTransactions( publishedBatches, callbackReads, receipts: [...transactions] - .flatMap(([transactionId, transaction]) => { + .map(([transactionId, transaction]) => { const state = transaction.state === `resolved` ? `resolved` : transaction.state === `rejected` ? `rejected` : `pending` - return [{ transactionId, state } as const] + return { transactionId, state } as const }) .sort((left, right) => left.transactionId.localeCompare(right.transactionId), @@ -644,6 +671,7 @@ export function projectReplayPublication( } case `requestDemand`: case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: case `releaseDemand`: case `restartSession`: case `cleanupSession`: @@ -686,7 +714,6 @@ export function projectSourceReadiness( const demands = new Map< string, { - sessionId: FullFlowSessionId sourceId: FullFlowSourceId state: `pending` | `resolved` | `rejected` } @@ -701,7 +728,6 @@ export function projectSourceReadiness( if (event.sessionId !== currentSession) break cleanedUp = false demands.set(`${event.sourceId}\u0000${event.demandId}`, { - sessionId: event.sessionId, sourceId: event.sourceId, state: `pending`, }) @@ -726,6 +752,7 @@ export function projectSourceReadiness( break case `requestDemand`: case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: case `releaseDemand`: case `advanceWindowRevision`: case `scheduleContinuation`: @@ -747,9 +774,7 @@ export function projectSourceReadiness( } } - const currentDemands = [...demands.values()].filter( - ({ sessionId }) => sessionId === currentSession, - ) + const currentDemands = [...demands.values()] const pendingSources = [ ...new Set( currentDemands diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 94b37db7b..6d86c25be 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1,5 +1,8 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { createLiveQueryCollection } from '../../src/query/index.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { projectAcquisitionSettlement, @@ -11,12 +14,26 @@ import { projectSyncTransactions, projectTransportLoads, } from '../load-subset-full-flow-model.js' -import { oraclePropertyOptions } from '../oracle-config.js' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' +import { flushPromises } from '../utils.js' import type { FullFlowVersionedRow, LoadSubsetFullFlowEvent, } from '../load-subset-full-flow-model.js' -import type { LoadSubsetOptions, LoadSubsetResult } from '../../src/types.js' +import type { LoadSubsetResult } from '../../src/types.js' + +function refinementCampaigns(fixedSeed: number) { + return [ + { + label: `fixed seed ${fixedSeed}`, + options: { numRuns: oracleRuns(50), seed: fixedSeed }, + }, + { + label: `random or replayed seed`, + options: oraclePropertyOptions(50), + }, + ] as const +} function successfulTransaction( transactionId: string, @@ -42,35 +59,43 @@ function successfulTransaction( ] } -fcTest.prop( - [ - fc.string({ minLength: 1, maxLength: 4 }), - fc.string({ minLength: 1, maxLength: 4 }), - ], - oraclePropertyOptions(50), -)( - `commuting independent transactions preserves final public state and receipts`, - (leftKey, rightKey) => { - const left = successfulTransaction(`left-tx`, `left-source`, leftKey) - const right = successfulTransaction(`right-tx`, `right-source`, rightKey) - const leftThenRight = projectSyncTransactions([...left, ...right]) - const rightThenLeft = projectSyncTransactions([...right, ...left]) - - // Event-batch order is intentionally observable and may differ. The - // metamorphic law concerns the final independent state and receipts. - expect(leftThenRight.visibleRows).toEqual(rightThenLeft.visibleRows) - expect(leftThenRight.receipts).toEqual(rightThenLeft.receipts) - }, -) - -function enumerateDemandLifecycles(): Array> { - const histories: Array> = [] +for (const campaign of refinementCampaigns(1_779_001)) { + fcTest.prop( + [ + fc.string({ minLength: 1, maxLength: 4 }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( + `commuting independent transactions preserves final public state and receipts (${campaign.label})`, + (leftKey, rightKey) => { + const left = successfulTransaction(`left-tx`, `left-source`, leftKey) + const right = successfulTransaction(`right-tx`, `right-source`, rightKey) + const leftThenRight = projectSyncTransactions([...left, ...right]) + const rightThenLeft = projectSyncTransactions([...right, ...left]) + + // Event-batch order is intentionally observable and may differ. The + // metamorphic law concerns the final independent state and receipts. + expect(leftThenRight.visibleRows).toEqual(rightThenLeft.visibleRows) + expect(leftThenRight.receipts).toEqual(rightThenLeft.receipts) + }, + ) +} + +type DemandLifecycleCase = { + history: Array + expected: Array<{ type: `invoke` | `release`; ownerId: string }> +} + +function enumerateDemandLifecycles(): Array { + const cases: Array = [] const visit = ( history: Array, + expected: DemandLifecycleCase[`expected`], unseenOwners: ReadonlyArray, activeOwners: ReadonlyArray, ) => { - histories.push(history) + cases.push({ history, expected }) if (history.length === 4) return for (const ownerId of unseenOwners) { @@ -86,6 +111,9 @@ function enumerateDemandLifecycles(): Array> { alreadyAborted, }, ], + alreadyAborted + ? expected + : [...expected, { type: `invoke`, ownerId }], unseenOwners.filter((owner) => owner !== ownerId), alreadyAborted ? activeOwners : [...activeOwners, ownerId], ) @@ -104,19 +132,21 @@ function enumerateDemandLifecycles(): Array> { invalidatesAdapterEvidence: false, }, ], + [...expected, { type: `release`, ownerId }], unseenOwners, activeOwners.filter((owner) => owner !== ownerId), ) } } - visit([], [`owner-a`, `owner-b`], []) - return histories + visit([], [], [`owner-a`, `owner-b`], []) + return cases } -it(`exhaustively conserves adapter starts and releases for two owners`, () => { - for (const history of enumerateDemandLifecycles()) { +it(`exhaustively projects exact adapter starts and releases for two owners`, () => { + for (const { history, expected } of enumerateDemandLifecycles()) { const lifecycle = projectAdapterLifecycle(history) + expect(lifecycle, JSON.stringify(history)).toEqual(expected) const activeOwners = new Set() for (const event of lifecycle) { @@ -135,6 +165,58 @@ it(`exhaustively conserves adapter starts and releases for two owners`, () => { } }) +it(`shares concurrent exact demand and retries after evidence-free settlement`, () => { + const request = (ownerId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + ownerId, + sessionId: `session`, + demandId: `exact-demand`, + alreadyAborted: false, + }) + const concurrent = [request(`owner-a`), request(`owner-b`)] + + expect(projectTransportLoads(concurrent)).toBe( + projectAcquisitionSettlement(acquisitionHistory(`shared`, [`row`])) + .physicalStarts.length, + ) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `releaseDemand`, + ownerId: `owner-a`, + demandId: `exact-demand`, + rowKeys: [], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + request(`owner-c`), + ]), + ).toBe(2) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `settleDemandWithoutEvidence`, + demandId: `exact-demand`, + }, + request(`owner-c`), + ]), + ).toBe(2) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `applyAuthoritativeRows`, + ownerId: `owner-a`, + demandId: `exact-demand`, + rowKeys: [`row`], + }, + request(`owner-c`), + ]), + ).toBe(1) +}) + function renameHistoryIds( history: ReadonlyArray, suffix: string, @@ -155,6 +237,11 @@ function renameHistoryIds( ownerId: `${event.ownerId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, } + case `settleDemandWithoutEvidence`: + return { + ...event, + demandId: `${event.demandId}-${suffix}`, + } case `registerSourceDemand`: case `settleSourceDemand`: return { @@ -217,149 +304,157 @@ function renameHistoryIds( }) } -fcTest.prop( - [fc.string({ minLength: 1, maxLength: 4 })], - oraclePropertyOptions(50), -)(`source demand names are observationally erased`, (suffix) => { - const history: Array = [ - { - type: `registerSourceDemand`, - sessionId: `session`, - sourceId: `source-a`, - demandId: `demand-a`, - }, - { - type: `registerSourceDemand`, - sessionId: `session`, - sourceId: `source-b`, - demandId: `demand-b`, - }, - { - type: `settleSourceDemand`, - sessionId: `session`, - sourceId: `source-a`, - demandId: `demand-a`, - outcome: `resolve`, - }, - ] +for (const campaign of refinementCampaigns(1_779_002)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `source demand names are observationally erased (${campaign.label})`, + (suffix) => { + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `demand-b`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + outcome: `resolve`, + }, + ] - expect(projectSourceReadiness(renameHistoryIds(history, suffix))).toEqual( - projectSourceReadiness(history), + expect(projectSourceReadiness(renameHistoryIds(history, suffix))).toEqual( + projectSourceReadiness(history), + ) + }, ) -}) +} -fcTest.prop( - [fc.string({ minLength: 1, maxLength: 4 })], - oraclePropertyOptions(50), -)( - `demand, owner, session, and task names preserve projected laws`, - (suffix) => { - const demandHistory: Array = [ - { - type: `requestDemand`, - ownerId: `owner`, - sessionId: `session`, - demandId: `demand`, - alreadyAborted: false, - }, - { - type: `applyAuthoritativeRows`, - ownerId: `owner`, - demandId: `demand`, - rowKeys: [`row`], - }, - { - type: `releaseDemand`, - ownerId: `owner`, - demandId: `demand`, - rowKeys: [`row`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, - }, - ] - const continuationHistory: Array = [ - { - type: `requestDemand`, - ownerId: `owner`, - sessionId: `session`, - demandId: `demand`, - alreadyAborted: false, - }, - { - type: `scheduleContinuation`, - taskId: `task`, - sessionId: `session`, - windowRevision: 0, - }, - { type: `runContinuation`, taskId: `task` }, - ] - - const renamedDemand = renameHistoryIds(demandHistory, suffix) - expect(projectTransportLoads(renamedDemand)).toBe( - projectTransportLoads(demandHistory), - ) - expect(projectRetainedRowKeys(renamedDemand)).toEqual( - projectRetainedRowKeys(demandHistory), - ) - expect( - projectAdapterLifecycle(renamedDemand).map(({ type }) => type), - ).toEqual(projectAdapterLifecycle(demandHistory).map(({ type }) => type)) - expect( - projectAuthorizedContinuationStarts( - renameHistoryIds(continuationHistory, suffix), - ), - ).toBe(projectAuthorizedContinuationStarts(continuationHistory)) - }, -) - -fcTest.prop( - [fc.string({ minLength: 1, maxLength: 4 })], - oraclePropertyOptions(50), -)(`transaction names do not change publication semantics`, (suffix) => { - const history = successfulTransaction(`transaction`, `source`, `row`) - const original = projectSyncTransactions(history) - const renamed = projectSyncTransactions(renameHistoryIds(history, suffix)) - - expect({ - visibleRows: renamed.visibleRows, - publishedBatches: renamed.publishedBatches, - callbackReads: renamed.callbackReads, - receiptStates: renamed.receipts.map(({ state }) => state), - }).toEqual({ - visibleRows: original.visibleRows, - publishedBatches: original.publishedBatches, - callbackReads: original.callbackReads, - receiptStates: original.receipts.map(({ state }) => state), - }) -}) +for (const campaign of refinementCampaigns(1_779_003)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `demand, owner, session, and task names preserve projected laws (${campaign.label})`, + (suffix) => { + const demandHistory: Array = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + ownerId: `owner`, + demandId: `demand`, + rowKeys: [`row`], + }, + { + type: `releaseDemand`, + ownerId: `owner`, + demandId: `demand`, + rowKeys: [`row`], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + ] + const continuationHistory: Array = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + alreadyAborted: false, + }, + { + type: `scheduleContinuation`, + taskId: `task`, + sessionId: `session`, + windowRevision: 0, + }, + { type: `runContinuation`, taskId: `task` }, + ] -fcTest.prop( - [ - fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { - minLength: 1, - maxLength: 3, - }), - fc.string({ minLength: 1, maxLength: 4 }), - ], - oraclePropertyOptions(50), -)(`acquisition and owner names are semantically erased`, (rowKeys, suffix) => { - const history = acquisitionHistory(`shared`, rowKeys) - const renamed = renameHistoryIds(history, suffix) - - const normalizeOwners = ( - observation: ReturnType, - ) => ({ - owners: observation.owners.map(({ state, rowKeys: keys }) => ({ - state, - rowKeys: keys, - })), - visibleRowKeys: observation.visibleRowKeys, - }) + const renamedDemand = renameHistoryIds(demandHistory, suffix) + expect(projectTransportLoads(renamedDemand)).toBe( + projectTransportLoads(demandHistory), + ) + expect(projectRetainedRowKeys(renamedDemand)).toEqual( + projectRetainedRowKeys(demandHistory), + ) + expect( + projectAdapterLifecycle(renamedDemand).map(({ type }) => type), + ).toEqual(projectAdapterLifecycle(demandHistory).map(({ type }) => type)) + expect( + projectAuthorizedContinuationStarts( + renameHistoryIds(continuationHistory, suffix), + ), + ).toBe(projectAuthorizedContinuationStarts(continuationHistory)) + }, + ) +} - expect(normalizeOwners(projectAcquisitionSettlement(renamed))).toEqual( - normalizeOwners(projectAcquisitionSettlement(history)), +for (const campaign of refinementCampaigns(1_779_004)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `transaction names do not change publication semantics (${campaign.label})`, + (suffix) => { + const history = successfulTransaction(`transaction`, `source`, `row`) + const original = projectSyncTransactions(history) + const renamed = projectSyncTransactions(renameHistoryIds(history, suffix)) + + expect({ + visibleRows: renamed.visibleRows, + publishedBatches: renamed.publishedBatches, + callbackReads: renamed.callbackReads, + receiptStates: renamed.receipts.map(({ state }) => state), + }).toEqual({ + visibleRows: original.visibleRows, + publishedBatches: original.publishedBatches, + callbackReads: original.callbackReads, + receiptStates: original.receipts.map(({ state }) => state), + }) + }, ) -}) +} + +for (const campaign of refinementCampaigns(1_779_005)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( + `acquisition and owner names are semantically erased (${campaign.label})`, + (rowKeys, suffix) => { + const history = acquisitionHistory(`shared`, rowKeys) + const renamed = renameHistoryIds(history, suffix) + + const normalizeOwners = ( + observation: ReturnType, + ) => ({ + owners: observation.owners.map(({ state, rowKeys: keys }) => ({ + state, + rowKeys: keys, + })), + visibleRowKeys: observation.visibleRowKeys, + }) + + expect(normalizeOwners(projectAcquisitionSettlement(renamed))).toEqual( + normalizeOwners(projectAcquisitionSettlement(history)), + ) + }, + ) +} function overlappingReplayHistory( baseline: FullFlowVersionedRow, @@ -420,80 +515,84 @@ function overlappingReplayHistory( ] } -fcTest.prop( - [fc.integer({ min: -10, max: 10 }), fc.integer({ min: -10, max: 10 })], - oraclePropertyOptions(50), -)( - `overlapping replay settlement order does not change the newest complete replacement`, - (baselineVersion, replacementVersion) => { - const baseline = { - sourceId: `source`, - rowKey: `row`, - version: baselineVersion, - } - const replacement = { - sourceId: `source`, - rowKey: `row`, - version: replacementVersion, - } +for (const campaign of refinementCampaigns(1_779_006)) { + fcTest.prop( + [fc.integer({ min: -10, max: 10 }), fc.integer({ min: -10, max: 10 })], + campaign.options, + )( + `overlapping replay settlement order does not change the newest complete replacement (${campaign.label})`, + (baselineVersion, replacementVersion) => { + const baseline = { + sourceId: `source`, + rowKey: `row`, + version: baselineVersion, + } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } - expect( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `old`, - `new`, - `old-first`, + expect( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `old-first`, + ), ), - ), - ).toEqual( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `old`, - `new`, - `new-first`, + ).toEqual( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `new-first`, + ), ), - ), - ) - }, -) - -fcTest.prop([fc.integer({ min: -10, max: 10 })], oraclePropertyOptions(50))( - `replay attempt names are observationally erased`, - (replacementVersion) => { - const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } - const replacement = { - sourceId: `source`, - rowKey: `row`, - version: replacementVersion, - } + ) + }, + ) +} - expect( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `attempt-a`, - `attempt-b`, - `new-first`, +for (const campaign of refinementCampaigns(1_779_007)) { + fcTest.prop([fc.integer({ min: -10, max: 10 })], campaign.options)( + `replay attempt names are observationally erased (${campaign.label})`, + (replacementVersion) => { + const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } + + expect( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ), ), - ), - ).toEqual( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `renamed-old`, - `renamed-new`, - `new-first`, + ).toEqual( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `renamed-old`, + `renamed-new`, + `new-first`, + ), ), - ), - ) - }, -) + ) + }, + ) +} type AcquisitionTopology = `shared` | `separate` @@ -550,80 +649,152 @@ async function runAcquisitionTopology( topology: AcquisitionTopology, rowKeys: ReadonlyArray, ) { + const runId = ++acquisitionRunId let physicalStarts = 0 - const createDeduplicator = () => - new DeduplicatedLoadSubset({ - loadSubset: () => { + const delivery = createDeferred() + const createSource = (suffix: string) => { + type Row = { id: string } + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: async () => { physicalStarts++ - return Promise.resolve({ + await delivery.promise + begin() + for (const id of rowKeys) write({ type: `insert`, value: { id } }) + const applied = commit() + if (applied !== true) await applied + return { hasMore: false, appliedRowKeys: rowKeys, - } satisfies LoadSubsetResult) + } satisfies LoadSubsetResult }, }) - const shared = createDeduplicator() - const ownerDeduplicators = - topology === `shared` ? [shared, shared] : [shared, createDeduplicator()] - const options: LoadSubsetOptions = { limit: rowKeys.length } - const results = await Promise.all( - ownerDeduplicators.map((deduplicator) => deduplicator.loadSubset(options)), + return createCollection({ + id: `refinement-acquisition-${runId}-${suffix}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: deduplicated.loadSubset, + unloadSubset: deduplicated.unloadSubset, + } + }, + }, + }) + } + const sharedSource = createSource(`shared`) + const ownerSources = + topology === `shared` + ? [sharedSource, sharedSource] + : [sharedSource, createSource(`separate`)] + const sources = [...new Set(ownerSources)] + const ownerIds = [`owner-a`, `owner-b`] as const + const liveQueries = ownerSources.map((source, index) => + createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-${ownerIds[index]}`, + query: (q) => q.from({ row: source }), + startSync: true, + }), + ) + const batches: Array>> = [[], []] + const callbackReads: Array>> = [[], []] + const subscriptions = liveQueries.map((live, index) => + live.subscribeChanges( + (changes) => { + batches[index]!.push(changes.map(({ key }) => String(key)).sort()) + callbackReads[index]!.push( + live.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ), ) + const preloads = liveQueries.map((live) => live.preload()) + const expectedPhysicalStarts = topology === `shared` ? 1 : 2 + + try { + for ( + let attempt = 0; + attempt < 20 && physicalStarts < expectedPhysicalStarts; + attempt++ + ) { + await flushPromises() + } + expect(physicalStarts).toBe(expectedPhysicalStarts) + delivery.resolve() + await Promise.all(preloads) - return { - physicalStarts, - owners: results.map((result, index) => ({ - ownerId: index === 0 ? `owner-a` : `owner-b`, + const owners = liveQueries.map((live, index) => ({ + ownerId: ownerIds[index]!, state: `resolved` as const, - rowKeys: - result === true || result === undefined - ? [] - : [...(result.appliedRowKeys ?? [])].map(String).sort(), - })), - visibleRowKeys: [ - ...new Set( - results.flatMap((result) => - result === true || result === undefined - ? [] - : [...(result.appliedRowKeys ?? [])].map(String), - ), - ), - ].sort(), + rowKeys: live.toArray.map(({ id }) => String(id)).sort(), + })) + return { + physicalStarts, + owners, + visibleRowKeys: [ + ...new Set(owners.flatMap(({ rowKeys: keys }) => keys)), + ].sort(), + batches, + callbackReads, + } + } finally { + delivery.resolve() + subscriptions.forEach((subscription) => subscription.unsubscribe()) + await Promise.all([ + ...liveQueries.map((live) => live.cleanup()), + ...sources.map((source) => source.cleanup()), + ]) } } -fcTest.prop( - [ - fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { - minLength: 1, - maxLength: 3, - }), - ], - oraclePropertyOptions(50), -)( - `sharing an exact physical acquisition changes work, not logical results`, - async (rowKeys) => { - const sharedHistory = acquisitionHistory(`shared`, rowKeys) - const separateHistory = acquisitionHistory(`separate`, rowKeys) - const sharedExpected = projectAcquisitionSettlement(sharedHistory) - const separateExpected = projectAcquisitionSettlement(separateHistory) - - expect(semanticAcquisitionResult(sharedHistory)).toEqual( - semanticAcquisitionResult(separateHistory), - ) - expect(sharedExpected.physicalStarts).toHaveLength(1) - expect(separateExpected.physicalStarts).toHaveLength(2) - - const sharedActual = await runAcquisitionTopology(`shared`, rowKeys) - const separateActual = await runAcquisitionTopology(`separate`, rowKeys) - expect({ - owners: sharedActual.owners, - visibleRowKeys: sharedActual.visibleRowKeys, - }).toEqual(semanticAcquisitionResult(sharedHistory)) - expect({ - owners: separateActual.owners, - visibleRowKeys: separateActual.visibleRowKeys, - }).toEqual(semanticAcquisitionResult(separateHistory)) - expect(sharedActual.physicalStarts).toBe(1) - expect(separateActual.physicalStarts).toBe(2) - }, -) +let acquisitionRunId = 0 + +for (const campaign of refinementCampaigns(1_779_008)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + ], + campaign.options, + )( + `sharing an exact physical acquisition changes work, not logical results (${campaign.label})`, + async (rowKeys) => { + const sharedHistory = acquisitionHistory(`shared`, rowKeys) + const separateHistory = acquisitionHistory(`separate`, rowKeys) + const sharedExpected = projectAcquisitionSettlement(sharedHistory) + const separateExpected = projectAcquisitionSettlement(separateHistory) + const sharedSemantic = semanticAcquisitionResult(sharedHistory) + const separateSemantic = semanticAcquisitionResult(separateHistory) + + expect(sharedSemantic).toEqual(separateSemantic) + expect(sharedExpected.physicalStarts).toHaveLength(1) + expect(separateExpected.physicalStarts).toHaveLength(2) + + const sharedActual = await runAcquisitionTopology(`shared`, rowKeys) + const separateActual = await runAcquisitionTopology(`separate`, rowKeys) + expect({ + owners: sharedActual.owners, + visibleRowKeys: sharedActual.visibleRowKeys, + }).toEqual(sharedSemantic) + expect({ + owners: separateActual.owners, + visibleRowKeys: separateActual.visibleRowKeys, + }).toEqual(separateSemantic) + expect(sharedActual.batches).toEqual(separateActual.batches) + expect(sharedActual.callbackReads).toEqual(separateActual.callbackReads) + expect(sharedActual.physicalStarts).toBe(1) + expect(separateActual.physicalStarts).toBe(2) + }, + ) +} diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts index d26472bef..8307ca262 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts @@ -75,19 +75,17 @@ describe(`loadSubset replay refinement`, () => { const callbackReads: Array> = [] const subscription = downstream.subscribeChanges( (changes: Array>) => { - const batch = changes.map((change) => { - return { - type: change.type, - row: { - sourceId, - rowKey: String(change.key), - version: change.value.version, - }, - ...(change.previousValue === undefined - ? {} - : { previousVersion: change.previousValue.version }), - } - }) + const batch = changes.map((change) => ({ + type: change.type, + row: { + sourceId, + rowKey: String(change.key), + version: change.value.version, + }, + ...(change.previousValue === undefined + ? {} + : { previousVersion: change.previousValue.version }), + })) if (batch.length > 0) { batches.push(batch) callbackReads.push( diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index 1bceff471..f0c8dd86b 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -1410,7 +1410,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { } finally { await Promise.all([ first.cleanup(), - second?.cleanup() ?? Promise.resolve(), + second?.cleanup(), electricCollection.cleanup(), ]) } diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 12dfcf1cc..837b01915 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -1801,6 +1801,14 @@ describe(`On-Demand Sync Mode`, () => { it(`matches the shared remount history after final-owner release`, async () => { const db = await createDatabase() await createTestProducts(db) + const expectedRowKeys = ( + await db.getAll<{ id: string }>( + `SELECT id FROM products WHERE category = 'electronics'`, + ) + ) + .map(({ id }) => String(id)) + .sort() + expect(expectedRowKeys).toHaveLength(3) let transportLoads = 0 const collection = createCollection( powerSyncCollectionOptions({ @@ -1834,13 +1842,15 @@ describe(`On-Demand Sync Mode`, () => { try { await first.preload() - const rowKeys = first.toArray.map(({ id }) => String(id)).sort() history.push({ type: `applyAuthoritativeRows`, ownerId: `owner-1`, demandId: `electronics`, - rowKeys, + rowKeys: expectedRowKeys, }) + expect(first.toArray.map(({ id }) => String(id)).sort()).toEqual( + projectRetainedRowKeys(history), + ) await first.cleanup() history.push( @@ -1848,7 +1858,7 @@ describe(`On-Demand Sync Mode`, () => { type: `releaseDemand`, ownerId: `owner-1`, demandId: `electronics`, - rowKeys, + rowKeys: expectedRowKeys, finalRowOwner: true, invalidatesAdapterEvidence: true, }, @@ -1874,7 +1884,7 @@ describe(`On-Demand Sync Mode`, () => { type: `applyAuthoritativeRows`, ownerId: `owner-2`, demandId: `electronics`, - rowKeys: reloadedKeys, + rowKeys: expectedRowKeys, }) expect(transportLoads).toBe(projectTransportLoads(history)) @@ -1882,7 +1892,7 @@ describe(`On-Demand Sync Mode`, () => { } finally { await Promise.all([ first.cleanup(), - second?.cleanup() ?? Promise.resolve(), + second?.cleanup(), collection.cleanup(), ]) } From 041560b9e2f7b5aff096b8abd63337e7e90fdc9e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 14:01:42 -0600 Subject: [PATCH 020/327] test(db): infer replay callback types --- .../load-subset-replay-refinement-oracle.property.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts index 8307ca262..c81af33ab 100644 --- a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts @@ -9,7 +9,6 @@ import type { LoadSubsetFullFlowEvent, } from '../load-subset-full-flow-model.js' import type { - ChangeMessage, ChangeMessageOrDeleteKeyMessage, LoadSubsetOptions, } from '../../src/types.js' @@ -74,7 +73,7 @@ describe(`loadSubset replay refinement`, () => { }) const callbackReads: Array> = [] const subscription = downstream.subscribeChanges( - (changes: Array>) => { + (changes) => { const batch = changes.map((change) => ({ type: change.type, row: { From b5fc012fd898bdecb04e6a012015dbbfe055c576 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 26 Aug 2026 16:15:02 -0600 Subject: [PATCH 021/327] test(db): cover filtered join transport starts --- ...d-subset-full-flow-oracle.property.test.ts | 96 ++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index e30a09ca1..199010ee6 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -2,7 +2,7 @@ import { expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BTreeIndex } from '../../src/index.js' -import { createLiveQueryCollection } from '../../src/query/index.js' +import { createLiveQueryCollection, eq, gte } from '../../src/query/index.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { projectAdapterLifecycle, @@ -30,6 +30,100 @@ function visibleRows( return Array.from(values, ({ id, value }) => ({ id, value })) } +it(`loads each side of a filtered inner join once`, async () => { + type Order = { + id: number + scheduledAt: string + status: string + addressId: number + } + type Charge = { id: number; addressId: number } + + const orderLoads: Array = [] + const chargeLoads: Array = [] + const orders = createCollection({ + id: `full-flow-filtered-join-orders`, + getKey: (order) => order.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id: 1, + scheduledAt: `2024-01-15`, + status: `queued`, + addressId: 1, + }, + }) + write({ + type: `insert`, + value: { + id: 2, + scheduledAt: `2024-01-10`, + status: `queued`, + addressId: 2, + }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + orderLoads.push(options) + return true + }, + } + }, + }, + }) + const charges = createCollection({ + id: `full-flow-filtered-join-charges`, + getKey: (charge) => charge.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: 10, addressId: 1 } }) + write({ type: `insert`, value: { id: 20, addressId: 2 } }) + commit() + markReady() + return { + loadSubset: (options) => { + chargeLoads.push(options) + return true + }, + } + }, + }, + }) + const query = createLiveQueryCollection((q) => + q + .from({ order: orders }) + .where(({ order }) => gte(order.scheduledAt, `2024-01-12`)) + .where(({ order }) => eq(order.status, `queued`)) + .innerJoin({ charge: charges }, ({ order, charge }) => + eq(order.addressId, charge.addressId), + ), + ) + + try { + await query.preload() + + expect( + [...query.values()].map(({ order, charge }) => [order.id, charge.id]), + ).toEqual([[1, 10]]) + expect(orderLoads).toHaveLength(1) + expect(chargeLoads).toHaveLength(1) + } finally { + await Promise.all([query.cleanup(), orders.cleanup(), charges.cleanup()]) + } +}) + it(`does not release physical work when an already-aborted demand skips adapter start`, async () => { const ownerId = `aborted-owner` const requestEvent: LoadSubsetFullFlowEvent = { From bda2303a339a8e16d2f861775c448149ef7e986f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:33:07 -0600 Subject: [PATCH 022/327] ci: Version Packages (#1784) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../lazy-runtime-reference-identities.md | 5 -- examples/angular/todos/package.json | 4 +- examples/electron/offline-first/package.json | 10 +-- .../offline-transactions/package.json | 10 +-- .../react-native/shopping-list/package.json | 10 +-- examples/react/next-ssr-e2e/package.json | 4 +- .../react/offline-transactions/package.json | 10 +-- .../react/paced-mutations-demo/package.json | 4 +- examples/react/projects/package.json | 4 +- examples/react/start-ssr-e2e/package.json | 2 +- examples/react/todo/package.json | 8 +- examples/solid/todo/package.json | 8 +- packages/angular-db/CHANGELOG.md | 7 ++ packages/angular-db/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../CHANGELOG.md | 7 ++ .../e2e/app/CHANGELOG.md | 8 ++ .../e2e/app/package.json | 2 +- .../package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../db-sqlite-persistence-core/CHANGELOG.md | 7 ++ .../db-sqlite-persistence-core/package.json | 2 +- packages/db/CHANGELOG.md | 6 ++ packages/db/package.json | 2 +- packages/electric-db-collection/CHANGELOG.md | 7 ++ packages/electric-db-collection/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../expo-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../e2e/expo-runtime-app/CHANGELOG.md | 8 ++ .../e2e/expo-runtime-app/package.json | 2 +- .../expo-db-sqlite-persistence/package.json | 2 +- .../node-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../node-db-sqlite-persistence/package.json | 2 +- packages/offline-transactions/CHANGELOG.md | 7 ++ packages/offline-transactions/package.json | 2 +- packages/powersync-db-collection/CHANGELOG.md | 7 ++ packages/powersync-db-collection/package.json | 2 +- packages/query-db-collection/CHANGELOG.md | 7 ++ packages/query-db-collection/package.json | 2 +- packages/react-db/CHANGELOG.md | 7 ++ packages/react-db/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- packages/rxdb-db-collection/CHANGELOG.md | 7 ++ packages/rxdb-db-collection/package.json | 2 +- packages/solid-db/CHANGELOG.md | 7 ++ packages/solid-db/package.json | 2 +- packages/svelte-db/CHANGELOG.md | 7 ++ packages/svelte-db/package.json | 2 +- .../tauri-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../e2e/app/CHANGELOG.md | 8 ++ .../e2e/app/package.json | 2 +- .../tauri-db-sqlite-persistence/package.json | 2 +- packages/trailbase-db-collection/CHANGELOG.md | 7 ++ packages/trailbase-db-collection/package.json | 2 +- packages/vue-db/CHANGELOG.md | 7 ++ packages/vue-db/package.json | 2 +- pnpm-lock.yaml | 74 +++++++++---------- 61 files changed, 268 insertions(+), 103 deletions(-) delete mode 100644 .changeset/lazy-runtime-reference-identities.md diff --git a/.changeset/lazy-runtime-reference-identities.md b/.changeset/lazy-runtime-reference-identities.md deleted file mode 100644 index 807b0f5ce..000000000 --- a/.changeset/lazy-runtime-reference-identities.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@tanstack/db': patch ---- - -Lazily initialize runtime reference identities to avoid generating random values during Cloudflare Worker module evaluation. diff --git a/examples/angular/todos/package.json b/examples/angular/todos/package.json index 26bd22f87..aad965305 100644 --- a/examples/angular/todos/package.json +++ b/examples/angular/todos/package.json @@ -28,8 +28,8 @@ "@angular/forms": "^20.3.16", "@angular/platform-browser": "^20.3.16", "@angular/router": "^20.3.16", - "@tanstack/angular-db": "^0.1.86", - "@tanstack/db": "^0.8.5", + "@tanstack/angular-db": "^0.1.87", + "@tanstack/db": "^0.8.6", "rxjs": "^7.8.2", "tslib": "^2.8.1", "zone.js": "~0.15.0" diff --git a/examples/electron/offline-first/package.json b/examples/electron/offline-first/package.json index 80e1cbdc1..04d41fbdb 100644 --- a/examples/electron/offline-first/package.json +++ b/examples/electron/offline-first/package.json @@ -13,11 +13,11 @@ "postinstall": "prebuild-install --runtime electron --target 40.2.1 --arch arm64 || echo 'prebuild-install failed, try: npx @electron/rebuild'" }, "dependencies": { - "@tanstack/electron-db-sqlite-persistence": "^0.1.30", - "@tanstack/node-db-sqlite-persistence": "^0.2.18", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/electron-db-sqlite-persistence": "^0.1.31", + "@tanstack/node-db-sqlite-persistence": "^0.2.19", + "@tanstack/offline-transactions": "^1.0.52", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/react-db": "^0.3.6", "@tanstack/react-query": "^5.90.20", "better-sqlite3": "^12.6.2", "react": "^19.2.4", diff --git a/examples/react-native/offline-transactions/package.json b/examples/react-native/offline-transactions/package.json index 59ea587b6..4a8cda4f8 100644 --- a/examples/react-native/offline-transactions/package.json +++ b/examples/react-native/offline-transactions/package.json @@ -15,11 +15,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.18", + "@tanstack/db": "^0.8.6", + "@tanstack/offline-transactions": "^1.0.52", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/react-db": "^0.3.6", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.19", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react-native/shopping-list/package.json b/examples/react-native/shopping-list/package.json index 1cce0f1a6..4e6943c15 100644 --- a/examples/react-native/shopping-list/package.json +++ b/examples/react-native/shopping-list/package.json @@ -18,11 +18,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.5", - "@tanstack/electric-db-collection": "^0.4.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/react-db": "^0.3.5", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.18", + "@tanstack/db": "^0.8.6", + "@tanstack/electric-db-collection": "^0.4.6", + "@tanstack/offline-transactions": "^1.0.52", + "@tanstack/react-db": "^0.3.6", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.19", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react/next-ssr-e2e/package.json b/examples/react/next-ssr-e2e/package.json index 3b584c5fd..b6420d761 100644 --- a/examples/react/next-ssr-e2e/package.json +++ b/examples/react/next-ssr-e2e/package.json @@ -9,8 +9,8 @@ "test:e2e": "pnpm --filter @tanstack/db build && pnpm --filter @tanstack/react-db build && playwright test" }, "dependencies": { - "@tanstack/db": "^0.8.5", - "@tanstack/react-db": "^0.3.5", + "@tanstack/db": "^0.8.6", + "@tanstack/react-db": "^0.3.6", "next": "^16.3.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/offline-transactions/package.json b/examples/react/offline-transactions/package.json index 6bb399af9..f1663d76c 100644 --- a/examples/react/offline-transactions/package.json +++ b/examples/react/offline-transactions/package.json @@ -8,11 +8,11 @@ "build": "vite build && tsc --noEmit" }, "dependencies": { - "@tanstack/browser-db-sqlite-persistence": "^0.2.18", - "@tanstack/db": "^0.8.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/browser-db-sqlite-persistence": "^0.2.19", + "@tanstack/db": "^0.8.6", + "@tanstack/offline-transactions": "^1.0.52", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/react-db": "^0.3.6", "@tanstack/react-query": "^5.90.20", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", diff --git a/examples/react/paced-mutations-demo/package.json b/examples/react/paced-mutations-demo/package.json index 318832aec..55d4e4cf2 100644 --- a/examples/react/paced-mutations-demo/package.json +++ b/examples/react/paced-mutations-demo/package.json @@ -9,8 +9,8 @@ "preview": "vite preview" }, "dependencies": { - "@tanstack/db": "^0.8.5", - "@tanstack/react-db": "^0.3.5", + "@tanstack/db": "^0.8.6", + "@tanstack/react-db": "^0.3.6", "mitt": "^3.0.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/projects/package.json b/examples/react/projects/package.json index 1454e644d..d605ed0db 100644 --- a/examples/react/projects/package.json +++ b/examples/react/projects/package.json @@ -17,8 +17,8 @@ "dependencies": { "@tailwindcss/vite": "^4.1.18", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/react-db": "^0.3.6", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", "@tanstack/react-router-with-query": "^1.130.17", diff --git a/examples/react/start-ssr-e2e/package.json b/examples/react/start-ssr-e2e/package.json index 3361ed012..439a5aba6 100644 --- a/examples/react/start-ssr-e2e/package.json +++ b/examples/react/start-ssr-e2e/package.json @@ -10,7 +10,7 @@ "test:e2e:hosted": "playwright test" }, "dependencies": { - "@tanstack/react-db": "^0.3.5", + "@tanstack/react-db": "^0.3.6", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-with-db": "^0.1.0", "@tanstack/react-start": "^1.159.5", diff --git a/examples/react/todo/package.json b/examples/react/todo/package.json index 4df1dea31..30392b3e3 100644 --- a/examples/react/todo/package.json +++ b/examples/react/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.1.27", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.5", + "@tanstack/electric-db-collection": "^0.4.6", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/react-db": "^0.3.6", "@tanstack/react-router": "^1.159.5", "@tanstack/react-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.104", + "@tanstack/trailbase-db-collection": "^0.1.105", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/examples/solid/todo/package.json b/examples/solid/todo/package.json index 2d6ae6a1a..6b5e333cb 100644 --- a/examples/solid/todo/package.json +++ b/examples/solid/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.0.36", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.5", + "@tanstack/electric-db-collection": "^0.4.6", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/solid-db": "^0.2.40", + "@tanstack/query-db-collection": "^1.2.11", + "@tanstack/solid-db": "^0.2.41", "@tanstack/solid-router": "^1.159.5", "@tanstack/solid-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.104", + "@tanstack/trailbase-db-collection": "^0.1.105", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/packages/angular-db/CHANGELOG.md b/packages/angular-db/CHANGELOG.md index 51d6114d3..d0d5462db 100644 --- a/packages/angular-db/CHANGELOG.md +++ b/packages/angular-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/angular-db +## 0.1.87 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.86 ### Patch Changes diff --git a/packages/angular-db/package.json b/packages/angular-db/package.json index 96cabe10b..7dec15154 100644 --- a/packages/angular-db/package.json +++ b/packages/angular-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/angular-db", - "version": "0.1.86", + "version": "0.1.87", "description": "Angular integration for @tanstack/db", "author": "Ethan McDaniel", "license": "MIT", diff --git a/packages/browser-db-sqlite-persistence/CHANGELOG.md b/packages/browser-db-sqlite-persistence/CHANGELOG.md index 56adbe19e..6a3ec3ead 100644 --- a/packages/browser-db-sqlite-persistence/CHANGELOG.md +++ b/packages/browser-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/browser-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 68ed0ff20..269ecef94 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/browser-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Browser wa-sqlite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md index ea93ca36b..7ece1d3c7 100644 --- a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/capacitor-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md index 58a7cb809..fa88a892a 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/capacitor-db-sqlite-persistence-e2e-app +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/capacitor-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json index f9098decc..30181f666 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.31", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/capacitor-db-sqlite-persistence/package.json b/packages/capacitor-db-sqlite-persistence/package.json index 07156ec2d..8f5fbe62e 100644 --- a/packages/capacitor-db-sqlite-persistence/package.json +++ b/packages/capacitor-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Capacitor SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md index 029aec5d2..886547c6a 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/cloudflare-durable-objects-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json index 77c98ed41..bbbd5a842 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/cloudflare-durable-objects-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Cloudflare Durable Object SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db-sqlite-persistence-core/CHANGELOG.md b/packages/db-sqlite-persistence-core/CHANGELOG.md index 06933cdf1..741329479 100644 --- a/packages/db-sqlite-persistence-core/CHANGELOG.md +++ b/packages/db-sqlite-persistence-core/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/db-sqlite-persistence-core +## 0.2.19 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.2.18 ### Patch Changes diff --git a/packages/db-sqlite-persistence-core/package.json b/packages/db-sqlite-persistence-core/package.json index 5dcc96de2..9dd360a64 100644 --- a/packages/db-sqlite-persistence-core/package.json +++ b/packages/db-sqlite-persistence-core/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db-sqlite-persistence-core", - "version": "0.2.18", + "version": "0.2.19", "description": "SQLite persisted collection core for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db/CHANGELOG.md b/packages/db/CHANGELOG.md index 6f0ecfe61..a1450fcdf 100644 --- a/packages/db/CHANGELOG.md +++ b/packages/db/CHANGELOG.md @@ -1,5 +1,11 @@ # @tanstack/db +## 0.8.6 + +### Patch Changes + +- Lazily initialize runtime reference identities to avoid generating random values during Cloudflare Worker module evaluation. ([#1782](https://github.com/TanStack/db/pull/1782)) + ## 0.8.5 ### Patch Changes diff --git a/packages/db/package.json b/packages/db/package.json index 1857935bb..4845820a7 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db", - "version": "0.8.5", + "version": "0.8.6", "description": "A reactive client store for building super fast apps on sync", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/electric-db-collection/CHANGELOG.md b/packages/electric-db-collection/CHANGELOG.md index e87bc9898..6d11766f8 100644 --- a/packages/electric-db-collection/CHANGELOG.md +++ b/packages/electric-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/electric-db-collection +## 0.4.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.4.5 ### Patch Changes diff --git a/packages/electric-db-collection/package.json b/packages/electric-db-collection/package.json index 0f5118b51..acc0ceb88 100644 --- a/packages/electric-db-collection/package.json +++ b/packages/electric-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electric-db-collection", - "version": "0.4.5", + "version": "0.4.6", "description": "ElectricSQL collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/electron-db-sqlite-persistence/CHANGELOG.md b/packages/electron-db-sqlite-persistence/CHANGELOG.md index e0e58b37d..909c5d0e8 100644 --- a/packages/electron-db-sqlite-persistence/CHANGELOG.md +++ b/packages/electron-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/electron-db-sqlite-persistence +## 0.1.31 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.1.30 ### Patch Changes diff --git a/packages/electron-db-sqlite-persistence/package.json b/packages/electron-db-sqlite-persistence/package.json index b7134749e..7f1e72e87 100644 --- a/packages/electron-db-sqlite-persistence/package.json +++ b/packages/electron-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electron-db-sqlite-persistence", - "version": "0.1.30", + "version": "0.1.31", "description": "Electron SQLite persisted collection bridge for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/expo-db-sqlite-persistence/CHANGELOG.md b/packages/expo-db-sqlite-persistence/CHANGELOG.md index f7dbde7b8..63e9334f8 100644 --- a/packages/expo-db-sqlite-persistence/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/expo-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md index 508b13390..feed398dc 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/expo-db-sqlite-persistence-e2e-app +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/expo-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json index a323c1055..f2c0a722f 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/expo-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.31", "main": "index.js", "scripts": { "start": "expo start", diff --git a/packages/expo-db-sqlite-persistence/package.json b/packages/expo-db-sqlite-persistence/package.json index 867f5c020..a44121bc2 100644 --- a/packages/expo-db-sqlite-persistence/package.json +++ b/packages/expo-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/expo-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/node-db-sqlite-persistence/CHANGELOG.md b/packages/node-db-sqlite-persistence/CHANGELOG.md index 2dc694656..3102ceb00 100644 --- a/packages/node-db-sqlite-persistence/CHANGELOG.md +++ b/packages/node-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/node-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/node-db-sqlite-persistence/package.json b/packages/node-db-sqlite-persistence/package.json index 87616d066..0ccb64255 100644 --- a/packages/node-db-sqlite-persistence/package.json +++ b/packages/node-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/node-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Node SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/offline-transactions/CHANGELOG.md b/packages/offline-transactions/CHANGELOG.md index 50db91818..71f29aa33 100644 --- a/packages/offline-transactions/CHANGELOG.md +++ b/packages/offline-transactions/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/offline-transactions +## 1.0.52 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 1.0.51 ### Patch Changes diff --git a/packages/offline-transactions/package.json b/packages/offline-transactions/package.json index 5194a4365..afe03a2ad 100644 --- a/packages/offline-transactions/package.json +++ b/packages/offline-transactions/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/offline-transactions", - "version": "1.0.51", + "version": "1.0.52", "description": "Offline-first transaction capabilities for TanStack DB", "author": "TanStack", "license": "MIT", diff --git a/packages/powersync-db-collection/CHANGELOG.md b/packages/powersync-db-collection/CHANGELOG.md index 71d922925..c1f400bb4 100644 --- a/packages/powersync-db-collection/CHANGELOG.md +++ b/packages/powersync-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/powersync-db-collection +## 0.1.65 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.64 ### Patch Changes diff --git a/packages/powersync-db-collection/package.json b/packages/powersync-db-collection/package.json index bff245fe8..e4b536b68 100644 --- a/packages/powersync-db-collection/package.json +++ b/packages/powersync-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/powersync-db-collection", - "version": "0.1.64", + "version": "0.1.65", "description": "PowerSync collection for TanStack DB", "author": "POWERSYNC", "license": "MIT", diff --git a/packages/query-db-collection/CHANGELOG.md b/packages/query-db-collection/CHANGELOG.md index 8add83780..326ec3134 100644 --- a/packages/query-db-collection/CHANGELOG.md +++ b/packages/query-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/query-db-collection +## 1.2.11 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 1.2.10 ### Patch Changes diff --git a/packages/query-db-collection/package.json b/packages/query-db-collection/package.json index 2ad2b4856..b5341773f 100644 --- a/packages/query-db-collection/package.json +++ b/packages/query-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/query-db-collection", - "version": "1.2.10", + "version": "1.2.11", "description": "TanStack Query collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/react-db/CHANGELOG.md b/packages/react-db/CHANGELOG.md index 940786b42..604690e64 100644 --- a/packages/react-db/CHANGELOG.md +++ b/packages/react-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-db +## 0.3.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.3.5 ### Patch Changes diff --git a/packages/react-db/package.json b/packages/react-db/package.json index 0dd6d60e6..e1b8a9c0c 100644 --- a/packages/react-db/package.json +++ b/packages/react-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-db", - "version": "0.3.5", + "version": "0.3.6", "description": "React integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/react-native-db-sqlite-persistence/CHANGELOG.md b/packages/react-native-db-sqlite-persistence/CHANGELOG.md index 46539d752..2927be705 100644 --- a/packages/react-native-db-sqlite-persistence/CHANGELOG.md +++ b/packages/react-native-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-native-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/react-native-db-sqlite-persistence/package.json b/packages/react-native-db-sqlite-persistence/package.json index a3ef7dc90..a47164344 100644 --- a/packages/react-native-db-sqlite-persistence/package.json +++ b/packages/react-native-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-native-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "React Native and Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/rxdb-db-collection/CHANGELOG.md b/packages/rxdb-db-collection/CHANGELOG.md index 934391326..13cc2d034 100644 --- a/packages/rxdb-db-collection/CHANGELOG.md +++ b/packages/rxdb-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/rxdb-db-collection +## 0.1.93 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.92 ### Patch Changes diff --git a/packages/rxdb-db-collection/package.json b/packages/rxdb-db-collection/package.json index 97cf5ba13..3db3b69c3 100644 --- a/packages/rxdb-db-collection/package.json +++ b/packages/rxdb-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/rxdb-db-collection", - "version": "0.1.92", + "version": "0.1.93", "description": "Reactive, Offline-First adapter for TanStack DB using RxDB. Sync, Replication and Local-First support.", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/solid-db/CHANGELOG.md b/packages/solid-db/CHANGELOG.md index 81957f00c..3091f962a 100644 --- a/packages/solid-db/CHANGELOG.md +++ b/packages/solid-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-db +## 0.2.41 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.2.40 ### Patch Changes diff --git a/packages/solid-db/package.json b/packages/solid-db/package.json index d0af6c422..2d1b4db39 100644 --- a/packages/solid-db/package.json +++ b/packages/solid-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/solid-db", - "version": "0.2.40", + "version": "0.2.41", "description": "Solid integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/svelte-db/CHANGELOG.md b/packages/svelte-db/CHANGELOG.md index 86729d816..bf2d8da46 100644 --- a/packages/svelte-db/CHANGELOG.md +++ b/packages/svelte-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/svelte-db +## 0.3.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.3.5 ### Patch Changes diff --git a/packages/svelte-db/package.json b/packages/svelte-db/package.json index c8c5a1e71..b1fa049b3 100644 --- a/packages/svelte-db/package.json +++ b/packages/svelte-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/svelte-db", - "version": "0.3.5", + "version": "0.3.6", "description": "Svelte integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/tauri-db-sqlite-persistence/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/CHANGELOG.md index 9456964a9..d764cd4a1 100644 --- a/packages/tauri-db-sqlite-persistence/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/tauri-db-sqlite-persistence +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md index f654a6452..0de77a1af 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/tauri-db-sqlite-persistence-e2e-app +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/tauri-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/package.json b/packages/tauri-db-sqlite-persistence/e2e/app/package.json index 26ae15919..6ed9b5f34 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/package.json +++ b/packages/tauri-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/tauri-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.31", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/tauri-db-sqlite-persistence/package.json b/packages/tauri-db-sqlite-persistence/package.json index 416536bf8..70b83ba0a 100644 --- a/packages/tauri-db-sqlite-persistence/package.json +++ b/packages/tauri-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/tauri-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.19", "description": "Tauri SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/trailbase-db-collection/CHANGELOG.md b/packages/trailbase-db-collection/CHANGELOG.md index 4d153b489..50dc48499 100644 --- a/packages/trailbase-db-collection/CHANGELOG.md +++ b/packages/trailbase-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/trailbase-db-collection +## 0.1.105 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.104 ### Patch Changes diff --git a/packages/trailbase-db-collection/package.json b/packages/trailbase-db-collection/package.json index ab06ace3f..1bba98931 100644 --- a/packages/trailbase-db-collection/package.json +++ b/packages/trailbase-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/trailbase-db-collection", - "version": "0.1.104", + "version": "0.1.105", "description": "TrailBase collection for TanStack DB", "author": "Sebastian Jeltsch", "license": "MIT", diff --git a/packages/vue-db/CHANGELOG.md b/packages/vue-db/CHANGELOG.md index 2d0753fbd..37684c9ed 100644 --- a/packages/vue-db/CHANGELOG.md +++ b/packages/vue-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/vue-db +## 0.1.8 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.7 ### Patch Changes diff --git a/packages/vue-db/package.json b/packages/vue-db/package.json index 92f956797..d03e5d57e 100644 --- a/packages/vue-db/package.json +++ b/packages/vue-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/vue-db", - "version": "0.1.7", + "version": "0.1.8", "description": "Vue integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943e48209..21c34e415 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,10 +148,10 @@ importers: specifier: ^20.3.16 version: 20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1)))(rxjs@7.8.2) '@tanstack/angular-db': - specifier: ^0.1.86 + specifier: ^0.1.87 version: link:../../../packages/angular-db '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db rxjs: specifier: ^7.8.2 @@ -209,19 +209,19 @@ importers: examples/electron/offline-first: dependencies: '@tanstack/electron-db-sqlite-persistence': - specifier: ^0.1.30 + specifier: ^0.1.31 version: link:../../../packages/electron-db-sqlite-persistence '@tanstack/node-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.19 version: link:../../../packages/node-db-sqlite-persistence '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.52 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -300,19 +300,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.52 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.19 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -397,19 +397,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.6 version: link:../../../packages/electric-db-collection '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.52 version: link:../../../packages/offline-transactions '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.19 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -482,10 +482,10 @@ importers: examples/react/next-ssr-e2e: dependencies: '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db next: specifier: ^16.3.1 @@ -516,19 +516,19 @@ importers: examples/react/offline-transactions: dependencies: '@tanstack/browser-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.19 version: link:../../../packages/browser-db-sqlite-persistence '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.52 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -586,10 +586,10 @@ importers: examples/react/paced-mutations-demo: dependencies: '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.6 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db mitt: specifier: ^3.0.1 @@ -626,10 +626,10 @@ importers: specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -759,7 +759,7 @@ importers: examples/react/start-ssr-e2e: dependencies: '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -805,16 +805,16 @@ importers: examples/react/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.6 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.6 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -823,7 +823,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.104 + specifier: ^0.1.105 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6 @@ -926,16 +926,16 @@ importers: examples/solid/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.6 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.11 version: link:../../../packages/query-db-collection '@tanstack/solid-db': - specifier: ^0.2.40 + specifier: ^0.2.41 version: link:../../../packages/solid-db '@tanstack/solid-router': specifier: ^1.159.5 @@ -944,7 +944,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(@tanstack/react-router@1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(solid-js@1.9.11)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.104 + specifier: ^0.1.105 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6 From 3eaf05674627a66e91a6f1d42b98366b526b0e67 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 11:19:17 -0600 Subject: [PATCH 023/327] test(db): bound locale refinement work --- packages/db/src/query/live/ARCHITECTURE.md | 6 ++++++ packages/db/tests/query/pagination-oracle.property.test.ts | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 401bfe341..1f99d4b53 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -388,6 +388,12 @@ prefix is only a candidate prefix. Core expands the complete source-order boundary class, then applies the public-key tie-break locally. Locale and reference orders that the predicate IR cannot express fetch the full filtered region and refine it locally. +That fallback issues no structural cursor: a lexical predicate is not a locale +boundary. Once the unbounded result proves the needed prefix, widening within +that result performs no more transport work. Bounded locale continuation would +require a future adapter capability with an opaque cursor that preserves the +provider's exact collation and snapshot. Until that contract exists, an +unbounded fetch is the only sound continuation. Every continuation boundary comes from rows established by the same ordered demand. Rows retained for another query, join, or window cannot move it. During diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index a78e13156..aa9c2821c 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -2099,10 +2099,12 @@ describe(`pagination recomputation oracle`, () => { expect(refinement.options.limit).toBeUndefined() expect(refinement.options.offset).toBeUndefined() + const transportCount = pending.length const widened = live.utils.setWindow({ offset: 0, limit: 2 }) expect(widened).toBe(true) expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) + expect(pending).toHaveLength(transportCount) } finally { for (const request of pending) request.deferred.resolve() live.cleanup() @@ -2962,6 +2964,7 @@ describe(`pagination recomputation oracle`, () => { expect(loads).toHaveLength(2) expect(loads[1]?.limit).toBeUndefined() expect(loads[1]?.offset).toBeUndefined() + expect(loads[1]?.cursor).toBeUndefined() }) it.each([`continues`, `unknown`] as const)( @@ -2985,6 +2988,9 @@ describe(`pagination recomputation oracle`, () => { expect(loads[1]?.limit).toBeUndefined() expect(loads[1]?.offset).toBeUndefined() expect(loads.map(({ limit }) => limit)).toEqual([1, undefined, undefined]) + expect(loads.slice(1).every(({ cursor }) => cursor === undefined)).toBe( + true, + ) }, ) From 09b29ea6ddb364c02f5dc3883830abcae467f63f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 11:34:13 -0600 Subject: [PATCH 024/327] perf(db): retain subset predicate evaluators --- packages/db/src/collection/subscription.ts | 23 +++--- packages/db/src/query/live/ARCHITECTURE.md | 4 + ...ubscription-replay-oracle.property.test.ts | 78 +++++++++++++++++++ 3 files changed, 91 insertions(+), 14 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 3678ab11b..0e2260132 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -116,6 +116,7 @@ type ReplayHandoffResult = type SubsetDemand = SubsetAcquisition & { requestOptions: LoadSubsetOptions + matchesWhere: (row: object) => boolean onLoadSubsetResult?: ( result: LoadSubsetRequestResult, demand: LoadSubsetOptions, @@ -227,6 +228,8 @@ function createSubsetCleanupError(errors: ReadonlyArray): unknown { return new SubsetCleanupAggregateError(errors) } +const matchesEveryRow = () => true + export class CollectionSubscription extends EventEmitter implements Subscription @@ -315,13 +318,7 @@ export class CollectionSubscription private activeAdditionalFilters(): Array<(row: object) => boolean> { return this.subsetDemands .filter((demand) => demand.active && demand.ordered === undefined) - .map((demand) => - demand.requestOptions.where - ? createFilterFunctionFromExpression( - demand.requestOptions.where, - ) - : () => true, - ) + .map((demand) => demand.matchesWhere) } private diffPublishedRows( @@ -1177,11 +1174,7 @@ export class CollectionSubscription const merged = [...session.buffer.flat(), ...retainedDeletes] const activeDemandFilters = this.subsetDemands .filter((demand) => demand.active) - .map((demand) => - demand.requestOptions.where - ? createFilterFunctionFromExpression(demand.requestOptions.where) - : undefined, - ) + .map((demand) => demand.matchesWhere) // The raw replay buffer can contain rows retained for another demand or // outside the ordered prefix. Publish the settled ordered reconciliation // as the replacement's one atomic batch. @@ -1190,8 +1183,7 @@ export class CollectionSubscription : this.createPublicationDiff( session.publicationState.publishedRows, merged, - (value) => - activeDemandFilters.some((filter) => filter?.(value) ?? true), + (value) => activeDemandFilters.some((filter) => filter(value)), ) if (replacement.length > 0) this.filteredCallback(replacement) // Buffering records every source key before active-demand filtering. Reset @@ -1915,6 +1907,9 @@ export class CollectionSubscription const demand: SubsetDemand = { requestOptions, options: requestOptions, + matchesWhere: requestOptions.where + ? createFilterFunctionFromExpression(requestOptions.where) + : matchesEveryRow, ...(ordered === undefined ? {} : { ordered }), pendingReplayAcquisitions: new Set(), active: true, diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 1f99d4b53..ccb1e75da 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -754,6 +754,10 @@ during adapter code cannot publish a later snapshot. A synchronous `loadSubset` throw that did not follow a failed release rolls the tentative owner back before it emits the error and without calling `unloadSubset`; a failed release keeps the owner so a later cleanup can retry the same acquisition identity. +The logical owner also retains its compiled predicate. Reconciliation reuses +that evaluator across source changes and truncate acquisition replacement. A +released owner cannot supply a predicate, and a later logical demand compiles +its own evaluator even when it reuses the same expression object. Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies coverage, or scans local rows. A callback may release or unsubscribe; obsolete diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index ecec05249..e0c809169 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5028,6 +5028,84 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`compiles an additional-demand predicate once per logical demand`, async () => { + type Row = { id: string; rank: number } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: () => void + const collection = createCollection({ + id: `additional-demand-predicate-compilation`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => true, + unloadSubset: () => {}, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderBy: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ] + let expressionReads = 0 + // Compilation reads the IR node type; the compiled evaluator does not. + // Count those reads without exposing test instrumentation in production. + const expression = new Proxy( + new Func(`eq`, [new PropRef([`id`]), new Value(`sibling`)]), + { + get(target, property, receiver) { + if (property === `type`) expressionReads++ + return Reflect.get(target, property, receiver) + }, + }, + ) + const subscription = collection.subscribeChanges(() => {}) + subscription.setOrderByIndex(index) + + const publish = (value: Row) => { + begin() + write({ type: `insert`, value }) + commit() + } + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + subscription.requestSnapshot({ where: expression }) + const firstDemandReads = expressionReads + expect(firstDemandReads).toBeGreaterThan(0) + + publish({ id: `sibling`, rank: 2 }) + publish({ id: `ordered`, rank: 1 }) + expect(expressionReads).toBe(firstDemandReads) + + subscription.releaseSnapshot(expression) + const beforeReplacement = expressionReads + subscription.requestSnapshot({ where: expression }) + expect(expressionReads).toBeGreaterThan(beforeReplacement) + const replacementDemandReads = expressionReads + + publish({ id: `later`, rank: 0 }) + expect(expressionReads).toBe(replacementDemandReads) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`releases ordered publication authority before retrying failed adapter cleanup`, async () => { type Row = { id: string; rank: number } let begin!: () => void From 60f018a7bf3e5995cfef625437523e415964db6d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 11:57:18 -0600 Subject: [PATCH 025/327] fix(db): snapshot logical subset demands --- packages/db/src/collection/subscription.ts | 20 +- packages/db/src/query/live/ARCHITECTURE.md | 33 +-- ...ubscription-replay-oracle.property.test.ts | 219 +++++++++++++----- 3 files changed, 193 insertions(+), 79 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 0e2260132..d76af1e1d 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -6,6 +6,7 @@ import { getSyncRequestProvenance, isLoadSubsetRequestSignalFor, } from '../load-subset-request-provenance.js' +import { cloneLoadSubsetOptions } from '../query/load-subset-options.js' import { buildCursor, buildCursorEquality, @@ -1898,17 +1899,19 @@ export class CollectionSubscription private startSubsetDemand( requestOptions: LoadSubsetOptions, ordered?: SubsetDemand[`ordered`], + releaseWhere = requestOptions.where, ): { demand: SubsetDemand acquisition: SubsetAcquisition & { abortController: AbortController } result: LoadSubsetRequestResult replayContext: TruncateReplayContext | undefined } { + const stableRequestOptions = cloneLoadSubsetOptions(requestOptions) const demand: SubsetDemand = { - requestOptions, - options: requestOptions, - matchesWhere: requestOptions.where - ? createFilterFunctionFromExpression(requestOptions.where) + requestOptions: stableRequestOptions, + options: stableRequestOptions, + matchesWhere: stableRequestOptions.where + ? createFilterFunctionFromExpression(stableRequestOptions.where) : matchesEveryRow, ...(ordered === undefined ? {} : { ordered }), pendingReplayAcquisitions: new Set(), @@ -1917,6 +1920,9 @@ export class CollectionSubscription releaseFailed: false, releaseSettled: false, } + if (releaseWhere) { + this.requestedSubsetWhere.set(stableRequestOptions, releaseWhere) + } const acquisition = this.createSubsetAcquisition(demand) demand.options = acquisition.options demand.ordered = acquisition.ordered @@ -2180,15 +2186,11 @@ export class CollectionSubscription limit: opts?.limit, } - // Reentrant adapter code must be able to release a request by the exact - // caller predicate even when the subscription predicate was combined into - // the transport predicate. - if (opts?.where) this.requestedSubsetWhere.set(loadOptions, opts.where) const { demand, result: syncResult, replayContext: startedReplayContext, - } = this.startSubsetDemand(loadOptions) + } = this.startSubsetDemand(loadOptions, undefined, opts?.where) const replayTracksCallback = this.retainReplayResultCallback(startedReplayContext) // Replay settlement owns the acquisition even if the result callback diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index ccb1e75da..245f2c4e2 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -743,21 +743,24 @@ shared abort lease. If one owner releases its lease, the source request remains active while another owner still needs its coverage. The source signal aborts only after every attached owner has released it. -A Collection subscription installs each logical subset owner before it calls -the source adapter. Reentrant release during `loadSubset` must therefore see and -release that exact acquisition. It also registers the caller's original -predicate before adapter entry, because the transport predicate may combine it -with the subscription predicate. After adapter return, both ordered and -unordered requests recheck logical ownership before they report results, track -loading state, establish coverage, or scan local state; a demand released -during adapter code cannot publish a later snapshot. A synchronous `loadSubset` -throw that did not follow a failed release rolls the tentative owner back before -it emits the error and without calling `unloadSubset`; a failed release keeps -the owner so a later cleanup can retry the same acquisition identity. -The logical owner also retains its compiled predicate. Reconciliation reuses -that evaluator across source changes and truncate acquisition replacement. A -released owner cannot supply a predicate, and a later logical demand compiles -its own evaluator even when it reuses the same expression object. +A Collection subscription snapshots each logical subset demand before it calls +the source adapter. That stable snapshot drives the adapter transport, +acquisition evidence, compiled predicate, and later truncate replay. The +caller's original predicate is retained only as a release handle, because the +transport predicate may combine it with the subscription predicate. The +subscription then installs the logical owner before adapter entry. Reentrant +release during `loadSubset` must therefore see and release that exact +acquisition. After adapter return, both ordered and unordered requests recheck +logical ownership before they report results, track loading state, establish +coverage, or scan local state; a demand released during adapter code cannot +publish a later snapshot. A synchronous `loadSubset` throw that did not follow +a failed release rolls the tentative owner back before it emits the error and +without calling `unloadSubset`; a failed release keeps the owner so a later +cleanup can retry the same acquisition identity. Reconciliation reuses the +logical owner's evaluator across source changes and truncate acquisition +replacement. A released owner cannot supply a predicate, and a later logical +demand compiles its own evaluator even when it reuses the same expression +object. Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies coverage, or scans local rows. A callback may release or unsubscribe; obsolete diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index e0c809169..083a8d4c4 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5,6 +5,7 @@ import { createDeferred } from '../src/deferred.js' import { BTreeIndex } from '../src/indexes/btree-index.js' import { ReverseIndex } from '../src/indexes/reverse-index.js' import { attachLoadSubsetRequestSignal } from '../src/load-subset-request-provenance.js' +import { getStableExpressionHash } from '../src/query/ir-stable-identity.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { DeduplicatedLoadSubset } from '../src/query/subset-dedupe.js' import { createTransaction } from '../src/transactions.js' @@ -294,7 +295,7 @@ async function exerciseReplayCallbackCleanup({ }, unloadSubset: (options) => { unloads.push(options) - if (cleanupArmed && options.where === whereB && !failedB) { + if (cleanupArmed && sameWhere(options.where, whereB) && !failedB) { failedB = true throw nestedFailure } @@ -629,8 +630,8 @@ function expectSameSubsetRequest( actual: LoadSubsetOptions, expected: LoadSubsetOptions, ): void { - expect(actual.where).toBe(expected.where) - expect(actual.orderBy).toBe(expected.orderBy) + expect(sameWhere(actual.where, expected.where)).toBe(true) + expect(actual.orderBy).toEqual(expected.orderBy) expect(actual.limit).toBe(expected.limit) expect(actual.cursor).toEqual(expected.cursor) expect(actual.offset).toBe(expected.offset) @@ -641,13 +642,23 @@ function expectReplayRequestToRestart( stored: LoadSubsetOptions, expectedOffset = 0, ): void { - expect(actual.where).toBe(stored.where) - expect(actual.orderBy).toBe(stored.orderBy) + expect(sameWhere(actual.where, stored.where)).toBe(true) + expect(actual.orderBy).toEqual(stored.orderBy) expect(actual.limit).toBe(stored.limit) expect(actual.cursor).toBeUndefined() expect(actual.offset).toBe(expectedOffset) } +function sameWhere( + actual: LoadSubsetOptions[`where`], + expected: LoadSubsetOptions[`where`], +): boolean { + if (actual === undefined || expected === undefined) { + return actual === expected + } + return getStableExpressionHash(actual) === getStableExpressionHash(expected) +} + async function runReplayScenario(scenario: ReplayScenario): Promise { let begin!: () => void let write!: ( @@ -673,10 +684,12 @@ async function runReplayScenario(scenario: ReplayScenario): Promise { new Func(`eq`, [new PropRef([`id`]), new Value(demandId)]), ]), ) - const demandIdByWhere = new Map< - NonNullable, - ReplayDemandId - >([...demandWheres].map(([demandId, where]) => [where, demandId])) + const demandIdByWhereHash = new Map( + [...demandWheres].map(([demandId, where]) => [ + getStableExpressionHash(where), + demandId, + ]), + ) const requestByDemand = new Map() const activeDemandIds = new Set(scenario.demandIds) @@ -744,7 +757,9 @@ async function runReplayScenario(scenario: ReplayScenario): Promise { const demandId = options.where === undefined ? undefined - : demandIdByWhere.get(options.where) + : demandIdByWhereHash.get( + getStableExpressionHash(options.where), + ) if (demandId === undefined) { throw new Error(`Subset request did not preserve its demand`) } @@ -3429,17 +3444,17 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereX) { + if (sameWhere(options.where, whereX)) { begin() write({ type: `insert`, value: { id: `x`, value: 3 } }) return commit(options.signal) } if (replaying) { - return options.where === whereA + return sameWhere(options.where, whereA) ? replayA.promise : replayB.promise } - const id = options.where === whereA ? `a` : `b` + const id = sameWhere(options.where, whereA) ? `a` : `b` begin() write({ type: `insert`, @@ -3694,11 +3709,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions.push(options) throw startError } - if (options.where === whereNestedSecond) { + if (sameWhere(options.where, whereNestedSecond)) { nestedOptions.push(options) throw secondStartError } @@ -3835,11 +3850,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { outerLoadCount++ if ( originContext === `replay-entry` && @@ -3854,7 +3869,7 @@ describe(`CollectionSubscription replay oracle`, () => { requestInner() } } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { if (propagation === `async`) { return (async () => { requestInner() @@ -3868,7 +3883,7 @@ describe(`CollectionSubscription replay oracle`, () => { unloadSubset: (options) => { if ( originContext === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { requestMiddle() } @@ -4002,12 +4017,12 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions = options throw startError } if ( - options.where === whereOuter || + sameWhere(options.where, whereOuter) || options.orderBy !== undefined ) { outerLoadCount++ @@ -4027,7 +4042,7 @@ describe(`CollectionSubscription replay oracle`, () => { unloadSubset: (options) => { if ( cleanupArmed && - options.where === whereCleanup && + sameWhere(options.where, whereCleanup) && cleanupThrowCount === 0 ) { cleanupThrowCount++ @@ -5106,6 +5121,94 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`snapshots a logical demand before caller-owned predicate mutation`, async () => { + type Row = { id: `a` | `b`; other: `a` | `b` } + type Outcome = { + hasMore: false + appliedRowKeys: ReadonlyArray + } + const rows: ReadonlyArray = [ + { id: `a`, other: `b` }, + { id: `b`, other: `a` }, + ] + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + const replay = createDeferred() + const loads: Array = [] + const unloads: Array = [] + const collection = createCollection({ + id: `logical-demand-predicate-snapshot`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + begin() + for (const row of rows) write({ type: `insert`, value: row }) + commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + return loads.length === 1 ? true : replay.promise + }, + unloadSubset: (options) => { + unloads.push(options) + }, + } + }, + }, + }) + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + const ref = new PropRef([`id`]) + const where = new Func(`eq`, [ref, new Value(`a`)]) + + try { + subscription.requestSnapshot({ where }) + expect([...visible.keys()]).toEqual([`a`]) + + ref.path[0] = `other` + begin() + truncate() + commit() + await flushPromises() + expect(loads).toHaveLength(2) + + begin() + for (const row of rows) write({ type: `insert`, value: row }) + const receipt = commit(loads[1]?.signal) + if (receipt !== true) await receipt + replay.resolve({ hasMore: false, appliedRowKeys: [`a`, `b`] }) + await flushPromises() + + expect(((loads[1]?.where as Func).args[0] as PropRef).path).toEqual([ + `id`, + ]) + expect([...visible.keys()]).toEqual([`a`]) + + subscription.releaseSnapshot(where) + expect(unloads.at(-1)).toBe(loads[1]) + } finally { + replay.resolve({ hasMore: false, appliedRowKeys: [] }) + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`releases ordered publication authority before retrying failed adapter cleanup`, async () => { type Row = { id: string; rank: number } let begin!: () => void @@ -6293,7 +6396,11 @@ describe(`CollectionSubscription replay oracle`, () => { }, unloadSubset: (options) => { unloads.push(options) - if (cleanupArmed && options.where === whereC && !cleanupFailed) { + if ( + cleanupArmed && + sameWhere(options.where, whereC) && + !cleanupFailed + ) { cleanupFailed = true throw cleanupFailure } @@ -6546,15 +6653,17 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions = options throw startFailure } - if (replaying && options.where === whereA) throw replayFailure + if (replaying && sameWhere(options.where, whereA)) { + throw replayFailure + } return true }, unloadSubset: (options) => { - if (options.where === whereC && !cleanupFailed) { + if (sameWhere(options.where, whereC) && !cleanupFailed) { cleanupFailed = true throw cleanupFailure } @@ -6659,12 +6768,12 @@ describe(`CollectionSubscription replay oracle`, () => { loadSubset: (options) => { loads.push(options) if (!replaying) return true - return options.where === whereA + return sameWhere(options.where, whereA) ? replayA.promise : replayB.promise }, unloadSubset: (options) => { - if (options.where !== whereA) return + if (!sameWhere(options.where, whereA)) return unloadAttempts++ if (unloadAttempts <= 2) { throw cleanupFailure @@ -6766,7 +6875,7 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { failedOptions = options throw failure } @@ -6842,7 +6951,7 @@ describe(`CollectionSubscription replay oracle`, () => { loadSubset: () => true, unloadSubset: (options) => { unloads.push(options) - if (armed && options.where === whereB && !failed) { + if (armed && sameWhere(options.where, whereB) && !failed) { failed = true failedOptions = options throw failure @@ -6970,31 +7079,31 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereAfterTeardown) { + if (sameWhere(options.where, whereAfterTeardown)) { postTeardownLoads++ } - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { failedOptions = options throw failure } if ( replaying && activeFrame === `adapter-entry` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { failWithinBoundary(options) } if ( replaying && activeFrame === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { subscription.releaseSnapshot(whereCleanup) } return true }, unloadSubset: (options) => { - if (options.where !== whereCleanup) return + if (!sameWhere(options.where, whereCleanup)) return cleanupUnloads.push(options) if (replaying && activeFrame === `cleanup`) { failWithinBoundary(options) @@ -7120,14 +7229,14 @@ describe(`CollectionSubscription replay oracle`, () => { if ( replaying && activeFrame === `adapter-entry` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { startTeardown() } if ( replaying && activeFrame === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { subscription.releaseSnapshot(whereActiveCleanup) } @@ -7137,11 +7246,11 @@ describe(`CollectionSubscription replay oracle`, () => { if ( replaying && activeFrame === `cleanup` && - options.where === whereActiveCleanup + sameWhere(options.where, whereActiveCleanup) ) { startTeardown() } - if (options.where !== whereTeardownCleanup) return + if (!sameWhere(options.where, whereTeardownCleanup)) return teardownCleanupOptions = options teardownCleanupUnloads++ if (!teardownCleanupFailed) { @@ -7252,11 +7361,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { subscription.releaseSnapshot(whereCleanup) return true } - if (options.where !== whereOuter) return true + if (!sameWhere(options.where, whereOuter)) return true outerLoads++ if (!replaying || outerLoads !== 2) return true @@ -7275,7 +7384,7 @@ describe(`CollectionSubscription replay oracle`, () => { throw outerFailure }, unloadSubset: (options) => { - if (options.where !== whereCleanup) return + if (!sameWhere(options.where, whereCleanup)) return cleanupUnloads++ cleanupOptions ??= options if (!cleanupFailed) { @@ -7415,7 +7524,7 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereB) { + if (sameWhere(options.where, whereB)) { nestedOptions = options throw failure } @@ -7423,7 +7532,7 @@ describe(`CollectionSubscription replay oracle`, () => { }, unloadSubset: (options) => { unloads.push(options) - if (options.where === whereA) { + if (sameWhere(options.where, whereA)) { try { owner.current!.requestSnapshot({ where: whereB }) } catch { @@ -7486,11 +7595,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { return (async () => { owner.current!.requestSnapshot({ where: whereInner }) await Promise.resolve() @@ -7499,7 +7608,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, @@ -7579,11 +7688,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { try { owner.current!.requestSnapshot({ where: whereInner }) } catch (error) { @@ -7592,8 +7701,8 @@ describe(`CollectionSubscription replay oracle`, () => { return true } if ( - options.where === whereLater || - (demandKind === `ordered` && options.orderBy === orderBy) + sameWhere(options.where, whereLater) || + (demandKind === `ordered` && options.orderBy !== undefined) ) { laterOptions = options if (laterFailure === `throw`) throw retainedCarrier @@ -7602,7 +7711,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, @@ -7675,11 +7784,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { middleOptions = options return (async () => { await Promise.resolve() @@ -7689,7 +7798,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, From efa40c397f09a359d8ae584fae6024d04a61d17d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 12:23:00 -0600 Subject: [PATCH 026/327] fix(db): isolate subset demand snapshots --- packages/db/src/collection/subscription.ts | 2 +- .../db/src/query/expression-value-context.ts | 45 ++++++++++ packages/db/src/query/ir-stable-identity.ts | 77 ++++++++++++----- packages/db/src/query/live/ARCHITECTURE.md | 8 +- packages/db/src/query/load-subset-options.ts | 29 +++---- ...ubscription-replay-oracle.property.test.ts | 84 ++++++++++++++++++- .../db/tests/query/ir-stable-identity.test.ts | 22 +++++ 7 files changed, 223 insertions(+), 44 deletions(-) create mode 100644 packages/db/src/query/expression-value-context.ts diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index d76af1e1d..2cf2769e0 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1688,7 +1688,7 @@ export class CollectionSubscription return { options: { - ...request.options, + ...cloneLoadSubsetOptions(request.options), signal: abortController.signal, }, ordered: request.ordered, diff --git a/packages/db/src/query/expression-value-context.ts b/packages/db/src/query/expression-value-context.ts new file mode 100644 index 000000000..a1f1468ab --- /dev/null +++ b/packages/db/src/query/expression-value-context.ts @@ -0,0 +1,45 @@ +export type ExpressionValueContext = + | `exact-output` + | `equality-operand` + | `ordering-operand` + | `structural-operand` + +/** Describe how each function argument contributes to its observable result. */ +export function getExpressionArgumentValueContext( + name: string, + index: number, + argumentCount: number, + resultContext: ExpressionValueContext, +): ExpressionValueContext { + if (name === `eq` || name === `in`) return `equality-operand` + if (isOrderingFunction(name)) return `ordering-operand` + + if ( + name === `concat` || + name === `length` || + name === `add` || + name === `subtract` || + name === `multiply` || + name === `divide` || + name === `date` || + name === `datetime` || + name === `strftime` + ) { + return `structural-operand` + } + + if (name === `coalesce` || name === `upper` || name === `lower`) { + return resultContext + } + + if (name === `caseWhen`) { + const isDefault = argumentCount % 2 === 1 && index === argumentCount - 1 + return isDefault || index % 2 === 1 ? resultContext : `exact-output` + } + + return `exact-output` +} + +function isOrderingFunction(name: string): boolean { + return name === `gt` || name === `gte` || name === `lt` || name === `lte` +} diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 425e35d5a..d49407f6e 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,7 +1,9 @@ import { normalizeValue } from '../utils/comparison.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/get-query-ir.js' +import { getExpressionArgumentValueContext } from './expression-value-context.js' import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' +import type { ExpressionValueContext } from './expression-value-context.js' import type { Aggregate, BasicExpression, @@ -26,11 +28,6 @@ type StableIdentityValue = | Array | { [key: string]: StableIdentityValue } -type ValueIdentityContext = - | `exact-output` - | `equality-operand` - | `ordering-operand` - type OpaqueValueIdentity = `reject` | `runtime-reference` type AliasScope = { @@ -568,7 +565,7 @@ function canonicalizeOrderBy( orderBy: OrderByClause, path: string, seen: WeakSet, - valueContext: ValueIdentityContext = `exact-output`, + valueContext: ExpressionValueContext = `exact-output`, scope?: AliasScope, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { @@ -597,7 +594,7 @@ function canonicalizeExpression( | ConditionalSelect, path: string, seen: WeakSet, - valueContext: ValueIdentityContext = `exact-output`, + valueContext: ExpressionValueContext = `exact-output`, scope?: AliasScope, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { @@ -644,12 +641,19 @@ function canonicalizeExpression( seen, opaqueValueIdentity, ) - : canonicalizeExactOutputRuntimeValue( - expression.value, - `${path}.value`, - seen, - opaqueValueIdentity, - ), + : valueContext === `structural-operand` + ? canonicalizeStructuralRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ) + : canonicalizeExactOutputRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ), } } @@ -687,21 +691,17 @@ function canonicalizeExpression( ]) } - const operandContext: ValueIdentityContext = - expression.name === `eq` - ? `equality-operand` - : expression.name === `gt` || - expression.name === `gte` || - expression.name === `lt` || - expression.name === `lte` - ? `ordering-operand` - : `exact-output` const args = expression.args.map((arg, index) => canonicalizeExpression( arg, `${path}.args[${index}]`, seen, - operandContext, + getExpressionArgumentValueContext( + expression.name, + index, + expression.args.length, + valueContext, + ), scope, opaqueValueIdentity, ), @@ -1080,6 +1080,37 @@ function canonicalizeEqualityRuntimeValue( return canonicalizeRuntimeValue(value, path, seen) } +function canonicalizeStructuralRuntimeValue( + value: unknown, + path: string, + seen: WeakSet, + opaqueValueIdentity: OpaqueValueIdentity = `reject`, +): StableIdentityValue { + if ( + opaqueValueIdentity === `runtime-reference` && + (typeof value === `function` || typeof value === `symbol`) + ) { + return getRuntimeReferenceIdentity(value) + } + + if (value instanceof Date && Number.isNaN(value.getTime())) { + return canonicalizeRuntimeValue(Number.NaN, path, seen) + } + + try { + return canonicalizeRuntimeValue(value, path, seen) + } catch (error) { + if ( + error instanceof UnhashableQueryIRError && + typeof value === `object` && + value !== null + ) { + return getRuntimeReferenceIdentity(value) + } + throw error + } +} + function canonicalizeOrderingRuntimeValue( value: unknown, path: string, diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 245f2c4e2..735c41317 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -744,8 +744,12 @@ active while another owner still needs its coverage. The source signal aborts only after every attached owner has released it. A Collection subscription snapshots each logical subset demand before it calls -the source adapter. That stable snapshot drives the adapter transport, -acquisition evidence, compiled predicate, and later truncate replay. The +the source adapter. That private snapshot drives acquisition evidence, the +compiled predicate, and later truncate replay. Each adapter acquisition gets a +separate clone derived from it, so neither caller nor adapter mutation can +rewrite the logical demand or another acquisition. Values observed by scalar +functions are snapshotted according to that function's semantics; opaque +values used by reference-sensitive equality retain their identity. The caller's original predicate is retained only as a release handle, because the transport predicate may combine it with the subscription predicate. The subscription then installs the logical owner before adapter entry. Reentrant diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index 92d56c895..26402f9c7 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,4 +1,6 @@ import { Func, PropRef, Value } from './ir.js' +import { getExpressionArgumentValueContext } from './expression-value-context.js' +import type { ExpressionValueContext } from './expression-value-context.js' import type { BasicExpression } from './ir.js' import type { LoadSubsetOptions } from '../types.js' @@ -44,14 +46,9 @@ export function snapshotLoadSubsetDemand( return demand } -type ExpressionCloneContext = - | `exact-output` - | `equality-operand` - | `ordering-operand` - function cloneBasicExpression( expression: BasicExpression, - context: ExpressionCloneContext = `exact-output`, + context: ExpressionValueContext = `exact-output`, ): BasicExpression { switch (expression.type) { case `ref`: @@ -62,7 +59,9 @@ function cloneBasicExpression( ? snapshotEqualityValue(expression.value) : context === `ordering-operand` ? snapshotStructuralValue(expression.value) - : expression.value, + : context === `structural-operand` + ? snapshotStructuralValue(expression.value) + : expression.value, ) case `func`: return new Func( @@ -79,22 +78,18 @@ function cloneBasicExpression( ) } - const argumentContext: ExpressionCloneContext = - expression.name === `eq` - ? `equality-operand` - : isOrderingFunction(expression.name) - ? `ordering-operand` - : `exact-output` + const argumentContext = getExpressionArgumentValueContext( + expression.name, + index, + expression.args.length, + context, + ) return cloneBasicExpression(arg, argumentContext) }), ) } } -function isOrderingFunction(name: string): boolean { - return name === `gt` || name === `gte` || name === `lt` || name === `lte` -} - function snapshotEqualityValue(value: T): T { if (value instanceof Date) { return new Date(value.getTime()) as T diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 083a8d4c4..70dc25ee8 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5157,7 +5157,13 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - return loads.length === 1 ? true : replay.promise + if (loads.length === 1) { + // Adapter code owns only this acquisition copy. Mutating it + // must not rewrite the private demand used by later replay. + ;((options.where as Func).args[0] as PropRef).path[0] = `other` + return true + } + return replay.promise }, unloadSubset: (options) => { unloads.push(options) @@ -5209,6 +5215,82 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`snapshots mutable values beneath output-producing predicate functions`, async () => { + type Row = { id: `row` } + type Outcome = { + hasMore: false + appliedRowKeys: ReadonlyArray + } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + const replay = createDeferred() + const loads: Array = [] + const collection = createCollection({ + id: `logical-demand-value-snapshot`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + begin() + write({ type: `insert`, value: { id: `row` } }) + commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + return loads.length === 1 ? true : replay.promise + }, + } + }, + }, + }) + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + const bytes = Buffer.from([65]) + const where = new Func(`eq`, [ + new Func(`concat`, [new Value(bytes)]), + new Value(`A`), + ]) + + try { + subscription.requestSnapshot({ where }) + expect([...visible.keys()]).toEqual([`row`]) + + bytes[0] = 66 + begin() + truncate() + commit() + await flushPromises() + + begin() + write({ type: `insert`, value: { id: `row` } }) + const receipt = commit(loads[1]?.signal) + if (receipt !== true) await receipt + replay.resolve({ hasMore: false, appliedRowKeys: [`row`] }) + await flushPromises() + + expect([...visible.keys()]).toEqual([`row`]) + } finally { + replay.resolve({ hasMore: false, appliedRowKeys: [] }) + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`releases ordered publication authority before retrying failed adapter cleanup`, async () => { type Row = { id: string; rank: number } let begin!: () => void diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index c3968b0f5..9695192f6 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -552,6 +552,28 @@ describe(`loadSubset demand identity`, () => { ).toThrow(/function value/) }) + it(`snapshots structural function operands without changing demand identity`, () => { + const bytes = Buffer.from([65]) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(bytes)]), + new Value(`A`), + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ( + ((snapshot.where as Func).args[0] as Func).args[0] as Value + ).value + + expect(snapshotBytes).not.toBe(bytes) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + + bytes[0] = 66 + expect(compileExpression(demand.where!)({})).toBe(false) + expect(compileExpression(snapshot.where!)({})).toBe(true) + }) + it.each([ [`signed zero`, -0, 0], [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], From 552635f1198ecd3d38c5405c02c2665a5d17ba7b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 12:53:09 -0600 Subject: [PATCH 027/327] fix(db): close subset snapshot grammar --- packages/db/src/collection/subscription.ts | 27 ++- .../db/src/query/expression-value-context.ts | 156 ++++++++++++++++++ packages/db/src/query/ir-stable-identity.ts | 6 +- packages/db/src/query/live/ARCHITECTURE.md | 31 ++-- packages/db/src/query/load-subset-options.ts | 12 +- ...ubscription-replay-oracle.property.test.ts | 117 +++++++++++++ .../db/tests/query/ir-stable-identity.test.ts | 76 +++++++++ 7 files changed, 400 insertions(+), 25 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 2cf2769e0..ea39efa46 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -2362,10 +2362,18 @@ export class CollectionSubscription ) } + const orderedRequest = cloneLoadSubsetOptions({ + where: this.options.whereExpression, + orderBy, + limit, + }) + orderBy = orderedRequest.orderBy! + const where = orderedRequest.where + this.orderedWindow ??= new WindowState( this.collection, orderBy, - this.options.whereExpression, + where, limit, ) @@ -2381,7 +2389,6 @@ export class CollectionSubscription } } - const where = this.options.whereExpression const retainedPublication = this.retainedOrderedPublication const activeReplacement = this.truncateReplaySession !== undefined const replayOwnsContinuation = @@ -2495,12 +2502,16 @@ export class CollectionSubscription acquisition, result: syncResult, replayContext: startedReplayContext, - } = this.startSubsetDemand(loadOptions, { - requestedPrefix, - hadBoundary: boundary !== undefined || refreshPrefix, - requiresUnboundedRefinement, - revision: this.orderedWindow.coverageRevision, - }) + } = this.startSubsetDemand( + loadOptions, + { + requestedPrefix, + hadBoundary: boundary !== undefined || refreshPrefix, + requiresUnboundedRefinement, + revision: this.orderedWindow.coverageRevision, + }, + this.options.whereExpression, + ) // A synchronous continuation can complete ordered coverage. Retain its // callback before applying that evidence so callback failure can still diff --git a/packages/db/src/query/expression-value-context.ts b/packages/db/src/query/expression-value-context.ts index a1f1468ab..387081dd1 100644 --- a/packages/db/src/query/expression-value-context.ts +++ b/packages/db/src/query/expression-value-context.ts @@ -40,6 +40,162 @@ export function getExpressionArgumentValueContext( return `exact-output` } +/** Reject values whose observable scalar behavior cannot be cloned exactly. */ +export function assertSnapshotCapableStructuralValue( + value: unknown, + path = `value`, +): void { + visitStructuralValue(value, path, new WeakSet(), new WeakSet()) +} + +function visitStructuralValue( + value: unknown, + path: string, + active: WeakSet, + complete: WeakSet, +): void { + if (typeof value === `function`) { + throwUnsupported(path, `functions may expose mutable coercion hooks`) + } + if (typeof value !== `object` || value === null) return + if (complete.has(value)) return + if (active.has(value)) throwUnsupported(path, `cyclic values are unsupported`) + active.add(value) + + if (value instanceof Date) { + assertPrototype(value, Date.prototype, path) + assertNoOwnProperties(value, path) + } else if (value instanceof ArrayBuffer) { + assertPrototype(value, ArrayBuffer.prototype, path) + assertNoOwnProperties(value, path) + } else if (ArrayBuffer.isView(value)) { + assertSupportedArrayBufferViewPrototype(value, path) + assertOnlyIndexedProperties(value, path, false) + } else if (Array.isArray(value)) { + assertPrototype(value, Array.prototype, path) + assertOnlyIndexedProperties(value, path, true) + value.forEach((entry, index) => + visitStructuralValue(entry, `${path}[${index}]`, active, complete), + ) + } else if (value instanceof Map) { + assertPrototype(value, Map.prototype, path) + assertNoOwnProperties(value, path) + let index = 0 + for (const [key, entryValue] of value) { + visitStructuralValue(key, `${path}.key[${index}]`, active, complete) + visitStructuralValue( + entryValue, + `${path}.value[${index}]`, + active, + complete, + ) + index++ + } + } else if (value instanceof Set) { + assertPrototype(value, Set.prototype, path) + assertNoOwnProperties(value, path) + let index = 0 + for (const entry of value) { + visitStructuralValue(entry, `${path}[${index}]`, active, complete) + index++ + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throwUnsupported(path, `opaque object prototypes are unsupported`) + } + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== `string`) { + throwUnsupported(path, `symbol properties are unsupported`) + } + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!descriptor.enumerable || !(`value` in descriptor)) { + throwUnsupported( + `${path}.${key}`, + `non-enumerable properties and accessors are unsupported`, + ) + } + visitStructuralValue(descriptor.value, `${path}.${key}`, active, complete) + } + } + + active.delete(value) + complete.add(value) +} + +function assertPrototype(value: object, expected: object, path: string): void { + if (Object.getPrototypeOf(value) !== expected) { + throwUnsupported(path, `built-in subclasses are unsupported`) + } +} + +function assertSupportedArrayBufferViewPrototype( + value: ArrayBufferView, + path: string, +): void { + const prototype = Object.getPrototypeOf(value) + const supported = [ + DataView.prototype, + Int8Array.prototype, + Uint8Array.prototype, + Uint8ClampedArray.prototype, + Int16Array.prototype, + Uint16Array.prototype, + Int32Array.prototype, + Uint32Array.prototype, + Float32Array.prototype, + Float64Array.prototype, + BigInt64Array.prototype, + BigUint64Array.prototype, + ...(typeof Buffer === `undefined` ? [] : [Buffer.prototype]), + ] + if (!supported.includes(prototype)) { + throwUnsupported(path, `built-in subclasses are unsupported`) + } +} + +function assertNoOwnProperties(value: object, path: string): void { + if (Reflect.ownKeys(value).length > 0) { + throwUnsupported(path, `custom properties are unsupported`) + } +} + +function assertOnlyIndexedProperties( + value: object, + path: string, + allowLength: boolean, +): void { + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== `string`) { + throwUnsupported(path, `custom properties are unsupported`) + } + + const isLength = allowLength && key === `length` + if (!isLength && !isArrayIndex(key)) { + throwUnsupported(path, `custom properties are unsupported`) + } + + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!(`value` in descriptor) || (!isLength && !descriptor.enumerable)) { + throwUnsupported( + `${path}.${key}`, + `non-enumerable indexed properties and accessors are unsupported`, + ) + } + } +} + +function isArrayIndex(key: string): boolean { + const index = Number(key) + return Number.isInteger(index) && index >= 0 && String(index) === key +} + +function throwUnsupported(path: string, reason: string): never { + throw new TypeError( + `Cannot snapshot structural expression value at ${path}: ${reason}`, + ) +} + function isOrderingFunction(name: string): boolean { return name === `gt` || name === `gte` || name === `lt` || name === `lte` } diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index d49407f6e..849a5c441 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,7 +1,10 @@ import { normalizeValue } from '../utils/comparison.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/get-query-ir.js' -import { getExpressionArgumentValueContext } from './expression-value-context.js' +import { + assertSnapshotCapableStructuralValue, + getExpressionArgumentValueContext, +} from './expression-value-context.js' import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' import type { ExpressionValueContext } from './expression-value-context.js' import type { @@ -1086,6 +1089,7 @@ function canonicalizeStructuralRuntimeValue( seen: WeakSet, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { + assertSnapshotCapableStructuralValue(value, path) if ( opaqueValueIdentity === `runtime-reference` && (typeof value === `function` || typeof value === `symbol`) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 735c41317..3436775a4 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -748,20 +748,23 @@ the source adapter. That private snapshot drives acquisition evidence, the compiled predicate, and later truncate replay. Each adapter acquisition gets a separate clone derived from it, so neither caller nor adapter mutation can rewrite the logical demand or another acquisition. Values observed by scalar -functions are snapshotted according to that function's semantics; opaque -values used by reference-sensitive equality retain their identity. The -caller's original predicate is retained only as a release handle, because the -transport predicate may combine it with the subscription predicate. The -subscription then installs the logical owner before adapter entry. Reentrant -release during `loadSubset` must therefore see and release that exact -acquisition. After adapter return, both ordered and unordered requests recheck -logical ownership before they report results, track loading state, establish -coverage, or scan local state; a demand released during adapter code cannot -publish a later snapshot. A synchronous `loadSubset` throw that did not follow -a failed release rolls the tentative owner back before it emits the error and -without calling `unloadSubset`; a failed release keeps the owner so a later -cleanup can retry the same acquisition identity. Reconciliation reuses the -logical owner's evaluator across source changes and truncate acquisition +functions use a closed snapshot-capable grammar; unsupported coercion hooks, +opaque structural objects, and cycles fail before the demand is retained. +Opaque values used by reference-sensitive equality retain their identity. +Ordered requests take this snapshot before constructing `WindowState`, so the +same stable order drives local reconciliation, boundaries, transport, replay, +and evidence. The caller's original predicate is retained only as a release +handle, because the transport predicate may combine it with the subscription +predicate. The subscription then installs the logical owner before adapter +entry. Reentrant release during `loadSubset` must therefore see and release that +exact acquisition. After adapter return, both ordered and unordered requests +recheck logical ownership before they report results, track loading state, +establish coverage, or scan local state; a demand released during adapter code +cannot publish a later snapshot. A synchronous `loadSubset` throw that did not +follow a failed release rolls the tentative owner back before it emits the +error and without calling `unloadSubset`; a failed release keeps the owner so a +later cleanup can retry the same acquisition identity. Reconciliation reuses +the logical owner's evaluator across source changes and truncate acquisition replacement. A released owner cannot supply a predicate, and a later logical demand compiles its own evaluator even when it reuses the same expression object. diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index 26402f9c7..f031926de 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,5 +1,8 @@ import { Func, PropRef, Value } from './ir.js' -import { getExpressionArgumentValueContext } from './expression-value-context.js' +import { + assertSnapshotCapableStructuralValue, + getExpressionArgumentValueContext, +} from './expression-value-context.js' import type { ExpressionValueContext } from './expression-value-context.js' import type { BasicExpression } from './ir.js' import type { LoadSubsetOptions } from '../types.js' @@ -60,7 +63,7 @@ function cloneBasicExpression( : context === `ordering-operand` ? snapshotStructuralValue(expression.value) : context === `structural-operand` - ? snapshotStructuralValue(expression.value) + ? snapshotStructuralOperand(expression.value) : expression.value, ) case `func`: @@ -90,6 +93,11 @@ function cloneBasicExpression( } } +function snapshotStructuralOperand(value: T): T { + assertSnapshotCapableStructuralValue(value) + return snapshotStructuralValue(value) +} + function snapshotEqualityValue(value: T): T { if (value instanceof Date) { return new Date(value.getTime()) as T diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 70dc25ee8..0ed2f8a8c 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5291,6 +5291,123 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it.each([`reference-path`, `direction`] as const)( + `snapshots ordered demand state before %s mutation`, + async (mutation) => { + type Row = { + id: `a` | `b` + rank: number + other: number + version: number + } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: () => void + const collection = createCollection({ + id: `ordered-demand-snapshot-${mutation}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + begin() + write({ + type: `insert`, + value: { id: `a`, rank: 1, other: 2, version: 0 }, + }) + write({ + type: `insert`, + value: { id: `b`, rank: 2, other: 1, version: 0 }, + }) + commit() + params.markReady() + return { loadSubset: () => true } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderRef = new PropRef([`rank`]) + const compareOptions: OrderBy[number][`compareOptions`] = { + direction: `asc`, + nulls: `first`, + stringSort: `lexical`, + } + const orderBy: OrderBy = [{ expression: orderRef, compareOptions }] + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + subscription.setOrderByIndex(index) + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + expect([...visible.keys()]).toEqual([`a`]) + + if (mutation === `reference-path`) orderRef.path[0] = `other` + else compareOptions.direction = `desc` + + begin() + write({ + type: `update`, + value: { id: `b`, rank: 2, other: 1, version: 1 }, + }) + commit() + + expect([...visible.keys()]).toEqual([`a`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + + it(`rejects unsupported structural demand constants before adapter entry`, async () => { + type Row = { id: string } + let loadCount = 0 + const collection = createCollection({ + id: `unsupported-structural-demand`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + params.markReady() + return { + loadSubset: () => { + loadCount++ + return true + }, + } + }, + }, + }) + const subscription = collection.subscribeChanges(() => {}) + const value = { [Symbol.toPrimitive]: () => `A` } + const where = new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(`A`), + ]) + + try { + expect(() => subscription.requestSnapshot({ where })).toThrow( + /snapshot structural expression value/i, + ) + expect(loadCount).toBe(0) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`releases ordered publication authority before retrying failed adapter cleanup`, async () => { type Row = { id: string; rank: number } let begin!: () => void diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 9695192f6..3fe0ce713 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -574,6 +574,82 @@ describe(`loadSubset demand identity`, () => { expect(compileExpression(snapshot.where!)({})).toBe(true) }) + it.each([ + [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => `A` }), `A`], + [ + `non-enumerable coercion`, + () => { + const value = {} + Object.defineProperty(value, `toString`, { + value: () => `A`, + }) + return value + }, + `A`, + ], + [ + `opaque mutable coercion`, + () => + new (class { + value = `A`; + [Symbol.toPrimitive]() { + return this.value + } + })(), + `A`, + ], + [ + `indexed accessor coercion`, + () => { + const value: Array = [] + Object.defineProperty(value, `0`, { + enumerable: true, + get: () => `A`, + }) + return value + }, + `A`, + ], + [ + `built-in subclass coercion`, + () => + new (class extends Array { + [Symbol.toPrimitive]() { + return `A` + } + })(), + `A`, + ], + [ + `cyclic structure`, + () => { + const value: { self?: unknown } = {} + value.self = value + return value + }, + `[object Object]`, + ], + ] as const)( + `rejects unsupported %s before retaining structural demand state`, + (_label, createValue, expected) => { + const value = createValue() + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(expected), + ]), + } + + expect(compileExpression(demand.where!)({})).toBe(true) + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /snapshot structural expression value/i, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /snapshot structural expression value/i, + ) + }, + ) + it.each([ [`signed zero`, -0, 0], [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], From f87a37484aa54960b335f419fb6e8554652033c1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 13:23:51 -0600 Subject: [PATCH 028/327] fix(db): freeze subset subscription semantics --- packages/db/src/collection/subscription.ts | 36 ++++-- packages/db/src/query/ir-stable-identity.ts | 81 ++++++++++--- packages/db/src/query/live/ARCHITECTURE.md | 49 ++++---- packages/db/src/query/load-subset-options.ts | 31 +++-- ...ubscription-replay-oracle.property.test.ts | 108 ++++++++++++++++++ .../db/tests/query/ir-stable-identity.test.ts | 59 ++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 20 ++++ 7 files changed, 324 insertions(+), 60 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index ea39efa46..4f1c3750e 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -266,9 +266,14 @@ export class CollectionSubscription private stalePublication: PublicationState | undefined private filteredCallback: (changes: Array>) => boolean + // Execution uses the frozen predicate; release keeps the caller's handle. + private readonly whereExpression: BasicExpression | undefined + private readonly releaseWhereExpression: BasicExpression | undefined private orderByIndex: IndexInterface | undefined private orderedWindow: WindowState | undefined + // The first ordered request fixes this subscription's total order. + private orderedRequestOptions: LoadSubsetOptions | undefined // Status tracking private _status: SubscriptionStatus = `ready` @@ -410,9 +415,13 @@ export class CollectionSubscription constructor( private collection: CollectionImpl, private callback: (changes: Array>) => void, - private options: CollectionSubscriptionOptions, + options: CollectionSubscriptionOptions, ) { super() + this.releaseWhereExpression = options.whereExpression + this.whereExpression = cloneLoadSubsetOptions({ + where: options.whereExpression, + }).where if (options.onUnsubscribe) { this.on(`unsubscribed`, options.onUnsubscribe) } @@ -421,8 +430,8 @@ export class CollectionSubscription } // Auto-index for where expressions if enabled - if (options.whereExpression) { - ensureIndexForExpression(options.whereExpression, this.collection) + if (this.whereExpression) { + ensureIndexForExpression(this.whereExpression, this.collection) } const callbackWithSentKeysTracking = ( @@ -437,8 +446,11 @@ export class CollectionSubscription this.callback = callbackWithSentKeysTracking // Create a filtered callback if where clause is provided - this.filteredCallback = options.whereExpression - ? createFilteredCallback(this.callback, options) + this.filteredCallback = this.whereExpression + ? createFilteredCallback(this.callback, { + ...options, + whereExpression: this.whereExpression, + }) : (changes) => { this.callback(changes) return true @@ -1360,8 +1372,8 @@ export class CollectionSubscription const window = this.orderedWindow if (!stalePublication || !ordered || !window) return [] - const orderedFilter = this.options.whereExpression - ? createFilterFunctionFromExpression(this.options.whereExpression) + const orderedFilter = this.whereExpression + ? createFilterFunctionFromExpression(this.whereExpression) : undefined const additionalFilters = this.activeAdditionalFilters() const isOrderedRow = (row: object) => orderedFilter?.(row) ?? true @@ -2154,7 +2166,7 @@ export class CollectionSubscription } const stateOpts: RequestSnapshotOptions = { - where: this.options.whereExpression, + where: this.whereExpression, optimizedOnly: opts?.optimizedOnly ?? false, } @@ -2362,11 +2374,11 @@ export class CollectionSubscription ) } - const orderedRequest = cloneLoadSubsetOptions({ - where: this.options.whereExpression, + this.orderedRequestOptions ??= cloneLoadSubsetOptions({ + where: this.whereExpression, orderBy, - limit, }) + const orderedRequest = this.orderedRequestOptions orderBy = orderedRequest.orderBy! const where = orderedRequest.where @@ -2510,7 +2522,7 @@ export class CollectionSubscription requiresUnboundedRefinement, revision: this.orderedWindow.coverageRevision, }, - this.options.whereExpression, + this.releaseWhereExpression, ) // A synchronous continuation can complete ordered coverage. Retain its diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 849a5c441..3939c7c71 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1087,32 +1087,77 @@ function canonicalizeStructuralRuntimeValue( value: unknown, path: string, seen: WeakSet, - opaqueValueIdentity: OpaqueValueIdentity = `reject`, + _opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { assertSnapshotCapableStructuralValue(value, path) - if ( - opaqueValueIdentity === `runtime-reference` && - (typeof value === `function` || typeof value === `symbol`) - ) { + return canonicalizeSnapshotStructuralValue(value, path, seen) +} + +function canonicalizeSnapshotStructuralValue( + value: unknown, + path: string, + seen: WeakSet, +): StableIdentityValue { + if (typeof value === `symbol`) { return getRuntimeReferenceIdentity(value) } - if (value instanceof Date && Number.isNaN(value.getTime())) { - return canonicalizeRuntimeValue(Number.NaN, path, seen) + if (value instanceof Date) { + return Number.isNaN(value.getTime()) + ? [`Date`, `Invalid`] + : canonicalizeRuntimeValue(value, path, seen) } - try { - return canonicalizeRuntimeValue(value, path, seen) - } catch (error) { - if ( - error instanceof UnhashableQueryIRError && - typeof value === `object` && - value !== null - ) { - return getRuntimeReferenceIdentity(value) - } - throw error + if (Array.isArray(value)) { + return withCircularGuard(value, path, seen, () => [ + `snapshotArray`, + value.length, + Object.keys(value).map((key) => [ + key, + canonicalizeSnapshotStructuralValue( + value[Number(key)], + `${path}[${key}]`, + seen, + ), + ]), + ]) + } + + if (value instanceof Map) { + return withCircularGuard(value, path, seen, () => [ + `snapshotMap`, + Array.from(value.entries(), ([key, entryValue], index) => [ + canonicalizeSnapshotStructuralValue(key, `${path}.key[${index}]`, seen), + canonicalizeSnapshotStructuralValue( + entryValue, + `${path}.value[${index}]`, + seen, + ), + ]), + ]) + } + + if (value instanceof Set) { + return withCircularGuard(value, path, seen, () => [ + `snapshotSet`, + Array.from(value, (entry, index) => + canonicalizeSnapshotStructuralValue(entry, `${path}[${index}]`, seen), + ), + ]) + } + + if (isPlainObject(value)) { + return withCircularGuard(value, path, seen, () => [ + `snapshotObject`, + Object.getPrototypeOf(value) === null ? `null` : `plain`, + Object.keys(value).map((key) => [ + key, + canonicalizeSnapshotStructuralValue(value[key], `${path}.${key}`, seen), + ]), + ]) } + + return canonicalizeRuntimeValue(value, path, seen) } function canonicalizeOrderingRuntimeValue( diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 3436775a4..a28e9e049 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -743,28 +743,35 @@ shared abort lease. If one owner releases its lease, the source request remains active while another owner still needs its coverage. The source signal aborts only after every attached owner has released it. -A Collection subscription snapshots each logical subset demand before it calls -the source adapter. That private snapshot drives acquisition evidence, the -compiled predicate, and later truncate replay. Each adapter acquisition gets a -separate clone derived from it, so neither caller nor adapter mutation can -rewrite the logical demand or another acquisition. Values observed by scalar -functions use a closed snapshot-capable grammar; unsupported coercion hooks, -opaque structural objects, and cycles fail before the demand is retained. -Opaque values used by reference-sensitive equality retain their identity. -Ordered requests take this snapshot before constructing `WindowState`, so the -same stable order drives local reconciliation, boundaries, transport, replay, -and evidence. The caller's original predicate is retained only as a release +A Collection subscription snapshots its predicate when it is constructed. Its +first ordered request also snapshots the total order. Later window requests may +change the requested size, but local reconciliation, boundaries, transport, +replay, and evidence all keep the same predicate and order for that +subscription. + +Each logical subset demand then gets a private snapshot before adapter entry. +Each adapter acquisition gets a separate clone derived from it, so neither +caller nor adapter mutation can rewrite the subscription machine, logical +demand, or another acquisition. Values observed by scalar functions use a +closed snapshot-capable grammar. Its identity preserves every observable part +of the clone, including prototype kind, property order, sparse-array holes, +invalid Dates, and symbol identity. Unsupported coercion hooks, opaque +structural objects, built-in subclasses, accessors, and cycles fail before the +demand is retained. Opaque values used by reference-sensitive equality retain +their identity. The caller's original predicate is retained only as a release handle, because the transport predicate may combine it with the subscription -predicate. The subscription then installs the logical owner before adapter -entry. Reentrant release during `loadSubset` must therefore see and release that -exact acquisition. After adapter return, both ordered and unordered requests -recheck logical ownership before they report results, track loading state, -establish coverage, or scan local state; a demand released during adapter code -cannot publish a later snapshot. A synchronous `loadSubset` throw that did not -follow a failed release rolls the tentative owner back before it emits the -error and without calling `unloadSubset`; a failed release keeps the owner so a -later cleanup can retry the same acquisition identity. Reconciliation reuses -the logical owner's evaluator across source changes and truncate acquisition +predicate. + +The subscription installs the logical owner before adapter entry. Reentrant +release during `loadSubset` must therefore see and release that exact +acquisition. After adapter return, both ordered and unordered requests recheck +logical ownership before they report results, track loading state, establish +coverage, or scan local state; a demand released during adapter code cannot +publish a later snapshot. A synchronous `loadSubset` throw that did not follow +a failed release rolls the tentative owner back before it emits the error and +without calling `unloadSubset`; a failed release keeps the owner so a later +cleanup can retry the same acquisition identity. Reconciliation reuses the +logical owner's evaluator across source changes and truncate acquisition replacement. A released owner cannot supply a predicate, and a later logical demand compiles its own evaluator even when it reuses the same expression object. diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index f031926de..837e02124 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,3 +1,4 @@ +import { normalizeValue } from '../utils/comparison.js' import { Func, PropRef, Value } from './ir.js' import { assertSnapshotCapableStructuralValue, @@ -103,12 +104,14 @@ function snapshotEqualityValue(value: T): T { return new Date(value.getTime()) as T } + // Large binaries use reference identity in indexes. Clone only values for + // which normalization establishes a content key. if (typeof Buffer !== `undefined` && value instanceof Buffer) { - return Buffer.from(value) as T + return (normalizeValue(value) === value ? value : Buffer.from(value)) as T } if (value instanceof Uint8Array) { - return value.slice() as T + return (normalizeValue(value) === value ? value : value.slice()) as T } // Other objects use reference equality in predicate identity and comparison. @@ -158,10 +161,15 @@ function snapshotStructuralValue( } if (Array.isArray(value)) { - const result: Array = [] + const result: Array = new Array(value.length) seen.set(value, result) - for (const item of value) { - result.push(snapshotStructuralValue(item, seen)) + for (const key of Object.keys(value)) { + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + writable: true, + value: snapshotStructuralValue(value[Number(key)], seen), + }) } return result as T } @@ -197,10 +205,15 @@ function snapshotStructuralValue( const result = Object.create(prototype) as Record seen.set(value, result) for (const key of Object.keys(value)) { - result[key] = snapshotStructuralValue( - (value as Record)[key], - seen, - ) + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + writable: true, + value: snapshotStructuralValue( + (value as Record)[key], + seen, + ), + }) } return result as T } diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 0ed2f8a8c..96304c4d8 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5371,6 +5371,114 @@ describe(`CollectionSubscription replay oracle`, () => { }, ) + it.each([`before-first-request`, `after-first-publication`] as const)( + `keeps one ordered machine when caller state mutates %s`, + async (timing) => { + type Row = { + id: `a` | `b` + group: `keep` | `drop` + alternate: `keep` | `drop` + rank: number + other: number + } + const loads: Array = [] + const collection = createCollection({ + id: `ordered-machine-${timing}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + params.begin() + params.write({ + type: `insert`, + value: { + id: `a`, + group: `keep`, + alternate: `drop`, + rank: 1, + other: 2, + }, + }) + params.write({ + type: `insert`, + value: { + id: `b`, + group: `drop`, + alternate: `keep`, + rank: 2, + other: 1, + }, + }) + params.commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + return true + }, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const whereRef = new PropRef([`group`]) + const where = new Func(`eq`, [ + whereRef, + new Value(`keep`), + ]) + const orderRef = new PropRef([`rank`]) + const compareOptions: OrderBy[number][`compareOptions`] = { + direction: `asc`, + nulls: `first`, + stringSort: `lexical`, + } + const orderBy: OrderBy = [{ expression: orderRef, compareOptions }] + const visible = new Map() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }, + { whereExpression: where }, + ) + subscription.setOrderByIndex(index) + const mutateCallerState = () => { + whereRef.path[0] = `alternate` + if (timing === `after-first-publication`) { + orderRef.path[0] = `other` + compareOptions.direction = `desc` + } + } + + try { + if (timing === `before-first-request`) mutateCallerState() + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + + if (timing === `after-first-publication`) { + expect([...visible.keys()]).toEqual([`a`]) + mutateCallerState() + subscription.requestLimitedSnapshot({ orderBy, limit: 2 }) + } + + const lastLoad = loads.at(-1)! + const loadedWhere = lastLoad.where as Func + const loadedOrder = lastLoad.orderBy![0]! + expect((loadedWhere.args[0] as PropRef).path).toEqual([`group`]) + expect((loadedOrder.expression as PropRef).path).toEqual([`rank`]) + expect(loadedOrder.compareOptions.direction).toBe(`asc`) + expect([...visible.keys()]).toEqual([`a`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + it(`rejects unsupported structural demand constants before adapter entry`, async () => { type Row = { id: string } let loadCount = 0 diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 3fe0ce713..146d0643c 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -574,6 +574,20 @@ describe(`loadSubset demand identity`, () => { expect(compileExpression(snapshot.where!)({})).toBe(true) }) + it(`retains reference-sensitive large binary equality values`, () => { + const bytes = new Uint8Array(129).fill(7) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotBytes).toBe(bytes) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + }) + it.each([ [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => `A` }), `A`], [ @@ -650,6 +664,51 @@ describe(`loadSubset demand identity`, () => { }, ) + it.each([ + [`nested invalid Date`, [new Date(Number.NaN)]], + [`nested symbol`, [Symbol(`immutable`)]], + [`sparse array`, new Array(1)], + ] as const)( + `preserves structural demand identity while cloning %s`, + (_label, value) => { + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value( + compileExpression(new Func(`concat`, [new Value(value)]))({}), + ), + ]), + } + const snapshot = cloneLoadSubsetOptions(demand) + + expect(compileExpression(snapshot.where!)({})).toBe(true) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + }, + ) + + it(`preserves an enumerable __proto__ data property while cloning`, () => { + const value: Record = {} + Object.defineProperty(value, `__proto__`, { + enumerable: true, + value: null, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(`[object Object]`), + ]), + } + const snapshot = cloneLoadSubsetOptions(demand) + + expect(compileExpression(demand.where!)({})).toBe(true) + expect(compileExpression(snapshot.where!)({})).toBe(true) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + }) + it.each([ [`signed zero`, -0, 0], [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 5ea056658..90aa4d1d9 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -45,6 +45,26 @@ function not(expression: BasicExpression): Func { } describe(`createDeduplicatedLoadSubset`, () => { + it(`does not deduplicate structural predicates with different observable key order`, () => { + const left = Object.create(null) as Record + left.a = 1 + left.b = 2 + const right = Object.create(null) as Record + right.b = 2 + right.a = 1 + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const expected = JSON.stringify(left) + const demand = (value: Record): LoadSubsetOptions => ({ + where: eq(new Func(`concat`, [val(value)]), val(expected)), + }) + + deduplicated.loadSubset(demand(left)) + deduplicated.loadSubset(demand(right)) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + it.each( [ { From c6ad294b801ee89e1082752fee4f5231840c581e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 13:51:00 -0600 Subject: [PATCH 029/327] fix(db): close retained demand value grammar --- packages/db/src/query/ir-stable-identity.ts | 1 + packages/db/src/query/load-subset-options.ts | 9 +- packages/db/src/utils/comparison.ts | 20 +--- packages/db/tests/comparison.property.test.ts | 5 +- .../uint8array-id-comparison.test.ts | 14 +-- packages/db/tests/load-subset-outcome.test.ts | 16 +-- .../db/tests/query/ir-stable-identity.test.ts | 70 +++++++++--- ...d-subset-full-flow-oracle.property.test.ts | 103 ++++++++++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 29 +++++ 9 files changed, 203 insertions(+), 64 deletions(-) diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 3939c7c71..fa2b36f67 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1166,6 +1166,7 @@ function canonicalizeOrderingRuntimeValue( seen: WeakSet, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { + assertSnapshotCapableStructuralValue(value, path) if ( opaqueValueIdentity === `runtime-reference` && (typeof value === `function` || typeof value === `symbol`) diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index 837e02124..c49d5f8af 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,4 +1,3 @@ -import { normalizeValue } from '../utils/comparison.js' import { Func, PropRef, Value } from './ir.js' import { assertSnapshotCapableStructuralValue, @@ -62,7 +61,7 @@ function cloneBasicExpression( context === `equality-operand` ? snapshotEqualityValue(expression.value) : context === `ordering-operand` - ? snapshotStructuralValue(expression.value) + ? snapshotStructuralOperand(expression.value) : context === `structural-operand` ? snapshotStructuralOperand(expression.value) : expression.value, @@ -104,14 +103,12 @@ function snapshotEqualityValue(value: T): T { return new Date(value.getTime()) as T } - // Large binaries use reference identity in indexes. Clone only values for - // which normalization establishes a content key. if (typeof Buffer !== `undefined` && value instanceof Buffer) { - return (normalizeValue(value) === value ? value : Buffer.from(value)) as T + return Buffer.from(value) as T } if (value instanceof Uint8Array) { - return (normalizeValue(value) === value ? value : value.slice()) as T + return value.slice() as T } // Other objects use reference equality in predicate identity and comparison. diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 26aa39943..204b07648 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -157,14 +157,6 @@ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { return true } -/** - * Threshold for normalizing Uint8Arrays to string representations. - * Arrays larger than this will use reference equality to avoid memory overhead. - * 128 bytes is enough for common ID formats (ULIDs are 16 bytes, UUIDs are 16 bytes) - * while avoiding excessive string allocation for large binary data. - */ -const UINT8ARRAY_NORMALIZE_THRESHOLD = 128 - /** * Sentinel value representing undefined in normalized form. * This allows distinguishing between "start from beginning" (undefined parameter) @@ -203,14 +195,10 @@ export function normalizeValue(value: any): any { value instanceof Uint8Array if (isUint8Array) { - // Only normalize small arrays to avoid memory overhead for large binary data - if (value.byteLength <= UINT8ARRAY_NORMALIZE_THRESHOLD) { - // Convert to a string representation that can be used as a Map key - // Use a special prefix to avoid collisions with user strings - return `__u8__${Array.from(value).join(`,`)}` - } - // For large arrays, fall back to reference equality - // Users working with large binary data should use a derived key if needed + // Convert to a string representation that can be used as a Map key. + // Equality compares every binary value by content, so index keys must not + // switch to reference identity at an arbitrary byte length. + return `__u8__${Array.from(value).join(`,`)}` } return value diff --git a/packages/db/tests/comparison.property.test.ts b/packages/db/tests/comparison.property.test.ts index dd6279001..cac34ccc4 100644 --- a/packages/db/tests/comparison.property.test.ts +++ b/packages/db/tests/comparison.property.test.ts @@ -384,10 +384,11 @@ describe(`normalizeValue property-based tests`, () => { ) fcTest.prop([fc.uint8Array({ minLength: 129, maxLength: 200 })])( - `large Uint8Arrays are not normalized`, + `large Uint8Arrays normalize to string representation`, (arr) => { const normalized = normalizeValue(arr) - expect(normalized).toBe(arr) + expect(typeof normalized).toBe(`string`) + expect(normalized).toMatch(/^__u8__/) }, ) diff --git a/packages/db/tests/integration/uint8array-id-comparison.test.ts b/packages/db/tests/integration/uint8array-id-comparison.test.ts index 7b13c04f6..481b7d465 100644 --- a/packages/db/tests/integration/uint8array-id-comparison.test.ts +++ b/packages/db/tests/integration/uint8array-id-comparison.test.ts @@ -79,8 +79,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { expect(resultByName?.name).toBe(makeItemName(selectedItemIndex)) }) - it(`should use reference equality for large Uint8Arrays (> 128 bytes)`, async () => { - // Create a large Uint8Array (> 128 bytes) that should use reference equality + it(`should use content equality for large Uint8Arrays`, async () => { const largeId = new Uint8Array(200).fill(42) interface LargeItem { @@ -102,7 +101,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { }), ) - // Query with the exact same reference - this should work + // The same reference works. const queryWithSameRef = createLiveQueryCollection((q) => q .from({ item: collection }) @@ -113,12 +112,10 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { await queryWithSameRef.preload() const resultWithSameRef = Array.from(queryWithSameRef.entries())[0]?.[1] - // Should find the item because we're using the same reference expect(resultWithSameRef).toBeDefined() expect(resultWithSameRef?.name).toBe(`Large Item`) - // Query with a different instance but same content - this will NOT work - // because large arrays use reference equality + // A different instance with the same bytes has the same value. const differentInstance = new Uint8Array(200).fill(42) const queryWithDifferentRef = createLiveQueryCollection((q) => q @@ -132,8 +129,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { queryWithDifferentRef.entries(), )[0]?.[1] - // Should NOT find the item because large arrays use reference equality - // This is expected behavior to avoid memory overhead - expect(resultWithDifferentRef).toBeUndefined() + expect(resultWithDifferentRef).toBeDefined() + expect(resultWithDifferentRef?.name).toBe(`Large Item`) }) }) diff --git a/packages/db/tests/load-subset-outcome.test.ts b/packages/db/tests/load-subset-outcome.test.ts index cfe048df0..6d869c912 100644 --- a/packages/db/tests/load-subset-outcome.test.ts +++ b/packages/db/tests/load-subset-outcome.test.ts @@ -935,7 +935,7 @@ describe(`loadSubset outcomes`, () => { }, ) - it(`tracks opaque demand values by runtime reference`, async () => { + it(`tracks opaque equality demand values by runtime reference`, async () => { const loadSubset = vi.fn((_options: LoadSubsetOptions) => Promise.resolve({ hasMore: false }), ) @@ -958,20 +958,6 @@ describe(`loadSubset outcomes`, () => { const createDemands = (value: unknown): Array => [ { where: new Func(`eq`, [field, new Value(value)]) }, { where: new Func(`in`, [field, new Value([value])]) }, - { - orderBy: [ - { - expression: new Func(`coalesce`, [field, new Value(value)]), - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - }, - { - cursor: { - whereFrom: new Func(`gt`, [field, new Value(value)]), - whereCurrent: new Func(`eq`, [field, new Value(value)]), - }, - }, ] const demands = [ ...createDemands(() => `opaque`), diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 146d0643c..6ab607780 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -47,6 +47,7 @@ import { } from '../../src/query/ir.js' import { compileExpression, + compileSingleRowExpression, toBooleanPredicate, } from '../../src/query/compiler/evaluators.js' import { isLoadSubsetRequestSubsumedBy } from '../../src/query/predicate-utils.js' @@ -508,20 +509,6 @@ describe(`loadSubset demand identity`, () => { const createDemands = (value: unknown): Array => [ { where: new Func(`eq`, [field, new Value(value)]) }, { where: new Func(`in`, [field, new Value([value])]) }, - { - orderBy: [ - { - expression: new Func(`coalesce`, [field, new Value(value)]), - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - }, - { - cursor: { - whereFrom: new Func(`gt`, [field, new Value(value)]), - whereCurrent: new Func(`eq`, [field, new Value(value)]), - }, - }, ] for (const [firstValue, secondValue] of [ @@ -574,7 +561,7 @@ describe(`loadSubset demand identity`, () => { expect(compileExpression(snapshot.where!)({})).toBe(true) }) - it(`retains reference-sensitive large binary equality values`, () => { + it(`snapshots large binary equality values without changing demand identity`, () => { const bytes = new Uint8Array(129).fill(7) const demand: LoadSubsetOptions = { where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), @@ -582,10 +569,61 @@ describe(`loadSubset demand identity`, () => { const snapshot = cloneLoadSubsetOptions(demand) const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value - expect(snapshotBytes).toBe(bytes) + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(bytes) expect(getLoadSubsetDemandKey(snapshot)).toBe( getLoadSubsetDemandKey(demand), ) + + bytes.fill(8) + expect( + compileSingleRowExpression(demand.where!)({ + id: new Uint8Array(129).fill(7), + }), + ).toBe(false) + expect( + compileSingleRowExpression(snapshot.where!)({ + id: new Uint8Array(129).fill(7), + }), + ).toBe(true) + }) + + it.each([ + [`function`, () => () => 1], + [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => 1 })], + [ + `indexed accessor`, + () => { + const value: Array = [] + Object.defineProperty(value, `0`, { + enumerable: true, + get: () => 1, + }) + return value + }, + ], + [ + `cycle`, + () => { + const value: Array = [] + value.push(value) + return value + }, + ], + ])(`rejects %s in ordering operands`, (_name, createValue) => { + const demand: LoadSubsetOptions = { + where: new Func(`gt`, [ + new PropRef([`value`]), + new Value(createValue()), + ]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot structural expression value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot structural expression value/, + ) }) it.each([ diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index c1116eca7..3a29b85dd 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -345,6 +345,109 @@ it(`does not release physical work when an already-aborted demand skips adapter } }) +it.each([127, 128, 129])( + `freezes a %i-byte equality constant across local filtering and adapter acquisition`, + async (byteLength) => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const originalToken = new Uint8Array(byteLength).fill(1) + const changedToken = new Uint8Array(byteLength).fill(2) + const callerToken = new Uint8Array(originalToken) + const rows: ReadonlyArray = [ + { id: `original`, token: originalToken }, + { id: `changed`, token: changedToken }, + ] + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-binary-equality-${byteLength}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + rows.forEach((value) => write({ type: `insert`, value })) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const visible = new Set() + const where = new Func(`eq`, [ + new PropRef([`token`]), + new Value(callerToken), + ]) + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { whereExpression: where }, + ) + + try { + callerToken.fill(2) + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredValue = ( + (acquired?.where as Func | undefined)?.args[1] as + | Value + | undefined + )?.value + expect(acquiredValue).toEqual(originalToken) + expect(acquiredValue).not.toBe(callerToken) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, +) + +it(`rejects unsupported relational coercion before adapter entry`, async () => { + let adapterCalls = 0 + const collection = createCollection<{ id: string; value: number }>({ + id: `unsupported-relational-coercion`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + const coercion = { [Symbol.toPrimitive]: () => 1 } + + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`gt`, [ + new PropRef([`value`]), + new Value(coercion), + ]), + }), + ).toThrow(/Cannot snapshot structural expression value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + it(`reloads authoritative rows after final-owner cleanup invalidates retained adapter coverage`, async () => { type Row = { id: string; value: number } const row: Row = { id: `row`, value: 1 } diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 90aa4d1d9..e39980815 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -45,6 +45,35 @@ function not(expression: BasicExpression): Func { } describe(`createDeduplicatedLoadSubset`, () => { + it(`does not let mutation rewrite settled large-binary coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const mutableToken = new Uint8Array(129).fill(1) + const demand = (token: Uint8Array): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(token)), + limit: 1, + }) + + deduplicated.loadSubset(demand(mutableToken)) + mutableToken.fill(2) + deduplicated.loadSubset(demand(new Uint8Array(129).fill(2))) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects unsupported relational coercion before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const coercion = { [Symbol.toPrimitive]: () => 1 } + + expect(() => + deduplicated.loadSubset({ + where: gt(ref(`value`), val(coercion)), + }), + ).toThrow(/Cannot snapshot structural expression value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + it(`does not deduplicate structural predicates with different observable key order`, () => { const left = Object.create(null) as Record left.a = 1 From 455aaf3a7175ec37c507a576d19bac4684aafb95 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 14:22:21 -0600 Subject: [PATCH 030/327] fix(db): freeze retained equality value domains --- .../db/src/query/expression-value-context.ts | 6 +- packages/db/src/query/ir-stable-identity.ts | 102 ++++++----- packages/db/src/query/load-subset-options.ts | 36 ++-- packages/db/src/utils/comparison.ts | 68 +++++++- packages/db/tests/comparison.property.test.ts | 24 +-- .../db/tests/query/ir-stable-identity.test.ts | 100 +++++++++++ ...d-subset-full-flow-oracle.property.test.ts | 160 ++++++++++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 23 +++ 8 files changed, 443 insertions(+), 76 deletions(-) diff --git a/packages/db/src/query/expression-value-context.ts b/packages/db/src/query/expression-value-context.ts index 387081dd1..07d1b3a68 100644 --- a/packages/db/src/query/expression-value-context.ts +++ b/packages/db/src/query/expression-value-context.ts @@ -1,6 +1,7 @@ export type ExpressionValueContext = | `exact-output` | `equality-operand` + | `membership-candidates` | `ordering-operand` | `structural-operand` @@ -11,7 +12,10 @@ export function getExpressionArgumentValueContext( argumentCount: number, resultContext: ExpressionValueContext, ): ExpressionValueContext { - if (name === `eq` || name === `in`) return `equality-operand` + if (name === `eq`) return `equality-operand` + if (name === `in`) { + return index === 0 ? `equality-operand` : `membership-candidates` + } if (isOrderingFunction(name)) return `ordering-operand` if ( diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index fa2b36f67..a6c0e1df3 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,4 +1,8 @@ -import { normalizeValue } from '../utils/comparison.js' +import { + normalizeValue, + snapshotTemporalEqualityValue, +} from '../utils/comparison.js' +import { isTemporal } from '../utils.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/get-query-ir.js' import { @@ -637,63 +641,38 @@ function canonicalizeExpression( scope, opaqueValueIdentity, ) - : valueContext === `ordering-operand` - ? canonicalizeOrderingRuntimeValue( + : valueContext === `membership-candidates` + ? canonicalizeMembershipCandidates( expression.value, `${path}.value`, seen, + scope, opaqueValueIdentity, ) - : valueContext === `structural-operand` - ? canonicalizeStructuralRuntimeValue( + : valueContext === `ordering-operand` + ? canonicalizeOrderingRuntimeValue( expression.value, `${path}.value`, seen, opaqueValueIdentity, ) - : canonicalizeExactOutputRuntimeValue( - expression.value, - `${path}.value`, - seen, - opaqueValueIdentity, - ), + : valueContext === `structural-operand` + ? canonicalizeStructuralRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ) + : canonicalizeExactOutputRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ), } } if (expression.type === `func`) { - if ( - expression.name === `in` && - expression.args.length === 2 && - expression.args[1]?.type === `val` && - Array.isArray(expression.args[1].value) - ) { - const candidates = expression.args[1].value.map((value, index) => - canonicalizeEqualityRuntimeValue( - value, - `${path}.args[1].value[${index}]`, - seen, - scope, - opaqueValueIdentity, - ), - ) - return canonicalizeFunction(expression.name, [ - canonicalizeExpression( - expression.args[0]!, - `${path}.args[0]`, - seen, - `equality-operand`, - scope, - opaqueValueIdentity, - ), - { - type: `val`, - // IN tests membership. Candidate order and duplicates do not change - // its result, but each candidate keeps its own equality semantics. - value: [`set`, sortUniqueStableIdentityValues(candidates)], - }, - ]) - } - const args = expression.args.map((arg, index) => canonicalizeExpression( arg, @@ -1071,6 +1050,11 @@ function canonicalizeEqualityRuntimeValue( return [`binary`, `Uint8Array`, Array.from(value as Uint8Array)] } + if (isTemporal(value)) { + const snapshot = snapshotTemporalEqualityValue(value) + return canonicalizeRuntimeValue(normalizeValue(snapshot), path, seen) + } + const normalized = normalizeValue(value) if (normalized !== value) { return canonicalizeRuntimeValue(normalized, path, seen) @@ -1083,6 +1067,36 @@ function canonicalizeEqualityRuntimeValue( return canonicalizeRuntimeValue(value, path, seen) } +function canonicalizeMembershipCandidates( + value: unknown, + path: string, + seen: WeakSet, + scope?: AliasScope, + opaqueValueIdentity: OpaqueValueIdentity = `reject`, +): StableIdentityValue { + if (!Array.isArray(value)) { + return canonicalizeExactOutputRuntimeValue( + value, + path, + seen, + opaqueValueIdentity, + ) + } + + const candidates = Array.from(value, (candidate, index) => + canonicalizeEqualityRuntimeValue( + candidate, + `${path}[${index}]`, + seen, + scope, + opaqueValueIdentity, + ), + ) + // IN tests membership. Candidate order and duplicates do not change its + // result, but each candidate keeps its own equality semantics. + return [`set`, sortUniqueStableIdentityValues(candidates)] +} + function canonicalizeStructuralRuntimeValue( value: unknown, path: string, diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index c49d5f8af..037e3a82e 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,3 +1,5 @@ +import { snapshotTemporalEqualityValue } from '../utils/comparison.js' +import { isTemporal } from '../utils.js' import { Func, PropRef, Value } from './ir.js' import { assertSnapshotCapableStructuralValue, @@ -60,27 +62,18 @@ function cloneBasicExpression( return new Value( context === `equality-operand` ? snapshotEqualityValue(expression.value) - : context === `ordering-operand` - ? snapshotStructuralOperand(expression.value) - : context === `structural-operand` + : context === `membership-candidates` + ? snapshotMembershipCandidates(expression.value) + : context === `ordering-operand` ? snapshotStructuralOperand(expression.value) - : expression.value, + : context === `structural-operand` + ? snapshotStructuralOperand(expression.value) + : expression.value, ) case `func`: return new Func( expression.name, expression.args.map((arg, index) => { - if ( - expression.name === `in` && - index === 1 && - arg.type === `val` && - Array.isArray(arg.value) - ) { - return new Value( - arg.value.map((value) => snapshotEqualityValue(value)), - ) - } - const argumentContext = getExpressionArgumentValueContext( expression.name, index, @@ -104,17 +97,26 @@ function snapshotEqualityValue(value: T): T { } if (typeof Buffer !== `undefined` && value instanceof Buffer) { - return Buffer.from(value) as T + return Buffer.from(new Uint8Array(value)) as T } if (value instanceof Uint8Array) { - return value.slice() as T + return new Uint8Array(value) as T + } + + if (isTemporal(value)) { + return snapshotTemporalEqualityValue(value) as T } // Other objects use reference equality in predicate identity and comparison. return value } +function snapshotMembershipCandidates(value: T): T { + if (!Array.isArray(value)) return value + return Array.from(value, (candidate) => snapshotEqualityValue(candidate)) as T +} + function snapshotStructuralValue( value: T, seen: WeakMap = new WeakMap(), diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 204b07648..43e8e8723 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -1,5 +1,6 @@ import { isTemporal } from '../utils' import type { CompareOptions } from '../query/builder/types' +import type { TemporalLike } from '../utils' // WeakMap to store stable IDs for objects const objectIds = new WeakMap() @@ -162,11 +163,61 @@ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { * This allows distinguishing between "start from beginning" (undefined parameter) * and "start from the key undefined" (actual undefined value in the tree). */ -export const UNDEFINED_SENTINEL = `__TS_DB_BTREE_UNDEFINED_VALUE__` +const NORMALIZED_KEY_PREFIX = `\u0000tanstack-db:` + +function normalizedKey(kind: string, value: string): string { + return `${NORMALIZED_KEY_PREFIX}${kind}:${value}` +} + +export const UNDEFINED_SENTINEL = normalizedKey(`undefined`, ``) const UNORDERABLE_BTREE_SENTINEL = Object.freeze({ kind: `tanstack-db-unorderable`, }) +/** Clone a Temporal equality value without trusting mutable brand lookalikes. */ +export function snapshotTemporalEqualityValue( + value: TemporalLike, +): TemporalLike { + const prototype = Object.getPrototypeOf(value) + const constructorDescriptor = + prototype === null + ? undefined + : Object.getOwnPropertyDescriptor(prototype, `constructor`) + const constructor = constructorDescriptor?.value + const fromDescriptor = + typeof constructor === `function` + ? Object.getOwnPropertyDescriptor(constructor, `from`) + : undefined + const toStringDescriptor = + prototype === null + ? undefined + : Object.getOwnPropertyDescriptor(prototype, `toString`) + if ( + typeof constructor !== `function` || + typeof fromDescriptor?.value !== `function` || + typeof toStringDescriptor?.value !== `function` + ) { + throw new TypeError( + `Cannot snapshot ${value[Symbol.toStringTag]} equality value`, + ) + } + + const serialized = Reflect.apply(toStringDescriptor.value, value, []) + const snapshot = Reflect.apply(fromDescriptor.value, constructor, [ + serialized, + ]) + if ( + snapshot === value || + !isTemporal(snapshot) || + snapshot[Symbol.toStringTag] !== value[Symbol.toStringTag] + ) { + throw new TypeError( + `Cannot snapshot ${value[Symbol.toStringTag]} equality value`, + ) + } + return snapshot +} + /** * Normalize a value for comparison and Map key usage * Converts values that can't be directly compared or used as Map keys @@ -176,6 +227,14 @@ const UNORDERABLE_BTREE_SENTINEL = Object.freeze({ * for BTree index operations that need to distinguish undefined values. */ export function normalizeValue(value: any): any { + // Internal normalized keys occupy a reserved string domain. Escape user + // strings in that domain so a literal cannot equal a binary or Temporal key. + if (typeof value === `string`) { + return value.startsWith(NORMALIZED_KEY_PREFIX) + ? normalizedKey(`string`, value) + : value + } + if (typeof value !== `object` || value === null) { return value } @@ -185,7 +244,10 @@ export function normalizeValue(value: any): any { } if (isTemporal(value)) { - return `__temporal__${value[Symbol.toStringTag]}__${value.toString()}` + return normalizedKey( + `temporal`, + `${value[Symbol.toStringTag]}:${value.toString()}`, + ) } // Normalize Uint8Arrays/Buffers to a string representation for Map key usage @@ -198,7 +260,7 @@ export function normalizeValue(value: any): any { // Convert to a string representation that can be used as a Map key. // Equality compares every binary value by content, so index keys must not // switch to reference identity at an arbitrary byte length. - return `__u8__${Array.from(value).join(`,`)}` + return normalizedKey(`binary`, Array.from(value).join(`,`)) } return value diff --git a/packages/db/tests/comparison.property.test.ts b/packages/db/tests/comparison.property.test.ts index cac34ccc4..ed0196791 100644 --- a/packages/db/tests/comparison.property.test.ts +++ b/packages/db/tests/comparison.property.test.ts @@ -375,37 +375,39 @@ describe(`normalizeValue property-based tests`, () => { }) fcTest.prop([fc.uint8Array({ minLength: 0, maxLength: 128 })])( - `small Uint8Arrays normalize to string representation`, + `small Uint8Arrays normalize to a stable key`, (arr) => { const normalized = normalizeValue(arr) expect(typeof normalized).toBe(`string`) - expect(normalized).toMatch(/^__u8__/) + expect(normalized).toBe(normalizeValue(new Uint8Array(arr))) }, ) fcTest.prop([fc.uint8Array({ minLength: 129, maxLength: 200 })])( - `large Uint8Arrays normalize to string representation`, + `large Uint8Arrays normalize to a stable key`, (arr) => { const normalized = normalizeValue(arr) expect(typeof normalized).toBe(`string`) - expect(normalized).toMatch(/^__u8__/) + expect(normalized).toBe(normalizeValue(new Uint8Array(arr))) }, ) - fcTest.prop([fc.string()])(`strings pass through unchanged`, (str) => { - expect(normalizeValue(str)).toBe(str) - }) + fcTest.prop([fc.string()])( + `strings preserve equality after normalization`, + (str) => { + expect(normalizeValue(str)).toBe(normalizeValue(`${str}`)) + }, + ) fcTest.prop([fc.integer()])(`integers pass through unchanged`, (n) => { expect(normalizeValue(n)).toBe(n) }) fcTest.prop([fc.uint8Array({ minLength: 0, maxLength: 128 })])( - `normalization is idempotent for Uint8Arrays`, + `binary keys cannot collide with user strings`, (arr) => { - const normalized1 = normalizeValue(arr) - // For strings (which small arrays become), normalizing again should be identity - expect(normalizeValue(normalized1)).toBe(normalized1) + const normalized = normalizeValue(arr) + expect(normalizeValue(normalized)).not.toBe(normalized) }, ) }) diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 6ab607780..a059cc9be 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -588,6 +588,106 @@ describe(`loadSubset demand identity`, () => { ).toBe(true) }) + it(`copies binary equality values without calling an overridden slice`, () => { + const bytes = new Uint8Array([1, 2, 3]) + Object.defineProperty(bytes, `slice`, { + value: () => bytes, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(new Uint8Array([1, 2, 3])) + + bytes.fill(9) + expect( + compileSingleRowExpression(snapshot.where!)({ + id: new Uint8Array([1, 2, 3]), + }), + ).toBe(true) + }) + + it.each([`coalesce`, `caseWhen`] as const)( + `snapshots equality candidates returned by %s`, + (wrapper) => { + const candidates = [new Uint8Array([1])] + const candidateExpression = + wrapper === `coalesce` + ? new Func(`coalesce`, [new Value(candidates)]) + : new Func(`caseWhen`, [ + new Value(true), + new Value(candidates), + new Value([]), + ]) + const demand: LoadSubsetOptions = { + where: new Func(`in`, [ + new PropRef([`token`]), + candidateExpression, + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + + candidates[0]![0] = 2 + candidates.push(new Uint8Array([3])) + + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + expect( + compileSingleRowExpression(snapshot.where!)({ + token: new Uint8Array([1]), + }), + ).toBe(true) + expect( + compileSingleRowExpression(snapshot.where!)({ + token: new Uint8Array([2]), + }), + ).toBe(false) + }, + ) + + it(`rejects mutable Temporal-branded equality lookalikes`, () => { + let callerDate = `2024-01-15` + const callerValue = { + [Symbol.toStringTag]: `Temporal.PlainDate`, + toString: () => callerDate, + } + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new PropRef([`date`]), + new Value(callerValue), + ]), + } + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + callerDate = `2024-01-16` + }) + + it(`clones genuine Temporal equality values without changing their type or identity`, () => { + const date = Temporal.PlainDate.from(`2024-01-15`) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`date`]), new Value(date)]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotDate = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotDate).not.toBe(date) + expect(snapshotDate).toBeInstanceOf(Temporal.PlainDate) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + expect( + compileSingleRowExpression(snapshot.where!)({ + date: Temporal.PlainDate.from(`2024-01-15`), + }), + ).toBe(true) + }) + it.each([ [`function`, () => () => 1], [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => 1 })], diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 3a29b85dd..bb58b215f 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -7,6 +7,7 @@ import { Func, PropRef, Value } from '../../src/query/ir.js' import { createEffect } from '../../src/query/effect.js' import { createLiveQueryCollection, eq } from '../../src/query/index.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' +import { normalizeValue } from '../../src/utils/comparison.js' import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' import { computeOrderedLoadCursor } from '../../src/query/live/utils.js' import { WindowState } from '../../src/query/live/window-state.js' @@ -352,6 +353,9 @@ it.each([127, 128, 129])( const originalToken = new Uint8Array(byteLength).fill(1) const changedToken = new Uint8Array(byteLength).fill(2) const callerToken = new Uint8Array(originalToken) + Object.defineProperty(callerToken, `slice`, { + value: () => callerToken, + }) const rows: ReadonlyArray = [ { id: `original`, token: originalToken }, { id: `changed`, token: changedToken }, @@ -412,6 +416,162 @@ it.each([127, 128, 129])( }, ) +it(`keeps binary equality distinct from a sentinel-looking string`, async () => { + type Row = { id: `binary` | `string`; token: Uint8Array | string } + const binary = new Uint8Array([1, 2, 3]) + const sentinel = normalizeValue(binary) as string + const collection = createCollection({ + id: `binary-string-normalization-domains`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `binary`, token: binary } }) + write({ type: `insert`, value: { id: `string`, token: sentinel } }) + commit() + markReady() + return { loadSubset: () => true } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(binary), + ]), + }, + ) + + try { + subscription.requestSnapshot({ optimizedOnly: false }) + expect([...visible]).toEqual([`binary`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`freezes computed membership candidates across local filtering and adapter acquisition`, async () => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const candidates = [new Uint8Array([1])] + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-computed-membership-candidates`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `original`, token: new Uint8Array([1]) }, + }) + write({ + type: `insert`, + value: { id: `changed`, token: new Uint8Array([2]) }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + }, + ) + + try { + candidates[0]![0] = 2 + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredCandidates = ( + ((acquired?.where as Func).args[1] as Func).args[0] as Value< + Array + > + ).value + expect(acquiredCandidates).toEqual([new Uint8Array([1])]) + expect(acquiredCandidates).not.toBe(candidates) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`rejects mutable Temporal-branded equality lookalikes before adapter acquisition`, async () => { + type TemporalValue = { + [Symbol.toStringTag]: string + toString: () => string + } + type Row = { id: string; date: TemporalValue } + let callerDate = `2024-01-15` + const createDate = (read: () => string): TemporalValue => ({ + [Symbol.toStringTag]: `Temporal.PlainDate`, + toString: read, + }) + const callerValue = createDate(() => callerDate) + let adapterCalls = 0 + const collection = createCollection({ + id: `frozen-temporal-branded-equality`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`eq`, [ + new PropRef([`date`]), + new Value(callerValue), + ]), + }), + ).toThrow(/Cannot snapshot Temporal.PlainDate equality value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + callerDate = `2024-01-16` + } finally { + await collection.cleanup() + } +}) + it(`rejects unsupported relational coercion before adapter entry`, async () => { let adapterCalls = 0 const collection = createCollection<{ id: string; value: number }>({ diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index e39980815..96d2e2aa0 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -61,6 +61,29 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it(`does not let mutation rewrite computed membership coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const candidates = [new Uint8Array([1])] + const demand = (): LoadSubsetOptions => ({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }) + + deduplicated.loadSubset(demand()) + candidates[0]![0] = 2 + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val([new Uint8Array([2])])]), + ]), + }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + it(`rejects unsupported relational coercion before adapter entry`, () => { const loadSubset = vi.fn(() => true as const) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) From a70457a3229beb48609e64a5b000a16e46d3feda Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 14:44:11 -0600 Subject: [PATCH 031/327] fix(db): use intrinsic retained value snapshots --- .../db/src/query/expression-value-context.ts | 48 +++++++ packages/db/src/query/ir-stable-identity.ts | 11 +- packages/db/src/query/load-subset-options.ts | 18 ++- packages/db/src/query/subset-dedupe.ts | 5 +- packages/db/src/utils/comparison.ts | 56 ++++++-- packages/db/tests/comparison.property.test.ts | 17 +++ .../db/tests/query/ir-stable-identity.test.ts | 97 +++++++++++++ ...d-subset-full-flow-oracle.property.test.ts | 129 ++++++++++++++++-- packages/db/tests/query/subset-dedupe.test.ts | 88 ++++++++++++ 9 files changed, 431 insertions(+), 38 deletions(-) diff --git a/packages/db/src/query/expression-value-context.ts b/packages/db/src/query/expression-value-context.ts index 07d1b3a68..29389268e 100644 --- a/packages/db/src/query/expression-value-context.ts +++ b/packages/db/src/query/expression-value-context.ts @@ -52,6 +52,48 @@ export function assertSnapshotCapableStructuralValue( visitStructuralValue(value, path, new WeakSet(), new WeakSet()) } +/** + * Read an IN candidate array without invoking caller-defined iteration or + * accessors. The plain result gives later identity and adapter paths one stable + * observation of the request. + */ +export function snapshotMembershipCandidateValues( + value: unknown, + path = `value`, +): Array | undefined { + if (!Array.isArray(value)) return undefined + if (Object.getPrototypeOf(value) !== Array.prototype) { + throwUnsupportedMembership(path, `array subclasses are unsupported`) + } + + const lengthDescriptor = Object.getOwnPropertyDescriptor(value, `length`) + const length = lengthDescriptor?.value + if (typeof length !== `number` || !Number.isInteger(length) || length < 0) { + throwUnsupportedMembership(path, `invalid array length`) + } + + const snapshot = new Array(length) + for (let index = 0; index < length; index++) snapshot[index] = undefined + + for (const key of Reflect.ownKeys(value)) { + if (key === `length`) continue + if (typeof key !== `string` || !isArrayIndex(key)) { + throwUnsupportedMembership(path, `custom properties are unsupported`) + } + + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!descriptor.enumerable || !(`value` in descriptor)) { + throwUnsupportedMembership( + `${path}.${key}`, + `non-enumerable indexed properties and accessors are unsupported`, + ) + } + snapshot[Number(key)] = descriptor.value + } + + return snapshot +} + function visitStructuralValue( value: unknown, path: string, @@ -200,6 +242,12 @@ function throwUnsupported(path: string, reason: string): never { ) } +function throwUnsupportedMembership(path: string, reason: string): never { + throw new TypeError( + `Cannot snapshot membership candidates at ${path}: ${reason}`, + ) +} + function isOrderingFunction(name: string): boolean { return name === `gt` || name === `gte` || name === `lt` || name === `lte` } diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index a6c0e1df3..553a4f0c9 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,6 +1,7 @@ import { normalizeValue, snapshotTemporalEqualityValue, + snapshotUint8ArrayBytes, } from '../utils/comparison.js' import { isTemporal } from '../utils.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' @@ -8,6 +9,7 @@ import { getQueryIR } from './builder/get-query-ir.js' import { assertSnapshotCapableStructuralValue, getExpressionArgumentValueContext, + snapshotMembershipCandidateValues, } from './expression-value-context.js' import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' import type { ExpressionValueContext } from './expression-value-context.js' @@ -1047,7 +1049,11 @@ function canonicalizeEqualityRuntimeValue( (typeof Buffer !== `undefined` && value instanceof Buffer) || value instanceof Uint8Array if (isUint8Array) { - return [`binary`, `Uint8Array`, Array.from(value as Uint8Array)] + return [ + `binary`, + `Uint8Array`, + Array.from(snapshotUint8ArrayBytes(value as Uint8Array)), + ] } if (isTemporal(value)) { @@ -1083,7 +1089,8 @@ function canonicalizeMembershipCandidates( ) } - const candidates = Array.from(value, (candidate, index) => + const candidateValues = snapshotMembershipCandidateValues(value, path)! + const candidates = candidateValues.map((candidate, index) => canonicalizeEqualityRuntimeValue( candidate, `${path}[${index}]`, diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index 037e3a82e..aad505fe0 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,9 +1,14 @@ -import { snapshotTemporalEqualityValue } from '../utils/comparison.js' +import { + readDateTimestamp, + snapshotTemporalEqualityValue, + snapshotUint8ArrayBytes, +} from '../utils/comparison.js' import { isTemporal } from '../utils.js' import { Func, PropRef, Value } from './ir.js' import { assertSnapshotCapableStructuralValue, getExpressionArgumentValueContext, + snapshotMembershipCandidateValues, } from './expression-value-context.js' import type { ExpressionValueContext } from './expression-value-context.js' import type { BasicExpression } from './ir.js' @@ -93,15 +98,15 @@ function snapshotStructuralOperand(value: T): T { function snapshotEqualityValue(value: T): T { if (value instanceof Date) { - return new Date(value.getTime()) as T + return new Date(readDateTimestamp(value)) as T } if (typeof Buffer !== `undefined` && value instanceof Buffer) { - return Buffer.from(new Uint8Array(value)) as T + return Buffer.from(snapshotUint8ArrayBytes(value)) as T } if (value instanceof Uint8Array) { - return new Uint8Array(value) as T + return snapshotUint8ArrayBytes(value) as T } if (isTemporal(value)) { @@ -113,8 +118,9 @@ function snapshotEqualityValue(value: T): T { } function snapshotMembershipCandidates(value: T): T { - if (!Array.isArray(value)) return value - return Array.from(value, (candidate) => snapshotEqualityValue(candidate)) as T + const candidates = snapshotMembershipCandidateValues(value) + if (candidates === undefined) return value + return candidates.map((candidate) => snapshotEqualityValue(candidate)) as T } function snapshotStructuralValue( diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index f342382e7..dc970bf4a 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -168,10 +168,7 @@ export class DeduplicatedLoadSubset { ...options, signal: lease.signal, }) - const loadOptions = cloneLoadSubsetOptions({ - ...options, - signal: lease.signal, - }) + const loadOptions = cloneLoadSubsetOptions(trackingOptions) if ( this.unlimitedWhere !== undefined && options.limit === undefined && diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 43e8e8723..1dbef3e2e 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -29,10 +29,20 @@ function getObjectId(obj: object): number { export function isUnorderable(value: any): boolean { return ( (typeof value === `number` && Number.isNaN(value)) || - (value instanceof Date && Number.isNaN(value.getTime())) + (value instanceof Date && Number.isNaN(readDateTimestamp(value))) ) } +/** Read a Date's internal timestamp without invoking an instance override. */ +export function readDateTimestamp(value: Date): number { + return Reflect.apply(Date.prototype.getTime, value, []) +} + +/** Copy a Uint8Array's internal bytes without invoking custom iteration. */ +export function snapshotUint8ArrayBytes(value: Uint8Array): Uint8Array { + return new Uint8Array(value) +} + /** * Universal comparison function for all data types * Handles null/undefined, strings, arrays, dates, objects, and primitives @@ -78,7 +88,7 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { // If both are dates, compare them if (a instanceof Date && b instanceof Date) { - return a.getTime() - b.getTime() + return readDateTimestamp(a) - readDateTimestamp(b) } // If both are Temporal objects, use compareTemporalValues for correct semantic ordering @@ -178,6 +188,7 @@ const UNORDERABLE_BTREE_SENTINEL = Object.freeze({ export function snapshotTemporalEqualityValue( value: TemporalLike, ): TemporalLike { + const tag = value[Symbol.toStringTag] const prototype = Object.getPrototypeOf(value) const constructorDescriptor = prototype === null @@ -192,16 +203,23 @@ export function snapshotTemporalEqualityValue( prototype === null ? undefined : Object.getOwnPropertyDescriptor(prototype, `toString`) + const brandAccessorName = TEMPORAL_BRAND_ACCESSORS[tag] + const brandAccessorDescriptor = + prototype === null || brandAccessorName === undefined + ? undefined + : Object.getOwnPropertyDescriptor(prototype, brandAccessorName) if ( typeof constructor !== `function` || typeof fromDescriptor?.value !== `function` || - typeof toStringDescriptor?.value !== `function` + typeof toStringDescriptor?.value !== `function` || + typeof brandAccessorDescriptor?.get !== `function` ) { - throw new TypeError( - `Cannot snapshot ${value[Symbol.toStringTag]} equality value`, - ) + throw new TypeError(`Cannot snapshot ${tag} equality value`) } + // Temporal accessors brand-check their receiver's internal slots. A tag plus + // constructor-shaped methods is not enough to establish a genuine value. + Reflect.apply(brandAccessorDescriptor.get, value, []) const serialized = Reflect.apply(toStringDescriptor.value, value, []) const snapshot = Reflect.apply(fromDescriptor.value, constructor, [ serialized, @@ -209,15 +227,26 @@ export function snapshotTemporalEqualityValue( if ( snapshot === value || !isTemporal(snapshot) || - snapshot[Symbol.toStringTag] !== value[Symbol.toStringTag] + snapshot[Symbol.toStringTag] !== tag || + Object.getPrototypeOf(snapshot) !== prototype ) { - throw new TypeError( - `Cannot snapshot ${value[Symbol.toStringTag]} equality value`, - ) + throw new TypeError(`Cannot snapshot ${tag} equality value`) } + Reflect.apply(brandAccessorDescriptor.get, snapshot, []) return snapshot } +const TEMPORAL_BRAND_ACCESSORS: Readonly> = { + 'Temporal.Duration': `years`, + 'Temporal.Instant': `epochNanoseconds`, + 'Temporal.PlainDate': `year`, + 'Temporal.PlainDateTime': `year`, + 'Temporal.PlainMonthDay': `day`, + 'Temporal.PlainTime': `hour`, + 'Temporal.PlainYearMonth': `year`, + 'Temporal.ZonedDateTime': `epochNanoseconds`, +} + /** * Normalize a value for comparison and Map key usage * Converts values that can't be directly compared or used as Map keys @@ -240,7 +269,7 @@ export function normalizeValue(value: any): any { } if (value instanceof Date) { - return value.getTime() + return readDateTimestamp(value) } if (isTemporal(value)) { @@ -260,7 +289,10 @@ export function normalizeValue(value: any): any { // Convert to a string representation that can be used as a Map key. // Equality compares every binary value by content, so index keys must not // switch to reference identity at an arbitrary byte length. - return normalizedKey(`binary`, Array.from(value).join(`,`)) + return normalizedKey( + `binary`, + Array.from(snapshotUint8ArrayBytes(value)).join(`,`), + ) } return value diff --git a/packages/db/tests/comparison.property.test.ts b/packages/db/tests/comparison.property.test.ts index ed0196791..2cff29760 100644 --- a/packages/db/tests/comparison.property.test.ts +++ b/packages/db/tests/comparison.property.test.ts @@ -410,6 +410,23 @@ describe(`normalizeValue property-based tests`, () => { expect(normalizeValue(normalized)).not.toBe(normalized) }, ) + + fcTest( + `reads binary keys from intrinsic bytes instead of custom iteration`, + () => { + const bytes = new Uint8Array([2]) + Object.defineProperty(bytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + + expect(normalizeValue(bytes)).toBe(normalizeValue(new Uint8Array([2]))) + expect(normalizeValue(bytes)).not.toBe( + normalizeValue(new Uint8Array([1])), + ) + }, + ) }) describe(`areValuesEqual property-based tests`, () => { diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index a059cc9be..fba19ea62 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -610,6 +610,36 @@ describe(`loadSubset demand identity`, () => { ).toBe(true) }) + it(`derives binary equality identity from intrinsic bytes`, () => { + const bytes = new Uint8Array([2]) + Object.defineProperty(bytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + const predicate = new Func(`eq`, [ + new PropRef([`id`]), + new Value(bytes), + ]) + + expect(getLoadSubsetDemandKey({ where: predicate })).toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [ + new PropRef([`id`]), + new Value(new Uint8Array([2])), + ]), + }), + ) + expect(getLoadSubsetDemandKey({ where: predicate })).not.toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [ + new PropRef([`id`]), + new Value(new Uint8Array([1])), + ]), + }), + ) + }) + it.each([`coalesce`, `caseWhen`] as const)( `snapshots equality candidates returned by %s`, (wrapper) => { @@ -648,6 +678,28 @@ describe(`loadSubset demand identity`, () => { }, ) + it(`rejects membership arrays with custom observation hooks`, () => { + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, + }) + const demand: LoadSubsetOptions = { + where: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot membership candidates/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot membership candidates/, + ) + }) + it(`rejects mutable Temporal-branded equality lookalikes`, () => { let callerDate = `2024-01-15` const callerValue = { @@ -669,6 +721,51 @@ describe(`loadSubset demand identity`, () => { callerDate = `2024-01-16` }) + it(`rejects constructor-shaped Temporal equality lookalikes`, () => { + class TemporalLookalike { + static shared = `2024-01-15` + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return TemporalLookalike.shared + } + } + const value = new TemporalLookalike() + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`date`]), new Value(value)]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + }) + + it(`reads Date equality values through the intrinsic getTime`, () => { + const date = new Date(2) + Object.defineProperty(date, `getTime`, { + value: () => 1, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`date`]), new Value(date)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotDate = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotDate.getTime()).toBe(2) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [new PropRef([`date`]), new Value(new Date(2))]), + }), + ) + }) + it(`clones genuine Temporal equality values without changing their type or identity`, () => { const date = Temporal.PlainDate.from(`2024-01-15`) const demand: LoadSubsetOptions = { diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index bb58b215f..9beedd170 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -526,21 +526,122 @@ it(`freezes computed membership candidates across local filtering and adapter ac } }) -it(`rejects mutable Temporal-branded equality lookalikes before adapter acquisition`, async () => { - type TemporalValue = { - [Symbol.toStringTag]: string - toString: () => string - } - type Row = { id: string; date: TemporalValue } - let callerDate = `2024-01-15` - const createDate = (read: () => string): TemporalValue => ({ - [Symbol.toStringTag]: `Temporal.PlainDate`, - toString: read, +it(`rejects custom membership observation before adapter acquisition`, async () => { + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, }) - const callerValue = createDate(() => callerDate) let adapterCalls = 0 + const collection = createCollection<{ id: string; token: Uint8Array }>({ + id: `reject-custom-membership-observation`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + }), + ).toThrow(/Cannot snapshot membership candidates/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + +it(`uses intrinsic Date state for local filtering and adapter acquisition`, async () => { + type Row = { id: `instance-hook` | `intrinsic`; date: Date } + const callerDate = new Date(2) + Object.defineProperty(callerDate, `getTime`, { value: () => 1 }) + let acquired: LoadSubsetOptions | undefined const collection = createCollection({ - id: `frozen-temporal-branded-equality`, + id: `intrinsic-date-equality`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `instance-hook`, date: new Date(1) }, + }) + write({ + type: `insert`, + value: { id: `intrinsic`, date: new Date(2) }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`date`]), + new Value(callerDate), + ]), + }, + ) + + try { + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`intrinsic`]) + const acquiredDate = ((acquired?.where as Func).args[1] as Value) + .value + expect(acquiredDate.getTime()).toBe(2) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`rejects constructor-shaped Temporal lookalikes before adapter acquisition`, async () => { + class TemporalLookalike { + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return `2024-01-15` + } + } + let adapterCalls = 0 + const collection = createCollection<{ id: string; date: TemporalLookalike }>({ + id: `reject-constructor-shaped-temporal`, getKey: (row) => row.id, syncMode: `on-demand`, sync: { @@ -555,18 +656,18 @@ it(`rejects mutable Temporal-branded equality lookalikes before adapter acquisit }, }, }) + try { expect(() => collection.subscribeChanges(() => {}, { whereExpression: new Func(`eq`, [ new PropRef([`date`]), - new Value(callerValue), + new Value(new TemporalLookalike()), ]), }), ).toThrow(/Cannot snapshot Temporal.PlainDate equality value/) expect(adapterCalls).toBe(0) expect(collection.subscriberCount).toBe(0) - callerDate = `2024-01-16` } finally { await collection.cleanup() } diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 96d2e2aa0..e66b63812 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -61,6 +61,94 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it(`does not let custom binary iteration alias intrinsic byte coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const customBytes = new Uint8Array([2]) + Object.defineProperty(customBytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + const demand = (token: Uint8Array): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(token)), + }) + + deduplicated.loadSubset(demand(new Uint8Array([1]))) + deduplicated.loadSubset(demand(customBytes)) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects custom membership observation before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, + }) + + expect(() => + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }), + ).toThrow(/Cannot snapshot membership candidates/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`uses intrinsic Date state for tracking and adapter acquisition`, () => { + const acquiredDates: Array = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquiredDates.push( + ((options.where as Func).args[1] as Value).value.getTime(), + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const date = new Date(2) + let observedTime = 0 + Object.defineProperty(date, `getTime`, { + value: () => ++observedTime, + }) + const demand = (value: Date): LoadSubsetOptions => ({ + where: eq(ref(`date`), val(value)), + }) + + deduplicated.loadSubset(demand(date)) + deduplicated.loadSubset(demand(new Date(1))) + + expect(acquiredDates).toEqual([2, 1]) + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects constructor-shaped Temporal lookalikes before adapter entry`, () => { + class TemporalLookalike { + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return `2024-01-15` + } + } + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + expect(() => + deduplicated.loadSubset({ + where: eq(ref(`date`), val(new TemporalLookalike())), + }), + ).toThrow(/Cannot snapshot Temporal.PlainDate equality value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + it(`does not let mutation rewrite computed membership coverage`, () => { const loadSubset = vi.fn(() => true as const) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) From 67e61f81bfdbcab52b66386f74d6d63811998750 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 15:04:26 -0600 Subject: [PATCH 032/327] fix(db): reject binary equality proxies --- packages/db/src/utils.ts | 4 ++ packages/db/src/utils/comparison.ts | 5 ++ .../db/tests/query/ir-stable-identity.test.ts | 68 ++++++++++++++----- ...d-subset-full-flow-oracle.property.test.ts | 43 ++++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 64 +++++++++++++++++ 5 files changed, 167 insertions(+), 17 deletions(-) diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index e65208741..925baece4 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -222,6 +222,10 @@ const temporalTypes = new Set([ `Temporal.ZonedDateTime`, ]) +/** + * A Temporal value. Objects that claim a Temporal tag are expected to obey the + * Temporal contract, including immutable value semantics. + */ export interface TemporalLike { [Symbol.toStringTag]: string toString: () => string diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 1dbef3e2e..0b3494887 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -40,6 +40,11 @@ export function readDateTimestamp(value: Date): number { /** Copy a Uint8Array's internal bytes without invoking custom iteration. */ export function snapshotUint8ArrayBytes(value: Uint8Array): Uint8Array { + if (!ArrayBuffer.isView(value)) { + throw new TypeError( + `Cannot snapshot binary equality value without intrinsic typed-array slots`, + ) + } return new Uint8Array(value) } diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index fba19ea62..30a7ac6a6 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -640,6 +640,27 @@ describe(`loadSubset demand identity`, () => { ) }) + it(`rejects binary values without intrinsic typed-array slots`, () => { + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot binary equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot binary equality value/, + ) + }) + it.each([`coalesce`, `caseWhen`] as const)( `snapshots equality candidates returned by %s`, (wrapper) => { @@ -766,24 +787,37 @@ describe(`loadSubset demand identity`, () => { ) }) - it(`clones genuine Temporal equality values without changing their type or identity`, () => { - const date = Temporal.PlainDate.from(`2024-01-15`) - const demand: LoadSubsetOptions = { - where: new Func(`eq`, [new PropRef([`date`]), new Value(date)]), - } - const demandKey = getLoadSubsetDemandKey(demand) - const snapshot = cloneLoadSubsetOptions(demand) - const snapshotDate = ((snapshot.where as Func).args[1] as Value).value + it.each([ + [`Duration`, Temporal.Duration.from(`P1DT2H`)], + [`Instant`, Temporal.Instant.from(`2024-01-15T12:00:00Z`)], + [`PlainDate`, Temporal.PlainDate.from(`2024-01-15`)], + [`PlainDateTime`, Temporal.PlainDateTime.from(`2024-01-15T12:00:00`)], + [`PlainMonthDay`, Temporal.PlainMonthDay.from(`01-15`)], + [`PlainTime`, Temporal.PlainTime.from(`12:00:00`)], + [`PlainYearMonth`, Temporal.PlainYearMonth.from(`2024-01`)], + [`ZonedDateTime`, Temporal.ZonedDateTime.from(`2024-01-15T12:00:00Z[UTC]`)], + ])( + `clones genuine Temporal.%s equality values without changing type or identity`, + (_name, value) => { + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new PropRef([`value`]), + new Value(value), + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotValue = ((snapshot.where as Func).args[1] as Value).value - expect(snapshotDate).not.toBe(date) - expect(snapshotDate).toBeInstanceOf(Temporal.PlainDate) - expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) - expect( - compileSingleRowExpression(snapshot.where!)({ - date: Temporal.PlainDate.from(`2024-01-15`), - }), - ).toBe(true) - }) + expect(snapshotValue).not.toBe(value) + expect(Object.getPrototypeOf(snapshotValue)).toBe( + Object.getPrototypeOf(value), + ) + expect(String(snapshotValue)).toBe(String(value)) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + expect(compileSingleRowExpression(snapshot.where!)({ value })).toBe(true) + }, + ) it.each([ [`function`, () => () => 1], diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 9beedd170..69946c82e 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -416,6 +416,49 @@ it.each([127, 128, 129])( }, ) +it(`rejects binary values without intrinsic typed-array slots before adapter acquisition`, async () => { + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + let adapterCalls = 0 + const collection = createCollection<{ id: string; token: Uint8Array }>({ + id: `reject-binary-proxy`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(bytes), + ]), + }), + ).toThrow(/Cannot snapshot binary equality value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + it(`keeps binary equality distinct from a sentinel-looking string`, async () => { type Row = { id: `binary` | `string`; token: Uint8Array | string } const binary = new Uint8Array([1, 2, 3]) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index e66b63812..a21afc7cf 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -80,6 +80,70 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it(`rejects binary proxies before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + + expect(() => + deduplicated.loadSubset({ where: eq(ref(`token`), val(bytes)) }), + ).toThrow(/Cannot snapshot binary equality value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`observes computed membership once for tracking and acquisition`, () => { + const first = new Uint8Array([1]) + const second = new Uint8Array([2]) + let observations = 0 + const candidates = new Proxy([first], { + getOwnPropertyDescriptor: (target, key) => { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key) + if (key !== `0` || descriptor === undefined) return descriptor + observations += 1 + return { + ...descriptor, + value: observations === 1 ? first : second, + } + }, + }) + const acquired: Array = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquired.push( + ...( + ((options.where as Func).args[1] as Func).args[0] as Value< + Array + > + ).value, + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }) + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val([first])]), + ]), + }) + + expect(observations).toBe(1) + expect(acquired).toEqual([first]) + expect(loadSubset).toHaveBeenCalledTimes(1) + }) + it(`rejects custom membership observation before adapter entry`, () => { const loadSubset = vi.fn(() => true as const) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) From b10e14623fdcc003c8786da7ec7aeb8aa84f1c6a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 15:22:00 -0600 Subject: [PATCH 033/327] fix(db): snapshot cross-realm binary values --- packages/db/src/query/ir-stable-identity.ts | 14 ++-- packages/db/src/query/load-subset-options.ts | 3 +- packages/db/src/utils/comparison.ts | 48 +++++++++---- .../db/tests/query/ir-stable-identity.test.ts | 23 +++++++ ...d-subset-full-flow-oracle.property.test.ts | 69 ++++++++++++++++++- packages/db/tests/query/subset-dedupe.test.ts | 23 +++++++ packages/db/tests/utils.ts | 9 +++ 7 files changed, 162 insertions(+), 27 deletions(-) diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 553a4f0c9..008beff0c 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,4 +1,5 @@ import { + isUint8ArrayCandidate, normalizeValue, snapshotTemporalEqualityValue, snapshotUint8ArrayBytes, @@ -1045,15 +1046,8 @@ function canonicalizeEqualityRuntimeValue( // Equality compares Uint8Array and Buffer values by content, independent of // their concrete constructor and size. - const isUint8Array = - (typeof Buffer !== `undefined` && value instanceof Buffer) || - value instanceof Uint8Array - if (isUint8Array) { - return [ - `binary`, - `Uint8Array`, - Array.from(snapshotUint8ArrayBytes(value as Uint8Array)), - ] + if (isUint8ArrayCandidate(value)) { + return [`binary`, `Uint8Array`, Array.from(snapshotUint8ArrayBytes(value))] } if (isTemporal(value)) { @@ -1203,7 +1197,7 @@ function canonicalizeOrderingRuntimeValue( } const normalized = normalizeValue(value) - if (normalized !== value && !(value instanceof Uint8Array)) { + if (normalized !== value && !isUint8ArrayCandidate(value)) { return canonicalizeRuntimeValue(normalized, path, seen) } diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index aad505fe0..691f41cda 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,4 +1,5 @@ import { + isUint8ArrayCandidate, readDateTimestamp, snapshotTemporalEqualityValue, snapshotUint8ArrayBytes, @@ -105,7 +106,7 @@ function snapshotEqualityValue(value: T): T { return Buffer.from(snapshotUint8ArrayBytes(value)) as T } - if (value instanceof Uint8Array) { + if (isUint8ArrayCandidate(value)) { return snapshotUint8ArrayBytes(value) as T } diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 0b3494887..3c6d22179 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -38,9 +38,33 @@ export function readDateTimestamp(value: Date): number { return Reflect.apply(Date.prototype.getTime, value, []) } +const typedArrayTagGetter = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), + Symbol.toStringTag, +)?.get + +/** Whether a value has intrinsic Uint8Array slots, independent of its realm. */ +export function hasIntrinsicUint8ArraySlots( + value: unknown, +): value is Uint8Array { + return ( + ArrayBuffer.isView(value) && + typedArrayTagGetter !== undefined && + Reflect.apply(typedArrayTagGetter, value, []) === `Uint8Array` + ) +} + +/** + * Whether a value must use binary equality semantics. Local prototype claims + * enter this path so slot-less proxies are rejected instead of becoming opaque. + */ +export function isUint8ArrayCandidate(value: unknown): value is Uint8Array { + return value instanceof Uint8Array || hasIntrinsicUint8ArraySlots(value) +} + /** Copy a Uint8Array's internal bytes without invoking custom iteration. */ export function snapshotUint8ArrayBytes(value: Uint8Array): Uint8Array { - if (!ArrayBuffer.isView(value)) { + if (!hasIntrinsicUint8ArraySlots(value)) { throw new TypeError( `Cannot snapshot binary equality value without intrinsic typed-array slots`, ) @@ -162,11 +186,13 @@ export const defaultComparator = makeComparator({ * Compare two Uint8Arrays for content equality */ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) { + const aBytes = snapshotUint8ArrayBytes(a) + const bBytes = snapshotUint8ArrayBytes(b) + if (aBytes.byteLength !== bBytes.byteLength) { return false } - for (let i = 0; i < a.byteLength; i++) { - if (a[i] !== b[i]) { + for (let i = 0; i < aBytes.byteLength; i++) { + if (aBytes[i] !== bBytes[i]) { return false } } @@ -286,11 +312,7 @@ export function normalizeValue(value: any): any { // Normalize Uint8Arrays/Buffers to a string representation for Map key usage // This enables content-based equality for binary data like ULIDs - const isUint8Array = - (typeof Buffer !== `undefined` && value instanceof Buffer) || - value instanceof Uint8Array - - if (isUint8Array) { + if (isUint8ArrayCandidate(value)) { // Convert to a string representation that can be used as a Map key. // Equality compares every binary value by content, so index keys must not // switch to reference identity at an arbitrary byte length. @@ -407,12 +429,8 @@ export function areValuesEqual(a: any, b: any): boolean { } // Check for Uint8Array/Buffer comparison - const aIsUint8Array = - (typeof Buffer !== `undefined` && a instanceof Buffer) || - a instanceof Uint8Array - const bIsUint8Array = - (typeof Buffer !== `undefined` && b instanceof Buffer) || - b instanceof Uint8Array + const aIsUint8Array = isUint8ArrayCandidate(a) + const bIsUint8Array = isUint8ArrayCandidate(b) // If both are Uint8Arrays, compare by content if (aIsUint8Array && bIsUint8Array) { diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 30a7ac6a6..78d584f4b 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -59,6 +59,8 @@ import { cloneLoadSubsetOptions, snapshotLoadSubsetDemand, } from '../../src/query/load-subset-options.js' +import { areValuesEqual, normalizeValue } from '../../src/utils/comparison.js' +import { createCrossRealmUint8Array } from '../utils.js' import type { BasicExpression, QueryIR } from '../../src/query/ir.js' import type { LoadSubsetOptions } from '../../src/types.js' @@ -661,6 +663,27 @@ describe(`loadSubset demand identity`, () => { ) }) + it(`snapshots intrinsic Uint8Array values across realms`, () => { + const bytes = createCrossRealmUint8Array([1, 2, 3]) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(areValuesEqual(bytes, new Uint8Array([1, 2, 3]))).toBe(true) + expect(normalizeValue(bytes)).toBe( + normalizeValue(new Uint8Array([1, 2, 3])), + ) + + bytes[0] = 9 + + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(new Uint8Array([1, 2, 3])) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + }) + it.each([`coalesce`, `caseWhen`] as const)( `snapshots equality candidates returned by %s`, (wrapper) => { diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 69946c82e..f116c6e68 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -23,7 +23,11 @@ import { projectReusableDemands, projectTransportLoads, } from '../load-subset-full-flow-model.js' -import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import { + createCrossRealmUint8Array, + flushPromises, + mockSyncCollectionOptions, +} from '../utils.js' import { oracleRandomParameters, readOracleRunConfig, @@ -459,6 +463,69 @@ it(`rejects binary values without intrinsic typed-array slots before adapter acq } }) +it(`freezes cross-realm binary equality across filtering and acquisition`, async () => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const rows: ReadonlyArray = [ + { id: `original`, token: new Uint8Array([1]) }, + { id: `changed`, token: new Uint8Array([2]) }, + ] + const callerToken = createCrossRealmUint8Array([1]) + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-cross-realm-binary-equality`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + rows.forEach((value) => write({ type: `insert`, value })) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(callerToken), + ]), + }, + ) + + try { + callerToken[0] = 2 + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredValue = ( + (acquired?.where as Func | undefined)?.args[1] as + | Value + | undefined + )?.value + expect(acquiredValue).toEqual(new Uint8Array([1])) + expect(acquiredValue).not.toBe(callerToken) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + it(`keeps binary equality distinct from a sentinel-looking string`, async () => { type Row = { id: `binary` | `string`; token: Uint8Array | string } const binary = new Uint8Array([1, 2, 3]) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index a21afc7cf..56ae5a12a 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -4,6 +4,7 @@ import { cloneOptions, } from '../../src/query/subset-dedupe' import { Func, PropRef, Value } from '../../src/query/ir' +import { createCrossRealmUint8Array } from '../utils' import type { BasicExpression, OrderBy } from '../../src/query/ir' import type { LoadSubsetOptions } from '../../src/types' @@ -98,6 +99,28 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).not.toHaveBeenCalled() }) + it(`retains cross-realm binary coverage by acquired bytes`, () => { + const acquired: Array> = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquired.push( + Array.from(((options.where as Func).args[1] as Value).value), + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const bytes = createCrossRealmUint8Array([1]) + const demand = (): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(bytes)), + }) + + deduplicated.loadSubset(demand()) + bytes[0] = 2 + deduplicated.loadSubset(demand()) + + expect(acquired).toEqual([[1], [2]]) + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + it(`observes computed membership once for tracking and acquisition`, () => { const first = new Uint8Array([1]) const second = new Uint8Array([2]) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index b31408d0b..fdb124f04 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -1,3 +1,4 @@ +import { runInNewContext } from 'node:vm' import { expect } from 'vitest' import { BTreeIndex } from '../src/indexes/btree-index' import { withCollectionConfigFactory } from '../src/client' @@ -12,6 +13,14 @@ import type { WithVirtualProps } from '../src/virtual-props.js' type OracleEnvironment = Record +export function createCrossRealmUint8Array( + values: ReadonlyArray, +): Uint8Array { + return runInNewContext(`new Uint8Array(values)`, { + values: Array.from(values), + }) as Uint8Array +} + export function readOracleRunConfig( environment: OracleEnvironment = process.env, ): { multiplier: number; replaySeed: number | undefined } { From 5f9778e52edcb8d606031640dff30fa2a57e57d5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 16:04:25 -0600 Subject: [PATCH 034/327] fix(db): refine joined ordered windows --- packages/db/src/collection/subscription.ts | 6 +- packages/db/src/query/compiler/order-by.ts | 25 ++- packages/db/src/query/effect.ts | 22 ++- packages/db/src/query/live/ARCHITECTURE.md | 8 + .../src/query/live/collection-subscriber.ts | 27 ++- packages/db/src/query/live/window-state.ts | 28 +++- packages/db/src/query/total-order.ts | 15 +- packages/db/tests/effect.test.ts | 82 ++++++++++ .../query/pagination-oracle.property.test.ts | 154 ++++++++++++++++++ 9 files changed, 350 insertions(+), 17 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 4f1c3750e..c4b0f4ec8 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1249,8 +1249,11 @@ export class CollectionSubscription return this.truncateReplaySession !== undefined } - setOrderByIndex(index: IndexInterface) { + private expandOrderedSourceTies = false + + setOrderByIndex(index: IndexInterface, expandSourceOrderTies = false) { this.orderByIndex = index + this.expandOrderedSourceTies = expandSourceOrderTies } /** @@ -2387,6 +2390,7 @@ export class CollectionSubscription orderBy, where, limit, + this.expandOrderedSourceTies, ) if (this.stalePublication && !this.stalePublication.ordered) { diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 0166cf44f..2f98d0b0b 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -3,7 +3,12 @@ import { orderByWithFractionalIndex, } from '@tanstack/db-ivm' import { defaultComparator, makeComparator } from '../../utils/comparison.js' -import { PropRef, collectCollectionSources, followRef } from '../ir.js' +import { + PropRef, + collectCollectionSources, + followRef, + isResidualWhere, +} from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { findIndexForField } from '../../utils/index-optimization.js' import { resolveCompareOptions, resolveOrderBy } from '../total-order.js' @@ -37,6 +42,10 @@ export type OrderByOptimizationInfo = { /** Index on the first orderBy column - used for lazy loading */ index?: IndexInterface dataNeeded?: () => number + /** D2 must see the complete source-order tie when later order terms are local. */ + expandSourceOrderTies: boolean + /** Upstream relational operators can discard source rows before top-K. */ + refillFromResultDeficit: boolean } /** @@ -276,6 +285,20 @@ export function processOrderBy( valueExtractorForRawRow: rawRowValueExtractor, index, orderBy: sourceOrderBy, + expandSourceOrderTies: sourceTerms.length < orderByClause.length, + refillFromResultDeficit: + rawQuery.from.type !== `collectionRef` || + rawQuery.from.sourceId !== orderBySourceId || + (rawQuery.join?.some( + ({ type }) => type === `inner` || type === `right`, + ) ?? + false) || + (rawQuery.where?.some(isResidualWhere) ?? false) || + (rawQuery.fnWhere?.length ?? 0) > 0 || + rawQuery.groupBy !== undefined || + rawQuery.having !== undefined || + rawQuery.fnHaving !== undefined || + rawQuery.distinct === true, } // Ordered loading is owned by one lexical source. A collection can occur diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 2390e96ac..f86bfd93d 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -959,7 +959,7 @@ class EffectPipelineRunner { const normalizedOrderBy = normalizeOrderByPaths(orderBy, alias) if (index) { - subscription.setOrderByIndex(index) + subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ limit: offset + limit, orderBy: normalizedOrderBy, @@ -1002,7 +1002,11 @@ class EffectPipelineRunner { subscription.ensureOrderedWindowSize( orderByInfo.offset + orderByInfo.limit, ) - if (subscription.hasOrderedCoverageForActiveWindow) { + const missingResultRows = orderByInfo.dataNeeded() + if ( + (!orderByInfo.refillFromResultDeficit || missingResultRows === 0) && + subscription.hasOrderedCoverageForActiveWindow + ) { continue } @@ -1011,10 +1015,16 @@ class EffectPipelineRunner { continue } - const n = Math.max( - orderByInfo.dataNeeded(), - subscription.orderedRowsNeeded, - ) + if (orderByInfo.refillFromResultDeficit && missingResultRows > 0) { + subscription.ensureOrderedWindowSize( + subscription.orderedRetainedWindowSize + missingResultRows, + ) + } + if (subscription.hasOrderedCoverageForActiveWindow) { + continue + } + + const n = Math.max(missingResultRows, subscription.orderedRowsNeeded) this.loadNextItems(orderByInfo, Math.max(1, n)) } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a28e9e049..f59a5c451 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -395,6 +395,14 @@ require a future adapter capability with an opaque cursor that preserves the provider's exact collation and snapshot. Until that contract exists, an unbounded fetch is the only sound continuation. +Test adapters must obey the same boundary contract as production adapters. A +mock that reports exhaustion must have made every matching source row readable +before its result settles. A mock that reports more data must honor later +offset, cursor, and boundary-class refinement requests. Every applied row key +must name a row established by that acquisition. Tests that withhold rows while +claiming exhaustion, or ignore a refinement request, do not model a valid +adapter and cannot establish a runtime defect. + Every continuation boundary comes from rows established by the same ordered demand. Rows retained for another query, join, or window cannot move it. During a failed truncate replay, the last complete publication remains the boundary; diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index dad987852..ad93e8b54 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -439,7 +439,7 @@ export class CollectionSubscriber< // under microtask timing (e.g., queueMicrotask delays in TanStack Query observers). if (index) { // We have an index on the first orderBy column - use lazy loading optimization - subscription.setOrderByIndex(index) + subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ limit: offset + limit, @@ -472,7 +472,8 @@ export class CollectionSubscriber< return true } - const { dataNeeded, index, offset, limit } = orderByInfo + const { dataNeeded, index, offset, limit, refillFromResultDeficit } = + orderByInfo if (!dataNeeded || !index) { // dataNeeded is not set when there's no index (e.g., non-ref expression @@ -482,7 +483,11 @@ export class CollectionSubscriber< } subscription.ensureOrderedWindowSize(offset + limit) - if (subscription.hasOrderedCoverageForActiveWindow) { + const missingResultRows = refillFromResultDeficit ? dataNeeded() : 0 + if ( + missingResultRows === 0 && + subscription.hasOrderedCoverageForActiveWindow + ) { return true } @@ -497,7 +502,21 @@ export class CollectionSubscriber< return true } - const n = Math.max(dataNeeded(), subscription.orderedRowsNeeded) + // A join or later predicate can discard source rows. Once the prior + // acquisition settles, grow the retained source prefix by the observed + // result deficit so already-local rows publish before another request. + // Never grow from callbacks while an acquisition is still pending: the + // same deficit can be observed more than once in that transaction. + if (missingResultRows > 0) { + subscription.ensureOrderedWindowSize( + subscription.orderedRetainedWindowSize + missingResultRows, + ) + } + if (subscription.hasOrderedCoverageForActiveWindow) { + return true + } + + const n = Math.max(missingResultRows, subscription.orderedRowsNeeded) const errorVersion = subscription.lastErrorVersion try { // Local rows may fill the visible window without proving its remote diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index 11335e9c2..6df3fe1a5 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -35,6 +35,7 @@ export class WindowState< orderBy: OrderBy, private readonly where: BasicExpression | undefined, targetSize: number, + private readonly expandSourceOrderTies = false, ) { this.totalOrder = new TotalOrder(orderBy, collection) const evaluateWhere = where && compileSingleRowExpression(where) @@ -397,7 +398,32 @@ export class WindowState< allowedKeys === undefined ? (rows ?? []) : (rows ?? []).filter((change) => allowedKeys.has(change.key)) - return limit === undefined ? allowed : allowed.slice(0, limit) + if (limit === undefined) return allowed + return this.expandSourceOrderTies + ? this.prefixThroughTieClass(allowed, limit) + : allowed.slice(0, limit) + } + + /** + * A provider orders only by the source-owned query terms. Keep the complete + * boundary equivalence class so D2 can apply the local key tie-breaker and + * any later joined or derived order terms without missing candidates. + */ + private prefixThroughTieClass( + rows: Array>, + limit: number, + ): Array> { + if (limit <= 0 || rows.length <= limit) return rows.slice(0, limit) + + const boundary = rows[limit - 1]! + let end = limit + while ( + end < rows.length && + this.totalOrder.compareRows(boundary.value, rows[end]!.value) === 0 + ) { + end++ + } + return rows.slice(0, end) } private readSourceRows( diff --git a/packages/db/src/query/total-order.ts b/packages/db/src/query/total-order.ts index 5a4629d69..0b720e6e8 100644 --- a/packages/db/src/query/total-order.ts +++ b/packages/db/src/query/total-order.ts @@ -69,14 +69,21 @@ export class TotalOrder< return { key, values: this.values(row) } } + /** Compare only the query-visible order terms, without the local key tie-breaker. */ + compareRows(left: TRow, right: TRow): number { + for (const { extract, compare } of this.terms) { + const result = compare(extract(left), extract(right)) + if (result !== 0) return result + } + return 0 + } + compareEntries( left: readonly [TKey, TRow], right: readonly [TKey, TRow], ): number { - for (const { extract, compare } of this.terms) { - const result = compare(extract(left[1]), extract(right[1])) - if (result !== 0) return result - } + const result = this.compareRows(left[1], right[1]) + if (result !== 0) return result return compareKeys(left[0], right[0]) } diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 7e5c12f67..c97516da8 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -1390,6 +1390,88 @@ describe(`createEffect`, () => { ) } + it(`refills a joined result window after source rows are rejected`, async () => { + type Parent = { id: number; rank: number; groupId: number } + type Child = { id: number; groupId: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] + const delivered = new Set() + let requestCount = 0 + const parents = createCollection({ + id: `effect-joined-underfill-parents`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: () => { + const requestNumber = ++requestCount + const requested = requestNumber === 1 ? rows.slice(0, 2) : rows + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit() + return Promise.resolve(receipt).then(() => ({ + hasMore: requestNumber === 1, + appliedRowKeys: requested.map(({ id }) => id), + })) + }, + } + }, + }, + }) + const children = createCollection( + mockSyncCollectionOptions({ + id: `effect-joined-underfill-children`, + getKey: (row) => row.id, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ], + }), + ) + const visible = new Set() + const effect = createEffect<{ id: number }, string | number>({ + query: (q) => + q + .from({ parent: parents }) + .innerJoin({ child: children }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + onEnter: ({ value }) => { + visible.add(value.id) + }, + onExit: ({ value }) => { + visible.delete(value.id) + }, + }) + + try { + await flushPromises() + expect([...visible]).toEqual([2, 3]) + expect(requestCount).toBe(2) + } finally { + await effect.dispose() + await Promise.all([parents.cleanup(), children.cleanup()]) + } + }) + it(`should load more data when pipeline filters items from the orderBy window`, async () => { // 6 users, ordered by name asc, limit 3 // But we filter on active=true, and Bob/Dave are inactive diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index aa9c2821c..0a6d08343 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -4,6 +4,7 @@ import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BTreeIndex } from '../../src/index.js' import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' +import { eq } from '../../src/query/builder/functions.js' import { PropRef } from '../../src/query/ir.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { makeComparator } from '../../src/utils/comparison.js' @@ -400,6 +401,55 @@ function withAppliedSubsetEvidence( }) } +function createConformingOrderedSource( + id: string, + rows: ReadonlyArray, +) { + const requests: Array = [] + const delivered = new Set() + const source = createCollection({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options: LoadSubsetOptions) => { + requests.push(options) + const requested = rowsForLoadSubset(rows, options) + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit(options.signal) + const hasMore = options.cursor + ? rows.filter((row) => + Boolean( + evaluateReferenceExpression(options.cursor!.whereFrom, row), + ), + ).length > (options.limit ?? Number.POSITIVE_INFINITY) + : rows.length > + (options.offset ?? 0) + + (options.limit ?? Number.POSITIVE_INFINITY) + return Promise.resolve(receipt).then(() => ({ + hasMore, + appliedRowKeys: requested.map(({ id: key }) => key), + })) + }, + } + }, + }, + }) + + return { requests, source } +} + async function runPaginationScenario( scenario: PaginationScenario, ): Promise { @@ -1625,6 +1675,110 @@ async function expectInflightRequestFillsNewWindow(): Promise { } describe(`pagination recomputation oracle`, () => { + it(`refills a joined result window through a contract-compliant source`, async () => { + type ParentRow = { id: number; rank: number; groupId: number } + type ChildRow = { id: number; groupId: number } + const parents = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-joined-underfill-source-${collectionSequence++}`, + parents, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-joined-underfill-child-${collectionSequence++}`, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .innerJoin({ child: childSource }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([2, 3]) + expect(requests).toHaveLength(2) + expect(requests[0]?.limit).toBe(2) + expect(requests[1]?.cursor).toBeDefined() + } finally { + live.cleanup() + childSource.cleanup() + parentSource.cleanup() + } + }) + + it(`refines a joined foreign order term through the source tie class`, async () => { + type ParentRow = { id: number; sourceRank: number; childId: number } + type ChildRow = { id: number; score: number } + const parents = [ + { id: 1, sourceRank: 0, childId: 1 }, + { id: 2, sourceRank: 0, childId: 2 }, + { id: 3, sourceRank: 0, childId: 3 }, + { id: 4, sourceRank: 0, childId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-joined-foreign-order-source-${collectionSequence++}`, + parents, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-joined-foreign-order-child-${collectionSequence++}`, + initialData: [ + { id: 1, score: 10 }, + { id: 2, score: 20 }, + { id: 3, score: 0 }, + { id: 4, score: 30 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .leftJoin({ child: childSource }, ({ parent, child }) => + eq(parent.childId, child.id), + ) + .orderBy(({ parent }) => parent.sourceRank, `asc`) + .orderBy(({ child }) => child.score, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([3, 1]) + expect(requests).toHaveLength(2) + expect(requests[0]?.orderBy).toHaveLength(1) + expect(requests[1]?.cursor).toBeDefined() + } finally { + live.cleanup() + childSource.cleanup() + parentSource.cleanup() + } + }) + it(`materializes an empty source window`, async () => { await runPaginationScenario({ ranks: [], From 6f275907cf6cbe39930ec7ecd2e56c3e1baa57a5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 16:22:16 -0600 Subject: [PATCH 035/327] fix(db): load ordered fallbacks unbounded --- packages/db/src/query/effect.ts | 7 +- packages/db/src/query/live/ARCHITECTURE.md | 6 ++ .../src/query/live/collection-subscriber.ts | 4 +- packages/db/tests/effect.test.ts | 86 +++++++++++++++++++ .../tests/query/live-query-collection.test.ts | 8 +- .../tests/query/load-subset-subquery.test.ts | 10 ++- .../query/pagination-oracle.property.test.ts | 54 +++++++++++- 7 files changed, 161 insertions(+), 14 deletions(-) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index f86bfd93d..4400d74b8 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -947,8 +947,8 @@ class EffectPipelineRunner { /** * Request the initial ordered snapshot for an alias. - * Uses requestLimitedSnapshot (index-based cursor) or requestSnapshot - * (full load with limit) depending on whether an index is available. + * Uses requestLimitedSnapshot (index-based cursor) or an unbounded + * requestSnapshot depending on whether an index is available. */ private requestInitialOrderedSnapshot( alias: string, @@ -968,9 +968,10 @@ class EffectPipelineRunner { this.trackOrderedLoad(result, orderByInfo.sourceId), }) } else { + // Without an index there is no sound cursor continuation. Load the full + // ordered source so later relational operators cannot underfill top-K. subscription.requestSnapshot({ orderBy: normalizedOrderBy, - limit: offset + limit, trackLoadSubsetPromise: false, }) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index f59a5c451..e96c9b792 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -395,6 +395,12 @@ require a future adapter capability with an opaque cursor that preserves the provider's exact collation and snapshot. Until that contract exists, an unbounded fetch is the only sound continuation. +The same rule applies when no range index can support ordered continuation. +Core issues one unbounded ordered acquisition, then lets D2 apply joins, +predicates, and top-K to the full readable source. It must not issue a limited +page and then disable continuation: later relational operators may reject that +page and leave the result window short. + Test adapters must obey the same boundary contract as production adapters. A mock that reports exhaustion must have made every matching source row readable before its result settles. A mock that reports more data must honor later diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index ad93e8b54..fd0439825 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -448,10 +448,10 @@ export class CollectionSubscriber< onLoadSubsetResult: handleLoadSubsetResult, }) } else { - // No index available (e.g., non-ref expression): pass orderBy/limit to loadSubset + // Without an index there is no sound cursor continuation. Load the full + // ordered source so later relational operators cannot underfill top-K. subscription.requestSnapshot({ orderBy: normalizedOrderBy, - limit: offset + limit, trackLoadSubsetPromise: false, onLoadSubsetResult: handleLoadSubsetResult, }) diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index c97516da8..4c26505ac 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -8,6 +8,7 @@ import { } from './utils.js' import type { DeltaEvent, + LoadSubsetOptions, SubscriptionLoadSubsetErrorEvent, } from '../src/index.js' @@ -1472,6 +1473,91 @@ describe(`createEffect`, () => { } }) + it(`loads the full joined ordered source without an index`, async () => { + type Parent = { id: number; rank: number; groupId: number } + type Child = { id: number; groupId: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] + const delivered = new Set() + const requests: Array = [] + const parents = createCollection({ + id: `effect-no-index-underfill-parents`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options) => { + requests.push(options) + const requested = + options.limit === undefined + ? rows + : rows.slice(0, options.limit) + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit() + return Promise.resolve(receipt).then(() => ({ + hasMore: requested.length < rows.length, + appliedRowKeys: requested.map(({ id }) => id), + })) + }, + } + }, + }, + }) + const children = createCollection( + mockSyncCollectionOptions({ + id: `effect-no-index-underfill-children`, + getKey: (row) => row.id, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ], + }), + ) + const visible = new Set() + const effect = createEffect<{ id: number }, string | number>({ + query: (q) => + q + .from({ parent: parents }) + .innerJoin({ child: children }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + onEnter: ({ value }) => { + visible.add(value.id) + }, + onExit: ({ value }) => { + visible.delete(value.id) + }, + }) + + try { + await flushPromises() + expect([...visible]).toEqual([2, 3]) + expect(requests).toHaveLength(1) + expect(requests[0]?.limit).toBeUndefined() + } finally { + await effect.dispose() + await Promise.all([parents.cleanup(), children.cleanup()]) + } + }) + it(`should load more data when pipeline filters items from the orderBy window`, async () => { // 6 users, ordered by name asc, limit 3 // But we filter on active=true, and Bob/Dave are inactive diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index fc4d7f06b..af9c06183 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -2836,7 +2836,7 @@ describe(`createLiveQueryCollection`, () => { } }) - it(`passes single orderBy clause to loadSubset when using limit`, async () => { + it(`loads an ordered source without a range index unbounded`, async () => { const capturedOptions: Array = [] let resolveLoadSubset: () => void const loadSubsetPromise = new Promise((resolve) => { @@ -2886,7 +2886,7 @@ describe(`createLiveQueryCollection`, () => { expect(callWithOrderBy).toBeDefined() expect(callWithOrderBy?.orderBy).toHaveLength(1) expect(callWithOrderBy?.orderBy?.[0]?.expression.type).toBe(`ref`) - expect(callWithOrderBy?.limit).toBe(10) + expect(callWithOrderBy?.limit).toBeUndefined() // Resolve the loadSubset promise so preload can complete resolveLoadSubset!() @@ -2894,7 +2894,7 @@ describe(`createLiveQueryCollection`, () => { await preloadPromise }) - it(`passes multiple orderBy columns to loadSubset when using limit`, async () => { + it(`loads a multi-column ordered source without an index unbounded`, async () => { const capturedOptions: Array = [] let resolveLoadSubset: () => void const loadSubsetPromise = new Promise((resolve) => { @@ -2950,7 +2950,7 @@ describe(`createLiveQueryCollection`, () => { expect(callWithMultiOrderBy?.orderBy).toHaveLength(2) expect(callWithMultiOrderBy?.orderBy?.[0]?.expression.type).toBe(`ref`) expect(callWithMultiOrderBy?.orderBy?.[1]?.expression.type).toBe(`ref`) - expect(callWithMultiOrderBy?.limit).toBe(10) + expect(callWithMultiOrderBy?.limit).toBeUndefined() // Resolve the loadSubset promise so preload can complete resolveLoadSubset!() diff --git a/packages/db/tests/query/load-subset-subquery.test.ts b/packages/db/tests/query/load-subset-subquery.test.ts index 2888753c3..002100dca 100644 --- a/packages/db/tests/query/load-subset-subquery.test.ts +++ b/packages/db/tests/query/load-subset-subquery.test.ts @@ -328,11 +328,12 @@ describe(`loadSubset with subqueries`, () => { // Verify loadSubset was called expect(loadSubsetCalls.length).toBeGreaterThan(0) - // Verify the last call has the orderBy clause and limit + // Without a range index, core asks the adapter for the full ordered source + // and applies the query limit locally. const lastCall = loadSubsetCalls[loadSubsetCalls.length - 1] expect(lastCall).toBeDefined() expect(lastCall!.orderBy).toBeDefined() - expect(lastCall!.limit).toBe(2) + expect(lastCall!.limit).toBeUndefined() const expectedOrderBy: OrderBy = [ { @@ -373,11 +374,12 @@ describe(`loadSubset with subqueries`, () => { // Verify loadSubset was called for the orders collection expect(loadSubsetCalls.length).toBeGreaterThan(0) - // Verify the last call has the orderBy clause and limit + // Without a range index, core asks the adapter for the full ordered source + // and applies the subquery limit locally. const lastCall = loadSubsetCalls[loadSubsetCalls.length - 1] expect(lastCall).toBeDefined() expect(lastCall!.orderBy).toBeDefined() - expect(lastCall!.limit).toBe(2) + expect(lastCall!.limit).toBeUndefined() const expectedOrderBy: OrderBy = [ { diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index 0a6d08343..fe939f92a 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -404,6 +404,7 @@ function withAppliedSubsetEvidence( function createConformingOrderedSource( id: string, rows: ReadonlyArray, + autoIndex: `eager` | `off` = `eager`, ) { const requests: Array = [] const delivered = new Set() @@ -412,7 +413,7 @@ function createConformingOrderedSource( getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, - autoIndex: `eager`, + autoIndex, defaultIndexType: BTreeIndex, sync: { sync: ({ begin, write, commit, markReady }) => { @@ -1726,6 +1727,57 @@ describe(`pagination recomputation oracle`, () => { } }) + it(`loads the full ordered source when no continuation index exists`, async () => { + type ParentRow = { id: number; rank: number; groupId: number } + type ChildRow = { id: number; groupId: number } + const parents = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-no-index-underfill-source-${collectionSequence++}`, + parents, + `off`, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-no-index-underfill-child-${collectionSequence++}`, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .innerJoin({ child: childSource }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([2, 3]) + expect(requests).toHaveLength(1) + expect(requests[0]?.limit).toBeUndefined() + } finally { + live.cleanup() + childSource.cleanup() + parentSource.cleanup() + } + }) + it(`refines a joined foreign order term through the source tie class`, async () => { type ParentRow = { id: number; sourceRank: number; childId: number } type ChildRow = { id: number; score: number } From f4ea3f93fe2361afb1e3cc60101dfa6fe9164fae Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 16:39:03 -0600 Subject: [PATCH 036/327] test(db): await pagination oracle teardown --- .../query/pagination-oracle.property.test.ts | 149 +++++++++++------- 1 file changed, 93 insertions(+), 56 deletions(-) diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index fe939f92a..6c00c2c58 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -323,6 +323,22 @@ const nullableCursorScenarioArbitrary: fc.Arbitrary = direction: fc.constantFrom(`asc` as const, `desc` as const), }) +type CleanupTarget = { + cleanup: () => unknown +} + +async function cleanupAll( + ...targets: ReadonlyArray +): Promise { + const results = await Promise.allSettled( + targets.map((target) => Promise.resolve().then(() => target.cleanup())), + ) + const rejection = results.find( + (result): result is PromiseRejectedResult => result.status === `rejected`, + ) + if (rejection) throw rejection.reason +} + const { multiplier, replaySeed } = readOracleRunConfig() const orderedScenarioRuns = 12 * multiplier const transitionScenarioRuns = 8 * multiplier @@ -489,8 +505,7 @@ async function runPaginationScenario( ) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -576,8 +591,7 @@ async function runMultiOrderScenario( throw new TraceAssertionError(0, error) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -665,8 +679,7 @@ async function runNullableCursorScenario( } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -738,8 +751,7 @@ async function runPaginationStateScenario( expectCurrentWindow(index + 1) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -845,8 +857,7 @@ async function runOnDemandPaginationScenario( for (const load of loads) expect(load.orderBy).toMatchObject(expectedOrderBy) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -939,9 +950,7 @@ async function expectOnDemandWindowsAreCompletionOrderIndependent( expect(Array.from(secondLive.values(), ({ id }) => id)).toEqual([1, 2, 3]) } finally { for (const request of pending) request.deferred.resolve() - firstLive.cleanup() - secondLive.cleanup() - source.cleanup() + await cleanupAll(firstLive, secondLive, source) } } @@ -1085,8 +1094,7 @@ async function runAdversarialOrderedProviderScenario(options: { // final refinement request. return [...loads] } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1304,8 +1312,7 @@ async function runPendingMutationScenario( } finally { for (const request of pending) request.deferred.resolve() await Promise.allSettled(outstanding) - await live.cleanup() - await source.cleanup() + await cleanupAll(live, source) } } @@ -1429,8 +1436,7 @@ async function runRejectedCursorRetryAfterMutation(): Promise { } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1573,8 +1579,7 @@ async function runPendingHistoryScenario( } finally { for (const request of pending) request.deferred.resolve() await Promise.allSettled(outstanding) - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1670,12 +1675,61 @@ async function expectInflightRequestFillsNewWindow(): Promise { } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } describe(`pagination recomputation oracle`, () => { + it(`observes cleanup failure after every teardown settles`, async () => { + const failure = new Error(`cleanup failed`) + const laterCleanup = createDeferred() + const events: Array = [] + const unhandled: Array = [] + const recordUnhandled = (reason: unknown) => unhandled.push(reason) + let cleanupFinished = false + process.on(`unhandledRejection`, recordUnhandled) + + try { + const cleanup = cleanupAll( + { + cleanup: () => { + events.push(`failed`) + return Promise.reject(failure) + }, + }, + { + cleanup: async () => { + await laterCleanup.promise + events.push(`settled`) + }, + }, + ).finally(() => { + cleanupFinished = true + }) + const observedFailure = cleanup.then( + () => { + throw new Error(`expected cleanup to reject`) + }, + (error: unknown) => expect(error).toBe(failure), + ) + + await flushPromises() + expect(events).toEqual([`failed`]) + expect(cleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + laterCleanup.resolve() + await observedFailure + await flushPromises() + expect(events).toEqual([`failed`, `settled`]) + expect(cleanupFinished).toBe(true) + expect(unhandled).toEqual([]) + } finally { + laterCleanup.resolve() + process.off(`unhandledRejection`, recordUnhandled) + } + }) + it(`refills a joined result window through a contract-compliant source`, async () => { type ParentRow = { id: number; rank: number; groupId: number } type ChildRow = { id: number; groupId: number } @@ -1721,9 +1775,7 @@ describe(`pagination recomputation oracle`, () => { expect(requests[0]?.limit).toBe(2) expect(requests[1]?.cursor).toBeDefined() } finally { - live.cleanup() - childSource.cleanup() - parentSource.cleanup() + await cleanupAll(live, childSource, parentSource) } }) @@ -1772,9 +1824,7 @@ describe(`pagination recomputation oracle`, () => { expect(requests).toHaveLength(1) expect(requests[0]?.limit).toBeUndefined() } finally { - live.cleanup() - childSource.cleanup() - parentSource.cleanup() + await cleanupAll(live, childSource, parentSource) } }) @@ -1825,9 +1875,7 @@ describe(`pagination recomputation oracle`, () => { expect(requests[0]?.orderBy).toHaveLength(1) expect(requests[1]?.cursor).toBeDefined() } finally { - live.cleanup() - childSource.cleanup() - parentSource.cleanup() + await cleanupAll(live, childSource, parentSource) } }) @@ -1949,8 +1997,7 @@ describe(`pagination recomputation oracle`, () => { expect(requests[1]).toMatchObject({ limit: 2, offset: 0 }) expect(requests[1]?.cursor).toBeUndefined() } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2005,8 +2052,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2121,24 +2167,22 @@ describe(`pagination recomputation oracle`, () => { expectedCovering.slice(0, 1), ) } finally { - covered.cleanup() + await cleanupAll(covered) } if (releaseFirst === `covering`) { - covering.cleanup() + await cleanupAll(covering) expect(Array.from(narrower.values(), ({ id }) => id)).toEqual( expectedCovering.slice(0, 2), ) } else { - narrower.cleanup() + await cleanupAll(narrower) expect(Array.from(covering.values(), ({ id }) => id)).toEqual( expectedCovering, ) } } finally { - covering.cleanup() - narrower.cleanup() - source.cleanup() + await cleanupAll(covering, narrower, source) } }, ) @@ -2223,8 +2267,7 @@ describe(`pagination recomputation oracle`, () => { if (widened instanceof Promise) await widened expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2313,8 +2356,7 @@ describe(`pagination recomputation oracle`, () => { expect(pending).toHaveLength(transportCount) } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2543,8 +2585,7 @@ describe(`pagination recomputation oracle`, () => { await live.utils.setWindow({ offset: 0, limit: 3 }) expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2, 9]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2645,8 +2686,7 @@ describe(`pagination recomputation oracle`, () => { expect(loads.at(-1)).toMatchObject({ offset: 0, limit: 2 }) expect(loads.at(-1)?.cursor).toBeUndefined() } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) @@ -2779,8 +2819,7 @@ describe(`pagination recomputation oracle`, () => { ) } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) @@ -3282,8 +3321,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([3, 1]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -3309,8 +3347,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([1]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) From a0767de4f341a459cabe9856aacec746339ee2be Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 16:48:15 -0600 Subject: [PATCH 037/327] test(db): cover hostile oracle teardown --- .../query/pagination-oracle.property.test.ts | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index 6c00c2c58..831820000 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -1681,49 +1681,70 @@ async function expectInflightRequestFillsNewWindow(): Promise { describe(`pagination recomputation oracle`, () => { it(`observes cleanup failure after every teardown settles`, async () => { - const failure = new Error(`cleanup failed`) + const firstFailure = new Error(`first cleanup failed`) + const secondFailure = new Error(`second cleanup failed`) const laterCleanup = createDeferred() const events: Array = [] const unhandled: Array = [] const recordUnhandled = (reason: unknown) => unhandled.push(reason) - let cleanupFinished = false - process.on(`unhandledRejection`, recordUnhandled) - - try { - const cleanup = cleanupAll( - { - cleanup: () => { - events.push(`failed`) - return Promise.reject(failure) - }, + const targets: ReadonlyArray = [ + { + cleanup: () => { + events.push(`first`) + throw firstFailure }, - { - cleanup: async () => { - await laterCleanup.promise - events.push(`settled`) - }, + }, + { + cleanup: async () => { + events.push(`second`) + await laterCleanup.promise + throw secondFailure }, - ).finally(() => { - cleanupFinished = true - }) - const observedFailure = cleanup.then( + }, + { + cleanup: () => { + events.push(`third`) + }, + }, + ] + const observeFirstFailure = (cleanup: Promise) => + cleanup.then( () => { throw new Error(`expected cleanup to reject`) }, - (error: unknown) => expect(error).toBe(failure), + (error: unknown) => expect(error).toBe(firstFailure), ) + let cleanupFinished = false + process.on(`unhandledRejection`, recordUnhandled) + + try { + const cleanup = cleanupAll(...targets).finally(() => { + cleanupFinished = true + }) + const observedFailure = observeFirstFailure(cleanup) await flushPromises() - expect(events).toEqual([`failed`]) + expect(events).toEqual([`first`, `second`, `third`]) expect(cleanupFinished).toBe(false) expect(unhandled).toEqual([]) laterCleanup.resolve() await observedFailure await flushPromises() - expect(events).toEqual([`failed`, `settled`]) expect(cleanupFinished).toBe(true) expect(unhandled).toEqual([]) + + await observeFirstFailure(cleanupAll(...targets)) + await flushPromises() + expect(events).toEqual([ + `first`, + `second`, + `third`, + `first`, + `second`, + `third`, + ]) + expect(unhandled).toEqual([]) } finally { laterCleanup.resolve() process.off(`unhandledRejection`, recordUnhandled) From 4fcd5ed4fed841a902df7bdd0aff326a64af1a4c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 16:57:05 -0600 Subject: [PATCH 038/327] test(db): distinguish cleanup failure order --- packages/db/tests/query/pagination-oracle.property.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index 831820000..fb9e9165b 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -1689,15 +1689,15 @@ describe(`pagination recomputation oracle`, () => { const recordUnhandled = (reason: unknown) => unhandled.push(reason) const targets: ReadonlyArray = [ { - cleanup: () => { + cleanup: async () => { events.push(`first`) + await laterCleanup.promise throw firstFailure }, }, { - cleanup: async () => { + cleanup: () => { events.push(`second`) - await laterCleanup.promise throw secondFailure }, }, From 64324bbbeac8263c2699388fd324cec6824f2f4d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 17:11:21 -0600 Subject: [PATCH 039/327] test(db): cover falsy cleanup failures --- .../query/pagination-oracle.property.test.ts | 136 ++++++++++-------- 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index fb9e9165b..95165eeae 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -1680,76 +1680,86 @@ async function expectInflightRequestFillsNewWindow(): Promise { } describe(`pagination recomputation oracle`, () => { - it(`observes cleanup failure after every teardown settles`, async () => { - const firstFailure = new Error(`first cleanup failed`) - const secondFailure = new Error(`second cleanup failed`) - const laterCleanup = createDeferred() - const events: Array = [] - const unhandled: Array = [] - const recordUnhandled = (reason: unknown) => unhandled.push(reason) - const targets: ReadonlyArray = [ - { - cleanup: async () => { - events.push(`first`) - await laterCleanup.promise - throw firstFailure - }, - }, - { - cleanup: () => { - events.push(`second`) - throw secondFailure + it.each([ + { label: `Error`, reason: new Error(`first cleanup failed`) }, + { label: `undefined`, reason: undefined }, + { label: `null`, reason: null }, + { label: `false`, reason: false }, + { label: `zero`, reason: 0 }, + { label: `NaN`, reason: Number.NaN }, + { label: `empty string`, reason: `` }, + ])( + `observes $label cleanup failure after every teardown settles`, + async ({ reason: firstFailure }) => { + const secondFailure = new Error(`second cleanup failed`) + const laterCleanup = createDeferred() + const events: Array = [] + const unhandled: Array = [] + const recordUnhandled = (reason: unknown) => unhandled.push(reason) + const targets: ReadonlyArray = [ + { + cleanup: async () => { + events.push(`first`) + await laterCleanup.promise + throw firstFailure + }, }, - }, - { - cleanup: () => { - events.push(`third`) + { + cleanup: () => { + events.push(`second`) + throw secondFailure + }, }, - }, - ] - const observeFirstFailure = (cleanup: Promise) => - cleanup.then( - () => { - throw new Error(`expected cleanup to reject`) + { + cleanup: () => { + events.push(`third`) + }, }, - (error: unknown) => expect(error).toBe(firstFailure), - ) - let cleanupFinished = false - process.on(`unhandledRejection`, recordUnhandled) + ] + const observeFirstFailure = (cleanup: Promise) => + cleanup.then( + () => { + throw new Error(`expected cleanup to reject`) + }, + (error: unknown) => expect(error).toBe(firstFailure), + ) + let cleanupFinished = false + process.on(`unhandledRejection`, recordUnhandled) - try { - const cleanup = cleanupAll(...targets).finally(() => { - cleanupFinished = true - }) - const observedFailure = observeFirstFailure(cleanup) + try { + const cleanup = cleanupAll(...targets).finally(() => { + cleanupFinished = true + }) + const observedFailure = observeFirstFailure(cleanup) - await flushPromises() - expect(events).toEqual([`first`, `second`, `third`]) - expect(cleanupFinished).toBe(false) - expect(unhandled).toEqual([]) + await flushPromises() + expect(events).toEqual([`first`, `second`, `third`]) + expect(cleanupFinished).toBe(false) + expect(unhandled).toEqual([]) - laterCleanup.resolve() - await observedFailure - await flushPromises() - expect(cleanupFinished).toBe(true) - expect(unhandled).toEqual([]) + laterCleanup.resolve() + await observedFailure + await flushPromises() + expect(cleanupFinished).toBe(true) + expect(unhandled).toEqual([]) - await observeFirstFailure(cleanupAll(...targets)) - await flushPromises() - expect(events).toEqual([ - `first`, - `second`, - `third`, - `first`, - `second`, - `third`, - ]) - expect(unhandled).toEqual([]) - } finally { - laterCleanup.resolve() - process.off(`unhandledRejection`, recordUnhandled) - } - }) + await observeFirstFailure(cleanupAll(...targets)) + await flushPromises() + expect(events).toEqual([ + `first`, + `second`, + `third`, + `first`, + `second`, + `third`, + ]) + expect(unhandled).toEqual([]) + } finally { + laterCleanup.resolve() + process.off(`unhandledRejection`, recordUnhandled) + } + }, + ) it(`refills a joined result window through a contract-compliant source`, async () => { type ParentRow = { id: number; rank: number; groupId: number } From ba76d9b478d157838932f232fbd47f3b0c774274 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 17:21:09 -0600 Subject: [PATCH 040/327] test(db): await every oracle cleanup --- .../query/pagination-oracle.property.test.ts | 55 ++++++++++++++++--- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index 95165eeae..3342cdeef 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -15,6 +15,7 @@ import { import { evaluateReferenceExpression } from '../reference-expression.js' import { TraceAssertionError } from '../trace-runner.js' import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import type { Deferred } from '../../src/deferred.js' import type { LoadSubsetOptions, LoadSubsetResult } from '../../src/types.js' type PageRow = { @@ -1692,15 +1693,21 @@ describe(`pagination recomputation oracle`, () => { `observes $label cleanup failure after every teardown settles`, async ({ reason: firstFailure }) => { const secondFailure = new Error(`second cleanup failed`) - const laterCleanup = createDeferred() + const firstFailureRelease = createDeferred() + const lastCleanupRelease = createDeferred() + const repeatedFirstFailureRelease = createDeferred() + const repeatedLastCleanupRelease = createDeferred() const events: Array = [] const unhandled: Array = [] const recordUnhandled = (reason: unknown) => unhandled.push(reason) - const targets: ReadonlyArray = [ + const createTargets = ( + firstRelease: Deferred, + lastRelease: Deferred, + ): ReadonlyArray => [ { cleanup: async () => { events.push(`first`) - await laterCleanup.promise + await firstRelease.promise throw firstFailure }, }, @@ -1711,8 +1718,9 @@ describe(`pagination recomputation oracle`, () => { }, }, { - cleanup: () => { + cleanup: async () => { events.push(`third`) + await lastRelease.promise }, }, ] @@ -1727,7 +1735,9 @@ describe(`pagination recomputation oracle`, () => { process.on(`unhandledRejection`, recordUnhandled) try { - const cleanup = cleanupAll(...targets).finally(() => { + const cleanup = cleanupAll( + ...createTargets(firstFailureRelease, lastCleanupRelease), + ).finally(() => { cleanupFinished = true }) const observedFailure = observeFirstFailure(cleanup) @@ -1737,13 +1747,27 @@ describe(`pagination recomputation oracle`, () => { expect(cleanupFinished).toBe(false) expect(unhandled).toEqual([]) - laterCleanup.resolve() + firstFailureRelease.resolve() + await flushPromises() + expect(cleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + lastCleanupRelease.resolve() await observedFailure await flushPromises() expect(cleanupFinished).toBe(true) expect(unhandled).toEqual([]) - await observeFirstFailure(cleanupAll(...targets)) + let repeatedCleanupFinished = false + const repeatedCleanup = cleanupAll( + ...createTargets( + repeatedFirstFailureRelease, + repeatedLastCleanupRelease, + ), + ).finally(() => { + repeatedCleanupFinished = true + }) + const repeatedObservedFailure = observeFirstFailure(repeatedCleanup) await flushPromises() expect(events).toEqual([ `first`, @@ -1753,9 +1777,24 @@ describe(`pagination recomputation oracle`, () => { `second`, `third`, ]) + expect(repeatedCleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + repeatedFirstFailureRelease.resolve() + await flushPromises() + expect(repeatedCleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + repeatedLastCleanupRelease.resolve() + await repeatedObservedFailure + await flushPromises() + expect(repeatedCleanupFinished).toBe(true) expect(unhandled).toEqual([]) } finally { - laterCleanup.resolve() + firstFailureRelease.resolve() + lastCleanupRelease.resolve() + repeatedFirstFailureRelease.resolve() + repeatedLastCleanupRelease.resolve() process.off(`unhandledRejection`, recordUnhandled) } }, From 33e0d43e6121bc222acc806e771b80738fe4f162 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 17:35:36 -0600 Subject: [PATCH 041/327] fix(electric): reject canceled refresh loads --- .../electric-db-collection/src/electric.ts | 12 ++++++--- .../tests/electric.test.ts | 26 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index c478fe59e..ddaf2349a 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -8,6 +8,7 @@ import { Store } from '@tanstack/store' import DebugModule from 'debug' import { DeduplicatedLoadSubset, + SyncTransactionAbortedError, and, withCollectionConfigFactory, } from '@tanstack/db' @@ -553,7 +554,6 @@ function createLoadSubsetDedupe>({ encodeColumnName?: ColumnEncoder /** * Abort signal to check if the stream has been aborted during cleanup. - * When aborted, errors from requestSnapshot are silently ignored. */ signal: AbortSignal }): DeduplicatedLoadSubset | null { @@ -581,7 +581,12 @@ function createLoadSubsetDedupe>({ const commitCursor = getCommitCursor() const isAborted = (): boolean => signal.aborted || opts.signal?.aborted === true - if (isAborted()) return + const throwIfAborted = () => { + if (isAborted()) { + throw new SyncTransactionAbortedError() + } + } + throwIfAborted() if (isBufferingInitialSync()) { const snapshotParams = compileSQL(opts, compileOptions) @@ -667,6 +672,7 @@ function createLoadSubsetDedupe>({ aborted, ]) } catch (error) { + throwIfAborted() if (handleSnapshotError(error, `forceDisconnectAndRefresh`)) { return } @@ -680,7 +686,7 @@ function createLoadSubsetDedupe>({ } } - if (isAborted()) return + throwIfAborted() // Upstream limitation: ShapeStream.requestSnapshot() publishes its rows // through the stream callback before its Promise resolves. It accepts no diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 6d3955943..26b41a1e5 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2925,21 +2925,26 @@ describe(`Electric Integration`, () => { let loadSettled = false const load = Promise.resolve( testCollection._sync.loadSubset({ limit: 10 }), - ).then(() => { + ).finally(() => { loadSettled = true }) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) await Promise.resolve() await testCollection.cleanup() await vi.advanceTimersByTimeAsync(0) expect(loadSettled).toBe(true) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) expect(mockRequestSnapshot).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) refresh.resolve() await refresh.promise - await load + await load.catch(() => undefined) expect(mockRequestSnapshot).not.toHaveBeenCalled() } finally { refresh.resolve() @@ -3001,7 +3006,7 @@ describe(`Electric Integration`, () => { expect(commit).not.toHaveBeenCalled() }) - it(`does not start a refresh when the collection signal is already aborted`, async () => { + it(`rejects before starting a refresh when the collection signal is already aborted`, async () => { mockStream.isUpToDate = true const abortController = new AbortController() abortController.abort() @@ -3019,7 +3024,9 @@ describe(`Electric Integration`, () => { }), ) - await testCollection._sync.loadSubset({ limit: 10 }) + await expect( + testCollection._sync.loadSubset({ limit: 10 }), + ).rejects.toMatchObject({ name: `AbortError` }) expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() expect(mockRequestSnapshot).not.toHaveBeenCalled() @@ -3044,15 +3051,22 @@ describe(`Electric Integration`, () => { limit: 10, signal: abortController.signal, }), - ).then(() => { + ).finally(() => { abortedLoadSettled = true }) + const abortedLoadError = abortedLoad.then( + () => undefined, + (error: unknown) => error, + ) await Promise.resolve() abortController.abort() await vi.advanceTimersByTimeAsync(0) expect(abortedLoadSettled).toBe(true) + await expect(abortedLoadError).resolves.toMatchObject({ + name: `AbortError`, + }) expect(mockRequestSnapshot).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) @@ -3062,7 +3076,7 @@ describe(`Electric Integration`, () => { expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(2) expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) await testCollection.cleanup() - await abortedLoad + await abortedLoad.catch(() => undefined) } finally { refresh.resolve() await vi.runOnlyPendingTimersAsync() From f62eaaf825372832d3c1e3835295ffa3cb4e4df7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 17:50:05 -0600 Subject: [PATCH 042/327] test(db): harden canceled load proofs --- packages/db/tests/query/subset-dedupe.test.ts | 24 +++ .../tests/electric.test.ts | 147 +++++++++++++++++- 2 files changed, 166 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index b76ba2963..04f3aded8 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -118,6 +118,30 @@ describe(`createDeduplicatedLoadSubset`, () => { await retry }) + it(`does not reuse an aborted in-flight lease while its work is still settling`, async () => { + const releases: Array<() => void> = [] + const loadSubset = vi.fn( + () => new Promise((resolve) => releases.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const owner = new AbortController() + const where = gt(ref(`age`), val(10)) + + const canceled = deduplicated.loadSubset({ + where, + signal: owner.signal, + }) + owner.abort() + + const retry = deduplicated.loadSubset({ where }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(retry).not.toBe(canceled) + + for (const release of releases) release() + await Promise.all([canceled, retry]) + }) + it(`keeps shared work active for a signal-less owner`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 26b41a1e5..d51a3c979 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3033,7 +3033,7 @@ describe(`Electric Integration`, () => { await testCollection.cleanup() }) - it(`should retry a refresh wait after the requesting demand is aborted`, async () => { + it(`retries immediately after the requesting demand is aborted`, async () => { vi.useFakeTimers() const refresh = createDeferred() @@ -3061,19 +3061,23 @@ describe(`Electric Integration`, () => { await Promise.resolve() abortController.abort() + + expect(mockRequestSnapshot).not.toHaveBeenCalled() + + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + const retry = testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(2) await vi.advanceTimersByTimeAsync(0) expect(abortedLoadSettled).toBe(true) await expect(abortedLoadError).resolves.toMatchObject({ name: `AbortError`, }) - expect(mockRequestSnapshot).not.toHaveBeenCalled() expect(vi.getTimerCount()).toBe(0) - mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) - await testCollection._sync.loadSubset({ limit: 10 }) + await retry - expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(2) expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) await testCollection.cleanup() await abortedLoad.catch(() => undefined) @@ -3084,6 +3088,139 @@ describe(`Electric Integration`, () => { } }) + it.each([`request`, `collection`] as const)( + `prefers AbortError when refresh rejection races %s cancellation`, + async (cancellationSource) => { + vi.useFakeTimers() + let rejectRefresh: (error: Error) => void = () => {} + const refresh = new Promise((_resolve, reject) => { + rejectRefresh = reject + }) + const request = new AbortController() + let testCollection: + | ReturnType + | undefined + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + testCollection = createOnDemandCollection( + `on-demand-refresh-${cancellationSource}-race-test`, + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + rejectRefresh(new Error(`refresh failed`)) + if (cancellationSource === `request`) { + request.abort() + } else { + await testCollection.cleanup() + } + + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + await load.catch(() => undefined) + } finally { + request.abort() + await testCollection?.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }, + ) + + it(`removes every abort listener installed by a canceled refresh wait`, async () => { + vi.useFakeTimers() + const refresh = createDeferred() + const request = new AbortController() + const added: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const removed: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const originalAdd = AbortSignal.prototype.addEventListener + const originalRemove = AbortSignal.prototype.removeEventListener + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) + const testCollection = createOnDemandCollection( + `on-demand-refresh-listener-cleanup-test`, + ) + + const addSpy = vi + .spyOn(AbortSignal.prototype, `addEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) added.push({ signal: this, listener }) + return originalAdd.call(this, type, listener, options) + }) + const removeSpy = vi + .spyOn(AbortSignal.prototype, `removeEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) removed.push({ signal: this, listener }) + return originalRemove.call(this, type, listener, options) + }) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + request.abort() + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + + expect(added.length).toBeGreaterThan(0) + for (const installed of added) { + expect( + removed.some( + (candidate) => + candidate.signal === installed.signal && + candidate.listener === installed.listener, + ), + ).toBe(true) + } + await load.catch(() => undefined) + } finally { + request.abort() + refresh.resolve() + await testCollection.cleanup() + addSpy.mockRestore() + removeSpy.mockRestore() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + it(`should clear the refresh timeout when refresh settles early`, async () => { vi.useFakeTimers() try { From 4d58b9f0cc86e4141bcfa86d3c4150e1da9f91f8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 18:00:23 -0600 Subject: [PATCH 043/327] fix(db): reject pre-aborted subset loads --- packages/db/src/collection/sync.ts | 3 +- packages/db/tests/collection.test.ts | 26 ++ .../tests/electric.test.ts | 237 ++++++++++-------- 3 files changed, 166 insertions(+), 100 deletions(-) diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index a14a357dd..a1ecb3ee4 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -5,6 +5,7 @@ import { NoPendingSyncTransactionCommitError, NoPendingSyncTransactionWriteError, SyncCleanupError, + SyncTransactionAbortedError, SyncTransactionAlreadyCommittedError, SyncTransactionAlreadyCommittedWriteError, } from '../errors' @@ -767,7 +768,7 @@ export class CollectionSyncManager< */ public loadSubset(options: LoadSubsetOptions): Promise | true { if (options.signal?.aborted) { - return true + return Promise.reject(new SyncTransactionAbortedError()) } // Bypass loadSubset when syncMode is 'eager' diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index 3ff8ede81..d2c4839b4 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -2329,4 +2329,30 @@ describe(`Collection isLoadingSubset property`, () => { expect(result).toBe(true) expect(collection.isLoadingSubset).toBe(false) }) + + it(`rejects an already-aborted subset request before calling the adapter`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-subset-request`, + getKey: (item) => item.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + await collection.cleanup() + }) }) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index d51a3c979..310c2a4e4 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3006,32 +3006,42 @@ describe(`Electric Integration`, () => { expect(commit).not.toHaveBeenCalled() }) - it(`rejects before starting a refresh when the collection signal is already aborted`, async () => { - mockStream.isUpToDate = true - const abortController = new AbortController() - abortController.abort() - const testCollection = createCollection( - electricCollectionOptions({ - id: `on-demand-refresh-already-aborted-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - signal: abortController.signal, - }, - syncMode: `on-demand`, - getKey: (item: Row) => item.id as number, - startSync: true, - }), - ) + it.each([`collection`, `request`] as const)( + `rejects before starting a refresh when the %s signal is already aborted`, + async (signalSource) => { + mockStream.isUpToDate = true + const abortController = new AbortController() + abortController.abort() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-${signalSource}-already-aborted-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: + signalSource === `collection` + ? abortController.signal + : undefined, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) - await expect( - testCollection._sync.loadSubset({ limit: 10 }), - ).rejects.toMatchObject({ name: `AbortError` }) + await expect( + testCollection._sync.loadSubset({ + limit: 10, + signal: + signalSource === `request` ? abortController.signal : undefined, + }), + ).rejects.toMatchObject({ name: `AbortError` }) - expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() - expect(mockRequestSnapshot).not.toHaveBeenCalled() - await testCollection.cleanup() - }) + expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() + expect(mockRequestSnapshot).not.toHaveBeenCalled() + await testCollection.cleanup() + }, + ) it(`retries immediately after the requesting demand is aborted`, async () => { vi.useFakeTimers() @@ -3140,86 +3150,115 @@ describe(`Electric Integration`, () => { }, ) - it(`removes every abort listener installed by a canceled refresh wait`, async () => { - vi.useFakeTimers() - const refresh = createDeferred() - const request = new AbortController() - const added: Array<{ - signal: AbortSignal - listener: EventListenerOrEventListenerObject - }> = [] - const removed: Array<{ - signal: AbortSignal - listener: EventListenerOrEventListenerObject - }> = [] - const originalAdd = AbortSignal.prototype.addEventListener - const originalRemove = AbortSignal.prototype.removeEventListener - mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) - const testCollection = createOnDemandCollection( - `on-demand-refresh-listener-cleanup-test`, - ) + it.each([ + `refresh`, + `rejection`, + `timeout`, + `request`, + `collection`, + ] as const)( + `removes every abort listener when %s settles the refresh wait`, + async (settlement) => { + vi.useFakeTimers() + const refresh = createDeferred() + const request = new AbortController() + const added: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const removed: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const originalAdd = AbortSignal.prototype.addEventListener + const originalRemove = AbortSignal.prototype.removeEventListener + mockStream.isUpToDate = true + if (settlement === `refresh`) { + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + } else if (settlement === `rejection`) { + mockForceDisconnectAndRefresh.mockRejectedValueOnce( + new Error(`refresh failed`), + ) + } else { + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) + } + const testCollection = createOnDemandCollection( + `on-demand-refresh-${settlement}-listener-cleanup-test`, + ) - const addSpy = vi - .spyOn(AbortSignal.prototype, `addEventListener`) - .mockImplementation(function ( - this: AbortSignal, - type, - listener, - options, - ) { - if (type === `abort`) added.push({ signal: this, listener }) - return originalAdd.call(this, type, listener, options) - }) - const removeSpy = vi - .spyOn(AbortSignal.prototype, `removeEventListener`) - .mockImplementation(function ( - this: AbortSignal, - type, - listener, - options, - ) { - if (type === `abort`) removed.push({ signal: this, listener }) - return originalRemove.call(this, type, listener, options) - }) + const addSpy = vi + .spyOn(AbortSignal.prototype, `addEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) added.push({ signal: this, listener }) + return originalAdd.call(this, type, listener, options) + }) + const removeSpy = vi + .spyOn(AbortSignal.prototype, `removeEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) removed.push({ signal: this, listener }) + return originalRemove.call(this, type, listener, options) + }) - try { - const load = Promise.resolve( - testCollection._sync.loadSubset({ - limit: 10, - signal: request.signal, - }), - ) - const loadError = load.then( - () => undefined, - (error: unknown) => error, - ) + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) - await Promise.resolve() - request.abort() - await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + if (settlement === `timeout`) { + await vi.advanceTimersByTimeAsync(250) + } else if (settlement === `request`) { + request.abort() + } else if (settlement === `collection`) { + await testCollection.cleanup() + } - expect(added.length).toBeGreaterThan(0) - for (const installed of added) { - expect( - removed.some( - (candidate) => - candidate.signal === installed.signal && - candidate.listener === installed.listener, - ), - ).toBe(true) + if (settlement === `request` || settlement === `collection`) { + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + } else { + await expect(loadError).resolves.toBeUndefined() + } + + expect(added.length).toBeGreaterThan(0) + for (const installed of added) { + expect( + removed.some( + (candidate) => + candidate.signal === installed.signal && + candidate.listener === installed.listener, + ), + ).toBe(true) + } + await load.catch(() => undefined) + } finally { + request.abort() + refresh.resolve() + await testCollection.cleanup() + addSpy.mockRestore() + removeSpy.mockRestore() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() } - await load.catch(() => undefined) - } finally { - request.abort() - refresh.resolve() - await testCollection.cleanup() - addSpy.mockRestore() - removeSpy.mockRestore() - await vi.runOnlyPendingTimersAsync() - vi.useRealTimers() - } - }) + }, + ) it(`should clear the refresh timeout when refresh settles early`, async () => { vi.useFakeTimers() From 262220d60dc96b7f17a0c58d9d6a66c5cd82ca0a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 18:17:23 -0600 Subject: [PATCH 044/327] test(db): cover abort boundary cross-products --- packages/db/tests/collection.test.ts | 56 +++++++- .../tests/electric.test.ts | 123 ++++++++++++++++-- 2 files changed, 168 insertions(+), 11 deletions(-) diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index d2c4839b4..c167a3a68 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -2330,7 +2330,7 @@ describe(`Collection isLoadingSubset property`, () => { expect(collection.isLoadingSubset).toBe(false) }) - it(`rejects an already-aborted subset request before calling the adapter`, async () => { + it(`rejects an already-aborted subset request before the adapter branch`, async () => { const loadSubset = vi.fn(() => true as const) const collection = createCollection<{ id: string; value: string }>({ id: `already-aborted-subset-request`, @@ -2355,4 +2355,58 @@ describe(`Collection isLoadingSubset property`, () => { expect(collection.isLoadingSubset).toBe(false) await collection.cleanup() }) + + it(`rejects an already-aborted subset request before the eager return`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-eager-subset-request`, + getKey: (item) => item.id, + syncMode: `eager`, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + await collection.cleanup() + }) + + it(`rejects an already-aborted subset request before deferred start`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-deferred-subset-request`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + expect(collection._deferSyncStart()).toBe(true) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + collection._resumeSyncStart() + expect(loadSubset).not.toHaveBeenCalled() + await collection.cleanup() + }) }) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 310c2a4e4..590c56b79 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3006,15 +3006,20 @@ describe(`Electric Integration`, () => { expect(commit).not.toHaveBeenCalled() }) - it.each([`collection`, `request`] as const)( - `rejects before starting a refresh when the %s signal is already aborted`, - async (signalSource) => { + it.each([ + { syncMode: `on-demand`, signalSource: `collection` }, + { syncMode: `on-demand`, signalSource: `request` }, + { syncMode: `progressive`, signalSource: `collection` }, + { syncMode: `progressive`, signalSource: `request` }, + ] as const)( + `rejects before starting $syncMode work when the $signalSource signal is already aborted`, + async ({ syncMode, signalSource }) => { mockStream.isUpToDate = true const abortController = new AbortController() abortController.abort() const testCollection = createCollection( electricCollectionOptions({ - id: `on-demand-refresh-${signalSource}-already-aborted-test`, + id: `${syncMode}-${signalSource}-already-aborted-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, @@ -3023,7 +3028,7 @@ describe(`Electric Integration`, () => { ? abortController.signal : undefined, }, - syncMode: `on-demand`, + syncMode, getKey: (item: Row) => item.id as number, startSync: true, }), @@ -3039,6 +3044,7 @@ describe(`Electric Integration`, () => { expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(mockFetchSnapshot).not.toHaveBeenCalled() await testCollection.cleanup() }, ) @@ -3098,9 +3104,14 @@ describe(`Electric Integration`, () => { } }) - it.each([`request`, `collection`] as const)( - `prefers AbortError when refresh rejection races %s cancellation`, - async (cancellationSource) => { + it.each([ + { cancellationSource: `request`, order: `rejection-first` }, + { cancellationSource: `request`, order: `cancellation-first` }, + { cancellationSource: `collection`, order: `rejection-first` }, + { cancellationSource: `collection`, order: `cancellation-first` }, + ] as const)( + `prefers AbortError for $cancellationSource cancellation in $order order`, + async ({ cancellationSource, order }) => { vi.useFakeTimers() let rejectRefresh: (error: Error) => void = () => {} const refresh = new Promise((_resolve, reject) => { @@ -3129,20 +3140,111 @@ describe(`Electric Integration`, () => { ) await Promise.resolve() - rejectRefresh(new Error(`refresh failed`)) + let cleanup: Promise | undefined + const cancel = () => { + if (cancellationSource === `request`) { + request.abort() + } else { + cleanup = testCollection?.cleanup() + } + } + const reject = () => rejectRefresh(new Error(`refresh failed`)) + if (order === `rejection-first`) { + reject() + cancel() + } else { + cancel() + reject() + } + await cleanup + + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + await load.catch(() => undefined) + } finally { + request.abort() + await testCollection?.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }, + ) + + it.each([ + { cancellationSource: `request`, lateSettlement: `fulfillment` }, + { cancellationSource: `request`, lateSettlement: `rejection` }, + { cancellationSource: `collection`, lateSettlement: `fulfillment` }, + { cancellationSource: `collection`, lateSettlement: `rejection` }, + ] as const)( + `keeps $cancellationSource cancellation final after late refresh $lateSettlement`, + async ({ cancellationSource, lateSettlement }) => { + vi.useFakeTimers() + let resolveRefresh: () => void = () => {} + let rejectRefresh: (error: Error) => void = () => {} + const refresh = new Promise((resolve, reject) => { + resolveRefresh = resolve + rejectRefresh = reject + }) + const refreshOutcome = refresh.then( + () => `fulfilled` as const, + () => `rejected` as const, + ) + const request = new AbortController() + let testCollection: + | ReturnType + | undefined + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + testCollection = createOnDemandCollection( + `on-demand-refresh-${cancellationSource}-late-${lateSettlement}-test`, + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + let cleanup: Promise | undefined if (cancellationSource === `request`) { request.abort() } else { - await testCollection.cleanup() + cleanup = testCollection.cleanup() + } + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + await cleanup + expect(mockRequestSnapshot).not.toHaveBeenCalled() + + if (lateSettlement === `fulfillment`) { + resolveRefresh() + } else { + rejectRefresh(new Error(`late refresh failure`)) } + await expect(refreshOutcome).resolves.toBe( + lateSettlement === `fulfillment` ? `fulfilled` : `rejected`, + ) + await Promise.resolve() await expect(loadError).resolves.toMatchObject({ name: `AbortError`, }) expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) await load.catch(() => undefined) } finally { request.abort() + resolveRefresh() await testCollection?.cleanup() await vi.runOnlyPendingTimersAsync() vi.useRealTimers() @@ -3236,6 +3338,7 @@ describe(`Electric Integration`, () => { } else { await expect(loadError).resolves.toBeUndefined() } + expect(vi.getTimerCount()).toBe(0) expect(added.length).toBeGreaterThan(0) for (const installed of added) { From 0287c2205880f558059bbfc55002b2a10b10d78c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 18:29:57 -0600 Subject: [PATCH 045/327] fix(electric): reject canceled progressive snapshots --- .../electric-db-collection/src/electric.ts | 6 +-- .../tests/electric.test.ts | 49 +++++++++++++------ 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index ddaf2349a..07a66e817 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -592,13 +592,13 @@ function createLoadSubsetDedupe>({ const snapshotParams = compileSQL(opts, compileOptions) try { const { data: rows } = await stream.fetchSnapshot(snapshotParams) - if (isAborted() || !isBufferingInitialSync()) { + throwIfAborted() + if (!isBufferingInitialSync()) { debug(`${logPrefix}Ignoring snapshot - sync completed while fetching`) return } if (rows.length > 0) { - if (isAborted()) return begin() for (const row of rows) { write({ @@ -611,7 +611,7 @@ function createLoadSubsetDedupe>({ debug(`${logPrefix}Applied snapshot with ${rows.length} rows`) } } catch (error) { - if (isAborted()) return + throwIfAborted() if (handleSnapshotError(error, `fetchSnapshot`)) { return } diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 590c56b79..08ec10563 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2953,7 +2953,7 @@ describe(`Electric Integration`, () => { } }) - it(`does not start buffered snapshot publication after adapter cleanup`, async () => { + it(`rejects buffered snapshot publication after adapter cleanup`, async () => { const snapshot = createDeferred<{ data: Array<{ key: string @@ -2988,7 +2988,11 @@ describe(`Electric Integration`, () => { throw new Error(`Expected progressive sync controls`) } - const load = controls.loadSubset({ limit: 10 }) + const load = Promise.resolve(controls.loadSubset({ limit: 10 })) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) controls.cleanup?.() snapshot.resolve({ data: [ @@ -2999,11 +3003,12 @@ describe(`Electric Integration`, () => { }, ], }) - if (load !== true) await load + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) expect(begin).not.toHaveBeenCalled() expect(write).not.toHaveBeenCalled() expect(commit).not.toHaveBeenCalled() + await load.catch(() => undefined) }) it.each([ @@ -3446,7 +3451,7 @@ describe(`Electric Integration`, () => { }) }) - it(`ignores a progressive snapshot after its subset request is aborted`, async () => { + it(`rejects a progressive snapshot aborted before application`, async () => { mockFetchSnapshot.mockReset() let resolveSnapshot!: (value: { metadata: Record @@ -3478,10 +3483,16 @@ describe(`Electric Integration`, () => { try { expect(mockFetchSnapshot).not.toHaveBeenCalled() - const load = testCollection._sync.loadSubset({ - limit: 1, - signal: abortController.signal, - }) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) expect(mockFetchSnapshot).toHaveBeenCalledOnce() expect(testCollection.has(2)).toBe(false) abortController.abort() @@ -3495,16 +3506,17 @@ describe(`Electric Integration`, () => { }, ], }) - if (load instanceof Promise) await load + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) } finally { resolveSnapshot({ metadata: {}, data: [] }) await testCollection.cleanup() } }) - it(`does not publish a progressive snapshot aborted while its commit is parked`, async () => { + it(`rejects a progressive snapshot aborted while its commit is parked`, async () => { mockFetchSnapshot.mockResolvedValue({ metadata: {}, data: [ @@ -3538,10 +3550,16 @@ describe(`Electric Integration`, () => { transaction.mutate(() => testCollection.insert({ id: 3, name: `Local row` }), ) - const load = testCollection._sync.loadSubset({ - limit: 1, - signal: abortController.signal, - }) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) await vi.waitFor(() => expect(mockFetchSnapshot).toHaveBeenCalledOnce()) await Promise.resolve() await Promise.resolve() @@ -3550,9 +3568,10 @@ describe(`Electric Integration`, () => { abortController.abort() persistence.resolve() await transaction.isPersisted.promise - if (load instanceof Promise) await load + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) } finally { abortController.abort() persistence.resolve() From 6032acac3e4b7242ba2868e6305f8e523b667ddc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 18:48:53 -0600 Subject: [PATCH 046/327] fix(electric): close progressive cancellation races --- .../electric-db-collection/src/electric.ts | 61 ++- .../tests/electric.test.ts | 494 ++++++++++++++---- 2 files changed, 449 insertions(+), 106 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 07a66e817..7b84bbe0f 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -561,6 +561,39 @@ function createLoadSubsetDedupe>({ return null } + const combineAbortSignals = ( + ...signals: Array + ): { signal: AbortSignal; cleanup: () => void } => { + const uniqueSignals = Array.from( + new Set( + signals.filter( + (candidate): candidate is AbortSignal => candidate !== undefined, + ), + ), + ) + if (uniqueSignals.length === 1) { + return { signal: uniqueSignals[0]!, cleanup: () => {} } + } + + const controller = new AbortController() + const abort = () => controller.abort() + for (const candidate of uniqueSignals) { + if (candidate.aborted) { + abort() + } else { + candidate.addEventListener(`abort`, abort, { once: true }) + } + } + return { + signal: controller.signal, + cleanup: () => { + for (const candidate of uniqueSignals) { + candidate.removeEventListener(`abort`, abort) + } + }, + } + } + const compileOptions = encodeColumnName ? { encodeColumnName } : undefined const logPrefix = collectionId ? `[${collectionId}] ` : `` @@ -607,7 +640,12 @@ function createLoadSubsetDedupe>({ metadata: { ...row.headers }, }) } - await commit(opts.signal) + const commitSignal = combineAbortSignals(signal, opts.signal) + try { + await commit(commitSignal.signal) + } finally { + commitSignal.cleanup() + } debug(`${logPrefix}Applied snapshot with ${rows.length} rows`) } } catch (error) { @@ -1608,19 +1646,21 @@ function createElectricSync>( // Abort controller for the stream - wraps the signal if provided const abortController = new AbortController() + let removeShapeAbortListener = () => {} if (shapeOptions.signal) { - shapeOptions.signal.addEventListener( - `abort`, - () => { - abortController.abort() - }, - { - once: true, - }, - ) + const abortFromShapeSignal = () => abortController.abort() if (shapeOptions.signal.aborted) { abortController.abort() + } else { + shapeOptions.signal.addEventListener(`abort`, abortFromShapeSignal, { + once: true, + }) + removeShapeAbortListener = () => + shapeOptions.signal?.removeEventListener( + `abort`, + abortFromShapeSignal, + ) } } @@ -2102,6 +2142,7 @@ function createElectricSync>( cleanup: () => { // Unsubscribe from the stream unsubscribeStream() + removeShapeAbortListener() // Abort the abort controller to stop the stream abortController.abort() // Reset deduplication tracking so collection can load fresh data if restarted diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 08ec10563..e6131b38a 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -26,12 +26,15 @@ const NativeAbortController = globalThis.AbortController function createDeferred(): { promise: Promise resolve: (value: T | PromiseLike) => void + reject: (reason?: unknown) => void } { let resolve!: (value: T | PromiseLike) => void - const promise = new Promise((resolvePromise) => { + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { resolve = resolvePromise + reject = rejectPromise }) - return { promise, resolve } + return { promise, resolve, reject } } // Mock the ShapeStream module @@ -2673,6 +2676,39 @@ describe(`Electric Integration`, () => { }), ) + it(`removes the external shape abort listener across cleanup and restart`, async () => { + const externalAbort = new NativeAbortController() + const addSpy = vi.spyOn(externalAbort.signal, `addEventListener`) + const removeSpy = vi.spyOn(externalAbort.signal, `removeEventListener`) + const testCollection = createCollection( + electricCollectionOptions({ + id: `shape-signal-listener-cleanup-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: externalAbort.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + await testCollection.cleanup() + const subscription = testCollection.subscribeChanges(() => {}) + await testCollection.cleanup() + subscription.unsubscribe() + + const addedListeners = addSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + const removedListeners = removeSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + expect(addedListeners).toHaveLength(2) + expect(removedListeners).toEqual(addedListeners) + }) + it(`should not request snapshots during subscription in eager mode`, () => { vi.clearAllMocks() @@ -3451,52 +3487,87 @@ describe(`Electric Integration`, () => { }) }) - it(`rejects a progressive snapshot aborted before application`, async () => { - mockFetchSnapshot.mockReset() - let resolveSnapshot!: (value: { - metadata: Record - data: Array<{ - key: string - value: Row - headers: { operation: `insert` } - }> - }) => void - mockFetchSnapshot.mockReturnValue( - new Promise((resolve) => { - resolveSnapshot = resolve - }), - ) - mockSubscribe.mockImplementation(() => () => {}) - const testCollection = createCollection( - electricCollectionOptions({ - id: `progressive-aborted-snapshot-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - }, - syncMode: `progressive`, - getKey: (item: Row) => item.id as number, - startSync: true, - }), - ) - const abortController = new AbortController() - - try { - expect(mockFetchSnapshot).not.toHaveBeenCalled() - const load = Promise.resolve( - testCollection._sync.loadSubset({ - limit: 1, - signal: abortController.signal, + it.each([ + { signalSource: `request`, result: `empty` }, + { signalSource: `request`, result: `rows` }, + { signalSource: `collection`, result: `empty` }, + { signalSource: `collection`, result: `rows` }, + ] as const)( + `rejects a progressive $result snapshot when the $signalSource signal aborts before application`, + async ({ signalSource, result }) => { + const snapshot = createDeferred<{ + metadata: Record + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + mockFetchSnapshot.mockReturnValue(snapshot.promise) + mockSubscribe.mockImplementation(() => () => {}) + const collectionAbortController = new AbortController() + const requestAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `progressive-${signalSource}-${result}-abort-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, }), ) - const loadError = load.then( - () => undefined, - (error: unknown) => error, - ) - expect(mockFetchSnapshot).toHaveBeenCalledOnce() - expect(testCollection.has(2)).toBe(false) - abortController.abort() - resolveSnapshot({ + const abortController = + signalSource === `request` + ? requestAbortController + : collectionAbortController + + try { + expect(mockFetchSnapshot).not.toHaveBeenCalled() + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + expect(mockFetchSnapshot).toHaveBeenCalledOnce() + abortController.abort() + snapshot.resolve({ + metadata: {}, + data: + result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Obsolete snapshot` }, + headers: { operation: `insert` }, + }, + ] + : [], + }) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + requestAbortController.abort() + snapshot.resolve({ metadata: {}, data: [] }) + await testCollection.cleanup() + } + }, + ) + + it.each([`request`, `collection`, `cleanup`] as const)( + `rejects a progressive snapshot when %s cancellation occurs while its commit is parked`, + async (cancellationSource) => { + mockFetchSnapshot.mockResolvedValue({ metadata: {}, data: [ { @@ -3506,31 +3577,92 @@ describe(`Electric Integration`, () => { }, ], }) - await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + mockSubscribe.mockImplementation(() => () => {}) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `progressive-parked-abort-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + const requestAbortController = new AbortController() + const abortController = + cancellationSource === `request` + ? requestAbortController + : collectionAbortController - expect(testCollection.has(2)).toBe(false) - await load.catch(() => undefined) - } finally { - resolveSnapshot({ metadata: {}, data: [] }) - await testCollection.cleanup() - } - }) + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockFetchSnapshot).toHaveBeenCalledOnce(), + ) + await Promise.resolve() + await Promise.resolve() - it(`rejects a progressive snapshot aborted while its commit is parked`, async () => { - mockFetchSnapshot.mockResolvedValue({ - metadata: {}, - data: [ - { - key: `2`, - value: { id: 2, name: `Obsolete snapshot` }, - headers: { operation: `insert` }, - }, - ], - }) - mockSubscribe.mockImplementation(() => () => {}) - const testCollection = createCollection( - electricCollectionOptions({ - id: `progressive-parked-abort-test`, + expect(testCollection.has(2)).toBe(false) + if (cancellationSource === `cleanup`) { + await testCollection.cleanup() + } else { + abortController.abort() + } + persistence.resolve() + await transaction.isPersisted.promise + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + abortController.abort() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }, + ) + + it.each([`fetch`, `commit`] as const)( + `propagates an uncanceled progressive %s error`, + async (failurePhase) => { + const failure = new Error(`${failurePhase} failed`) + if (failurePhase === `fetch`) { + mockFetchSnapshot.mockRejectedValue(failure) + } else { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + } + const options = electricCollectionOptions({ + id: `progressive-${failurePhase}-error-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, @@ -3538,45 +3670,215 @@ describe(`Electric Integration`, () => { syncMode: `progressive`, getKey: (item: Row) => item.id as number, startSync: true, - }), - ) - const persistence = createDeferred() - const transaction = createTransaction({ - mutationFn: () => persistence.promise, - }) - const abortController = new AbortController() + }) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: + failurePhase === `commit` + ? vi.fn(() => Promise.reject(failure)) + : vi.fn(() => true as const), + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } - try { - transaction.mutate(() => - testCollection.insert({ id: 3, name: `Local row` }), - ) + try { + await expect( + Promise.resolve(controls.loadSubset({ limit: 1 })), + ).rejects.toBe(failure) + } finally { + controls.cleanup?.() + } + }, + ) + + it.each([ + { failurePhase: `fetch`, signalSource: `request`, order: `cancel-first` }, + { failurePhase: `fetch`, signalSource: `request`, order: `error-first` }, + { + failurePhase: `fetch`, + signalSource: `collection`, + order: `cancel-first`, + }, + { + failurePhase: `fetch`, + signalSource: `collection`, + order: `error-first`, + }, + { + failurePhase: `commit`, + signalSource: `request`, + order: `cancel-first`, + }, + { failurePhase: `commit`, signalSource: `request`, order: `error-first` }, + { + failurePhase: `commit`, + signalSource: `collection`, + order: `cancel-first`, + }, + { + failurePhase: `commit`, + signalSource: `collection`, + order: `error-first`, + }, + ] as const)( + `prefers AbortError when $signalSource cancellation races a progressive $failurePhase error in $order order`, + async ({ failurePhase, signalSource, order }) => { + const failure = new Error(`${failurePhase} failed`) + const fetch = createDeferred<{ + metadata: Record + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + const commit = createDeferred() + if (failurePhase === `fetch`) { + mockFetchSnapshot.mockReturnValue(fetch.promise) + } else { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + } + const collectionAbortController = new AbortController() + const requestAbortController = new AbortController() + const options = electricCollectionOptions({ + id: `progressive-${failurePhase}-${signalSource}-${order}-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = + failurePhase === `commit` + ? vi.fn(() => commit.promise) + : vi.fn(() => true as const) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } + const abortController = + signalSource === `request` + ? requestAbortController + : collectionAbortController const load = Promise.resolve( - testCollection._sync.loadSubset({ + controls.loadSubset({ limit: 1, - signal: abortController.signal, + signal: requestAbortController.signal, }), ) const loadError = load.then( () => undefined, (error: unknown) => error, ) - await vi.waitFor(() => expect(mockFetchSnapshot).toHaveBeenCalledOnce()) - await Promise.resolve() - await Promise.resolve() - expect(testCollection.has(2)).toBe(false) - abortController.abort() - persistence.resolve() - await transaction.isPersisted.promise - await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + try { + if (failurePhase === `commit`) { + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + } + if (order === `cancel-first`) abortController.abort() + if (failurePhase === `fetch`) fetch.reject(failure) + else commit.reject(failure) + if (order === `error-first`) abortController.abort() - expect(testCollection.has(2)).toBe(false) - await load.catch(() => undefined) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + requestAbortController.abort() + fetch.resolve({ metadata: {}, data: [] }) + commit.resolve() + controls.cleanup?.() + } + }, + ) + + it(`keeps a progressive snapshot applied before cancellation`, async () => { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Applied snapshot` }, + headers: { operation: `insert` }, + }, + ], + }) + const requestAbortController = new AbortController() + const stagedRows: Array = [] + const appliedRows: Array = [] + const options = electricCollectionOptions({ + id: `progressive-applied-before-abort-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn((change: { value: Row }) => stagedRows.push(change.value)), + commit: vi.fn(() => { + appliedRows.push(...stagedRows) + requestAbortController.abort() + return true as const + }), + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!controls || typeof controls === `function` || !controls.loadSubset) { + throw new Error(`Expected progressive sync controls`) + } + + try { + await expect( + Promise.resolve( + controls.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ), + ).resolves.toBeUndefined() + expect(appliedRows).toEqual([{ id: 2, name: `Applied snapshot` }]) } finally { - abortController.abort() - persistence.resolve() - await transaction.isPersisted.promise.catch(() => undefined) - await testCollection.cleanup() + controls.cleanup?.() } }) From af1d7523a73ce6a5075b92bd2cbadd3e2ec6c82c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 19:01:11 -0600 Subject: [PATCH 047/327] test(electric): prove progressive abort resources --- .../tests/electric.test.ts | 235 ++++++++++++++---- 1 file changed, 186 insertions(+), 49 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index e6131b38a..73b952d2c 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3826,61 +3826,198 @@ describe(`Electric Integration`, () => { }, ) - it(`keeps a progressive snapshot applied before cancellation`, async () => { - mockFetchSnapshot.mockResolvedValue({ - metadata: {}, - data: [ - { - key: `2`, - value: { id: 2, name: `Applied snapshot` }, - headers: { operation: `insert` }, + it.each([`request`, `collection`, `cleanup`] as const)( + `keeps a progressive snapshot applied before %s cancellation`, + async (cancellationSource) => { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Applied snapshot` }, + headers: { operation: `insert` }, + }, + ], + }) + const requestAbortController = new AbortController() + const collectionAbortController = new AbortController() + const stagedRows: Array = [] + const appliedRows: Array = [] + let cleanup = () => {} + const options = electricCollectionOptions({ + id: `progressive-applied-before-${cancellationSource}-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, }, - ], - }) - const requestAbortController = new AbortController() - const stagedRows: Array = [] - const appliedRows: Array = [] - const options = electricCollectionOptions({ - id: `progressive-applied-before-abort-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - }, - syncMode: `progressive`, - getKey: (item: Row) => item.id as number, - startSync: true, - }) - const controls = options.sync.sync({ - collection: { id: options.id, status: `loading` }, - begin: vi.fn(), - write: vi.fn((change: { value: Row }) => stagedRows.push(change.value)), - commit: vi.fn(() => { - appliedRows.push(...stagedRows) - requestAbortController.abort() - return true as const - }), - markReady: vi.fn(), - markError: vi.fn(), - truncate: vi.fn(), - } as never) - if (!controls || typeof controls === `function` || !controls.loadSubset) { - throw new Error(`Expected progressive sync controls`) - } + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn((change: { value: Row }) => + stagedRows.push(change.value), + ), + commit: vi.fn(() => { + appliedRows.push(...stagedRows) + if (cancellationSource === `request`) { + requestAbortController.abort() + } else if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + cleanup() + } + return true as const + }), + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } + cleanup = controls.cleanup ?? (() => {}) - try { - await expect( - Promise.resolve( + try { + await expect( + Promise.resolve( + controls.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ), + ).resolves.toBeUndefined() + expect(appliedRows).toEqual([{ id: 2, name: `Applied snapshot` }]) + } finally { + controls.cleanup?.() + } + }, + ) + + it.each([`success`, `rejection`, `request-abort`, `cleanup`] as const)( + `removes combined commit abort listeners after %s`, + async (settlement) => { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + const commit = createDeferred() + const requestAbortController = new AbortController() + const options = electricCollectionOptions({ + id: `progressive-combined-listener-${settlement}-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = vi.fn(() => commit.promise) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } + + const added: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const removed: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const originalAdd = AbortSignal.prototype.addEventListener + const originalRemove = AbortSignal.prototype.removeEventListener + const addSpy = vi + .spyOn(AbortSignal.prototype, `addEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + listenerOptions, + ) { + if (type === `abort`) added.push({ signal: this, listener }) + return originalAdd.call(this, type, listener, listenerOptions) + }) + const removeSpy = vi + .spyOn(AbortSignal.prototype, `removeEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + listenerOptions, + ) { + if (type === `abort`) removed.push({ signal: this, listener }) + return originalRemove.call(this, type, listener, listenerOptions) + }) + const failure = new Error(`commit failed`) + + try { + const load = Promise.resolve( controls.loadSubset({ limit: 1, signal: requestAbortController.signal, }), - ), - ).resolves.toBeUndefined() - expect(appliedRows).toEqual([{ id: 2, name: `Applied snapshot` }]) - } finally { - controls.cleanup?.() - } - }) + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + + if (settlement === `request-abort`) { + requestAbortController.abort() + } else if (settlement === `cleanup`) { + controls.cleanup?.() + } + if (settlement === `rejection`) commit.reject(failure) + else commit.resolve() + + if (settlement === `rejection`) { + await expect(loadError).resolves.toBe(failure) + } else { + await expect(loadError).resolves.toBeUndefined() + } + await load.catch(() => undefined) + + for (const installed of added) { + expect(removed).toContainEqual(installed) + } + } finally { + addSpy.mockRestore() + removeSpy.mockRestore() + requestAbortController.abort() + commit.resolve() + controls.cleanup?.() + } + }, + ) it(`should not request snapshots when loadSubset is called in eager mode`, async () => { vi.clearAllMocks() From 01c26195f3617c1a8527ae1dcddc43f6df23c717 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 19:10:00 -0600 Subject: [PATCH 048/327] test(electric): cover single abort signal path --- .../tests/electric.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 73b952d2c..5e45b77a6 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3564,9 +3564,14 @@ describe(`Electric Integration`, () => { }, ) - it.each([`request`, `collection`, `cleanup`] as const)( - `rejects a progressive snapshot when %s cancellation occurs while its commit is parked`, - async (cancellationSource) => { + it.each([ + { cancellationSource: `request`, requestSignal: `present` }, + { cancellationSource: `collection`, requestSignal: `present` }, + { cancellationSource: `collection`, requestSignal: `absent` }, + { cancellationSource: `cleanup`, requestSignal: `present` }, + ] as const)( + `rejects a progressive snapshot when $cancellationSource cancellation occurs with the request signal $requestSignal while its commit is parked`, + async ({ cancellationSource, requestSignal }) => { mockFetchSnapshot.mockResolvedValue({ metadata: {}, data: [ @@ -3609,7 +3614,10 @@ describe(`Electric Integration`, () => { const load = Promise.resolve( testCollection._sync.loadSubset({ limit: 1, - signal: requestAbortController.signal, + signal: + requestSignal === `present` + ? requestAbortController.signal + : undefined, }), ) const loadError = load.then( From 0766e9839573a2163ac64b88aab25ab3daedd3af Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 19:21:06 -0600 Subject: [PATCH 049/327] test(electric): prove on-demand abort boundary --- .../tests/electric.test.ts | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 5e45b77a6..903c615cb 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2799,6 +2799,103 @@ describe(`Electric Integration`, () => { } }) + it.each([`before-publication`, `after-publication`] as const)( + `keeps on-demand rows applied %s cancellation but retries the canceled demand`, + async (abortPhase) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-${abortPhase}-abort-boundary-test`, + ) + const abortController = new AbortController() + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + + if (abortPhase === `before-publication`) abortController.abort() + subscriber([ + { + key: `2`, + value: { id: 2, name: `Applied on-demand row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + if (abortPhase === `after-publication`) abortController.abort() + request.resolve() + + await expect(loadError).resolves.toBeUndefined() + expect(stripVirtualProps(testCollection.get(2))).toEqual({ + id: 2, + name: `Applied on-demand row`, + }) + await load + + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + subscriber([{ headers: { control: `subset-end` } }]) + await retry + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + + it(`keeps on-demand coverage when cancellation happens after settlement`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-post-settlement-abort-test`, + ) + const abortController = new AbortController() + const options = { limit: 10, signal: abortController.signal } + + try { + const load = Promise.resolve(testCollection._sync.loadSubset(options)) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Settled on-demand row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + request.resolve() + await load + + abortController.abort() + expect(testCollection.has(2)).toBe(true) + await testCollection._sync.loadSubset({ limit: 10 }) + expect(mockRequestSnapshot).toHaveBeenCalledOnce() + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }) + it(`should refresh the stream before requesting on-demand snapshots when already up-to-date`, async () => { vi.clearAllMocks() From 15e25e56d47653a001f311109585ab138d463dbd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 19:36:46 -0600 Subject: [PATCH 050/327] fix(electric): preserve on-demand request errors --- .../electric-db-collection/src/electric.ts | 4 +- .../tests/electric.test.ts | 336 +++++++++++++++++- 2 files changed, 326 insertions(+), 14 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 7b84bbe0f..f242304ed 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -764,7 +764,9 @@ function createLoadSubsetDedupe>({ await stream.requestSnapshot(snapshotParams) } } catch (error) { - if (opts.signal?.aborted) return + if (signal.aborted) { + throw new SyncTransactionAbortedError() + } if (handleSnapshotError(error, `requestSnapshot`)) { return } diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 903c615cb..ad171a617 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ShapeStream } from '@electric-sql/client' import { CollectionImpl, + IR, createCollection, createTransaction, } from '@tanstack/db' @@ -2799,9 +2800,14 @@ describe(`Electric Integration`, () => { } }) - it.each([`before-publication`, `after-publication`] as const)( - `keeps on-demand rows applied %s cancellation but retries the canceled demand`, - async (abortPhase) => { + it.each([ + { abortPhase: `before-publication`, result: `empty` }, + { abortPhase: `before-publication`, result: `rows` }, + { abortPhase: `after-publication`, result: `empty` }, + { abortPhase: `after-publication`, result: `rows` }, + ] as const)( + `keeps an on-demand $result result applied $abortPhase cancellation but retries the canceled demand`, + async ({ abortPhase, result }) => { const request = createDeferred() mockRequestSnapshot.mockReturnValueOnce(request.promise) const testCollection = createOnDemandCollection( @@ -2826,22 +2832,30 @@ describe(`Electric Integration`, () => { if (abortPhase === `before-publication`) abortController.abort() subscriber([ - { - key: `2`, - value: { id: 2, name: `Applied on-demand row` }, - headers: { operation: `insert` }, - }, + ...(result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Applied on-demand row` }, + headers: { operation: `insert` as const }, + }, + ] + : []), { headers: { control: `subset-end` } }, ]) - await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await vi.waitFor(() => + expect(testCollection.has(2)).toBe(result === `rows`), + ) if (abortPhase === `after-publication`) abortController.abort() request.resolve() await expect(loadError).resolves.toBeUndefined() - expect(stripVirtualProps(testCollection.get(2))).toEqual({ - id: 2, - name: `Applied on-demand row`, - }) + if (result === `rows`) { + expect(stripVirtualProps(testCollection.get(2))).toEqual({ + id: 2, + name: `Applied on-demand row`, + }) + } await load const retry = Promise.resolve( @@ -2860,6 +2874,115 @@ describe(`Electric Integration`, () => { }, ) + it.each([`collection`, `cleanup`] as const)( + `rejects with AbortError when %s cancellation ends an active on-demand request`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-${cancellationSource}-active-request-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const failure = new Error(`request failed after cancellation`) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } + request.reject(failure) + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + + it.each([`success`, `rejection`, `cancellation`] as const)( + `removes the on-demand request lease listener after %s`, + async (settlement) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-request-listener-${settlement}-test`, + ) + const abortController = new AbortController() + const addSpy = vi.spyOn(abortController.signal, `addEventListener`) + const removeSpy = vi.spyOn( + abortController.signal, + `removeEventListener`, + ) + const failure = new Error(`request failed`) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (settlement === `cancellation`) abortController.abort() + subscriber([{ headers: { control: `subset-end` } }]) + if (settlement === `rejection`) request.reject(failure) + else request.resolve() + + if (settlement === `rejection`) { + await expect(loadError).resolves.toBe(failure) + } else { + await expect(loadError).resolves.toBeUndefined() + } + await load.catch(() => undefined) + + const addedListeners = addSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + const removedListeners = removeSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + expect(addedListeners.length).toBeGreaterThan(0) + expect(removedListeners).toEqual(addedListeners) + } finally { + addSpy.mockRestore() + removeSpy.mockRestore() + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + it(`keeps on-demand coverage when cancellation happens after settlement`, async () => { const request = createDeferred() mockRequestSnapshot.mockReturnValueOnce(request.promise) @@ -2896,6 +3019,193 @@ describe(`Electric Integration`, () => { } }) + it.each([ + { cancellation: `none`, result: `empty` }, + { cancellation: `none`, result: `rows` }, + { cancellation: `request`, result: `empty` }, + { cancellation: `request`, result: `rows` }, + ] as const)( + `propagates an on-demand request error with $result after $cancellation cancellation`, + async ({ cancellation, result }) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-${cancellation}-${result}-request-error-test`, + ) + const abortController = new AbortController() + const failure = new Error(`request failed`) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (cancellation === `request`) abortController.abort() + subscriber([ + ...(result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Partial on-demand row` }, + headers: { operation: `insert` as const }, + }, + ] + : []), + { headers: { control: `subset-end` } }, + ]) + request.reject(failure) + + await expect(loadError).resolves.toBe(failure) + expect(testCollection.has(2)).toBe(result === `rows`) + await load.catch(() => undefined) + + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + subscriber([{ headers: { control: `subset-end` } }]) + await retry + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + + it(`does not fulfill a failed on-demand request before its published receipt applies`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-parked-request-error-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + const abortController = new AbortController() + const failure = new Error(`request failed`) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Parked on-demand row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + abortController.abort() + request.reject(failure) + + await expect(loadError).resolves.toBe(failure) + expect(testCollection.has(2)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await load.catch(() => undefined) + } finally { + abortController.abort() + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }) + + it(`propagates one failed cursor request while its sibling can still publish`, async () => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-request-error-test`, + ) + const abortController = new AbortController() + const failure = new Error(`cursor request failed`) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + abortController.abort() + whereCurrent.reject(failure) + + await expect(loadError).resolves.toBe(failure) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Late cursor row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + whereFrom.resolve() + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await load.catch(() => undefined) + } finally { + abortController.abort() + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }) + it(`should refresh the stream before requesting on-demand snapshots when already up-to-date`, async () => { vi.clearAllMocks() From 6a25cd6cab6f4e22feaabc91eebb8ae8f4f1990b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 19:57:26 -0600 Subject: [PATCH 051/327] fix(electric): cancel active on-demand loads --- .../electric-db-collection/src/electric.ts | 13 +- .../tests/electric.test.ts | 137 +++++++++++++++++- 2 files changed, 143 insertions(+), 7 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index f242304ed..0bb72282f 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -614,6 +614,11 @@ function createLoadSubsetDedupe>({ const commitCursor = getCommitCursor() const isAborted = (): boolean => signal.aborted || opts.signal?.aborted === true + const throwIfCollectionAborted = () => { + if (signal.aborted) { + throw new SyncTransactionAbortedError() + } + } const throwIfAborted = () => { if (isAborted()) { throw new SyncTransactionAbortedError() @@ -772,7 +777,9 @@ function createLoadSubsetDedupe>({ } throw error } + throwIfCollectionAborted() await waitForCommitsAfter(commitCursor) + throwIfCollectionAborted() } return new DeduplicatedLoadSubset({ loadSubset }) @@ -2063,7 +2070,7 @@ function createElectricSync>( // Commit the atomic swap stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) // Exit buffering phase by marking that we've received up-to-date // isBufferingInitialSync() will now return false @@ -2077,12 +2084,12 @@ function createElectricSync>( // Both up-to-date and subset-end trigger a commit if (transactionStarted) { stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) transactionStarted = false } else if (commitPoint === `up-to-date` && metadata) { begin() stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) } } const readyErrorVersion = streamErrorVersion diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index ad171a617..446c49e68 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2874,9 +2874,14 @@ describe(`Electric Integration`, () => { }, ) - it.each([`collection`, `cleanup`] as const)( - `rejects with AbortError when %s cancellation ends an active on-demand request`, - async (cancellationSource) => { + it.each([ + { cancellationSource: `collection`, requestOutcome: `fulfillment` }, + { cancellationSource: `collection`, requestOutcome: `rejection` }, + { cancellationSource: `cleanup`, requestOutcome: `fulfillment` }, + { cancellationSource: `cleanup`, requestOutcome: `rejection` }, + ] as const)( + `rejects with AbortError when $cancellationSource cancellation ends an active on-demand request before $requestOutcome`, + async ({ cancellationSource, requestOutcome }) => { const request = createDeferred() mockRequestSnapshot.mockReturnValueOnce(request.promise) const collectionAbortController = new AbortController() @@ -2911,7 +2916,8 @@ describe(`Electric Integration`, () => { } else { await testCollection.cleanup() } - request.reject(failure) + if (requestOutcome === `rejection`) request.reject(failure) + else request.resolve() await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) await load.catch(() => undefined) @@ -2923,6 +2929,129 @@ describe(`Electric Integration`, () => { }, ) + it.each([`collection`, `cleanup`] as const)( + `cancels a parked on-demand commit after %s cancellation`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-${cancellationSource}-parked-commit-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Canceled parked row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }, + ) + + it(`waits for a successful on-demand commit to apply`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-successful-parked-commit-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + let loadSettled = false + void load.finally(() => { + loadSettled = true + }) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Applied parked row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + request.resolve() + await Promise.resolve() + await Promise.resolve() + + expect(loadSettled).toBe(false) + expect(testCollection.has(2)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await load + expect(stripVirtualProps(testCollection.get(2))).toEqual({ + id: 2, + name: `Applied parked row`, + }) + } finally { + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }) + it.each([`success`, `rejection`, `cancellation`] as const)( `removes the on-demand request lease listener after %s`, async (settlement) => { From d78f2c4238233c0f02699d378435f7685b3fa4e9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 21:13:45 -0600 Subject: [PATCH 052/327] test(electric): prove on-demand settlement gates --- .../tests/electric.test.ts | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 446c49e68..a22f86775 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3015,10 +3015,6 @@ describe(`Electric Integration`, () => { const load = Promise.resolve( testCollection._sync.loadSubset({ limit: 10 }), ) - let loadSettled = false - void load.finally(() => { - loadSettled = true - }) await vi.waitFor(() => expect(mockRequestSnapshot).toHaveBeenCalledOnce(), ) @@ -3031,10 +3027,13 @@ describe(`Electric Integration`, () => { { headers: { control: `subset-end` } }, ]) request.resolve() - await Promise.resolve() - await Promise.resolve() - expect(loadSettled).toBe(false) + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([load.then(() => `load-settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) expect(testCollection.has(2)).toBe(false) persistence.resolve() @@ -3052,6 +3051,43 @@ describe(`Electric Integration`, () => { } }) + it(`rejects when collection cancellation lands after request fulfillment but before applied settlement`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-post-request-collection-cancel-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + + request.resolve() + queueMicrotask(() => collectionAbortController.abort()) + + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }) + it.each([`success`, `rejection`, `cancellation`] as const)( `removes the on-demand request lease listener after %s`, async (settlement) => { From b736e2c09a3de67bd62b70f7bc054bccf4967829 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 21:35:57 -0600 Subject: [PATCH 053/327] fix(electric): retain request commit receipts --- .../electric-db-collection/src/electric.ts | 133 ++++++++++-------- .../tests/electric.test.ts | 125 ++++++++++++++++ 2 files changed, 202 insertions(+), 56 deletions(-) diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 0bb72282f..60605a048 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -528,8 +528,7 @@ function createLoadSubsetDedupe>({ begin, write, commit, - getCommitCursor, - waitForCommitsAfter, + captureCommits, collectionId, encodeColumnName, signal, @@ -544,8 +543,10 @@ function createLoadSubsetDedupe>({ metadata: Record }) => void commit: (signal?: AbortSignal) => SyncAppliedReceipt - getCommitCursor: () => number - waitForCommitsAfter: (cursor: number) => Promise + captureCommits: () => { + wait: () => Promise + dispose: () => void + } collectionId?: string /** * Optional function to encode column names (e.g., camelCase to snake_case). @@ -611,7 +612,6 @@ function createLoadSubsetDedupe>({ } const loadSubset = async (opts: LoadSubsetOptions) => { - const commitCursor = getCommitCursor() const isAborted = (): boolean => signal.aborted || opts.signal?.aborted === true const throwIfCollectionAborted = () => { @@ -737,49 +737,58 @@ function createLoadSubsetDedupe>({ // aborted request can already have installed rows before the check below. // Full request-scoped cancellation requires support in the Electric client; // matching snapshots by parameters is unsafe for overlapping equal requests. + const commitCapture = captureCommits() try { - if (cursor) { - const whereCurrentOpts: LoadSubsetOptions = { - where: where ? and(where, cursor.whereCurrent) : cursor.whereCurrent, - orderBy, - } - const whereCurrentParams = compileSQL( - whereCurrentOpts, - compileOptions, - ) + try { + if (cursor) { + const whereCurrentOpts: LoadSubsetOptions = { + where: where + ? and(where, cursor.whereCurrent) + : cursor.whereCurrent, + orderBy, + } + const whereCurrentParams = compileSQL( + whereCurrentOpts, + compileOptions, + ) - const whereFromOpts: LoadSubsetOptions = { - where: where ? and(where, cursor.whereFrom) : cursor.whereFrom, - orderBy, - limit, - } - const whereFromParams = compileSQL(whereFromOpts, compileOptions) + const whereFromOpts: LoadSubsetOptions = { + where: where ? and(where, cursor.whereFrom) : cursor.whereFrom, + orderBy, + limit, + } + const whereFromParams = compileSQL(whereFromOpts, compileOptions) - debug(`${logPrefix}Requesting cursor.whereCurrent snapshot (all ties)`) - debug( - `${logPrefix}Requesting cursor.whereFrom snapshot (with limit ${limit})`, - ) + debug( + `${logPrefix}Requesting cursor.whereCurrent snapshot (all ties)`, + ) + debug( + `${logPrefix}Requesting cursor.whereFrom snapshot (with limit ${limit})`, + ) - await Promise.all([ - stream.requestSnapshot(whereCurrentParams), - stream.requestSnapshot(whereFromParams), - ]) - } else { - const snapshotParams = compileSQL(opts, compileOptions) - await stream.requestSnapshot(snapshotParams) - } - } catch (error) { - if (signal.aborted) { - throw new SyncTransactionAbortedError() - } - if (handleSnapshotError(error, `requestSnapshot`)) { - return + await Promise.all([ + stream.requestSnapshot(whereCurrentParams), + stream.requestSnapshot(whereFromParams), + ]) + } else { + const snapshotParams = compileSQL(opts, compileOptions) + await stream.requestSnapshot(snapshotParams) + } + } catch (error) { + if (signal.aborted) { + throw new SyncTransactionAbortedError() + } + if (handleSnapshotError(error, `requestSnapshot`)) { + return + } + throw error } - throw error + throwIfCollectionAborted() + await commitCapture.wait() + throwIfCollectionAborted() + } finally { + commitCapture.dispose() } - throwIfCollectionAborted() - await waitForCommitsAfter(commitCursor) - throwIfCollectionAborted() } return new DeduplicatedLoadSubset({ loadSubset }) @@ -1583,25 +1592,38 @@ function createElectricSync>( collection, metadata, } = params - let commitSequence = 0 - const pendingAppliedReceipts = new Map>() + const activeCommitCaptures = new Set>>() const commit = (signal?: AbortSignal): SyncAppliedReceipt => { - const sequence = ++commitSequence const applied = commitSyncTransaction(signal) if (applied === true) { return true } - pendingAppliedReceipts.set(sequence, applied) - const removeReceipt = () => pendingAppliedReceipts.delete(sequence) - void applied.then(removeReceipt, removeReceipt) + if (activeCommitCaptures.size > 0) { + for (const receipts of activeCommitCaptures) { + receipts.add(applied) + } + // A receipt can reject before its request Promise settles. Observe it + // now while retaining the original Promise for the capture to await. + void applied.catch(() => undefined) + } return applied } - const waitForCommitsAfter = async (cursor: number): Promise => { - await Promise.all( - Array.from(pendingAppliedReceipts, ([sequence, applied]) => - sequence > cursor ? applied : undefined, - ), - ) + const captureCommits = () => { + const receipts = new Set>() + let active = true + const dispose = () => { + if (!active) return + active = false + activeCommitCaptures.delete(receipts) + } + activeCommitCaptures.add(receipts) + return { + wait: async () => { + dispose() + await Promise.all(receipts) + }, + dispose, + } } const readPersistedResumeState = (): ElectricResumeState | undefined => { const persistedResumeState = metadata?.collection.get(`electric:resume`) @@ -1850,8 +1872,7 @@ function createElectricSync>( begin, write, commit, - getCommitCursor: () => commitSequence, - waitForCommitsAfter, + captureCommits, collectionId, // Pass the columnMapper's encode function to transform column names // (e.g., camelCase to snake_case) when compiling SQL for subset queries diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index a22f86775..a7958d3df 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3051,6 +3051,78 @@ describe(`Electric Integration`, () => { } }) + it(`retains every applied receipt until the on-demand request settles`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-retained-receipts-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Applied row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + subscriber([ + { + key: `4`, + value: { id: 4, name: `Canceled row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + expect(testCollection._state.pendingSyncedTransactions).toHaveLength(2) + const canceledReceipt = + testCollection._state.pendingSyncedTransactions[1]! + testCollection._state.cancelPendingSyncedTransaction(canceledReceipt) + await Promise.resolve() + + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(true) + expect(testCollection.has(4)).toBe(false) + await load.catch(() => undefined) + + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + await retry + } finally { + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }) + it(`rejects when collection cancellation lands after request fulfillment but before applied settlement`, async () => { const request = createDeferred() mockRequestSnapshot.mockReturnValueOnce(request.promise) @@ -3371,6 +3443,59 @@ describe(`Electric Integration`, () => { } }) + it(`waits for both cursor snapshot requests before settling`, async () => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-all-requests-test`, + ) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + + whereCurrent.resolve() + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([load.then(() => `load-settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + whereFrom.resolve() + await load + } finally { + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }) + it(`should refresh the stream before requesting on-demand snapshots when already up-to-date`, async () => { vi.clearAllMocks() From 7ef216bd629e794dad80bb8acc74bb8df16a2fd8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 21:56:10 -0600 Subject: [PATCH 054/327] fix(electric): settle complete snapshot batches --- .../src/applied-commit-capture.ts | 67 ++++++ .../electric-db-collection/src/electric.ts | 48 ++--- .../tests/applied-commit-capture.test.ts | 117 +++++++++++ .../tests/electric.test.ts | 190 ++++++++++++------ 4 files changed, 329 insertions(+), 93 deletions(-) create mode 100644 packages/electric-db-collection/src/applied-commit-capture.ts create mode 100644 packages/electric-db-collection/tests/applied-commit-capture.test.ts diff --git a/packages/electric-db-collection/src/applied-commit-capture.ts b/packages/electric-db-collection/src/applied-commit-capture.ts new file mode 100644 index 000000000..2b38728fb --- /dev/null +++ b/packages/electric-db-collection/src/applied-commit-capture.ts @@ -0,0 +1,67 @@ +import type { SyncAppliedReceipt } from '@tanstack/db' + +export type AppliedCommitCapture = { + wait: () => Promise + dispose: () => void +} + +export type AppliedCommitCaptureRegistry = { + capture: (signal?: AbortSignal) => AppliedCommitCapture + record: (receipt: SyncAppliedReceipt) => void + readonly activeCount: number +} + +/** + * Captures every asynchronous commit receipt produced during an Electric + * request. A capture is sealed before waiting, so later stream work cannot + * become part of an already-settled request. + */ +export function createAppliedCommitCaptureRegistry( + onActiveCountChange?: (activeCount: number) => void, +): AppliedCommitCaptureRegistry { + const activeCaptures = new Set>>() + const notifyActiveCount = () => onActiveCountChange?.(activeCaptures.size) + + return { + capture: (signal) => { + const receipts = new Set>() + let active = true + const dispose = () => { + if (!active) return + active = false + signal?.removeEventListener(`abort`, dispose) + activeCaptures.delete(receipts) + notifyActiveCount() + } + + activeCaptures.add(receipts) + notifyActiveCount() + if (signal?.aborted) { + dispose() + } else { + signal?.addEventListener(`abort`, dispose, { once: true }) + } + + return { + wait: async () => { + dispose() + await Promise.all(receipts) + }, + dispose, + } + }, + record: (receipt) => { + if (receipt === true || activeCaptures.size === 0) return + + for (const receipts of activeCaptures) { + receipts.add(receipt) + } + // A receipt can reject before its request Promise settles. Observe it + // now while retaining the original Promise for every capture to await. + void receipt.catch(() => undefined) + }, + get activeCount() { + return activeCaptures.size + }, + } +} diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index 60605a048..a721ef777 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -18,6 +18,7 @@ import { TimeoutWaitingForMatchError, TimeoutWaitingForTxIdError, } from './errors' +import { createAppliedCommitCaptureRegistry } from './applied-commit-capture' import { compileSQL } from './sql-compiler' import { addTagToIndex, @@ -86,6 +87,8 @@ export interface ElectricTestHooks { * Allows tests to pause and validate snapshot phase before atomic swap completes */ beforeMarkingReady?: () => Promise + /** Reports the number of active on-demand applied-receipt captures. */ + onActiveCommitCapturesChange?: (activeCount: number) => void } /** @@ -543,7 +546,7 @@ function createLoadSubsetDedupe>({ metadata: Record }) => void commit: (signal?: AbortSignal) => SyncAppliedReceipt - captureCommits: () => { + captureCommits: (signal?: AbortSignal) => { wait: () => Promise dispose: () => void } @@ -737,7 +740,7 @@ function createLoadSubsetDedupe>({ // aborted request can already have installed rows before the check below. // Full request-scoped cancellation requires support in the Electric client; // matching snapshots by parameters is unsafe for overlapping equal requests. - const commitCapture = captureCommits() + const commitCapture = captureCommits(signal) try { try { if (cursor) { @@ -766,10 +769,14 @@ function createLoadSubsetDedupe>({ `${logPrefix}Requesting cursor.whereFrom snapshot (with limit ${limit})`, ) - await Promise.all([ + const requestResults = await Promise.allSettled([ stream.requestSnapshot(whereCurrentParams), stream.requestSnapshot(whereFromParams), ]) + const failedRequest = requestResults.find( + (result) => result.status === `rejected`, + ) + if (failedRequest) throw failedRequest.reason } else { const snapshotParams = compileSQL(opts, compileOptions) await stream.requestSnapshot(snapshotParams) @@ -1592,39 +1599,14 @@ function createElectricSync>( collection, metadata, } = params - const activeCommitCaptures = new Set>>() + const commitCaptures = createAppliedCommitCaptureRegistry( + testHooks?.onActiveCommitCapturesChange, + ) const commit = (signal?: AbortSignal): SyncAppliedReceipt => { const applied = commitSyncTransaction(signal) - if (applied === true) { - return true - } - if (activeCommitCaptures.size > 0) { - for (const receipts of activeCommitCaptures) { - receipts.add(applied) - } - // A receipt can reject before its request Promise settles. Observe it - // now while retaining the original Promise for the capture to await. - void applied.catch(() => undefined) - } + commitCaptures.record(applied) return applied } - const captureCommits = () => { - const receipts = new Set>() - let active = true - const dispose = () => { - if (!active) return - active = false - activeCommitCaptures.delete(receipts) - } - activeCommitCaptures.add(receipts) - return { - wait: async () => { - dispose() - await Promise.all(receipts) - }, - dispose, - } - } const readPersistedResumeState = (): ElectricResumeState | undefined => { const persistedResumeState = metadata?.collection.get(`electric:resume`) return parseElectricResumeState(persistedResumeState) @@ -1872,7 +1854,7 @@ function createElectricSync>( begin, write, commit, - captureCommits, + captureCommits: commitCaptures.capture, collectionId, // Pass the columnMapper's encode function to transform column names // (e.g., camelCase to snake_case) when compiling SQL for subset queries diff --git a/packages/electric-db-collection/tests/applied-commit-capture.test.ts b/packages/electric-db-collection/tests/applied-commit-capture.test.ts new file mode 100644 index 000000000..63abe5e11 --- /dev/null +++ b/packages/electric-db-collection/tests/applied-commit-capture.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, it, vi } from 'vitest' +import { createAppliedCommitCaptureRegistry } from '../src/applied-commit-capture' + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +describe(`applied commit capture`, () => { + it(`waits for every recorded receipt before settling`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const first = createDeferred() + const second = createDeferred() + registry.record(first.promise) + registry.record(second.promise) + + const wait = capture.wait() + second.resolve() + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + + await expect( + Promise.race([wait.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + expect(registry.activeCount).toBe(0) + + first.resolve() + await wait + }) + + it.each([`first`, `second`] as const)( + `propagates a settled %s receipt failure`, + async (failedReceipt) => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const first = createDeferred() + const second = createDeferred() + const failure = new Error(`${failedReceipt} receipt failed`) + registry.record(first.promise) + registry.record(second.promise) + + if (failedReceipt === `first`) { + first.reject(failure) + second.resolve() + } else { + first.resolve() + second.reject(failure) + } + + await expect(capture.wait()).rejects.toBe(failure) + expect(registry.activeCount).toBe(0) + }, + ) + + it(`records one receipt for every concurrent capture`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const firstCapture = registry.capture() + const secondCapture = registry.capture() + const receipt = createDeferred() + const failure = new Error(`shared receipt failed`) + registry.record(receipt.promise) + receipt.reject(failure) + + const errors = await Promise.all([ + firstCapture.wait().catch((error: unknown) => error), + secondCapture.wait().catch((error: unknown) => error), + ]) + expect(errors).toEqual([failure, failure]) + expect(registry.activeCount).toBe(0) + }) + + it(`disposes a capture as soon as its lifetime signal aborts`, () => { + const registry = createAppliedCommitCaptureRegistry() + const controller = new AbortController() + const addSpy = vi.spyOn(controller.signal, `addEventListener`) + const removeSpy = vi.spyOn(controller.signal, `removeEventListener`) + registry.capture(controller.signal) + + expect(registry.activeCount).toBe(1) + expect(addSpy).toHaveBeenCalledOnce() + + controller.abort() + + expect(registry.activeCount).toBe(0) + expect(removeSpy).toHaveBeenCalledOnce() + }) + + it(`does not retain a capture for an already-aborted lifetime`, () => { + const registry = createAppliedCommitCaptureRegistry() + const controller = new AbortController() + controller.abort() + + registry.capture(controller.signal) + + expect(registry.activeCount).toBe(0) + }) + + it.each([`wait`, `dispose`] as const)( + `removes a capture after %s`, + async (settlement) => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + + if (settlement === `wait`) await capture.wait() + else capture.dispose() + + expect(registry.activeCount).toBe(0) + }, + ) +}) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index a7958d3df..5a0177c5d 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -7,7 +7,11 @@ import { createTransaction, } from '@tanstack/db' import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' -import { electricCollectionOptions, isChangeMessage } from '../src/electric' +import { + ELECTRIC_TEST_HOOKS, + electricCollectionOptions, + isChangeMessage, +} from '../src/electric' import { stripVirtualProps } from '../../db/tests/utils' import type { ElectricCollectionUtils } from '../src/electric' import type { @@ -2997,6 +3001,57 @@ describe(`Electric Integration`, () => { }, ) + it.each([`collection`, `cleanup`] as const)( + `disposes the active commit capture during %s cancellation while the request remains pending`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const activeCaptureCounts: Array = [] + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-${cancellationSource}-pending-capture-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + [ELECTRIC_TEST_HOOKS]: { + onActiveCommitCapturesChange: (activeCount) => + activeCaptureCounts.push(activeCount), + }, + }), + ) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + expect(activeCaptureCounts.at(-1)).toBe(1) + + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } + + expect(activeCaptureCounts.at(-1)).toBe(0) + request.resolve() + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + it(`waits for a successful on-demand commit to apply`, async () => { const request = createDeferred() mockRequestSnapshot.mockReturnValueOnce(request.promise) @@ -3378,70 +3433,85 @@ describe(`Electric Integration`, () => { } }) - it(`propagates one failed cursor request while its sibling can still publish`, async () => { - const whereCurrent = createDeferred() - const whereFrom = createDeferred() - mockRequestSnapshot - .mockReturnValueOnce(whereCurrent.promise) - .mockReturnValueOnce(whereFrom.promise) - const testCollection = createOnDemandCollection( - `on-demand-cursor-request-error-test`, - ) - const abortController = new AbortController() - const failure = new Error(`cursor request failed`) - const id = new IR.PropRef([`id`]) + it.each([`whereCurrent`, `whereFrom`] as const)( + `waits for the cursor sibling after $failedRequest rejects`, + async (failedRequest) => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-${failedRequest}-error-test`, + ) + const abortController = new AbortController() + const failure = new Error(`${failedRequest} request failed`) + const id = new IR.PropRef([`id`]) - try { - const load = Promise.resolve( - testCollection._sync.loadSubset({ - limit: 10, - orderBy: [ - { - expression: id, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `lexical`, + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, }, - ], - cursor: { - whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), - whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), - lastKey: 1, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + abortController.abort() + const failed = + failedRequest === `whereCurrent` ? whereCurrent : whereFrom + const sibling = + failedRequest === `whereCurrent` ? whereFrom : whereCurrent + failed.reject(failure) + + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([loadError.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Late cursor row` }, + headers: { operation: `insert` }, }, - signal: abortController.signal, - }), - ) - const loadError = load.then( - () => undefined, - (error: unknown) => error, - ) - await vi.waitFor(() => - expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), - ) - abortController.abort() - whereCurrent.reject(failure) + { headers: { control: `subset-end` } }, + ]) + sibling.resolve() - await expect(loadError).resolves.toBe(failure) - subscriber([ - { - key: `2`, - value: { id: 2, name: `Late cursor row` }, - headers: { operation: `insert` }, - }, - { headers: { control: `subset-end` } }, - ]) - whereFrom.resolve() - await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) - await load.catch(() => undefined) - } finally { - abortController.abort() - whereCurrent.resolve() - whereFrom.resolve() - await testCollection.cleanup() - } - }) + await expect(loadError).resolves.toBe(failure) + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await load.catch(() => undefined) + } finally { + abortController.abort() + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }, + ) it(`waits for both cursor snapshot requests before settling`, async () => { const whereCurrent = createDeferred() From b90254cf63510b7ed454dbb28aa8103235861664 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Fri, 28 Aug 2026 23:58:02 -0600 Subject: [PATCH 055/327] test(electric): prove capture settlement boundaries --- .../tests/applied-commit-capture.test.ts | 15 ++++ .../tests/electric.test.ts | 72 +++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/packages/electric-db-collection/tests/applied-commit-capture.test.ts b/packages/electric-db-collection/tests/applied-commit-capture.test.ts index 63abe5e11..f315190ae 100644 --- a/packages/electric-db-collection/tests/applied-commit-capture.test.ts +++ b/packages/electric-db-collection/tests/applied-commit-capture.test.ts @@ -35,6 +35,19 @@ describe(`applied commit capture`, () => { await wait }) + it(`seals the receipt set before waiting`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const lateReceipt = createDeferred() + + const wait = capture.wait() + registry.record(lateReceipt.promise) + + expect(registry.activeCount).toBe(0) + await expect(wait).resolves.toBeUndefined() + lateReceipt.resolve() + }) + it.each([`first`, `second`] as const)( `propagates a settled %s receipt failure`, async (failedReceipt) => { @@ -96,10 +109,12 @@ describe(`applied commit capture`, () => { const registry = createAppliedCommitCaptureRegistry() const controller = new AbortController() controller.abort() + const addSpy = vi.spyOn(controller.signal, `addEventListener`) registry.capture(controller.signal) expect(registry.activeCount).toBe(0) + expect(addSpy).not.toHaveBeenCalled() }) it.each([`wait`, `dispose`] as const)( diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 5a0177c5d..6052639a2 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3513,6 +3513,78 @@ describe(`Electric Integration`, () => { }, ) + it.each([`whereCurrent`, `whereFrom`] as const)( + `uses stable cursor error priority when $firstFailure rejects first`, + async (firstFailure) => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-${firstFailure}-first-double-error-test`, + ) + const currentFailure = new Error(`whereCurrent request failed`) + const fromFailure = new Error(`whereFrom request failed`) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + + const first = + firstFailure === `whereCurrent` ? whereCurrent : whereFrom + const second = + firstFailure === `whereCurrent` ? whereFrom : whereCurrent + first.reject( + firstFailure === `whereCurrent` ? currentFailure : fromFailure, + ) + + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([loadError.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + second.reject( + firstFailure === `whereCurrent` ? fromFailure : currentFailure, + ) + await expect(loadError).resolves.toBe(currentFailure) + await load.catch(() => undefined) + } finally { + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }, + ) + it(`waits for both cursor snapshot requests before settling`, async () => { const whereCurrent = createDeferred() const whereFrom = createDeferred() From 0a23452ade2042e0fbf27f35dad2af906b73fab5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:08:10 -0600 Subject: [PATCH 056/327] test(electric): observe early receipt failures --- .../tests/applied-commit-capture.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/electric-db-collection/tests/applied-commit-capture.test.ts b/packages/electric-db-collection/tests/applied-commit-capture.test.ts index f315190ae..88f6dd3bb 100644 --- a/packages/electric-db-collection/tests/applied-commit-capture.test.ts +++ b/packages/electric-db-collection/tests/applied-commit-capture.test.ts @@ -72,6 +72,19 @@ describe(`applied commit capture`, () => { }, ) + it(`observes a receipt failure before waiting begins`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const receipt = createDeferred() + const failure = new Error(`receipt failed before wait`) + registry.record(receipt.promise) + + receipt.reject(failure) + await new Promise((resolve) => setTimeout(resolve, 0)) + + await expect(capture.wait()).rejects.toBe(failure) + }) + it(`records one receipt for every concurrent capture`, async () => { const registry = createAppliedCommitCaptureRegistry() const firstCapture = registry.capture() From 187dff5f588d254aacb7aaa1f0acd4d70c5108e0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:17:09 -0600 Subject: [PATCH 057/327] fix(db): release subset leases on reset --- packages/db/src/query/subset-dedupe.ts | 1 + packages/db/tests/query/subset-dedupe.test.ts | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index f5b98b253..8c7037a0b 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -228,6 +228,7 @@ export class DeduplicatedLoadSubset { this.unlimitedWhere = undefined this.hasLoadedAllData = false this.limitedCalls = [] + for (const inflight of this.inflightCalls) inflight.lease.dispose() this.inflightCalls = [] // Increment generation to invalidate any in-flight completion handlers // This ensures requests that were started before reset() don't repopulate the state diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 04f3aded8..50bb9df4f 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -172,6 +172,52 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(deduplicated.loadSubset({ where })).toBe(true) }) + it(`releases in-flight cancellation owners when reset`, async () => { + const releases: Array<() => void> = [] + const sharedSignals: Array = [] + const loadSubset = vi.fn( + (options: LoadSubsetOptions) => + new Promise((resolve) => { + sharedSignals.push(options.signal) + releases.push(resolve) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const owners = [new AbortController(), new AbortController()] + const addSpies = owners.map((owner) => + vi.spyOn(owner.signal, `addEventListener`), + ) + const removeSpies = owners.map((owner) => + vi.spyOn(owner.signal, `removeEventListener`), + ) + + const loads = [ + deduplicated.loadSubset({ + where: gt(ref(`age`), val(10)), + signal: owners[0]!.signal, + }), + deduplicated.loadSubset({ + where: lt(ref(`age`), val(0)), + signal: owners[1]!.signal, + }), + ] + expect(loadSubset).toHaveBeenCalledTimes(2) + for (const addSpy of addSpies) expect(addSpy).toHaveBeenCalledOnce() + + deduplicated.reset() + + for (const removeSpy of removeSpies) + expect(removeSpy).toHaveBeenCalledOnce() + for (const signal of sharedSignals) expect(signal?.aborted).toBe(false) + for (const owner of owners) owner.abort() + for (const signal of sharedSignals) expect(signal?.aborted).toBe(false) + + for (const release of releases) release() + await Promise.all(loads) + for (const removeSpy of removeSpies) + expect(removeSpy).toHaveBeenCalledOnce() + }) + it(`should call underlying loadSubset on first call`, async () => { let callCount = 0 const mockLoadSubset = () => { From 184d0d46397b2c7d9cde13ddf159dbf827cae029 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:25:20 -0600 Subject: [PATCH 058/327] test(db): prove subset reset isolation --- packages/db/tests/query/subset-dedupe.test.ts | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 50bb9df4f..3098a391b 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -172,7 +172,7 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(deduplicated.loadSubset({ where })).toBe(true) }) - it(`releases in-flight cancellation owners when reset`, async () => { + it(`releases every owner from every in-flight lease when reset`, async () => { const releases: Array<() => void> = [] const sharedSignals: Array = [] const loadSubset = vi.fn( @@ -183,7 +183,7 @@ describe(`createDeduplicatedLoadSubset`, () => { }), ) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) - const owners = [new AbortController(), new AbortController()] + const owners = Array.from({ length: 4 }, () => new AbortController()) const addSpies = owners.map((owner) => vi.spyOn(owner.signal, `addEventListener`), ) @@ -197,9 +197,17 @@ describe(`createDeduplicatedLoadSubset`, () => { signal: owners[0]!.signal, }), deduplicated.loadSubset({ - where: lt(ref(`age`), val(0)), + where: gt(ref(`age`), val(10)), signal: owners[1]!.signal, }), + deduplicated.loadSubset({ + where: lt(ref(`age`), val(0)), + signal: owners[2]!.signal, + }), + deduplicated.loadSubset({ + where: lt(ref(`age`), val(0)), + signal: owners[3]!.signal, + }), ] expect(loadSubset).toHaveBeenCalledTimes(2) for (const addSpy of addSpies) expect(addSpy).toHaveBeenCalledOnce() @@ -218,6 +226,32 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(removeSpy).toHaveBeenCalledOnce() }) + it(`starts new work immediately after reset and protects it from old completion`, async () => { + const releases: Array<() => void> = [] + const loadSubset = vi.fn( + () => new Promise((resolve) => releases.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const where = gt(ref(`age`), val(10)) + + const oldLoad = deduplicated.loadSubset({ where }) + deduplicated.reset() + const currentLoad = deduplicated.loadSubset({ where }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(currentLoad).not.toBe(oldLoad) + + releases[0]?.() + await oldLoad + + const joinedLoad = deduplicated.loadSubset({ where }) + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(joinedLoad).toBe(currentLoad) + + releases[1]?.() + await Promise.all([currentLoad, joinedLoad]) + }) + it(`should call underlying loadSubset on first call`, async () => { let callCount = 0 const mockLoadSubset = () => { From 7b90273faad31e993e8506735818d141794a1e51 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:34:53 -0600 Subject: [PATCH 059/327] test(electric): prove commit lifetime boundaries --- .../tests/electric.test.ts | 130 ++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 6052639a2..00e1e9519 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3215,6 +3215,66 @@ describe(`Electric Integration`, () => { } }) + it(`prefers collection cancellation over an already-rejected applied receipt`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const receiptFailure = new Error(`applied receipt failed`) + const options = electricCollectionOptions({ + id: `on-demand-pre-wait-collection-cancel-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = vi.fn(() => Promise.reject(receiptFailure)) + const controls = options.sync.sync({ + collection: { + id: options.id, + status: `loading`, + getKeyFromItem: (item: Row) => item.id, + }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!controls || typeof controls === `function` || !controls.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + try { + const load = Promise.resolve(controls.loadSubset({ limit: 10 })) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Rejected receipt row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + + collectionAbortController.abort() + request.resolve() + + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + controls.cleanup?.() + } + }) + it.each([`success`, `rejection`, `cancellation`] as const)( `removes the on-demand request lease listener after %s`, async (settlement) => { @@ -4866,6 +4926,76 @@ describe(`Electric Integration`, () => { }, ) + it.each([`progressive atomic-swap`, `metadata-only`] as const)( + `binds the %s commit to collection lifetime`, + (commitPath) => { + const collectionAbortController = new AbortController() + const receipt = createDeferred() + const metadataHarness = createInMemorySyncMetadataApi() + const isProgressive = commitPath === `progressive atomic-swap` + let commitSignal: AbortSignal | undefined + const options = electricCollectionOptions({ + id: `${commitPath.replaceAll(` `, `-`)}-commit-signal-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: isProgressive ? `progressive` : `eager`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = vi.fn((signal?: AbortSignal) => { + commitSignal = signal + return receipt.promise + }) + const controls = options.sync.sync({ + collection: { + id: options.id, + status: `loading`, + getKeyFromItem: (item: Row) => item.id, + }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + metadata: isProgressive ? undefined : metadataHarness.api, + } as never) + if (!controls || typeof controls === `function`) { + throw new Error(`Expected sync controls`) + } + + try { + if (isProgressive) { + subscriber([ + { + key: `2`, + value: { id: 2, name: `Buffered row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `up-to-date` } }, + ]) + } else { + subscriber([{ headers: { control: `up-to-date` } }]) + } + + expect(commitMock).toHaveBeenCalledOnce() + expect(commitSignal).toBeDefined() + expect(commitSignal?.aborted).toBe(false) + + collectionAbortController.abort() + + expect(commitSignal?.aborted).toBe(true) + } finally { + collectionAbortController.abort() + receipt.resolve() + controls.cleanup?.() + } + }, + ) + it(`should not request snapshots when loadSubset is called in eager mode`, async () => { vi.clearAllMocks() From 70b10d97163146aaa884055cd60f83f64739e585 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:44:08 -0600 Subject: [PATCH 060/327] test(electric): cover cleanup lifetime boundaries --- .../tests/electric.test.ts | 142 ++++++++++-------- 1 file changed, 82 insertions(+), 60 deletions(-) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 00e1e9519..d122d0ac0 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -3215,65 +3215,76 @@ describe(`Electric Integration`, () => { } }) - it(`prefers collection cancellation over an already-rejected applied receipt`, async () => { - const request = createDeferred() - mockRequestSnapshot.mockReturnValueOnce(request.promise) - const collectionAbortController = new AbortController() - const receiptFailure = new Error(`applied receipt failed`) - const options = electricCollectionOptions({ - id: `on-demand-pre-wait-collection-cancel-test`, - shapeOptions: { - url: `http://test-url`, - params: { table: `test_table` }, - signal: collectionAbortController.signal, - }, - syncMode: `on-demand`, - getKey: (item: Row) => item.id as number, - startSync: true, - }) - const commitMock = vi.fn(() => Promise.reject(receiptFailure)) - const controls = options.sync.sync({ - collection: { - id: options.id, - status: `loading`, - getKeyFromItem: (item: Row) => item.id, - }, - begin: vi.fn(), - write: vi.fn(), - commit: commitMock, - markReady: vi.fn(), - markError: vi.fn(), - truncate: vi.fn(), - } as never) - if (!controls || typeof controls === `function` || !controls.loadSubset) { - throw new Error(`Expected on-demand sync controls`) - } - - try { - const load = Promise.resolve(controls.loadSubset({ limit: 10 })) - await vi.waitFor(() => - expect(mockRequestSnapshot).toHaveBeenCalledOnce(), - ) - subscriber([ - { - key: `2`, - value: { id: 2, name: `Rejected receipt row` }, - headers: { operation: `insert` }, + it.each([`external abort`, `cleanup`] as const)( + `prefers %s over an already-rejected applied receipt`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const receiptFailure = new Error(`applied receipt failed`) + const options = electricCollectionOptions({ + id: `on-demand-pre-wait-collection-cancel-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, }, - { headers: { control: `subset-end` } }, - ]) - await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = vi.fn(() => Promise.reject(receiptFailure)) + const controls = options.sync.sync({ + collection: { + id: options.id, + status: `loading`, + getKeyFromItem: (item: Row) => item.id, + }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } - collectionAbortController.abort() - request.resolve() + try { + const load = Promise.resolve(controls.loadSubset({ limit: 10 })) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Rejected receipt row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) - await expect(load).rejects.toMatchObject({ name: `AbortError` }) - } finally { - collectionAbortController.abort() - request.resolve() - controls.cleanup?.() - } - }) + if (cancellationSource === `external abort`) { + collectionAbortController.abort() + } else { + controls.cleanup?.() + } + request.resolve() + + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + controls.cleanup?.() + } + }, + ) it.each([`success`, `rejection`, `cancellation`] as const)( `removes the on-demand request lease listener after %s`, @@ -4926,9 +4937,16 @@ describe(`Electric Integration`, () => { }, ) - it.each([`progressive atomic-swap`, `metadata-only`] as const)( - `binds the %s commit to collection lifetime`, - (commitPath) => { + it.each( + ([`progressive atomic-swap`, `metadata-only`] as const).flatMap( + (commitPath) => + ([`external abort`, `cleanup`] as const).map( + (cancellationSource) => [commitPath, cancellationSource] as const, + ), + ), + )( + `binds the %s commit to collection lifetime through %s`, + (commitPath, cancellationSource) => { const collectionAbortController = new AbortController() const receipt = createDeferred() const metadataHarness = createInMemorySyncMetadataApi() @@ -4985,7 +5003,11 @@ describe(`Electric Integration`, () => { expect(commitSignal).toBeDefined() expect(commitSignal?.aborted).toBe(false) - collectionAbortController.abort() + if (cancellationSource === `external abort`) { + collectionAbortController.abort() + } else { + controls.cleanup?.() + } expect(commitSignal?.aborted).toBe(true) } finally { From 61f66ef9112d7cf5bfaa33372930d05bd23cc020 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 03:51:07 -0600 Subject: [PATCH 061/327] test(powersync): restore startup error spy --- .../powersync-db-collection/tests/on-demand-sync.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index ea54dbe1c..fcf95476b 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2604,7 +2604,10 @@ describe(`On-Demand Sync Mode`, () => { const db = await createDatabase() const startupError = new Error(`change observation failed`) vi.spyOn(db.logger, `error`).mockImplementation(() => {}) - vi.spyOn(console, `error`).mockImplementation(() => {}) + const consoleError = vi + .spyOn(console, `error`) + .mockImplementation(() => {}) + onTestFinished(() => consoleError.mockRestore()) vi.spyOn(db, `onChangeWithCallback`).mockImplementation(() => { throw startupError }) From 84c6b1f560d17644612dbd869fe58b7947008019 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 04:00:36 -0600 Subject: [PATCH 062/327] test(powersync): prove applied subset outcomes --- .../tests/on-demand-sync.test.ts | 123 ++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index fcf95476b..9ea96829a 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' import { + IR, and, createCollection, createLiveQueryCollection, @@ -218,6 +219,128 @@ describe(`On-Demand Sync Mode`, () => { } }) + it.each([ + { source: `rows`, settlement: `fulfill` }, + { source: `empty`, settlement: `fulfill` }, + { source: `rows`, settlement: `reject` }, + { source: `empty`, settlement: `reject` }, + ] as const)( + `settles a $source subset only through an applied $settlement outcome`, + async ({ source, settlement }) => { + type ProductRow = { + id: string + name: string + price: number + category: string + } + type StagedChange = { + type: `insert` | `update` | `delete` + value?: ProductRow + key?: string + } + + const db = await createDatabase() + await createTestProducts(db) + const category = source === `rows` ? `electronics` : `furniture` + const authoritativeRows = await db.getAll( + `SELECT id, name, price, category FROM products WHERE category = ?`, + [category], + ) + expect(authoritativeRows.length > 0).toBe(source === `rows`) + const receipt = pDefer() + const receiptFailure = new Error(`applied receipt failed`) + const readableRows = new Map() + let stagedChanges: Array = [] + const commit = vi.fn(() => { + const changes = stagedChanges + stagedChanges = [] + return receipt.promise.then(() => { + for (const change of changes) { + if (change.type === `delete`) { + if (!change.key) throw new Error(`Delete requires a key`) + readableRows.delete(change.key) + } else { + if (!change.value) throw new Error(`Write requires a value`) + readableRows.set(change.value.id, change.value) + } + } + }) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { + status: `ready`, + has: (key: string) => readableRows.has(key), + }, + begin: vi.fn(() => { + stagedChanges = [] + }), + write: vi.fn((change: StagedChange) => { + stagedChanges.push(change) + }), + commit, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + let settled = false + const where = new IR.Func(`eq`, [ + new IR.PropRef([`category`]), + new IR.Value(category), + ]) + const observed = Promise.resolve( + sync.loadSubset({ where }), + ).then( + () => { + settled = true + return { status: `fulfilled` } as const + }, + (reason: unknown) => { + settled = true + return { status: `rejected`, reason } as const + }, + ) + + try { + await vi.waitFor(() => expect(commit).toHaveBeenCalledOnce()) + expect(settled).toBe(false) + expect(readableRows.size).toBe(0) + + if (settlement === `reject`) { + receipt.reject(receiptFailure) + } else { + receipt.resolve() + } + + const result = await observed + if (settlement === `reject`) { + expect(result).toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect(readableRows.size).toBe(0) + } else { + expect(result).toEqual({ status: `fulfilled` }) + expect( + Array.from(readableRows.values(), (row) => row.name).sort(), + ).toEqual(authoritativeRows.map((row) => row.name).sort()) + } + } finally { + receipt.resolve() + sync.cleanup?.() + await observed + } + }, + ) + it(`should reactively update live query when new matching data is inserted into SQLite`, async () => { const db = await createDatabase() await createTestProducts(db) From 382156e8540d7efc2ef0112c2ea668dc09c807eb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 06:27:40 -0600 Subject: [PATCH 063/327] test(powersync): cover multi-batch applied outcomes --- .../tests/on-demand-sync.test.ts | 281 ++++++++++++------ 1 file changed, 189 insertions(+), 92 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 9ea96829a..5390a4546 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -58,6 +58,118 @@ describe(`On-Demand Sync Mode`, () => { `) } + type ProductRow = { + id: string + name: string + price: number + category: string + } + + type StagedChange = { + type: `insert` | `update` | `delete` + value?: ProductRow + key?: string + } + + type ControlledReceipt = { + promise: Promise + resolve: () => void + reject: (reason: unknown) => void + } + + async function startAppliedOutcomeLoad( + source: `rows` | `empty`, + syncBatchSize?: number, + receiptMode: `controlled` | `immediate` = `controlled`, + ) { + const db = await createDatabase() + await createTestProducts(db) + const category = source === `rows` ? `electronics` : `furniture` + const authoritativeRows = await db.getAll( + `SELECT id, name, price, category FROM products WHERE category = ?`, + [category], + ) + const receipts: Array = [] + const readableRows = new Map() + let stagedChanges: Array = [] + const applyChanges = (changes: Array) => { + for (const change of changes) { + if (change.type === `delete`) { + if (!change.key) throw new Error(`Delete requires a key`) + readableRows.delete(change.key) + } else { + if (!change.value) throw new Error(`Write requires a value`) + readableRows.set(change.value.id, change.value) + } + } + } + const commit = vi.fn(() => { + const changes = stagedChanges + stagedChanges = [] + if (receiptMode === `immediate`) { + applyChanges(changes) + return true + } + const receipt = pDefer() + receipts.push(receipt) + return receipt.promise.then(() => applyChanges(changes)) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + ...(syncBatchSize === undefined ? {} : { syncBatchSize }), + }) + const sync = config.sync.sync({ + collection: { + status: `ready`, + has: (key: string) => readableRows.has(key), + }, + begin: vi.fn(() => { + stagedChanges = [] + }), + write: vi.fn((change: StagedChange) => { + stagedChanges.push(change) + }), + commit, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + let settled = false + const where = new IR.Func(`eq`, [ + new IR.PropRef([`category`]), + new IR.Value(category), + ]) + const observed = Promise.resolve(sync.loadSubset({ where })).then( + () => { + settled = true + return { status: `fulfilled` } as const + }, + (reason: unknown) => { + settled = true + return { status: `rejected`, reason } as const + }, + ) + + return { + authoritativeRows, + readableRows, + receipts, + observed, + isSettled: () => settled, + cleanup: async () => { + receipts.forEach((receipt) => receipt.resolve()) + sync.cleanup?.() + await observed + }, + } + } + it(`should not load any data initially in on-demand mode`, async () => { const db = await createDatabase() await createTestProducts(db) @@ -227,116 +339,101 @@ describe(`On-Demand Sync Mode`, () => { ] as const)( `settles a $source subset only through an applied $settlement outcome`, async ({ source, settlement }) => { - type ProductRow = { - id: string - name: string - price: number - category: string - } - type StagedChange = { - type: `insert` | `update` | `delete` - value?: ProductRow - key?: string - } - - const db = await createDatabase() - await createTestProducts(db) - const category = source === `rows` ? `electronics` : `furniture` - const authoritativeRows = await db.getAll( - `SELECT id, name, price, category FROM products WHERE category = ?`, - [category], - ) - expect(authoritativeRows.length > 0).toBe(source === `rows`) - const receipt = pDefer() + const harness = await startAppliedOutcomeLoad(source) const receiptFailure = new Error(`applied receipt failed`) - const readableRows = new Map() - let stagedChanges: Array = [] - const commit = vi.fn(() => { - const changes = stagedChanges - stagedChanges = [] - return receipt.promise.then(() => { - for (const change of changes) { - if (change.type === `delete`) { - if (!change.key) throw new Error(`Delete requires a key`) - readableRows.delete(change.key) - } else { - if (!change.value) throw new Error(`Write requires a value`) - readableRows.set(change.value.id, change.value) - } - } - }) - }) - const config = powerSyncCollectionOptions({ - database: db, - table: APP_SCHEMA.props.products, - syncMode: `on-demand`, - }) - const sync = config.sync.sync({ - collection: { - status: `ready`, - has: (key: string) => readableRows.has(key), - }, - begin: vi.fn(() => { - stagedChanges = [] - }), - write: vi.fn((change: StagedChange) => { - stagedChanges.push(change) - }), - commit, - markReady: vi.fn(), - markError: vi.fn(), - truncate: vi.fn(), - } as never) - if (!sync || typeof sync === `function` || !sync.loadSubset) { - throw new Error(`Expected on-demand sync controls`) - } - - let settled = false - const where = new IR.Func(`eq`, [ - new IR.PropRef([`category`]), - new IR.Value(category), - ]) - const observed = Promise.resolve( - sync.loadSubset({ where }), - ).then( - () => { - settled = true - return { status: `fulfilled` } as const - }, - (reason: unknown) => { - settled = true - return { status: `rejected`, reason } as const - }, - ) + expect(harness.authoritativeRows.length > 0).toBe(source === `rows`) try { - await vi.waitFor(() => expect(commit).toHaveBeenCalledOnce()) - expect(settled).toBe(false) - expect(readableRows.size).toBe(0) + await vi.waitFor(() => expect(harness.receipts).toHaveLength(1)) + expect(harness.isSettled()).toBe(false) + expect(harness.readableRows.size).toBe(0) if (settlement === `reject`) { - receipt.reject(receiptFailure) + harness.receipts[0]!.reject(receiptFailure) } else { - receipt.resolve() + harness.receipts[0]!.resolve() } - const result = await observed + const result = await harness.observed if (settlement === `reject`) { expect(result).toEqual({ status: `rejected`, reason: receiptFailure, }) - expect(readableRows.size).toBe(0) + expect(harness.readableRows.size).toBe(0) } else { expect(result).toEqual({ status: `fulfilled` }) expect( - Array.from(readableRows.values(), (row) => row.name).sort(), - ).toEqual(authoritativeRows.map((row) => row.name).sort()) + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) } } finally { - receipt.resolve() - sync.cleanup?.() - await observed + await harness.cleanup() + } + }, + ) + + it.each([`fulfill`, `reject`] as const)( + `waits for every applied receipt when a multi-batch subset will %s`, + async (settlement) => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const laterFailure = new Error(`later applied receipt failed`) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + expect(harness.isSettled()).toBe(false) + expect(harness.readableRows.size).toBe(0) + + harness.receipts[0]!.resolve() + await vi.waitFor(() => expect(harness.readableRows.size).toBe(1)) + expect(harness.isSettled()).toBe(false) + + if (settlement === `reject`) { + harness.receipts[1]!.reject(laterFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: laterFailure, + }) + expect(harness.readableRows.size).toBe(1) + } else { + harness.receipts.slice(1).forEach((receipt) => receipt.resolve()) + await expect(harness.observed).resolves.toEqual({ + status: `fulfilled`, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } + } finally { + await harness.cleanup() + } + }, + ) + + it.each([`rows`, `empty`] as const)( + `accepts an immediate applied outcome for a %s subset`, + async (source) => { + const harness = await startAppliedOutcomeLoad( + source, + undefined, + `immediate`, + ) + + try { + expect(harness.authoritativeRows.length > 0).toBe(source === `rows`) + await expect(harness.observed).resolves.toEqual({ + status: `fulfilled`, + }) + expect(harness.receipts).toHaveLength(0) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() } }, ) From 80350d7e060015292efb6a4ad778c7f9dced82ef Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 08:02:37 -0600 Subject: [PATCH 064/327] test(powersync): cover every applied receipt --- .../tests/on-demand-sync.test.ts | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 5390a4546..aac379e43 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -414,6 +414,68 @@ describe(`On-Demand Sync Mode`, () => { }, ) + it(`waits for the first applied receipt after every later batch applies`, async () => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + + harness.receipts.slice(1).forEach((receipt) => receipt.resolve()) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe( + harness.authoritativeRows.length - 1, + ), + ) + expect(harness.isSettled()).toBe(false) + + harness.receipts[0]!.resolve() + await expect(harness.observed).resolves.toEqual({ + status: `fulfilled`, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }) + + it(`preserves rejection from the terminal empty-batch receipt`, async () => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const terminalFailure = new Error(`terminal applied receipt failed`) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + + harness.receipts.slice(0, -1).forEach((receipt) => receipt.resolve()) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe( + harness.authoritativeRows.length, + ), + ) + expect(harness.isSettled()).toBe(false) + + harness.receipts.at(-1)!.reject(terminalFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: terminalFailure, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }) + it.each([`rows`, `empty`] as const)( `accepts an immediate applied outcome for a %s subset`, async (source) => { From 94a29fcb89bb9edb278add52f3be68dae674107e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 08:14:04 -0600 Subject: [PATCH 065/327] test(powersync): quantify applied receipt positions --- .../tests/on-demand-sync.test.ts | 136 ++++++++---------- 1 file changed, 57 insertions(+), 79 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index aac379e43..83104cd0a 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -373,48 +373,7 @@ describe(`On-Demand Sync Mode`, () => { }, ) - it.each([`fulfill`, `reject`] as const)( - `waits for every applied receipt when a multi-batch subset will %s`, - async (settlement) => { - const harness = await startAppliedOutcomeLoad(`rows`, 1) - const laterFailure = new Error(`later applied receipt failed`) - - try { - await vi.waitFor(() => - expect(harness.receipts).toHaveLength( - harness.authoritativeRows.length + 1, - ), - ) - expect(harness.isSettled()).toBe(false) - expect(harness.readableRows.size).toBe(0) - - harness.receipts[0]!.resolve() - await vi.waitFor(() => expect(harness.readableRows.size).toBe(1)) - expect(harness.isSettled()).toBe(false) - - if (settlement === `reject`) { - harness.receipts[1]!.reject(laterFailure) - await expect(harness.observed).resolves.toEqual({ - status: `rejected`, - reason: laterFailure, - }) - expect(harness.readableRows.size).toBe(1) - } else { - harness.receipts.slice(1).forEach((receipt) => receipt.resolve()) - await expect(harness.observed).resolves.toEqual({ - status: `fulfilled`, - }) - expect( - Array.from(harness.readableRows.values(), (row) => row.name).sort(), - ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) - } - } finally { - await harness.cleanup() - } - }, - ) - - it(`waits for the first applied receipt after every later batch applies`, async () => { + it(`waits for every applied receipt before fulfilling a multi-batch subset`, async () => { const harness = await startAppliedOutcomeLoad(`rows`, 1) try { @@ -424,15 +383,17 @@ describe(`On-Demand Sync Mode`, () => { ), ) - harness.receipts.slice(1).forEach((receipt) => receipt.resolve()) - await vi.waitFor(() => - expect(harness.readableRows.size).toBe( - harness.authoritativeRows.length - 1, - ), - ) - expect(harness.isSettled()).toBe(false) - - harness.receipts[0]!.resolve() + for (const [index, receipt] of harness.receipts.entries()) { + receipt.resolve() + await vi.waitFor(() => + expect(harness.readableRows.size).toBe( + Math.min(index + 1, harness.authoritativeRows.length), + ), + ) + if (index < harness.receipts.length - 1) { + expect(harness.isSettled()).toBe(false) + } + } await expect(harness.observed).resolves.toEqual({ status: `fulfilled`, }) @@ -444,37 +405,54 @@ describe(`On-Demand Sync Mode`, () => { } }) - it(`preserves rejection from the terminal empty-batch receipt`, async () => { - const harness = await startAppliedOutcomeLoad(`rows`, 1) - const terminalFailure = new Error(`terminal applied receipt failed`) - - try { - await vi.waitFor(() => - expect(harness.receipts).toHaveLength( - harness.authoritativeRows.length + 1, - ), + it.each([ + { receiptIndex: 0 }, + { receiptIndex: 1 }, + { receiptIndex: 2 }, + { receiptIndex: 3 }, + ])( + `keeps applied receipt $receiptIndex independent in a multi-batch subset`, + async ({ receiptIndex }) => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const receiptFailure = new Error( + `applied receipt ${receiptIndex} failed`, ) - harness.receipts.slice(0, -1).forEach((receipt) => receipt.resolve()) - await vi.waitFor(() => - expect(harness.readableRows.size).toBe( - harness.authoritativeRows.length, - ), - ) - expect(harness.isSettled()).toBe(false) + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + expect(receiptIndex).toBeLessThan(harness.receipts.length) - harness.receipts.at(-1)!.reject(terminalFailure) - await expect(harness.observed).resolves.toEqual({ - status: `rejected`, - reason: terminalFailure, - }) - expect( - Array.from(harness.readableRows.values(), (row) => row.name).sort(), - ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) - } finally { - await harness.cleanup() - } - }) + harness.receipts.forEach((receipt, index) => { + if (index !== receiptIndex) receipt.resolve() + }) + const expectedRows = harness.authoritativeRows.filter( + (_row, index) => index !== receiptIndex, + ) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe(expectedRows.length), + ) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + expect(harness.isSettled()).toBe(false) + + harness.receipts[receiptIndex]!.reject(receiptFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }, + ) it.each([`rows`, `empty`] as const)( `accepts an immediate applied outcome for a %s subset`, From 854730e028751021f355fc07d06b0ae15afcaa0a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 08:29:16 -0600 Subject: [PATCH 066/327] test(powersync): preserve applied receipt fail-fast --- .../tests/on-demand-sync.test.ts | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 83104cd0a..e97ab08c5 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -454,6 +454,49 @@ describe(`On-Demand Sync Mode`, () => { }, ) + it.each([ + { receiptIndex: 0 }, + { receiptIndex: 1 }, + { receiptIndex: 2 }, + { receiptIndex: 3 }, + ])( + `fails fast at applied receipt $receiptIndex while later receipts remain pending`, + async ({ receiptIndex }) => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const receiptFailure = new Error( + `applied receipt ${receiptIndex} failed before its suffix settled`, + ) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + expect(receiptIndex).toBeLessThan(harness.receipts.length) + + harness.receipts + .slice(0, receiptIndex) + .forEach((receipt) => receipt.resolve()) + const expectedRows = harness.authoritativeRows.slice(0, receiptIndex) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe(expectedRows.length), + ) + + harness.receipts[receiptIndex]!.reject(receiptFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }, + ) + it.each([`rows`, `empty`] as const)( `accepts an immediate applied outcome for a %s subset`, async (source) => { From 51d60ad17f6ada9781ee4c4c21666b331cea6b6d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 08:55:54 -0600 Subject: [PATCH 067/327] fix(powersync): reconcile current tracking revision --- .../powersync-db-collection/src/powersync.ts | 138 ++++++++------ .../tests/on-demand-sync.test.ts | 180 ++++++++++++++++++ 2 files changed, 257 insertions(+), 61 deletions(-) diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index c5bac6fcd..e8a5336cc 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -621,6 +621,8 @@ function createPowerSyncCollectionConfig< let stopped = false let lifecycleGeneration = 0 let trackingRevision = 0 + let reconciledTrackingRevision = 0 + let rebuildPromise: Promise | null = null let drainingReleases = false let releaseRetryTimer: ReturnType | undefined const hasStopped = () => stopped @@ -638,66 +640,83 @@ function createPowerSyncCollectionConfig< .filter((demand) => demand.state === `active`) .map((demand) => demand.options.where) - const rebuildTracking = async (): Promise => { - const generation = lifecycleGeneration - const revision = trackingRevision - const isCurrent = () => + // One reconciliation owns every queued revision so callers cannot + // settle against a stale trigger configuration. + const reconcileTracking = async (): Promise => { + while ( !hasStopped() && - lifecycleGeneration === generation && - trackingRevision === revision - const appliedReceipts: Array = [] - - await database.writeLock(async (ctx) => { - if (!isCurrent()) return - await flushDiffRecordsWithContext(ctx, appliedReceipts) - if (!isCurrent()) return - await safelyDisposeTracking(ctx) - if (!isCurrent()) return - - const active = activeWhereExpressions() - if (active.length === 0) return - const combinedWhere = - active.length === 1 - ? active[0] - : or(active[0], active[1], ...active.slice(2)) - const compiledNewData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, - ) - const compiledOldData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, - ) - const compiledView = compileSQLite({ where: combinedWhere }) - const newDataWhenClause = toInlinedWhereClause(compiledNewData) - const oldDataWhenClause = toInlinedWhereClause(compiledOldData) - const viewWhereClause = toInlinedWhereClause(compiledView) - - await establishTracking( - { - setupContext: ctx, - when: { - [DiffTriggerOperation.INSERT]: newDataWhenClause, - [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, - [DiffTriggerOperation.DELETE]: oldDataWhenClause, + reconciledTrackingRevision !== trackingRevision + ) { + const generation = lifecycleGeneration + const revision = trackingRevision + const isCurrent = () => + !hasStopped() && + lifecycleGeneration === generation && + trackingRevision === revision + const appliedReceipts: Array = [] + + await database.writeLock(async (ctx) => { + if (!isCurrent()) return + await flushDiffRecordsWithContext(ctx, appliedReceipts) + if (!isCurrent()) return + await safelyDisposeTracking(ctx) + if (!isCurrent()) return + + const active = activeWhereExpressions() + if (active.length === 0) return + const combinedWhere = + active.length === 1 + ? active[0] + : or(active[0], active[1], ...active.slice(2)) + const compiledNewData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'NEW.data' }, + ) + const compiledOldData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'OLD.data' }, + ) + const compiledView = compileSQLite({ where: combinedWhere }) + const newDataWhenClause = toInlinedWhereClause(compiledNewData) + const oldDataWhenClause = toInlinedWhereClause(compiledOldData) + const viewWhereClause = toInlinedWhereClause(compiledView) + + await establishTracking( + { + setupContext: ctx, + when: { + [DiffTriggerOperation.INSERT]: newDataWhenClause, + [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, + [DiffTriggerOperation.DELETE]: oldDataWhenClause, + }, + writeType: (rowId: string) => + collection.has(rowId) ? `update` : `insert`, + batchQuery: ( + lockContext: LockContext, + batchSize: number, + cursor: number, + ) => + lockContext.getAll( + `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, + [batchSize, cursor], + ), }, - writeType: (rowId: string) => - collection.has(rowId) ? `update` : `insert`, - batchQuery: ( - lockContext: LockContext, - batchSize: number, - cursor: number, - ) => - lockContext.getAll( - `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, - [batchSize, cursor], - ), - }, - appliedReceipts, - ) - if (!isCurrent()) await safelyDisposeTracking(ctx) + appliedReceipts, + ) + if (!isCurrent()) await safelyDisposeTracking(ctx) + }) + await Promise.all(appliedReceipts) + if (isCurrent()) { + reconciledTrackingRevision = revision + } + } + } + + const rebuildTracking = (): Promise => { + rebuildPromise ??= reconcileTracking().finally(() => { + rebuildPromise = null }) - await Promise.all(appliedReceipts) + return rebuildPromise } const loadSubset = async ( @@ -716,14 +735,12 @@ function createPowerSyncCollectionConfig< const demand: DemandRecord = { options, state: `provisional` } demands.set(options, demand) - trackingRevision++ try { const cleanup = await restConfig.onLoadSubset?.(options) if (cleanup) demand.cleanup = cleanup } catch (error) { demand.state = `failed` demands.delete(options) - trackingRevision++ throw error } @@ -735,7 +752,6 @@ function createPowerSyncCollectionConfig< ) { demand.state = `released` demands.delete(options) - trackingRevision++ demand.cleanup?.() return } @@ -848,7 +864,7 @@ function createPowerSyncCollectionConfig< const wasActive = demand.state === `active` demand.state = `released` demands.delete(options) - trackingRevision++ + if (wasActive) trackingRevision++ try { demand.cleanup?.() } catch (error) { diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index e97ab08c5..409efa54e 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2533,6 +2533,88 @@ describe(`On-Demand Sync Mode`, () => { }) } + function queueWriteLocks(db: PowerSyncDatabase) { + const queued: Array<() => Promise> = [] + vi.spyOn(db, `writeLock`).mockImplementation( + (callback) => + new Promise((resolve, reject) => { + let started = false + queued.push(async () => { + if (started) return + started = true + try { + const result = await callback({} as never) + resolve(result as never) + } catch (error) { + reject(error) + } + }) + }) as never, + ) + return queued + } + + async function startConcurrentLifecycleHarness() { + const db = await createDatabase() + const hooks: Array>> = [] + const hookCleanups: Array> = [] + const onLoadSubset = vi.fn(() => { + const hook = pDefer() + hooks.push(hook) + const cleanup = vi.fn() + hookCleanups.push(cleanup) + return hook.promise.then(() => cleanup) + }) + const queuedLocks = queueWriteLocks(db) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const first = { where: eq(`category`, `electronics`) } + const second = { where: eq(`category`, `clothing`) } + const loadSubset = sync.loadSubset + const unloadSubset = sync.unloadSubset + + return { + sync, + loadSubset, + unloadSubset, + first, + second, + hooks, + hookCleanups, + queuedLocks, + createDiffTrigger, + cleanup: async () => { + hooks.forEach((hook) => hook.resolve()) + sync.cleanup?.() + await Promise.all(queuedLocks.map((run) => run())) + }, + } + } + it(`does not acquire a subset released while tracking startup is suspended`, async () => { const db = await createDatabase() const onLoadSubset = vi.fn() @@ -2578,6 +2660,104 @@ describe(`On-Demand Sync Mode`, () => { } }) + it.each([`reject`, `release`] as const)( + `keeps an active rebuild current when a provisional hook will %s`, + async (secondOutcome) => { + const harness = await startConcurrentLifecycleHarness() + const hookFailure = new Error(`second hook failed`) + let firstSettled = false + let secondLoad: Promise | undefined + + try { + const firstLoad = Promise.resolve( + harness.loadSubset(harness.first), + ).then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve( + harness.loadSubset(harness.second), + ).then(() => undefined) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + + await harness.queuedLocks[0]!() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(firstSettled).toBe(true) + expect(harness.createDiffTrigger).toHaveBeenCalledOnce() + const when = harness.createDiffTrigger.mock.calls[0]?.[0].when + expect(when?.INSERT).toContain(`electronics`) + expect(when?.INSERT).not.toContain(`clothing`) + + if (secondOutcome === `reject`) { + harness.hooks[1]!.reject(hookFailure) + await expect(secondLoad).rejects.toBe(hookFailure) + } else { + harness.unloadSubset(harness.second) + harness.hooks[1]!.resolve() + await secondLoad + } + + await firstLoad + expect(harness.queuedLocks).toHaveLength(1) + expect(harness.hookCleanups[1]).toHaveBeenCalledTimes( + secondOutcome === `release` ? 1 : 0, + ) + } finally { + await harness.cleanup() + await secondLoad?.catch(() => undefined) + } + }, + ) + + it(`does not settle a superseded rebuild before its replacement publishes`, async () => { + const harness = await startConcurrentLifecycleHarness() + let firstSettled = false + let firstLoad: Promise | undefined + let secondLoad: Promise | undefined + + try { + firstLoad = Promise.resolve( + harness.loadSubset(harness.first), + ).then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve( + harness.loadSubset(harness.second), + ).then(() => undefined) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + harness.hooks[1]!.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) + + await harness.queuedLocks[0]!() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(firstSettled).toBe(false) + expect(harness.createDiffTrigger).not.toHaveBeenCalled() + + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(2)) + await harness.queuedLocks[1]!() + await Promise.all([firstLoad, secondLoad]) + + expect(harness.queuedLocks).toHaveLength(2) + expect(harness.createDiffTrigger).toHaveBeenCalledOnce() + const when = harness.createDiffTrigger.mock.calls[0]?.[0].when + expect(when?.INSERT).toContain(`electronics`) + expect(when?.INSERT).toContain(`clothing`) + } finally { + await harness.cleanup() + await Promise.all([ + firstLoad?.catch(() => undefined), + secondLoad?.catch(() => undefined), + ]) + } + }) + it(`does not start queued tracking after collection cleanup`, async () => { const db = await createDatabase() const queued = pDefer() From f1e8f3d54376fcc1dd1cf5e851ec087ba8ba64f1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 09:05:04 -0600 Subject: [PATCH 068/327] test(powersync): prove joined rebuild settlement --- .../powersync-db-collection/tests/on-demand-sync.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 409efa54e..97eba6cd9 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2715,6 +2715,7 @@ describe(`On-Demand Sync Mode`, () => { it(`does not settle a superseded rebuild before its replacement publishes`, async () => { const harness = await startConcurrentLifecycleHarness() let firstSettled = false + let secondSettled = false let firstLoad: Promise | undefined let secondLoad: Promise | undefined @@ -2730,14 +2731,18 @@ describe(`On-Demand Sync Mode`, () => { secondLoad = Promise.resolve( harness.loadSubset(harness.second), - ).then(() => undefined) + ).then(() => { + secondSettled = true + }) await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) harness.hooks[1]!.resolve() await new Promise((resolve) => setTimeout(resolve, 0)) + expect(secondSettled).toBe(false) await harness.queuedLocks[0]!() await new Promise((resolve) => setTimeout(resolve, 0)) expect(firstSettled).toBe(false) + expect(secondSettled).toBe(false) expect(harness.createDiffTrigger).not.toHaveBeenCalled() await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(2)) From fd49442fecc8c88c27ef110652d53c63fd046e92 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 09:23:57 -0600 Subject: [PATCH 069/327] test(powersync): schedule tracking lifecycle races --- .../tests/on-demand-sync.test.ts | 175 +++++++++++++++++- 1 file changed, 169 insertions(+), 6 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 97eba6cd9..a025c8ae1 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' +import { fc, test as fcTest } from '@fast-check/vitest' import { IR, and, @@ -16,6 +17,7 @@ import { import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' +import type { Scheduler } from 'fast-check' const APP_SCHEMA = new Schema({ products: new Table({ @@ -2533,13 +2535,16 @@ describe(`On-Demand Sync Mode`, () => { }) } - function queueWriteLocks(db: PowerSyncDatabase) { + function queueWriteLocks( + db: PowerSyncDatabase, + scheduler?: Scheduler, + ) { const queued: Array<() => Promise> = [] vi.spyOn(db, `writeLock`).mockImplementation( (callback) => new Promise((resolve, reject) => { let started = false - queued.push(async () => { + const run = async () => { if (started) return started = true try { @@ -2548,13 +2553,17 @@ describe(`On-Demand Sync Mode`, () => { } catch (error) { reject(error) } - }) + } + queued.push(run) + if (scheduler) { + void scheduler.scheduleFunction(run)() + } }) as never, ) return queued } - async function startConcurrentLifecycleHarness() { + async function startConcurrentLifecycleHarness(scheduler?: Scheduler) { const db = await createDatabase() const hooks: Array>> = [] const hookCleanups: Array> = [] @@ -2565,10 +2574,22 @@ describe(`On-Demand Sync Mode`, () => { hookCleanups.push(cleanup) return hook.promise.then(() => cleanup) }) - const queuedLocks = queueWriteLocks(db) + const queuedLocks = queueWriteLocks(db, scheduler) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + const trackingHandles: Array<{ + when: Record<`INSERT` | `UPDATE` | `DELETE`, string> + dispose: ReturnType + }> = [] const createDiffTrigger = vi .spyOn(db.triggers, `createDiffTrigger`) - .mockResolvedValue(vi.fn()) + .mockImplementation(({ when }) => { + const dispose = vi.fn(() => Promise.resolve()) + trackingHandles.push({ + when: when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + dispose, + }) + return Promise.resolve(dispose) + }) const config = powerSyncCollectionOptions({ database: db, table: APP_SCHEMA.props.products, @@ -2607,6 +2628,7 @@ describe(`On-Demand Sync Mode`, () => { hookCleanups, queuedLocks, createDiffTrigger, + trackingHandles, cleanup: async () => { hooks.forEach((hook) => hook.resolve()) sync.cleanup?.() @@ -2615,6 +2637,131 @@ describe(`On-Demand Sync Mode`, () => { } } + type ScheduledSecondOutcome = + | `activate` + | `reject` + | `release-during-hook` + | `release-after-publication` + | `cleanup-during-hook` + | `cleanup-after-publication` + + async function drainScheduledLifecycle(scheduler: Scheduler) { + let quietTurns = 0 + while (quietTurns < 2) { + if (scheduler.count() > 0) { + quietTurns = 0 + await scheduler.waitAll() + } else { + quietTurns++ + await Promise.resolve() + } + } + } + + async function expectScheduledLifecycleMatches( + scheduler: Scheduler, + secondOutcome: ScheduledSecondOutcome, + ) { + const harness = await startConcurrentLifecycleHarness(scheduler) + const hookFailure = new Error(`scheduled hook failure`) + let firstError: unknown + let secondError: unknown + + const firstLoad = Promise.resolve( + harness.loadSubset(harness.first), + ) + .then(() => undefined) + .catch((error: unknown) => { + firstError = error + }) + let secondLoad: Promise | undefined + + try { + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve( + harness.loadSubset(harness.second), + ) + .then(() => undefined) + .catch((error: unknown) => { + secondError = error + }) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + + const schedule = (action: () => void) => { + void scheduler.scheduleFunction(() => Promise.resolve(action()))() + } + const endsInRelease = secondOutcome.startsWith(`release-`) + const endsInCleanup = secondOutcome.startsWith(`cleanup-`) + const actsAfterPublication = secondOutcome.endsWith( + `after-publication`, + ) + + if (secondOutcome === `reject`) { + schedule(() => harness.hooks[1]!.reject(hookFailure)) + } else { + schedule(() => harness.hooks[1]!.resolve()) + if (secondOutcome === `release-during-hook`) { + schedule(() => harness.unloadSubset(harness.second)) + } else if (secondOutcome === `cleanup-during-hook`) { + schedule(() => harness.sync.cleanup?.()) + } + } + + await scheduler.waitFor(Promise.all([firstLoad, secondLoad])) + await drainScheduledLifecycle(scheduler) + if (actsAfterPublication) { + if (endsInRelease) { + harness.unloadSubset(harness.second) + } else { + harness.sync.cleanup?.() + } + await drainScheduledLifecycle(scheduler) + } + + expect(firstError).toBeUndefined() + expect(secondError).toBe( + secondOutcome === `reject` ? hookFailure : undefined, + ) + expect(harness.hookCleanups[0]).toHaveBeenCalledTimes( + endsInCleanup ? 1 : 0, + ) + expect(harness.hookCleanups[1]).toHaveBeenCalledTimes( + endsInRelease || endsInCleanup ? 1 : 0, + ) + + const liveTracking = harness.trackingHandles.filter( + ({ dispose }) => dispose.mock.calls.length === 0, + ) + if (endsInCleanup) { + expect(liveTracking).toEqual([]) + return + } + + expect(liveTracking).toHaveLength(1) + const finalInsert = liveTracking[0]!.when.INSERT + expect(finalInsert).toContain(`electronics`) + if (secondOutcome === `activate`) { + expect(finalInsert).toContain(`clothing`) + } else { + expect(finalInsert).not.toContain(`clothing`) + } + if (secondOutcome === `reject`) { + expect( + harness.trackingHandles.every( + ({ when }) => !when.INSERT.includes(`clothing`), + ), + ).toBe(true) + } + } finally { + await harness.cleanup() + if (scheduler.count() > 0) await scheduler.waitAll() + await Promise.allSettled([firstLoad, secondLoad]) + } + } + it(`does not acquire a subset released while tracking startup is suspended`, async () => { const db = await createDatabase() const onLoadSubset = vi.fn() @@ -2763,6 +2910,22 @@ describe(`On-Demand Sync Mode`, () => { } }) + for (const secondOutcome of [ + `activate`, + `reject`, + `release-during-hook`, + `release-after-publication`, + `cleanup-during-hook`, + `cleanup-after-publication`, + ] as const) { + fcTest.prop([fc.scheduler()], { numRuns: 8 })( + `keeps tracking coherent when concurrent lifecycle tasks end in ${secondOutcome}`, + async (scheduler) => { + await expectScheduledLifecycleMatches(scheduler, secondOutcome) + }, + ) + } + it(`does not start queued tracking after collection cleanup`, async () => { const db = await createDatabase() const queued = pDefer() From 191eaa13c613f876458337d57b3f640a1fc611b6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 09:45:11 -0600 Subject: [PATCH 070/327] test(powersync): prove lifecycle scheduler phases --- .../tests/on-demand-sync.test.ts | 188 ++++++++++++++++-- 1 file changed, 174 insertions(+), 14 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index a025c8ae1..4f37cf6dc 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2556,7 +2556,9 @@ describe(`On-Demand Sync Mode`, () => { } queued.push(run) if (scheduler) { - void scheduler.scheduleFunction(run)() + void scheduler + .schedule(Promise.resolve(), `write-lock-${queued.length}`) + .then(run) } }) as never, ) @@ -2661,9 +2663,11 @@ describe(`On-Demand Sync Mode`, () => { async function expectScheduledLifecycleMatches( scheduler: Scheduler, secondOutcome: ScheduledSecondOutcome, + expectedActionOrder?: ReadonlyArray, ) { const harness = await startConcurrentLifecycleHarness(scheduler) const hookFailure = new Error(`scheduled hook failure`) + const actionOrder: Array = [] let firstError: unknown let secondError: unknown @@ -2690,8 +2694,11 @@ describe(`On-Demand Sync Mode`, () => { }) await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) - const schedule = (action: () => void) => { - void scheduler.scheduleFunction(() => Promise.resolve(action()))() + const schedule = (label: string, action: () => void) => { + void scheduler.schedule(Promise.resolve(), label).then(() => { + actionOrder.push(label) + action() + }) } const endsInRelease = secondOutcome.startsWith(`release-`) const endsInCleanup = secondOutcome.startsWith(`cleanup-`) @@ -2700,13 +2707,17 @@ describe(`On-Demand Sync Mode`, () => { ) if (secondOutcome === `reject`) { - schedule(() => harness.hooks[1]!.reject(hookFailure)) + schedule(`reject-second-hook`, () => + harness.hooks[1]!.reject(hookFailure), + ) } else { - schedule(() => harness.hooks[1]!.resolve()) + schedule(`resolve-second-hook`, () => harness.hooks[1]!.resolve()) if (secondOutcome === `release-during-hook`) { - schedule(() => harness.unloadSubset(harness.second)) + schedule(`release-second-demand`, () => + harness.unloadSubset(harness.second), + ) } else if (secondOutcome === `cleanup-during-hook`) { - schedule(() => harness.sync.cleanup?.()) + schedule(`cleanup-sync`, () => harness.sync.cleanup?.()) } } @@ -2721,6 +2732,10 @@ describe(`On-Demand Sync Mode`, () => { await drainScheduledLifecycle(scheduler) } + if (expectedActionOrder) { + expect(actionOrder).toEqual(expectedActionOrder) + } + expect(firstError).toBeUndefined() expect(secondError).toBe( secondOutcome === `reject` ? hookFailure : undefined, @@ -2741,17 +2756,22 @@ describe(`On-Demand Sync Mode`, () => { } expect(liveTracking).toHaveLength(1) - const finalInsert = liveTracking[0]!.when.INSERT - expect(finalInsert).toContain(`electronics`) - if (secondOutcome === `activate`) { - expect(finalInsert).toContain(`clothing`) - } else { - expect(finalInsert).not.toContain(`clothing`) + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + const finalClause = liveTracking[0]!.when[operation] + expect(finalClause).toContain(`electronics`) + if (secondOutcome === `activate`) { + expect(finalClause).toContain(`clothing`) + } else { + expect(finalClause).not.toContain(`clothing`) + } } if (secondOutcome === `reject`) { expect( harness.trackingHandles.every( - ({ when }) => !when.INSERT.includes(`clothing`), + ({ when }) => + ([`INSERT`, `UPDATE`, `DELETE`] as const).every( + (operation) => !when[operation].includes(`clothing`), + ), ), ).toBe(true) } @@ -2910,6 +2930,104 @@ describe(`On-Demand Sync Mode`, () => { } }) + it(`disposes superseded tracking before its replacement starts`, async () => { + const db = await createDatabase() + const hooks: Array>> = [] + const onLoadSubset = vi.fn(() => { + const hook = pDefer() + hooks.push(hook) + return hook.promise.then(() => vi.fn()) + }) + const queuedLocks = queueWriteLocks(db) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + + const triggerStarted = pDefer() + const finishTrigger = pDefer() + const staleDispose = vi.fn(() => Promise.resolve()) + const currentDispose = vi.fn(() => Promise.resolve()) + const triggerClauses: Array< + Record<`INSERT` | `UPDATE` | `DELETE`, string> + > = [] + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockImplementation(async ({ when }) => { + triggerClauses.push( + when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + ) + if (triggerClauses.length === 1) { + triggerStarted.resolve() + await finishTrigger.promise + return staleDispose + } + return currentDispose + }) + + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const first = { where: eq(`category`, `electronics`) } + const second = { where: eq(`category`, `clothing`) } + let firstLoad: Promise | undefined + let secondLoad: Promise | undefined + + try { + firstLoad = Promise.resolve(sync.loadSubset(first)).then( + () => undefined, + ) + await vi.waitFor(() => expect(hooks).toHaveLength(1)) + hooks[0]!.resolve() + await vi.waitFor(() => expect(queuedLocks).toHaveLength(1)) + + const staleRebuild = queuedLocks[0]!() + await triggerStarted.promise + + secondLoad = Promise.resolve(sync.loadSubset(second)).then( + () => undefined, + ) + await vi.waitFor(() => expect(hooks).toHaveLength(2)) + hooks[1]!.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) + + finishTrigger.resolve() + await staleRebuild + + expect(staleDispose).toHaveBeenCalledOnce() + expect(createDiffTrigger).toHaveBeenCalledOnce() + + await vi.waitFor(() => expect(queuedLocks).toHaveLength(2)) + await queuedLocks[1]!() + await Promise.all([firstLoad, secondLoad]) + + expect(createDiffTrigger).toHaveBeenCalledTimes(2) + expect(currentDispose).not.toHaveBeenCalled() + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + expect(triggerClauses[1]![operation]).toContain(`electronics`) + expect(triggerClauses[1]![operation]).toContain(`clothing`) + } + } finally { + hooks.forEach((hook) => hook.resolve()) + sync.cleanup?.() + await Promise.all(queuedLocks.map((run) => run())) + await Promise.allSettled([firstLoad, secondLoad]) + } + }) + for (const secondOutcome of [ `activate`, `reject`, @@ -2926,6 +3044,48 @@ describe(`On-Demand Sync Mode`, () => { ) } + it.each([ + { + name: `release before hook resolution`, + outcome: `release-during-hook` as const, + order: [3, 2, 1], + expectedActionOrder: [ + `release-second-demand`, + `resolve-second-hook`, + ], + }, + { + name: `hook resolution before release`, + outcome: `release-during-hook` as const, + order: [2, 3, 1, 4], + expectedActionOrder: [ + `resolve-second-hook`, + `release-second-demand`, + ], + }, + { + name: `cleanup before hook resolution`, + outcome: `cleanup-during-hook` as const, + order: [3, 2, 1], + expectedActionOrder: [`cleanup-sync`, `resolve-second-hook`], + }, + { + name: `hook resolution before cleanup`, + outcome: `cleanup-during-hook` as const, + order: [2, 3, 1], + expectedActionOrder: [`resolve-second-hook`, `cleanup-sync`], + }, + ])( + `keeps tracking coherent when $name`, + async ({ outcome, order, expectedActionOrder }) => { + await expectScheduledLifecycleMatches( + fc.schedulerFor(order), + outcome, + expectedActionOrder, + ) + }, + ) + it(`does not start queued tracking after collection cleanup`, async () => { const db = await createDatabase() const queued = pDefer() From 181d813dab373643dcc60053d82d72de958a3584 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 10:02:19 -0600 Subject: [PATCH 071/327] test(powersync): isolate restarted sync lifecycle --- .../tests/on-demand-sync.test.ts | 135 +++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 4f37cf6dc..a7cf400e3 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -2538,15 +2538,18 @@ describe(`On-Demand Sync Mode`, () => { function queueWriteLocks( db: PowerSyncDatabase, scheduler?: Scheduler, + invocationOrder?: Array, ) { const queued: Array<() => Promise> = [] vi.spyOn(db, `writeLock`).mockImplementation( (callback) => new Promise((resolve, reject) => { let started = false + const label = `write-lock-${queued.length + 1}` const run = async () => { if (started) return started = true + invocationOrder?.push(label) try { const result = await callback({} as never) resolve(result as never) @@ -2557,7 +2560,7 @@ describe(`On-Demand Sync Mode`, () => { queued.push(run) if (scheduler) { void scheduler - .schedule(Promise.resolve(), `write-lock-${queued.length}`) + .schedule(Promise.resolve(), label) .then(run) } }) as never, @@ -3086,6 +3089,136 @@ describe(`On-Demand Sync Mode`, () => { }, ) + it.each([ + { + name: `the stopped callback runs before the restarted callback`, + order: [1, 2], + expectedInvocationOrder: [`write-lock-1`, `write-lock-2`], + }, + { + name: `the restarted callback runs before the stopped callback`, + order: [2, 1], + expectedInvocationOrder: [`write-lock-2`, `write-lock-1`], + }, + ])( + `keeps a restarted sync isolated when $name`, + async ({ order, expectedInvocationOrder }) => { + const scheduler = fc.schedulerFor(order) + const db = await createDatabase() + const invocationOrder: Array = [] + queueWriteLocks(db, scheduler, invocationOrder) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + + const hookCleanups: Array> = [] + const onLoadSubset = vi.fn(() => { + const cleanup = vi.fn() + hookCleanups.push(cleanup) + return cleanup + }) + const trackingHandles: Array<{ + when: Record<`INSERT` | `UPDATE` | `DELETE`, string> + dispose: ReturnType + }> = [] + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockImplementation(({ when }) => { + const dispose = vi.fn(() => Promise.resolve()) + trackingHandles.push({ + when: when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + dispose, + }) + return Promise.resolve(dispose) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const startSync = () => { + const started = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!started || typeof started === `function` || !started.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + return started + } + + const stoppedSync = startSync() + let stoppedSettled = false + let restartedSettled = false + const stoppedLoad = Promise.resolve( + stoppedSync.loadSubset!({ + where: eq(`category`, `electronics`), + }), + ).then(() => { + stoppedSettled = true + }) + let restartedSync: ReturnType | undefined + let restartedLoad: Promise | undefined + + try { + await vi.waitFor(() => expect(scheduler.count()).toBe(1)) + stoppedSync.cleanup?.() + + restartedSync = startSync() + restartedLoad = Promise.resolve( + restartedSync.loadSubset!({ + where: eq(`category`, `clothing`), + }), + ).then(() => { + restartedSettled = true + }) + await vi.waitFor(() => expect(scheduler.count()).toBe(2)) + expect(stoppedSettled).toBe(false) + expect(restartedSettled).toBe(false) + + await scheduler.waitOne() + const stoppedRunsFirst = order[0] === 1 + await vi.waitFor(() => { + expect(stoppedSettled).toBe(stoppedRunsFirst) + expect(restartedSettled).toBe(!stoppedRunsFirst) + }) + + await scheduler.waitFor( + Promise.all([stoppedLoad, restartedLoad]), + ) + await drainScheduledLifecycle(scheduler) + + expect(invocationOrder).toEqual(expectedInvocationOrder) + expect(hookCleanups[0]).toHaveBeenCalledOnce() + expect(hookCleanups[1]).not.toHaveBeenCalled() + expect(createDiffTrigger).toHaveBeenCalledOnce() + expect(trackingHandles).toHaveLength(1) + expect(trackingHandles[0]!.dispose).not.toHaveBeenCalled() + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + expect(trackingHandles[0]!.when[operation]).toContain(`clothing`) + expect(trackingHandles[0]!.when[operation]).not.toContain( + `electronics`, + ) + } + + restartedSync.cleanup?.() + await vi.waitFor(() => { + expect(hookCleanups[1]).toHaveBeenCalledOnce() + expect(trackingHandles[0]!.dispose).toHaveBeenCalledOnce() + }) + } finally { + stoppedSync.cleanup?.() + restartedSync?.cleanup?.() + if (scheduler.count() > 0) await scheduler.waitAll() + await Promise.allSettled([stoppedLoad, restartedLoad]) + } + }, + ) + it(`does not start queued tracking after collection cleanup`, async () => { const db = await createDatabase() const queued = pDefer() From 349719922738651dbeddefa55d711bd79fced8e5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 10:20:31 -0600 Subject: [PATCH 072/327] test(powersync): assert stable restart cleanup --- .../tests/on-demand-sync.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index a7cf400e3..0c3e67a07 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -3163,6 +3163,7 @@ describe(`On-Demand Sync Mode`, () => { }) let restartedSync: ReturnType | undefined let restartedLoad: Promise | undefined + let restartedCleaned = false try { await vi.waitFor(() => expect(scheduler.count()).toBe(1)) @@ -3206,13 +3207,14 @@ describe(`On-Demand Sync Mode`, () => { } restartedSync.cleanup?.() - await vi.waitFor(() => { - expect(hookCleanups[1]).toHaveBeenCalledOnce() - expect(trackingHandles[0]!.dispose).toHaveBeenCalledOnce() - }) + restartedSync.cleanup?.() + restartedCleaned = true + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(hookCleanups[1]).toHaveBeenCalledOnce() + expect(trackingHandles[0]!.dispose).toHaveBeenCalledOnce() } finally { stoppedSync.cleanup?.() - restartedSync?.cleanup?.() + if (!restartedCleaned) restartedSync?.cleanup?.() if (scheduler.count() > 0) await scheduler.waitAll() await Promise.allSettled([stoppedLoad, restartedLoad]) } From bba4754177a11219a8fa4e913aa1be7bff817f06 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 10:32:08 -0600 Subject: [PATCH 073/327] test(powersync): exhaust restart cleanup timers --- .../powersync-db-collection/tests/on-demand-sync.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 0c3e67a07..264afa0b6 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -3164,6 +3164,7 @@ describe(`On-Demand Sync Mode`, () => { let restartedSync: ReturnType | undefined let restartedLoad: Promise | undefined let restartedCleaned = false + let usingFakeTimers = false try { await vi.waitFor(() => expect(scheduler.count()).toBe(1)) @@ -3206,13 +3207,18 @@ describe(`On-Demand Sync Mode`, () => { ) } + vi.useFakeTimers() + usingFakeTimers = true restartedSync.cleanup?.() restartedSync.cleanup?.() restartedCleaned = true - await new Promise((resolve) => setTimeout(resolve, 0)) + await vi.runAllTimersAsync() expect(hookCleanups[1]).toHaveBeenCalledOnce() expect(trackingHandles[0]!.dispose).toHaveBeenCalledOnce() + vi.useRealTimers() + usingFakeTimers = false } finally { + if (usingFakeTimers) vi.useRealTimers() stoppedSync.cleanup?.() if (!restartedCleaned) restartedSync?.cleanup?.() if (scheduler.count() > 0) await scheduler.waitAll() From 2a75046045090b6af56f83ad78047bc84c02897f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 13:19:48 -0600 Subject: [PATCH 074/327] test(db): distinguish demand attempts --- ...ubscription-replay-oracle.property.test.ts | 1 + .../db/tests/load-subset-full-flow-model.ts | 58 +++--- ...d-subset-full-flow-oracle.property.test.ts | 23 +++ ...d-subset-refinement-model.property.test.ts | 176 +++++++++++++++++- .../tests/electric-live-query.test.ts | 5 + .../tests/on-demand-sync.test.ts | 5 + 6 files changed, 245 insertions(+), 23 deletions(-) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 96304c4d8..04ab87945 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -8954,6 +8954,7 @@ describe(`CollectionSubscription replay oracle`, () => { ownerId: `other-owner`, sessionId: `session`, demandId: `other`, + attemptId: `other-attempt`, alreadyAborted: false, }, { diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 0517b8627..ea524338c 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -10,6 +10,7 @@ export type FullFlowOwnerId = string export type FullFlowSessionId = string export type FullFlowDemandId = string +export type FullFlowAttemptId = string export type FullFlowSourceId = string export type FullFlowTransactionId = string export type FullFlowAcquisitionId = string @@ -124,28 +125,33 @@ export type LoadSubsetFullFlowEvent = ownerId: FullFlowOwnerId sessionId: FullFlowSessionId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId alreadyAborted: boolean } | { type: `applyAuthoritativeRows` ownerId: FullFlowOwnerId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId rowKeys: ReadonlyArray } | { type: `settleDemandWithoutEvidence` demandId: FullFlowDemandId + attemptId: FullFlowAttemptId } | { type: `applyUnprovenRows` ownerId: FullFlowOwnerId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId rowKeys: ReadonlyArray } | { type: `rejectDemand` ownerId: FullFlowOwnerId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId } | { type: `truncateSource` @@ -155,6 +161,7 @@ export type LoadSubsetFullFlowEvent = type: `releaseDemand` ownerId: FullFlowOwnerId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId rowKeys: ReadonlyArray finalRowOwner: boolean invalidatesAdapterEvidence: boolean @@ -341,9 +348,9 @@ export function projectAdapterLifecycle( export function projectTransportLoads( history: ReadonlyArray, ): number { - const reusableDemands = new Set() - const inFlightDemands = new Set() - const requestEpochs = new Map() + const reusableDemands = new Map() + const inFlightDemands = new Map() + const attemptEpochs = new Map() let sourceEpoch = 0 let loads = 0 @@ -356,18 +363,18 @@ export function projectTransportLoads( !inFlightDemands.has(event.demandId) ) { loads++ - inFlightDemands.add(event.demandId) - } - if (!event.alreadyAborted) { - requestEpochs.set(event.ownerId, sourceEpoch) + inFlightDemands.set(event.demandId, event.attemptId) + attemptEpochs.set(event.attemptId, sourceEpoch) } break - case `applyAuthoritativeRows`: + case `applyAuthoritativeRows`: { + if (inFlightDemands.get(event.demandId) !== event.attemptId) break inFlightDemands.delete(event.demandId) - if (requestEpochs.get(event.ownerId) === sourceEpoch) { - reusableDemands.add(event.demandId) + if (attemptEpochs.get(event.attemptId) === sourceEpoch) { + reusableDemands.set(event.demandId, event.attemptId) } break + } case `truncateSource`: sourceEpoch++ reusableDemands.clear() @@ -375,15 +382,19 @@ export function projectTransportLoads( break case `applyUnprovenRows`: case `rejectDemand`: - inFlightDemands.delete(event.demandId) - break case `settleDemandWithoutEvidence`: - inFlightDemands.delete(event.demandId) + if (inFlightDemands.get(event.demandId) === event.attemptId) { + inFlightDemands.delete(event.demandId) + } break case `releaseDemand`: if (event.invalidatesAdapterEvidence) { - reusableDemands.delete(event.demandId) - inFlightDemands.delete(event.demandId) + if (reusableDemands.get(event.demandId) === event.attemptId) { + reusableDemands.delete(event.demandId) + } + if (inFlightDemands.get(event.demandId) === event.attemptId) { + inFlightDemands.delete(event.demandId) + } } break case `restartSession`: @@ -511,20 +522,20 @@ export function projectAuthorizedContinuationStarts( export function projectReusableDemands( history: ReadonlyArray, ): Array { - const reusableDemands = new Set() - const requestEpochs = new Map() + const reusableDemands = new Map() + const attemptEpochs = new Map() let sourceEpoch = 0 for (const event of history) { switch (event.type) { case `requestDemand`: if (!event.alreadyAborted) { - requestEpochs.set(event.ownerId, sourceEpoch) + attemptEpochs.set(event.attemptId, sourceEpoch) } break case `applyAuthoritativeRows`: - if (requestEpochs.get(event.ownerId) === sourceEpoch) { - reusableDemands.add(event.demandId) + if (attemptEpochs.get(event.attemptId) === sourceEpoch) { + reusableDemands.set(event.demandId, event.attemptId) } break case `truncateSource`: @@ -532,7 +543,10 @@ export function projectReusableDemands( reusableDemands.clear() break case `releaseDemand`: - if (event.invalidatesAdapterEvidence) { + if ( + event.invalidatesAdapterEvidence && + reusableDemands.get(event.demandId) === event.attemptId + ) { reusableDemands.delete(event.demandId) } break @@ -553,7 +567,7 @@ export function projectReusableDemands( } } - return [...reusableDemands].sort() + return [...reusableDemands.keys()].sort() } /** diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index c724551a4..a333ab581 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -236,6 +236,7 @@ async function runTruncateCoverageScenario( ownerId, sessionId: `session`, demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, alreadyAborted: false, }) activeOptions.push(options) @@ -263,6 +264,7 @@ async function runTruncateCoverageScenario( hasMore === undefined ? `applyUnprovenRows` : `applyAuthoritativeRows`, ownerId, demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, rowKeys: rows.map(({ id }) => id), }) } @@ -273,6 +275,7 @@ async function runTruncateCoverageScenario( type: `rejectDemand`, ownerId, demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, }) } @@ -349,6 +352,13 @@ async function runTruncateCoverageScenario( ? `old` : `fresh`, demandId: `prefix-${options.limit}`, + attemptId: `${ + options === initialOptions + ? `initial` + : options === oldOptions + ? `old` + : `fresh` + }-attempt`, rowKeys: options === initialOptions ? [`initial`] @@ -379,6 +389,7 @@ it(`does not release physical work when an already-aborted demand skips adapter ownerId, sessionId: `session-1`, demandId: `all-rows`, + attemptId: `aborted-attempt`, alreadyAborted: true, } const history: ReadonlyArray = [ @@ -387,6 +398,7 @@ it(`does not release physical work when an already-aborted demand skips adapter type: `releaseDemand`, ownerId, demandId: `all-rows`, + attemptId: `aborted-attempt`, rowKeys: [], finalRowOwner: false, invalidatesAdapterEvidence: false, @@ -921,18 +933,21 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad ownerId: `owner-1`, sessionId: `session-1`, demandId: `all-rows`, + attemptId: `attempt-1`, alreadyAborted: false, }, { type: `applyAuthoritativeRows`, ownerId: `owner-1`, demandId: `all-rows`, + attemptId: `attempt-1`, rowKeys: [row.id], }, { type: `releaseDemand`, ownerId: `owner-1`, demandId: `all-rows`, + attemptId: `attempt-1`, rowKeys: [row.id], finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -947,12 +962,14 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad ownerId: `owner-2`, sessionId: `session-2`, demandId: `all-rows`, + attemptId: `attempt-2`, alreadyAborted: false, }, { type: `applyAuthoritativeRows`, ownerId: `owner-2`, demandId: `all-rows`, + attemptId: `attempt-2`, rowKeys: [row.id], }, ] @@ -1030,6 +1047,7 @@ it(`does not let an ordered continuation from a cleaned session start new work a ownerId: `owner-1`, sessionId: `session-1`, demandId: `top-1`, + attemptId: `attempt-1`, alreadyAborted: false, }, { @@ -1049,6 +1067,7 @@ it(`does not let an ordered continuation from a cleaned session start new work a ownerId: `owner-2`, sessionId: `session-2`, demandId: `top-1`, + attemptId: `attempt-2`, alreadyAborted: false, }, { type: `runContinuation`, taskId: `load-1-settlement` }, @@ -2656,6 +2675,7 @@ async function runOrderedBoundaryProvenanceScenario( type: `rejectDemand`, ownerId: `ordered-owner`, demandId: `ordered-window`, + attemptId: `ordered-attempt`, }, ] const expectedBoundary = projectOrderedPublicationBoundary(history, { @@ -3254,6 +3274,7 @@ async function runAtomicOrderedReplayScenario( type: `releaseDemand`, ownerId: `other-owner`, demandId: `other`, + attemptId: `other-attempt`, rowKeys: [replacementOtherRow.id], finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -3278,6 +3299,7 @@ async function runAtomicOrderedReplayScenario( ownerId: `other-owner`, sessionId: `atomic-session`, demandId: `other`, + attemptId: `other-attempt`, alreadyAborted: false, }) subscription.requestSnapshot({ where: otherWhere }) @@ -3341,6 +3363,7 @@ async function runAtomicOrderedReplayScenario( type: `releaseDemand`, ownerId: `other-owner`, demandId: `other`, + attemptId: `other-attempt`, rowKeys: [replacementOtherRow.id], finalRowOwner: true, invalidatesAdapterEvidence: true, diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 6d86c25be..db00f4f89 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -10,6 +10,7 @@ import { projectAuthorizedContinuationStarts, projectReplayPublication, projectRetainedRowKeys, + projectReusableDemands, projectSourceReadiness, projectSyncTransactions, projectTransportLoads, @@ -108,6 +109,7 @@ function enumerateDemandLifecycles(): Array { ownerId, sessionId: `session`, demandId: `demand`, + attemptId: `${ownerId}-attempt`, alreadyAborted, }, ], @@ -127,6 +129,7 @@ function enumerateDemandLifecycles(): Array { type: `releaseDemand`, ownerId, demandId: `demand`, + attemptId: `${ownerId}-attempt`, rowKeys: [], finalRowOwner: false, invalidatesAdapterEvidence: false, @@ -166,11 +169,15 @@ it(`exhaustively projects exact adapter starts and releases for two owners`, () }) it(`shares concurrent exact demand and retries after evidence-free settlement`, () => { - const request = (ownerId: string): LoadSubsetFullFlowEvent => ({ + const request = ( + ownerId: string, + attemptId = `${ownerId}-attempt`, + ): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, ownerId, sessionId: `session`, demandId: `exact-demand`, + attemptId, alreadyAborted: false, }) const concurrent = [request(`owner-a`), request(`owner-b`)] @@ -186,6 +193,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, type: `releaseDemand`, ownerId: `owner-a`, demandId: `exact-demand`, + attemptId: `owner-a-attempt`, rowKeys: [], finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -199,6 +207,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, { type: `settleDemandWithoutEvidence`, demandId: `exact-demand`, + attemptId: `owner-a-attempt`, }, request(`owner-c`), ]), @@ -210,6 +219,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, type: `applyAuthoritativeRows`, ownerId: `owner-a`, demandId: `exact-demand`, + attemptId: `owner-a-attempt`, rowKeys: [`row`], }, request(`owner-c`), @@ -217,6 +227,161 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ).toBe(1) }) +it.each([ + { + name: `authoritative`, + event: { + type: `applyAuthoritativeRows`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + }, + { + name: `unproven`, + event: { + type: `applyUnprovenRows`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + }, + { + name: `rejected`, + event: { + type: `rejectDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + }, + { + name: `evidence-free`, + event: { + type: `settleDemandWithoutEvidence`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + }, + { + name: `released`, + event: { + type: `releaseDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + }, +] satisfies ReadonlyArray<{ + name: string + event: LoadSubsetFullFlowEvent +}>)( + `keeps fresh same-demand work shared when an old attempt is $name after truncate`, + ({ event }) => { + expect( + projectTransportLoads([ + { + type: `requestDemand`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + }, + { type: `truncateSource`, sessionId: `session` }, + { + type: `requestDemand`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + }, + event, + { + type: `requestDemand`, + ownerId: `peer-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `peer-attempt`, + alreadyAborted: false, + }, + ]), + ).toBe(2) + }, +) + +it(`scopes reusable evidence to the physical attempt when an owner is reused`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + ownerId: `stable-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + ...oldRequest, + attemptId: `fresh-attempt`, + } + const oldSettlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + ownerId: `stable-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + } + const freshSettlement: LoadSubsetFullFlowEvent = { + ...oldSettlement, + attemptId: `fresh-attempt`, + rowKeys: [`fresh-row`], + } + const staleRelease: LoadSubsetFullFlowEvent = { + type: `releaseDemand`, + ownerId: `stable-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + } + const beforeFreshSettlement = [ + oldRequest, + { type: `truncateSource`, sessionId: `session` } as const, + freshRequest, + oldSettlement, + ] + + expect(projectReusableDemands(beforeFreshSettlement)).toEqual([]) + expect( + projectReusableDemands([...beforeFreshSettlement, freshSettlement]), + ).toEqual([`exact-demand`]) + expect( + projectReusableDemands([ + ...beforeFreshSettlement, + freshSettlement, + staleRelease, + ]), + ).toEqual([`exact-demand`]) + expect( + projectTransportLoads([ + ...beforeFreshSettlement, + freshSettlement, + staleRelease, + { + ...freshRequest, + ownerId: `peer-owner`, + attemptId: `peer-attempt`, + }, + ]), + ).toBe(2) +}) + function renameHistoryIds( history: ReadonlyArray, suffix: string, @@ -229,18 +394,23 @@ function renameHistoryIds( ownerId: `${event.ownerId}-${suffix}`, sessionId: `${event.sessionId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, } case `applyAuthoritativeRows`: + case `applyUnprovenRows`: + case `rejectDemand`: case `releaseDemand`: return { ...event, ownerId: `${event.ownerId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, } case `settleDemandWithoutEvidence`: return { ...event, demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, } case `registerSourceDemand`: case `settleSourceDemand`: @@ -347,18 +517,21 @@ for (const campaign of refinementCampaigns(1_779_003)) { ownerId: `owner`, sessionId: `session`, demandId: `demand`, + attemptId: `attempt`, alreadyAborted: false, }, { type: `applyAuthoritativeRows`, ownerId: `owner`, demandId: `demand`, + attemptId: `attempt`, rowKeys: [`row`], }, { type: `releaseDemand`, ownerId: `owner`, demandId: `demand`, + attemptId: `attempt`, rowKeys: [`row`], finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -370,6 +543,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { ownerId: `owner`, sessionId: `session`, demandId: `demand`, + attemptId: `attempt`, alreadyAborted: false, }, { diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index f0c8dd86b..cfce12133 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -1331,6 +1331,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { ownerId: `owner-1`, sessionId: `session-1`, demandId: `active-users`, + attemptId: `attempt-1`, alreadyAborted: false, }, ] @@ -1362,6 +1363,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { type: `applyAuthoritativeRows`, ownerId: `owner-1`, demandId: `active-users`, + attemptId: `attempt-1`, rowKeys: [String(row.id)], }) expect(first.toArray.map(({ id }) => String(id))).toEqual([ @@ -1374,6 +1376,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { type: `releaseDemand`, ownerId: `owner-1`, demandId: `active-users`, + attemptId: `attempt-1`, rowKeys: [String(row.id)], finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -1388,6 +1391,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { ownerId: `owner-2`, sessionId: `session-2`, demandId: `active-users`, + attemptId: `attempt-2`, alreadyAborted: false, }, ) @@ -1398,6 +1402,7 @@ describe(`Electric Collection - loadSubset deduplication`, () => { type: `applyAuthoritativeRows`, ownerId: `owner-2`, demandId: `active-users`, + attemptId: `attempt-2`, rowKeys: [String(row.id)], }) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 837b01915..7ee920907 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -1836,6 +1836,7 @@ describe(`On-Demand Sync Mode`, () => { ownerId: `owner-1`, sessionId: `session-1`, demandId: `electronics`, + attemptId: `attempt-1`, alreadyAborted: false, }, ] @@ -1846,6 +1847,7 @@ describe(`On-Demand Sync Mode`, () => { type: `applyAuthoritativeRows`, ownerId: `owner-1`, demandId: `electronics`, + attemptId: `attempt-1`, rowKeys: expectedRowKeys, }) expect(first.toArray.map(({ id }) => String(id)).sort()).toEqual( @@ -1858,6 +1860,7 @@ describe(`On-Demand Sync Mode`, () => { type: `releaseDemand`, ownerId: `owner-1`, demandId: `electronics`, + attemptId: `attempt-1`, rowKeys: expectedRowKeys, finalRowOwner: true, invalidatesAdapterEvidence: true, @@ -1872,6 +1875,7 @@ describe(`On-Demand Sync Mode`, () => { ownerId: `owner-2`, sessionId: `session-2`, demandId: `electronics`, + attemptId: `attempt-2`, alreadyAborted: false, }, ) @@ -1884,6 +1888,7 @@ describe(`On-Demand Sync Mode`, () => { type: `applyAuthoritativeRows`, ownerId: `owner-2`, demandId: `electronics`, + attemptId: `attempt-2`, rowKeys: expectedRowKeys, }) From 6fa6d6671a4dacb1d9ecf4366b22550943ce704c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 13:41:14 -0600 Subject: [PATCH 075/327] test(db): harden demand attempt boundaries --- .../db/tests/load-subset-full-flow-model.ts | 18 ++-- ...d-subset-full-flow-oracle.property.test.ts | 98 +++++++++++++++++++ ...d-subset-refinement-model.property.test.ts | 50 +++++++++- 3 files changed, 153 insertions(+), 13 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index ea524338c..9a6384485 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -350,8 +350,6 @@ export function projectTransportLoads( ): number { const reusableDemands = new Map() const inFlightDemands = new Map() - const attemptEpochs = new Map() - let sourceEpoch = 0 let loads = 0 for (const event of history) { @@ -364,19 +362,15 @@ export function projectTransportLoads( ) { loads++ inFlightDemands.set(event.demandId, event.attemptId) - attemptEpochs.set(event.attemptId, sourceEpoch) } break case `applyAuthoritativeRows`: { if (inFlightDemands.get(event.demandId) !== event.attemptId) break inFlightDemands.delete(event.demandId) - if (attemptEpochs.get(event.attemptId) === sourceEpoch) { - reusableDemands.set(event.demandId, event.attemptId) - } + reusableDemands.set(event.demandId, event.attemptId) break } case `truncateSource`: - sourceEpoch++ reusableDemands.clear() inFlightDemands.clear() break @@ -543,11 +537,11 @@ export function projectReusableDemands( reusableDemands.clear() break case `releaseDemand`: - if ( - event.invalidatesAdapterEvidence && - reusableDemands.get(event.demandId) === event.attemptId - ) { - reusableDemands.delete(event.demandId) + if (event.invalidatesAdapterEvidence) { + attemptEpochs.delete(event.attemptId) + if (reusableDemands.get(event.demandId) === event.attemptId) { + reusableDemands.delete(event.demandId) + } } break case `applyUnprovenRows`: diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index a333ab581..b4785e60d 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -382,6 +382,104 @@ async function runTruncateCoverageScenario( await source.cleanup() } } + +it.each([`authoritative`, `unproven`, `rejected`] as const)( + `keeps fresh exact-demand work shared after a pre-truncate %s request settles`, + async (oldOutcome) => { + type Row = { id: string; value: number } + type AdapterResult = { + hasMore: boolean | undefined + appliedRowKeys: ReadonlyArray + } + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const pending: Array>> = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + }) + const source = createCollection({ + id: `same-demand-truncate-${oldOutcome}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: deduplicated.loadSubset, + } + }, + }, + }) + const oldOptions = { limit: 2 } + const freshOptions = { limit: 2 } + const peerOptions = { limit: 2 } + const applyRows = async (rows: ReadonlyArray) => { + begin() + rows.forEach((row) => write({ type: `insert`, value: row })) + const applied = commit() + if (applied !== true) await applied + } + + try { + const oldLoad = source._sync.loadSubset(oldOptions) + if (oldLoad === true) throw new Error(`Expected an async old request`) + expect(pending).toHaveLength(1) + + begin() + truncate() + const truncated = commit() + if (truncated !== true) await truncated + deduplicated.reset() + + const freshLoad = source._sync.loadSubset(freshOptions) + if (freshLoad === true) throw new Error(`Expected an async fresh request`) + expect(pending).toHaveLength(2) + + if (oldOutcome === `rejected`) { + const rejection = expect(oldLoad).rejects.toThrow(`old request failed`) + pending[0]!.reject(new Error(`old request failed`)) + await rejection + } else { + await applyRows([{ id: `old-row`, value: 1 }]) + pending[0]!.resolve({ + hasMore: oldOutcome === `authoritative` ? false : undefined, + appliedRowKeys: [`old-row`], + }) + await oldLoad + } + + expect(source._sync.getLoadSubsetOutcome(freshOptions)).toBeUndefined() + const peerLoad = source._sync.loadSubset(peerOptions) + if (peerLoad === true) throw new Error(`Expected a shared peer request`) + expect(pending).toHaveLength(2) + + await applyRows([{ id: `fresh-row`, value: 2 }]) + pending[1]!.resolve({ + hasMore: false, + appliedRowKeys: [`fresh-row`], + }) + await Promise.all([freshLoad, peerLoad]) + expect(source._sync.getLoadSubsetOutcome(peerOptions)).toBeDefined() + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await source.cleanup() + } + }, +) + it(`does not release physical work when an already-aborted demand skips adapter start`, async () => { const ownerId = `aborted-owner` const requestEvent: LoadSubsetFullFlowEvent = { diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index db00f4f89..2cd0994e0 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -382,6 +382,45 @@ it(`scopes reusable evidence to the physical attempt when an owner is reused`, ( ).toBe(2) }) +it(`does not rebuild coverage when a released attempt settles after its replacement starts`, () => { + expect( + projectReusableDemands([ + { + type: `requestDemand`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + { + type: `requestDemand`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + ]), + ).toEqual([]) +}) + function renameHistoryIds( history: ReadonlyArray, suffix: string, @@ -509,7 +548,7 @@ for (const campaign of refinementCampaigns(1_779_002)) { for (const campaign of refinementCampaigns(1_779_003)) { fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( - `demand, owner, session, and task names preserve projected laws (${campaign.label})`, + `demand, attempt, owner, session, and task names preserve projected laws (${campaign.label})`, (suffix) => { const demandHistory: Array = [ { @@ -556,6 +595,15 @@ for (const campaign of refinementCampaigns(1_779_003)) { ] const renamedDemand = renameHistoryIds(demandHistory, suffix) + expect( + renamedDemand.flatMap((event) => + `attemptId` in event ? [event.attemptId] : [], + ), + ).toEqual( + demandHistory.flatMap((event) => + `attemptId` in event ? [`${event.attemptId}-${suffix}`] : [], + ), + ) expect(projectTransportLoads(renamedDemand)).toBe( projectTransportLoads(demandHistory), ) From e549aa28043f6036bf76621639047cb6679c1bc6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 14:11:30 -0600 Subject: [PATCH 076/327] fix(db): preserve newer subset attempts --- packages/db/src/query/live/ARCHITECTURE.md | 5 +- packages/db/src/query/subset-dedupe.ts | 123 ++++++++++++--- .../db/tests/load-subset-full-flow-model.ts | 74 +++++++++ ...d-subset-full-flow-oracle.property.test.ts | 30 +++- ...d-subset-refinement-model.property.test.ts | 145 ++++++++++++++++++ packages/db/tests/query/subset-dedupe.test.ts | 73 +++++++++ .../tests/electric-live-query.test.ts | 12 +- 7 files changed, 430 insertions(+), 32 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a56199775..9c0209561 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -915,7 +915,10 @@ Collection sync boundary; shared rows remain until their final owner retires. An adapter that uses `DeduplicatedLoadSubset` across live-query lifetimes must also return the helper's paired `unloadSubset` callback. That callback invalidates remembered request coverage when core may delete its establishing -rows. A dedupe hit cannot outlive the evidence it claims to reuse. +rows. Core pairs each accepted load with one release of the same options object. +The helper keeps those logical owner reservations across resets so a late +release cannot retire newer-generation work or work still shared by another +owner. A dedupe hit cannot outlive the evidence it claims to reuse. An eager Query DB collection owns its base query for the Collection lifetime. If TanStack Query removes that cache entry while the Collection has no public diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index dc970bf4a..d9de6c938 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -24,11 +24,19 @@ type SharedAbortLease = { dispose: () => void } +type LogicalLoadReservation = { + generation: number + inflight?: InflightCall +} + type InflightCall = { options: LoadSubsetOptions promise: Promise lease: SharedAbortLease matchesPhysicalRequest: (options: LoadSubsetOptions) => boolean + generation: number + trackable: boolean + reservations: Set } /** @@ -83,6 +91,13 @@ export class DeduplicatedLoadSubset { // check if their captured generation matches before updating tracking state private generation = 0 + // Core releases the exact options object that it passed to loadSubset. + // A queue preserves that identity when one object is reused across calls. + private ownerReservations = new WeakMap< + LoadSubsetOptions, + Array + >() + constructor(opts: { loadSubset: LoadSubsetFn onDeduplicate?: (options: LoadSubsetOptions) => void @@ -104,6 +119,19 @@ export class DeduplicatedLoadSubset { loadSubset = ( options: LoadSubsetOptions, ): true | Promise => { + const reservation = this.reserveOwner(options) + try { + return this.loadSubsetRequest(options, reservation) + } catch (error) { + this.removeOwnerReservation(options, reservation) + throw error + } + } + + private loadSubsetRequest( + options: LoadSubsetOptions, + reservation: LogicalLoadReservation, + ): true | Promise { // If we've loaded all data, everything is covered if (this.hasLoadedAllData) { this.onDeduplicate?.(options) @@ -140,6 +168,8 @@ export class DeduplicatedLoadSubset { ) if (matchingInflight !== undefined) { + matchingInflight.reservations.add(reservation) + reservation.inflight = matchingInflight matchingInflight.lease.attach(options.signal) // An in-flight call will load data that covers this request // Every requester shares the physical work and cancellation lease. A @@ -202,23 +232,24 @@ export class DeduplicatedLoadSubset { lease.dispose() return true } else { - // Async return - track the promise and update tracking after it resolves - - // Capture the current generation - this lets us detect if reset() was called - // while this request was in-flight, so we can skip updating tracking state - const capturedGeneration = this.generation - // We need to create a reference to the in-flight entry so we can remove it later - const inflightEntry = { + const inflightEntry: InflightCall = { options: trackingOptions, lease, matchesPhysicalRequest, + generation: this.generation, + trackable: true, + reservations: new Set([reservation]), promise: resultPromise .then((result) => { // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared - if (capturedGeneration === this.generation && !lease.aborted) { + if ( + inflightEntry.trackable && + inflightEntry.generation === this.generation && + !lease.aborted + ) { this.updateTracking(trackingOptions) } return recordLoadSubsetResultDemandMatcher( @@ -236,6 +267,7 @@ export class DeduplicatedLoadSubset { lease.dispose() }), } + reservation.inflight = inflightEntry recordLoadSubsetPromiseDemandMatcher( inflightEntry.promise, @@ -261,15 +293,28 @@ export class DeduplicatedLoadSubset { * across live-query lifetimes must return this method as their unloadSubset * callback. * - * The reset is intentionally conservative. One released request may clear - * evidence still useful to another owner, causing a later refetch, but it can - * never reuse evidence for rows that core no longer retains. Until adapters - * report which retained rows came from which demand, the settled-case cost is - * bounded to one new physical request for each distinct demand revisited - * before deduplication state is rebuilt. + * Settled evidence is invalidated conservatively. In-flight work is tracked + * by exact logical owner, so a late release cannot retire a newer generation + * or work that another owner still needs. Core must release the same options + * object that it passed to loadSubset; unmatched releases are no-ops. */ - unloadSubset = (_options: LoadSubsetOptions): void => { - this.reset() + unloadSubset = (options: LoadSubsetOptions): void => { + const reservation = this.shiftOwnerReservation(options) + // A synchronous adapter throw never established helper state. Core may + // still release that logical demand later, but it must not invalidate a + // newer request that happens to use equivalent options. + if (!reservation || reservation.generation !== this.generation) return + + this.clearLoadedTracking() + const inflight = reservation.inflight + if (!inflight) return + + inflight.reservations.delete(reservation) + if (inflight.reservations.size > 0) return + + inflight.trackable = false + const index = this.inflightCalls.indexOf(inflight) + if (index !== -1) this.inflightCalls.splice(index, 1) } /** @@ -281,15 +326,55 @@ export class DeduplicatedLoadSubset { * state after the reset. This prevents old requests from repopulating cleared state. */ reset(): void { - this.unlimitedWhere = undefined - this.hasLoadedAllData = false - this.limitedCalls = [] + this.clearLoadedTracking() + for (const inflight of this.inflightCalls) inflight.trackable = false this.inflightCalls = [] // Increment generation to invalidate any in-flight completion handlers // This ensures requests that were started before reset() don't repopulate the state this.generation++ } + private reserveOwner(options: LoadSubsetOptions): LogicalLoadReservation { + const reservation = { generation: this.generation } + const reservations = this.ownerReservations.get(options) + if (reservations) reservations.push(reservation) + else this.ownerReservations.set(options, [reservation]) + return reservation + } + + private shiftOwnerReservation( + options: LoadSubsetOptions, + ): LogicalLoadReservation | undefined { + const reservations = this.ownerReservations.get(options) + const reservation = reservations?.shift() + if (reservations?.length === 0) this.ownerReservations.delete(options) + return reservation + } + + private removeOwnerReservation( + options: LoadSubsetOptions, + reservation: LogicalLoadReservation, + ): void { + const reservations = this.ownerReservations.get(options) + const reservationIndex = reservations?.indexOf(reservation) ?? -1 + if (reservationIndex !== -1) reservations!.splice(reservationIndex, 1) + if (reservations?.length === 0) this.ownerReservations.delete(options) + + const inflight = reservation.inflight + if (!inflight) return + inflight.reservations.delete(reservation) + if (inflight.reservations.size > 0) return + inflight.trackable = false + const inflightIndex = this.inflightCalls.indexOf(inflight) + if (inflightIndex !== -1) this.inflightCalls.splice(inflightIndex, 1) + } + + private clearLoadedTracking(): void { + this.unlimitedWhere = undefined + this.hasLoadedAllData = false + this.limitedCalls = [] + } + private updateTracking(options: LoadSubsetOptions): void { // Update tracking based on whether this was a limited or unlimited call if (options.limit === undefined && options.cursor === undefined) { diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 9a6384485..a2145d02a 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -311,6 +311,78 @@ export type ExpectedAdapterLifecycleEvent = { ownerId: FullFlowOwnerId } +type DemandAttemptRecord = { + ownerId: FullFlowOwnerId + demandId: FullFlowDemandId + settled: boolean + released: boolean +} + +/** Reject histories that cannot name logical demand attempts unambiguously. */ +function assertWellFormedDemandAttempts( + history: ReadonlyArray, +): void { + const attempts = new Map() + + for (const event of history) { + if (event.type === `requestDemand`) { + if (attempts.has(event.attemptId)) { + throw new Error( + `Demand attempt "${event.attemptId}" was requested more than once`, + ) + } + attempts.set(event.attemptId, { + ownerId: event.ownerId, + demandId: event.demandId, + settled: false, + released: false, + }) + continue + } + + const usesDemandAttempt = + event.type === `applyAuthoritativeRows` || + event.type === `applyUnprovenRows` || + event.type === `rejectDemand` || + event.type === `settleDemandWithoutEvidence` || + event.type === `releaseDemand` + if (!usesDemandAttempt) continue + + const attempt = attempts.get(event.attemptId) + if (!attempt) { + throw new Error( + `Demand attempt "${event.attemptId}" was used before it was requested`, + ) + } + if (attempt.demandId !== event.demandId) { + throw new Error( + `Demand attempt "${event.attemptId}" changed its demand identity`, + ) + } + if (`ownerId` in event && attempt.ownerId !== event.ownerId) { + throw new Error( + `Demand attempt "${event.attemptId}" changed its owner identity`, + ) + } + + if (event.type === `releaseDemand`) { + if (attempt.released) { + throw new Error( + `Demand attempt "${event.attemptId}" was released more than once`, + ) + } + attempt.released = true + } else { + if (attempt.settled) { + throw new Error( + `Demand attempt "${event.attemptId}" settled more than once`, + ) + } + attempt.settled = true + } + } +} + /** * Projects logical adapter callback obligations. * @@ -348,6 +420,7 @@ export function projectAdapterLifecycle( export function projectTransportLoads( history: ReadonlyArray, ): number { + assertWellFormedDemandAttempts(history) const reusableDemands = new Map() const inFlightDemands = new Map() let loads = 0 @@ -516,6 +589,7 @@ export function projectAuthorizedContinuationStarts( export function projectReusableDemands( history: ReadonlyArray, ): Array { + assertWellFormedDemandAttempts(history) const reusableDemands = new Map() const attemptEpochs = new Map() let sourceEpoch = 0 diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index b4785e60d..5de6c38fd 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -383,14 +383,22 @@ async function runTruncateCoverageScenario( } } -it.each([`authoritative`, `unproven`, `rejected`] as const)( +it.each([ + `authoritative`, + `unproven`, + `rejected`, + `evidence-free`, + `released`, +] as const)( `keeps fresh exact-demand work shared after a pre-truncate %s request settles`, async (oldOutcome) => { type Row = { id: string; value: number } - type AdapterResult = { - hasMore: boolean | undefined - appliedRowKeys: ReadonlyArray - } + type AdapterResult = + | { + hasMore: boolean | undefined + appliedRowKeys: ReadonlyArray + } + | undefined let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void let commit!: () => true | Promise @@ -417,6 +425,7 @@ it.each([`authoritative`, `unproven`, `rejected`] as const)( params.markReady() return { loadSubset: deduplicated.loadSubset, + unloadSubset: deduplicated.unloadSubset, } }, }, @@ -446,10 +455,15 @@ it.each([`authoritative`, `unproven`, `rejected`] as const)( if (freshLoad === true) throw new Error(`Expected an async fresh request`) expect(pending).toHaveLength(2) - if (oldOutcome === `rejected`) { + if (oldOutcome === `released`) { + source._sync.unloadSubset(oldOptions) + } else if (oldOutcome === `rejected`) { const rejection = expect(oldLoad).rejects.toThrow(`old request failed`) pending[0]!.reject(new Error(`old request failed`)) await rejection + } else if (oldOutcome === `evidence-free`) { + pending[0]!.resolve(undefined) + await oldLoad } else { await applyRows([{ id: `old-row`, value: 1 }]) pending[0]!.resolve({ @@ -471,6 +485,10 @@ it.each([`authoritative`, `unproven`, `rejected`] as const)( }) await Promise.all([freshLoad, peerLoad]) expect(source._sync.getLoadSubsetOutcome(peerOptions)).toBeDefined() + if (oldOutcome === `released`) { + pending[0]!.resolve(undefined) + await oldLoad + } } finally { for (const request of pending) { request.reject(new Error(`test cleanup`)) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 2cd0994e0..3584ea499 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -421,6 +421,127 @@ it(`does not rebuild coverage when a released attempt settles after its replacem ).toEqual([]) }) +it(`keeps fresh same-epoch work shared after an older rejected attempt releases`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + } + + expect( + projectTransportLoads([ + oldRequest, + { + type: `rejectDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + freshRequest, + { + type: `releaseDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + { + ...freshRequest, + ownerId: `peer-owner`, + attemptId: `peer-attempt`, + }, + ]), + ).toBe(2) +}) + +it(`rejects histories that reuse one demand attempt identity`, () => { + const history: Array = [ + { + type: `requestDemand`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + rowKeys: [], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + { + type: `requestDemand`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + rowKeys: [`stale-row`], + }, + ] + + expect(() => projectTransportLoads(history)).toThrow( + `Demand attempt "reused-attempt" was requested more than once`, + ) + expect(() => projectReusableDemands(history)).toThrow( + `Demand attempt "reused-attempt" was requested more than once`, + ) +}) + +it(`rejects histories that settle one demand attempt twice`, () => { + const history: Array = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `settleDemandWithoutEvidence`, + demandId: `demand`, + attemptId: `attempt`, + }, + { + type: `rejectDemand`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + }, + ] + + expect(() => projectTransportLoads(history)).toThrow( + `Demand attempt "attempt" settled more than once`, + ) + expect(() => projectReusableDemands(history)).toThrow( + `Demand attempt "attempt" settled more than once`, + ) +}) + function renameHistoryIds( history: ReadonlyArray, suffix: string, @@ -594,6 +715,22 @@ for (const campaign of refinementCampaigns(1_779_003)) { { type: `runContinuation`, taskId: `task` }, ] + const evidenceFreeHistory: Array = [ + { + type: `requestDemand`, + ownerId: `evidence-free-owner`, + sessionId: `session`, + demandId: `evidence-free-demand`, + attemptId: `evidence-free-attempt`, + alreadyAborted: false, + }, + { + type: `settleDemandWithoutEvidence`, + demandId: `evidence-free-demand`, + attemptId: `evidence-free-attempt`, + }, + ] + const renamedDemand = renameHistoryIds(demandHistory, suffix) expect( renamedDemand.flatMap((event) => @@ -604,6 +741,14 @@ for (const campaign of refinementCampaigns(1_779_003)) { `attemptId` in event ? [`${event.attemptId}-${suffix}`] : [], ), ) + expect( + renameHistoryIds(evidenceFreeHistory, suffix).flatMap((event) => + `attemptId` in event ? [event.attemptId] : [], + ), + ).toEqual([ + `evidence-free-attempt-${suffix}`, + `evidence-free-attempt-${suffix}`, + ]) expect(projectTransportLoads(renamedDemand)).toBe( projectTransportLoads(demandHistory), ) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 56ae5a12a..e72acf4c5 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -384,6 +384,79 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it.each([`reset`, `rejection`] as const)( + `keeps newer exact in-flight work when an older owner unloads after %s`, + async (oldOutcome) => { + const pending: Array<{ + resolve: () => void + reject: (error: Error) => void + }> = [] + const loadSubset = vi.fn( + () => + new Promise((resolve, reject) => { + pending.push({ resolve, reject }) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + const oldLoad = deduplicated.loadSubset(reusedOptions) + + if (oldOutcome === `reset`) { + deduplicated.reset() + } else { + const rejected = expect(oldLoad).rejects.toThrow(`old failed`) + pending[0]!.reject(new Error(`old failed`)) + await rejected + } + + const freshLoad = deduplicated.loadSubset(reusedOptions) + deduplicated.unloadSubset(reusedOptions) + const peerLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + pending[1]!.resolve() + if (oldOutcome === `reset`) { + pending[0]!.resolve() + await oldLoad + } + await Promise.all([freshLoad, peerLoad]) + }, + ) + + it(`keeps shared exact in-flight work while another logical owner remains`, async () => { + let resolveLoad: (() => void) | undefined + const loadSubset = vi.fn( + () => new Promise((resolve) => (resolveLoad = resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const firstOptions = { limit: 2 } + const first = deduplicated.loadSubset(firstOptions) + const second = deduplicated.loadSubset({ limit: 2 }) + + deduplicated.unloadSubset(firstOptions) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + resolveLoad?.() + await Promise.all([first, second, peer]) + }) + + it(`ignores an unload that has no matching logical owner`, async () => { + let resolveLoad: (() => void) | undefined + const loadSubset = vi.fn( + () => new Promise((resolve) => (resolveLoad = resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const load = deduplicated.loadSubset({ limit: 2 }) + + deduplicated.unloadSubset({ limit: 2 }) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + resolveLoad?.() + await Promise.all([load, peer]) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index cfce12133..e7e9211c0 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -1211,9 +1211,9 @@ describe(`Electric Collection - loadSubset deduplication`, () => { // Wait for the existing live query to re-request data after truncate await new Promise((resolve) => setTimeout(resolve, 0)) - // Truncate replays the exact demand once. Electric does not yet return an - // applied outcome, so the empty local prefix then requests one refill. - expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) + // Truncate replays the exact demand once. Releasing the old acquisition + // must not discard that replacement while it is still owned. + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2) // Create the same live query again after reset // This should NOT be deduped because the reset cleared the deduplication state, @@ -1231,9 +1231,9 @@ describe(`Electric Collection - loadSubset deduplication`, () => { await new Promise((resolve) => setTimeout(resolve, 0)) - // Should have more calls - the different query triggered a new request - // 1 initial + 1 replay + 1 outcome-free refill + 1 new query = 4 - expect(mockRequestSnapshot).toHaveBeenCalledTimes(4) + // The different query triggers one more physical request. + // 1 initial + 1 replay + 1 new query = 3 + expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) }) it(`should deduplicate unlimited queries regardless of orderBy`, async () => { From 6634e62ec5f2dd0a901097ce1e394c8f6e0e566f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 14:24:08 -0600 Subject: [PATCH 077/327] test(db): cover settled attempt releases --- ...d-subset-full-flow-oracle.property.test.ts | 33 +++++++++++++++---- packages/db/tests/query/subset-dedupe.test.ts | 23 +++++++++++++ 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 5de6c38fd..b52ba9091 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -384,14 +384,15 @@ async function runTruncateCoverageScenario( } it.each([ - `authoritative`, - `unproven`, - `rejected`, - `evidence-free`, - `released`, + { oldOutcome: `authoritative`, freshSettlesFirst: false }, + { oldOutcome: `unproven`, freshSettlesFirst: false }, + { oldOutcome: `rejected`, freshSettlesFirst: false }, + { oldOutcome: `evidence-free`, freshSettlesFirst: false }, + { oldOutcome: `released`, freshSettlesFirst: false }, + { oldOutcome: `released`, freshSettlesFirst: true }, ] as const)( - `keeps fresh exact-demand work shared after a pre-truncate %s request settles`, - async (oldOutcome) => { + `keeps fresh exact-demand work shared after a pre-truncate $oldOutcome request (freshSettlesFirst=$freshSettlesFirst)`, + async ({ oldOutcome, freshSettlesFirst }) => { type Row = { id: string; value: number } type AdapterResult = | { @@ -455,6 +456,24 @@ it.each([ if (freshLoad === true) throw new Error(`Expected an async fresh request`) expect(pending).toHaveLength(2) + if (freshSettlesFirst) { + await applyRows([{ id: `fresh-row`, value: 2 }]) + pending[1]!.resolve({ + hasMore: false, + appliedRowKeys: [`fresh-row`], + }) + await freshLoad + + source._sync.unloadSubset(oldOptions) + expect(source._sync.loadSubset(peerOptions)).toBe(true) + expect(pending).toHaveLength(2) + expect(source._sync.getLoadSubsetOutcome(peerOptions)).toBeDefined() + + pending[0]!.resolve(undefined) + await oldLoad + return + } + if (oldOutcome === `released`) { source._sync.unloadSubset(oldOptions) } else if (oldOutcome === `rejected`) { diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index e72acf4c5..0f830cf47 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -457,6 +457,29 @@ describe(`createDeduplicatedLoadSubset`, () => { await Promise.all([load, peer]) }) + it(`rolls back only the reservation whose adapter start throws`, async () => { + const pending: Array<() => void> = [] + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + throw new Error(`start failed`) + }) + .mockImplementation( + () => new Promise((resolve) => pending.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow(`start failed`) + const accepted = deduplicated.loadSubset(reusedOptions) + deduplicated.unloadSubset(reusedOptions) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(3) + pending.forEach((resolve) => resolve()) + await Promise.all([accepted, peer]) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 83e964d88ede0af8bf83eeb7c820fb30ab231115 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 14:40:19 -0600 Subject: [PATCH 078/327] fix(db): fence reentrant subset resets --- packages/db/src/query/live/ARCHITECTURE.md | 5 +- packages/db/src/query/subset-dedupe.ts | 11 ++-- packages/db/tests/query/subset-dedupe.test.ts | 53 ++++++++++++++++++- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 9c0209561..e267daed0 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -918,7 +918,10 @@ invalidates remembered request coverage when core may delete its establishing rows. Core pairs each accepted load with one release of the same options object. The helper keeps those logical owner reservations across resets so a late release cannot retire newer-generation work or work still shared by another -owner. A dedupe hit cannot outlive the evidence it claims to reuse. +owner. Adapter entry is a reentrancy boundary: capture the request generation +before calling adapter code, and do not publish coverage or in-flight work if a +reentrant reset has retired that generation. A dedupe hit cannot outlive the +evidence it claims to reuse. An eager Query DB collection owns its base query for the Collection lifetime. If TanStack Query removes that cache entry while the Collection has no public diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index d9de6c938..67c79beed 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -218,6 +218,7 @@ export class DeduplicatedLoadSubset { isLoadSubsetRequestSubsumedBy(physicalRequest, candidate) // Call underlying loadSubset to load the missing data + const requestGeneration = this.generation let resultPromise: true | Promise try { resultPromise = this._loadSubset(loadOptions) @@ -228,7 +229,9 @@ export class DeduplicatedLoadSubset { // Handle both sync (true) and async (Promise) return values if (resultPromise === true) { - if (!lease.aborted) this.updateTracking(trackingOptions) + if (requestGeneration === this.generation && !lease.aborted) { + this.updateTracking(trackingOptions) + } lease.dispose() return true } else { @@ -237,7 +240,7 @@ export class DeduplicatedLoadSubset { options: trackingOptions, lease, matchesPhysicalRequest, - generation: this.generation, + generation: requestGeneration, trackable: true, reservations: new Set([reservation]), promise: resultPromise @@ -275,7 +278,9 @@ export class DeduplicatedLoadSubset { ) // Store the in-flight entry so concurrent subset calls can wait for it - this.inflightCalls.push(inflightEntry) + if (requestGeneration === this.generation) { + this.inflightCalls.push(inflightEntry) + } return projectLoadSubsetResultForCaller( inflightEntry.promise, options, diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 0f830cf47..108725957 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -6,7 +6,7 @@ import { import { Func, PropRef, Value } from '../../src/query/ir' import { createCrossRealmUint8Array } from '../utils' import type { BasicExpression, OrderBy } from '../../src/query/ir' -import type { LoadSubsetOptions } from '../../src/types' +import type { LoadSubsetFn, LoadSubsetOptions } from '../../src/types' // Helper functions to build expressions more easily function ref(path: string | Array): PropRef { @@ -480,6 +480,57 @@ describe(`createDeduplicatedLoadSubset`, () => { await Promise.all([accepted, peer]) }) + it(`does not cache synchronous work from before a reentrant reset`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return true + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`does not share pending work from before a reentrant reset`, async () => { + let resolveOld!: () => void + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return new Promise((resolve) => (resolveOld = resolve)) + }) + .mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + const oldLoad = deduplicated.loadSubset({ limit: 2 }) + const freshLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + resolveOld() + await Promise.all([oldLoad, freshLoad]) + }) + + it(`does not cache settled work from before a reentrant reset`, async () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return Promise.resolve() + }) + .mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset({ limit: 2 }) + await deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 463b457d541528a952f424298fbdd3a890848e1b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 14:55:05 -0600 Subject: [PATCH 079/327] test(db): close reentrant subset boundaries --- packages/db/tests/query/subset-dedupe.test.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 108725957..b80b5f174 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -531,6 +531,86 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it(`rolls back a stale reservation when adapter reset precedes a throw`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + throw new Error(`start failed after reset`) + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow( + `start failed after reset`, + ) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + + it(`consumes a stale owner before releasing a fresh reused owner`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return true + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + + it(`does not share work reset while installing Promise handlers`, async () => { + let resolveOld!: () => void + class ResetOnThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + deduplicated.reset() + return super.then(onfulfilled, onrejected) + } + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + if (loadSubsetCalls === 1) { + return new ResetOnThenPromise((resolve) => { + resolveOld = resolve + }) + } + return Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + const oldLoad = deduplicated.loadSubset({ limit: 2 }) + const freshLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubsetCalls).toBe(2) + resolveOld() + await Promise.all([oldLoad, freshLoad]) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From ae60e8e681cff81c97b3b39a2140ed90a4b9d06f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 15:02:58 -0600 Subject: [PATCH 080/327] test(db): pin exact subset rollback --- packages/db/tests/query/subset-dedupe.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index b80b5f174..558e90d3d 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -552,6 +552,29 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(3) }) + it(`rolls back the exact throw behind an older stale owner`, () => { + const loadSubset = vi + .fn() + .mockReturnValueOnce(true) + .mockImplementationOnce(() => { + throw new Error(`replacement start failed`) + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.reset() + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow( + `replacement start failed`, + ) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + it(`consumes a stale owner before releasing a fresh reused owner`, () => { const loadSubset = vi .fn() From b259ea3cd49ebd598289d301bf4e29674e4b0fd6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 15:22:39 -0600 Subject: [PATCH 081/327] test(db): model multi-source ordered windows --- packages/db/src/query/live/ARCHITECTURE.md | 51 ++- .../db/tests/load-subset-full-flow-model.ts | 55 +++ ...d-subset-full-flow-oracle.property.test.ts | 406 +++++++++++++++++- 3 files changed, 488 insertions(+), 24 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index e267daed0..0f37ee943 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -401,6 +401,15 @@ predicates, and top-K to the full readable source. It must not issue a limited page and then disable continuation: later relational operators may reject that page and leave the result window short. +For an indexed ordered source above a join or later predicate, the visible +window is the direct relational result: source order, then downstream +operators, then top-K. Core must establish at least the shortest ordered source +prefix that contains that result, advancing its cursor across source rows that +the later relation rejects. A second source may settle after a continuation is +already in flight, so safe extra primary rows may become readable. They do not +change the top-K result or permit a shorter required prefix. Exhaustion may +leave the window short; a non-exhausted source may not. + Test adapters must obey the same boundary contract as production adapters. A mock that reports exhaustion must have made every matching source row readable before its result settles. A mock that reports more data must honor later @@ -1070,27 +1079,27 @@ create recursive Collection machinery. ## Executable contracts -| Contract | Test suite | -| --------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | -| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | -| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | -| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | -| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | -| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | -| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | -| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | -| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | -| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | -| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | -| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | -| End-to-end demand, continuation, and outcome-free boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | -| Subset acquisition, readiness, receipt, and replay refinement laws | `packages/db/tests/query/load-subset-refinement-*.property.test.ts` | -| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.property.test.ts` | -| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | -| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | -| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | +| Contract | Test suite | +| ---------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | +| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | +| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | +| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | +| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | +| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | +| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | +| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | +| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | +| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | +| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | +| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | +| End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | +| Subset acquisition, readiness, receipt, and replay refinement laws | `packages/db/tests/query/load-subset-refinement-*.property.test.ts` | +| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.property.test.ts` | +| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | +| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | +| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | Each oracle identifies the first divergent checkpoint and compares either the whole result or one exact structural difference. Correlated-materialization diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index a2145d02a..48434f0c7 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -40,6 +40,61 @@ export type OrderedContinuationEvidence = { rowsNeeded: number } +export type MultiSourceOrderedRow = { + key: string + joinKey: string +} + +export type MultiSourceOrderedWindow = { + visibleKeys: ReadonlyArray + scannedPrimaryKeys: ReadonlyArray + primaryCursorKeys: ReadonlyArray + demandedJoinKeys: ReadonlyArray + rowsNeeded: number + sourceExhausted: boolean +} + +/** + * Projects the smallest primary-source prefix needed to fill a joined window. + * The caller supplies primary rows in total order, so this projector only owns + * the cross-source law: every scanned primary row advances source progress, + * while only rows admitted by the secondary source fill the visible window. + */ +export function projectMultiSourceOrderedWindow(options: { + primaryOrder: ReadonlyArray + secondaryJoinKeys: ReadonlySet + targetSize: number +}): MultiSourceOrderedWindow { + const scannedPrimaryKeys: Array = [] + const visibleKeys: Array = [] + const demandedJoinKeys: Array = [] + const seenJoinKeys = new Set() + + for (const row of options.primaryOrder) { + if (visibleKeys.length >= options.targetSize) break + + scannedPrimaryKeys.push(row.key) + if (!seenJoinKeys.has(row.joinKey)) { + seenJoinKeys.add(row.joinKey) + demandedJoinKeys.push(row.joinKey) + } + if (options.secondaryJoinKeys.has(row.joinKey)) { + visibleKeys.push(row.key) + } + } + + return { + visibleKeys, + scannedPrimaryKeys, + primaryCursorKeys: scannedPrimaryKeys.map((_, index) => + index === 0 ? undefined : scannedPrimaryKeys[index - 1], + ), + demandedJoinKeys, + rowsNeeded: Math.max(0, options.targetSize - visibleKeys.length), + sourceExhausted: scannedPrimaryKeys.length === options.primaryOrder.length, + } +} + /** * Projects ordered evidence from request receipts alone. Requested size and * source progress are independent inputs; only eligible applied rows count diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index b52ba9091..c54d1731b 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -17,6 +17,7 @@ import { projectAtomicOrderedPublicationState, projectAtomicOrderedPublications, projectAuthorizedContinuationStarts, + projectMultiSourceOrderedWindow, projectOrderedContinuationEvidence, projectOrderedPublicationBoundary, projectRetainedRowKeys, @@ -146,6 +147,408 @@ it(`loads each side of a filtered inner join once`, async () => { } }) +const { multiplier: fullFlowMultiplier, replaySeed: fullFlowReplaySeed } = + readOracleRunConfig() + +type MultiSourceOrderedScenario = { + primaryRows: ReadonlyArray<{ + id: string + rank: number + joinKey: string + }> + secondaryJoinKeys: ReadonlyArray + targetSize: number + direction: `asc` | `desc` + secondaryPublication: `preloaded` | `on-demand` +} + +const multiSourceJoinKeyArbitrary = fc.constantFrom(`x`, `y`, `z`) +const multiSourceOrderedScenarioArbitrary: fc.Arbitrary = + fc + .record({ + ranks: fc.tuple( + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + ), + joinKeys: fc.tuple( + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + ), + secondaryJoinKeys: fc.uniqueArray(multiSourceJoinKeyArbitrary, { + maxLength: 3, + }), + targetSize: fc.integer({ min: 1, max: 3 }), + direction: fc.constantFrom(`asc` as const, `desc` as const), + secondaryPublication: fc.constantFrom( + `preloaded` as const, + `on-demand` as const, + ), + }) + .map(({ ranks, joinKeys, ...scenario }) => ({ + ...scenario, + primaryRows: [`a`, `b`, `c`, `d`].map((id, index) => ({ + id, + rank: ranks[index]!, + joinKey: joinKeys[index]!, + })), + })) + +if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + fc.statistics( + multiSourceOrderedScenarioArbitrary, + ({ + primaryRows, + secondaryJoinKeys, + targetSize, + direction, + secondaryPublication, + }) => [ + `direction=${direction}`, + `target=${targetSize}`, + `secondary=${secondaryPublication}`, + `exhaustion=${ + primaryRows.filter(({ joinKey }) => secondaryJoinKeys.includes(joinKey)) + .length < targetSize + }`, + `leading-exclusion=${!secondaryJoinKeys.includes( + orderedPrimaryRows({ + primaryRows, + secondaryJoinKeys, + targetSize, + direction, + secondaryPublication, + })[0]!.joinKey, + )}`, + `tied=${new Set(primaryRows.map(({ rank }) => rank)).size < primaryRows.length}`, + ], + oracleRandomParameters(1_000, fullFlowReplaySeed), + ) +} + +function orderedPrimaryRows( + scenario: MultiSourceOrderedScenario, +): Array { + return [...scenario.primaryRows].sort((left, right) => { + const rankOrder = + scenario.direction === `asc` + ? left.rank - right.rank + : right.rank - left.rank + return rankOrder || left.id.localeCompare(right.id) + }) +} + +function containsOrderedSubsequence( + values: ReadonlyArray, + subsequence: ReadonlyArray, +): boolean { + let expectedIndex = 0 + for (const value of values) { + if (expectedIndex === subsequence.length) return true + if (Object.is(value, subsequence[expectedIndex])) expectedIndex++ + } + return expectedIndex === subsequence.length +} + +let multiSourceOrderedHarnessId = 0 + +async function runMultiSourceOrderedScenario( + scenario: MultiSourceOrderedScenario, +): Promise { + type PrimaryRow = MultiSourceOrderedScenario[`primaryRows`][number] + type SecondaryRow = { id: string; joinKey: string } + + const primaryOrder = orderedPrimaryRows(scenario) + const projection = projectMultiSourceOrderedWindow({ + primaryOrder: primaryOrder.map(({ id, joinKey }) => ({ + key: id, + joinKey, + })), + secondaryJoinKeys: new Set(scenario.secondaryJoinKeys), + targetSize: scenario.targetSize, + }) + const primaryCalls: Array = [] + const primaryAppliedKeys: Array = [] + const secondaryCalls: Array = [] + let primaryBegin!: () => void + let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void + let primaryCommit!: () => true | Promise + const primary = createCollection({ + id: `multi-source-ordered-primary-${multiSourceOrderedHarnessId}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + primaryBegin = params.begin + primaryWrite = params.write + primaryCommit = params.commit + params.markReady() + return { + loadSubset: async (options) => { + primaryCalls.push(options) + const lastKey = options.cursor?.lastKey + const previousIndex = + lastKey === undefined + ? -1 + : primaryOrder.findIndex(({ id }) => id === lastKey) + const row = primaryOrder[previousIndex + 1] + if (row) { + primaryAppliedKeys.push(row.id) + primaryBegin() + primaryWrite({ type: `insert`, value: row }) + const applied = primaryCommit() + if (applied !== true) await applied + } + return { + hasMore: previousIndex + 1 < primaryOrder.length - 1, + appliedRowKeys: row ? [row.id] : [], + } + }, + unloadSubset: () => {}, + } + }, + }, + }) + + let secondaryBegin!: () => void + let secondaryWrite!: (message: { + type: `insert` + value: SecondaryRow + }) => void + let secondaryCommit!: () => true | Promise + const secondaryRows = scenario.secondaryJoinKeys.map((joinKey) => ({ + id: `secondary-${joinKey}`, + joinKey, + })) + const secondary = createCollection({ + id: `multi-source-ordered-secondary-${multiSourceOrderedHarnessId}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + secondaryBegin = params.begin + secondaryWrite = params.write + secondaryCommit = params.commit + if ( + scenario.secondaryPublication === `preloaded` && + secondaryRows.length > 0 + ) { + secondaryBegin() + for (const row of secondaryRows) { + secondaryWrite({ type: `insert`, value: row }) + } + const applied = secondaryCommit() + if (applied !== true) { + throw new Error(`Expected synchronous initial secondary rows`) + } + } + params.markReady() + return { + loadSubset: async (options) => { + secondaryCalls.push(options) + if ( + scenario.secondaryPublication === `on-demand` && + secondaryRows.length > 0 + ) { + secondaryBegin() + for (const row of secondaryRows) { + secondaryWrite({ type: `insert`, value: row }) + } + const applied = secondaryCommit() + if (applied !== true) await applied + } + return { + hasMore: false, + appliedRowKeys: secondaryRows.map(({ id }) => id), + } + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `multi-source-ordered-live-${multiSourceOrderedHarnessId++}`, + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank, scenario.direction) + .limit(scenario.targetSize), + startSync: true, + }) + + try { + await live.preload() + await flushPromises() + + expect(live.toArray.map(({ primaryRow }) => primaryRow.id)).toEqual( + projection.visibleKeys, + ) + const distinctPrimaryKeys = [...new Set(primaryAppliedKeys)] + expect(distinctPrimaryKeys).toEqual( + primaryOrder.slice(0, distinctPrimaryKeys.length).map(({ id }) => id), + ) + expect( + distinctPrimaryKeys.slice(0, projection.scannedPrimaryKeys.length), + ).toEqual(projection.scannedPrimaryKeys) + expect( + containsOrderedSubsequence( + primaryCalls + .filter(({ orderBy }) => orderBy !== undefined) + .map(({ cursor }) => cursor?.lastKey as string | undefined), + projection.primaryCursorKeys, + ), + ).toBe(true) + expect(secondaryCalls.length).toBeGreaterThan(0) + } finally { + await Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]) + } +} + +it(`continues an ordered primary source until a joined window is full`, async () => { + await runMultiSourceOrderedScenario({ + primaryRows: [ + { id: `a`, rank: 1, joinKey: `a` }, + { id: `b`, rank: 2, joinKey: `b` }, + { id: `c`, rank: 3, joinKey: `c` }, + { id: `d`, rank: 4, joinKey: `d` }, + ], + secondaryJoinKeys: [`c`, `d`], + targetSize: 2, + direction: `asc`, + secondaryPublication: `preloaded`, + }) +}) + +it(`projects the minimal primary prefix needed by a joined window`, () => { + const projection = projectMultiSourceOrderedWindow({ + primaryOrder: [ + { key: `a`, joinKey: `x` }, + { key: `b`, joinKey: `y` }, + { key: `c`, joinKey: `z` }, + { key: `d`, joinKey: `x` }, + ], + secondaryJoinKeys: new Set([`x`, `z`]), + targetSize: 2, + }) + + expect(projection).toEqual({ + visibleKeys: [`a`, `c`], + scannedPrimaryKeys: [`a`, `b`, `c`], + primaryCursorKeys: [undefined, `a`, `b`], + demandedJoinKeys: [`x`, `y`, `z`], + rowsNeeded: 0, + sourceExhausted: false, + }) +}) + +it(`erases join-key spelling and ignores unreachable secondary rows`, () => { + const original = projectMultiSourceOrderedWindow({ + primaryOrder: [ + { key: `a`, joinKey: `x` }, + { key: `b`, joinKey: `y` }, + { key: `c`, joinKey: `x` }, + ], + secondaryJoinKeys: new Set([`x`, `unused`]), + targetSize: 2, + }) + const renamed = projectMultiSourceOrderedWindow({ + primaryOrder: [ + { key: `a`, joinKey: `renamed-x` }, + { key: `b`, joinKey: `renamed-y` }, + { key: `c`, joinKey: `renamed-x` }, + ], + secondaryJoinKeys: new Set([`renamed-x`]), + targetSize: 2, + }) + + expect({ + visibleKeys: original.visibleKeys, + scannedPrimaryKeys: original.scannedPrimaryKeys, + primaryCursorKeys: original.primaryCursorKeys, + rowsNeeded: original.rowsNeeded, + sourceExhausted: original.sourceExhausted, + }).toEqual({ + visibleKeys: renamed.visibleKeys, + scannedPrimaryKeys: renamed.scannedPrimaryKeys, + primaryCursorKeys: renamed.primaryCursorKeys, + rowsNeeded: renamed.rowsNeeded, + sourceExhausted: renamed.sourceExhausted, + }) +}) + +it(`exhausts the bounded multi-source ordered-window model`, () => { + const rows = [ + { key: `a`, joinKey: `x` }, + { key: `b`, joinKey: `y` }, + { key: `c`, joinKey: `z` }, + ] + const joinKeys = [`x`, `y`, `z`] as const + + for (let mask = 0; mask < 1 << joinKeys.length; mask++) { + const secondaryJoinKeys = new Set( + joinKeys.filter((_, index) => (mask & (1 << index)) !== 0), + ) + for (const targetSize of [1, 2, 3]) { + const projection = projectMultiSourceOrderedWindow({ + primaryOrder: rows, + secondaryJoinKeys, + targetSize, + }) + const direct = rows + .filter(({ joinKey }) => secondaryJoinKeys.has(joinKey)) + .slice(0, targetSize) + .map(({ key }) => key) + + expect(projection.visibleKeys).toEqual(direct) + expect(projection.rowsNeeded).toBe( + Math.max(0, targetSize - direct.length), + ) + if (projection.scannedPrimaryKeys.length < rows.length) { + const shorterPrefix = rows.slice( + 0, + projection.scannedPrimaryKeys.length - 1, + ) + expect( + shorterPrefix.filter(({ joinKey }) => secondaryJoinKeys.has(joinKey)), + ).toHaveLength(targetSize - 1) + } else { + expect(projection.sourceExhausted).toBe(true) + } + } + } +}) + +fcTest.prop([multiSourceOrderedScenarioArbitrary], { + numRuns: 12 * fullFlowMultiplier, + seed: 17802, +})( + `fills joined ordered windows for a fixed seed`, + runMultiSourceOrderedScenario, +) + +fcTest.prop( + [multiSourceOrderedScenarioArbitrary], + oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), +)( + `fills joined ordered windows for a random or replayed seed`, + runMultiSourceOrderedScenario, +) + type TruncateCoverageScenario = { oldRequest: `none` | `settles-late` freshResult: `authoritative` | `unknown` | `reject` @@ -179,9 +582,6 @@ const exhaustiveTruncateCoverageScenarios: Array = [ ), ) -const { multiplier: fullFlowMultiplier, replaySeed: fullFlowReplaySeed } = - readOracleRunConfig() - let truncateCoverageHarnessId = 0 async function runTruncateCoverageScenario( From 93af99b53ce2c66cbd773ceff270a1397c557508 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 15:50:23 -0600 Subject: [PATCH 082/327] test(db): complete multi-source ordered oracle --- packages/db/src/query/live/ARCHITECTURE.md | 15 +- .../db/tests/load-subset-full-flow-model.ts | 45 +- ...d-subset-full-flow-oracle.property.test.ts | 446 +++++++++++++----- 3 files changed, 364 insertions(+), 142 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 0f37ee943..3bfa950c2 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -403,12 +403,15 @@ page and leave the result window short. For an indexed ordered source above a join or later predicate, the visible window is the direct relational result: source order, then downstream -operators, then top-K. Core must establish at least the shortest ordered source -prefix that contains that result, advancing its cursor across source rows that -the later relation rejects. A second source may settle after a continuation is -already in flight, so safe extra primary rows may become readable. They do not -change the top-K result or permit a shorter required prefix. Exhaustion may -leave the window short; a non-exhausted source may not. +operators, then offset and top-K. Core may prove that result with the shortest +ordered source prefix, or with other authoritative active demands that +establish all contributors which can precede the boundary. A forward scan +advances its cursor across source rows that the later relation rejects. A +reverse join demand can instead make a later matching row readable without +claiming reusable ordered-prefix coverage for skipped rows. A second source may +settle after a continuation is already in flight, so safe extra primary rows +may become readable. None of these paths may change the direct result or let an +exhaustible source leave a provable window under-filled. Test adapters must obey the same boundary contract as production adapters. A mock that reports exhaustion must have made every matching source row readable diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 48434f0c7..5a2ce3c2f 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -45,8 +45,13 @@ export type MultiSourceOrderedRow = { joinKey: string } +export type MultiSourceSecondaryRow = { + key: string + joinKey: string +} + export type MultiSourceOrderedWindow = { - visibleKeys: ReadonlyArray + visiblePairKeys: ReadonlyArray scannedPrimaryKeys: ReadonlyArray primaryCursorKeys: ReadonlyArray demandedJoinKeys: ReadonlyArray @@ -55,42 +60,56 @@ export type MultiSourceOrderedWindow = { } /** - * Projects the smallest primary-source prefix needed to fill a joined window. - * The caller supplies primary rows in total order, so this projector only owns - * the cross-source law: every scanned primary row advances source progress, - * while only rows admitted by the secondary source fill the visible window. + * Projects the smallest forward primary-source scan that fills a joined + * window. The caller supplies primary rows in total order, so this projector + * only owns the cross-source relational law: every scanned primary row advances + * source progress, while joined pair multiplicity fills offset plus limit. + * Production may prove the same result with reverse authoritative demands; the + * boundary harness compares the public result and each transport law separately. */ export function projectMultiSourceOrderedWindow(options: { primaryOrder: ReadonlyArray - secondaryJoinKeys: ReadonlySet - targetSize: number + secondaryRows: ReadonlyArray + offset: number + limit: number }): MultiSourceOrderedWindow { const scannedPrimaryKeys: Array = [] - const visibleKeys: Array = [] + const joinedPairKeys: Array = [] const demandedJoinKeys: Array = [] const seenJoinKeys = new Set() + const targetSize = options.offset + options.limit + const secondaryRows = [...options.secondaryRows].sort((left, right) => + left.key.localeCompare(right.key), + ) for (const row of options.primaryOrder) { - if (visibleKeys.length >= options.targetSize) break + if (joinedPairKeys.length >= targetSize) break scannedPrimaryKeys.push(row.key) if (!seenJoinKeys.has(row.joinKey)) { seenJoinKeys.add(row.joinKey) demandedJoinKeys.push(row.joinKey) } - if (options.secondaryJoinKeys.has(row.joinKey)) { - visibleKeys.push(row.key) + for (const secondaryRow of secondaryRows) { + if (secondaryRow.joinKey === row.joinKey) { + joinedPairKeys.push(`${row.key}:${secondaryRow.key}`) + } } } + const visiblePairKeys = joinedPairKeys.slice( + options.offset, + options.offset + options.limit, + ) + return { - visibleKeys, + visiblePairKeys, scannedPrimaryKeys, primaryCursorKeys: scannedPrimaryKeys.map((_, index) => index === 0 ? undefined : scannedPrimaryKeys[index - 1], ), demandedJoinKeys, - rowsNeeded: Math.max(0, options.targetSize - visibleKeys.length), + rowsNeeded: Math.max(0, options.limit - visiblePairKeys.length), sourceExhausted: scannedPrimaryKeys.length === options.primaryOrder.length, } } diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index c54d1731b..8d8af7155 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -156,10 +156,14 @@ type MultiSourceOrderedScenario = { rank: number joinKey: string }> - secondaryJoinKeys: ReadonlyArray - targetSize: number + secondaryRows: ReadonlyArray<{ id: string; joinKey: string }> + offset: number + limit: number direction: `asc` | `desc` - secondaryPublication: `preloaded` | `on-demand` + secondaryPublication: + | `preloaded` + | `after-primary-continuation` + | `after-primary-exhaustion` } const multiSourceJoinKeyArbitrary = fc.constantFrom(`x`, `y`, `z`) @@ -178,23 +182,36 @@ const multiSourceOrderedScenarioArbitrary: fc.Arbitrary ({ + .map(({ ranks, joinKeys, secondaryMatchCounts, ...scenario }) => ({ ...scenario, primaryRows: [`a`, `b`, `c`, `d`].map((id, index) => ({ id, rank: ranks[index]!, joinKey: joinKeys[index]!, })), + secondaryRows: [`x`, `y`, `z`].flatMap((joinKey, index) => + Array.from( + { length: secondaryMatchCounts[index]! }, + (_, matchIndex) => ({ + id: `${joinKey}-${matchIndex}`, + joinKey, + }), + ), + ), })) if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { @@ -202,27 +219,38 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { multiSourceOrderedScenarioArbitrary, ({ primaryRows, - secondaryJoinKeys, - targetSize, + secondaryRows, + offset, + limit, direction, secondaryPublication, }) => [ `direction=${direction}`, - `target=${targetSize}`, + `offset=${offset}`, + `limit=${limit}`, `secondary=${secondaryPublication}`, `exhaustion=${ - primaryRows.filter(({ joinKey }) => secondaryJoinKeys.includes(joinKey)) - .length < targetSize + primaryRows.reduce( + (count, { joinKey }) => + count + + secondaryRows.filter((row) => row.joinKey === joinKey).length, + 0, + ) < + offset + limit }`, - `leading-exclusion=${!secondaryJoinKeys.includes( - orderedPrimaryRows({ - primaryRows, - secondaryJoinKeys, - targetSize, - direction, - secondaryPublication, - })[0]!.joinKey, + `leading-exclusion=${!secondaryRows.some( + ({ joinKey }) => + joinKey === + orderedPrimaryRows({ + primaryRows, + secondaryRows, + offset, + limit, + direction, + secondaryPublication, + })[0]!.joinKey, )}`, + `multiplicity=${new Set(secondaryRows.map(({ joinKey }) => joinKey)).size < secondaryRows.length}`, `tied=${new Set(primaryRows.map(({ rank }) => rank)).size < primaryRows.length}`, ], oracleRandomParameters(1_000, fullFlowReplaySeed), @@ -241,18 +269,6 @@ function orderedPrimaryRows( }) } -function containsOrderedSubsequence( - values: ReadonlyArray, - subsequence: ReadonlyArray, -): boolean { - let expectedIndex = 0 - for (const value of values) { - if (expectedIndex === subsequence.length) return true - if (Object.is(value, subsequence[expectedIndex])) expectedIndex++ - } - return expectedIndex === subsequence.length -} - let multiSourceOrderedHarnessId = 0 async function runMultiSourceOrderedScenario( @@ -267,15 +283,41 @@ async function runMultiSourceOrderedScenario( key: id, joinKey, })), - secondaryJoinKeys: new Set(scenario.secondaryJoinKeys), - targetSize: scenario.targetSize, + secondaryRows: scenario.secondaryRows.map(({ id, joinKey }) => ({ + key: id, + joinKey, + })), + offset: scenario.offset, + limit: scenario.limit, }) const primaryCalls: Array = [] - const primaryAppliedKeys: Array = [] + const primaryOrderedVisitedKeys: Array = [] const secondaryCalls: Array = [] + const secondaryReceipts: Array> = [] + const secondaryPublicationGate = createDeferred() + const establishedPrimaryKeys = new Set() + const establishedSecondaryKeys = new Set() + let primaryOrderedCallCount = 0 + let primaryKeysBeforeSecondaryPublication: ReadonlyArray | undefined let primaryBegin!: () => void let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void let primaryCommit!: () => true | Promise + + const applyPrimaryRows = async ( + rows: ReadonlyArray, + ): Promise> => { + const freshRows = rows.filter(({ id }) => !establishedPrimaryKeys.has(id)) + if (freshRows.length === 0) return [] + primaryBegin() + for (const row of freshRows) { + establishedPrimaryKeys.add(row.id) + primaryWrite({ type: `insert`, value: row }) + } + const applied = primaryCommit() + if (applied !== true) await applied + return freshRows.map(({ id }) => id) + } + const primary = createCollection({ id: `multi-source-ordered-primary-${multiSourceOrderedHarnessId}`, getKey: (row) => row.id, @@ -292,22 +334,49 @@ async function runMultiSourceOrderedScenario( return { loadSubset: async (options) => { primaryCalls.push(options) + if (!options.orderBy) { + const rows = primaryOrder.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + return { + hasMore: false, + appliedRowKeys: await applyPrimaryRows(rows), + } + } + + primaryOrderedCallCount++ const lastKey = options.cursor?.lastKey const previousIndex = lastKey === undefined ? -1 : primaryOrder.findIndex(({ id }) => id === lastKey) + if (lastKey !== undefined && previousIndex < 0) { + throw new Error(`Unknown primary cursor ${String(lastKey)}`) + } const row = primaryOrder[previousIndex + 1] + let appliedRowKeys: Array = [] if (row) { - primaryAppliedKeys.push(row.id) - primaryBegin() - primaryWrite({ type: `insert`, value: row }) - const applied = primaryCommit() - if (applied !== true) await applied + primaryOrderedVisitedKeys.push(row.id) + appliedRowKeys = await applyPrimaryRows([row]) + } + const hasMore = previousIndex + 1 < primaryOrder.length - 1 + if ( + scenario.secondaryPublication === `after-primary-continuation` && + primaryOrderedCallCount >= 2 + ) { + secondaryPublicationGate.resolve() + } + if ( + scenario.secondaryPublication === `after-primary-exhaustion` && + !hasMore + ) { + secondaryPublicationGate.resolve() } return { - hasMore: previousIndex + 1 < primaryOrder.length - 1, - appliedRowKeys: row ? [row.id] : [], + hasMore, + appliedRowKeys, } }, unloadSubset: () => {}, @@ -322,10 +391,21 @@ async function runMultiSourceOrderedScenario( value: SecondaryRow }) => void let secondaryCommit!: () => true | Promise - const secondaryRows = scenario.secondaryJoinKeys.map((joinKey) => ({ - id: `secondary-${joinKey}`, - joinKey, - })) + const secondaryRows = scenario.secondaryRows + const applySecondaryRows = async ( + rows: ReadonlyArray, + ): Promise> => { + const freshRows = rows.filter(({ id }) => !establishedSecondaryKeys.has(id)) + if (freshRows.length === 0) return [] + secondaryBegin() + for (const row of freshRows) { + establishedSecondaryKeys.add(row.id) + secondaryWrite({ type: `insert`, value: row }) + } + const applied = secondaryCommit() + if (applied !== true) await applied + return freshRows.map(({ id }) => id) + } const secondary = createCollection({ id: `multi-source-ordered-secondary-${multiSourceOrderedHarnessId}`, getKey: (row) => row.id, @@ -344,6 +424,7 @@ async function runMultiSourceOrderedScenario( ) { secondaryBegin() for (const row of secondaryRows) { + establishedSecondaryKeys.add(row.id) secondaryWrite({ type: `insert`, value: row }) } const applied = secondaryCommit() @@ -355,20 +436,22 @@ async function runMultiSourceOrderedScenario( return { loadSubset: async (options) => { secondaryCalls.push(options) - if ( - scenario.secondaryPublication === `on-demand` && - secondaryRows.length > 0 - ) { - secondaryBegin() - for (const row of secondaryRows) { - secondaryWrite({ type: `insert`, value: row }) - } - const applied = secondaryCommit() - if (applied !== true) await applied + if (scenario.secondaryPublication !== `preloaded`) { + await secondaryPublicationGate.promise + primaryKeysBeforeSecondaryPublication ??= [ + ...new Set(primaryOrderedVisitedKeys), + ] } + const matchingRows = secondaryRows.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + const appliedRowKeys = await applySecondaryRows(matchingRows) + secondaryReceipts.push(appliedRowKeys) return { hasMore: false, - appliedRowKeys: secondaryRows.map(({ id }) => id), + appliedRowKeys, } }, unloadSubset: () => {}, @@ -387,7 +470,8 @@ async function runMultiSourceOrderedScenario( eq(primaryRow.joinKey, secondaryRow.joinKey), ) .orderBy(({ primaryRow }) => primaryRow.rank, scenario.direction) - .limit(scenario.targetSize), + .offset(scenario.offset) + .limit(scenario.limit), startSync: true, }) @@ -395,42 +479,130 @@ async function runMultiSourceOrderedScenario( await live.preload() await flushPromises() - expect(live.toArray.map(({ primaryRow }) => primaryRow.id)).toEqual( - projection.visibleKeys, - ) - const distinctPrimaryKeys = [...new Set(primaryAppliedKeys)] - expect(distinctPrimaryKeys).toEqual( - primaryOrder.slice(0, distinctPrimaryKeys.length).map(({ id }) => id), - ) - expect( - distinctPrimaryKeys.slice(0, projection.scannedPrimaryKeys.length), - ).toEqual(projection.scannedPrimaryKeys) expect( - containsOrderedSubsequence( - primaryCalls - .filter(({ orderBy }) => orderBy !== undefined) - .map(({ cursor }) => cursor?.lastKey as string | undefined), - projection.primaryCursorKeys, + live.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, ), - ).toBe(true) + ).toEqual(projection.visiblePairKeys) + expect(primaryCalls.some(({ orderBy }) => orderBy !== undefined)).toBe(true) + expect(primaryCalls.length).toBeLessThanOrEqual(32) expect(secondaryCalls.length).toBeGreaterThan(0) + + const claimedSecondaryKeys = secondaryReceipts.flat() + expect(new Set(claimedSecondaryKeys).size).toBe(claimedSecondaryKeys.length) + const expectedClaimedSecondaryKeys = + scenario.secondaryPublication === `preloaded` + ? [] + : scenario.secondaryRows + .filter((row) => + secondaryCalls.some( + ({ where }) => + where === undefined || + evaluateReferenceExpression(where, row), + ), + ) + .map(({ id }) => id) + .sort() + expect([...claimedSecondaryKeys].sort()).toEqual( + expectedClaimedSecondaryKeys, + ) + + const primaryJoinKeys = new Set( + scenario.primaryRows.map(({ joinKey }) => joinKey), + ) + const probedJoinKeys = [...primaryJoinKeys, `never-demanded`] + const joinCalls = secondaryCalls.filter(({ where }) => where !== undefined) + if (scenario.secondaryPublication === `preloaded`) { + expect(joinCalls.length).toBeGreaterThan(0) + } + if (joinCalls.length > 0) { + const requestedJoinKeys = new Set( + joinCalls.flatMap(({ where }) => + probedJoinKeys.filter((joinKey) => + evaluateReferenceExpression(where!, { + id: `probe-${joinKey}`, + joinKey, + }), + ), + ), + ) + expect(requestedJoinKeys.has(`never-demanded`)).toBe(false) + for (const joinKey of projection.demandedJoinKeys) { + expect(requestedJoinKeys.has(joinKey)).toBe(true) + } + expect( + [...requestedJoinKeys].every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) + } + + if (scenario.secondaryPublication === `after-primary-continuation`) { + expect( + primaryKeysBeforeSecondaryPublication?.length, + ).toBeGreaterThanOrEqual(1) + expect(primaryOrderedCallCount).toBeGreaterThanOrEqual(2) + } + if (scenario.secondaryPublication === `after-primary-exhaustion`) { + expect(primaryKeysBeforeSecondaryPublication).toEqual( + primaryOrder.map(({ id }) => id), + ) + } } finally { await Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]) } } -it(`continues an ordered primary source until a joined window is full`, async () => { - await runMultiSourceOrderedScenario({ - primaryRows: [ - { id: `a`, rank: 1, joinKey: `a` }, - { id: `b`, rank: 2, joinKey: `b` }, - { id: `c`, rank: 3, joinKey: `c` }, - { id: `d`, rank: 4, joinKey: `d` }, +const orderedPrimaryFixture = [ + { id: `a`, rank: 1, joinKey: `a` }, + { id: `b`, rank: 2, joinKey: `b` }, + { id: `c`, rank: 3, joinKey: `c` }, + { id: `d`, rank: 4, joinKey: `d` }, +] + +it.each([ + { + name: `continues across rejected rows when the secondary is preloaded`, + secondaryRows: [ + { id: `c-0`, joinKey: `c` }, + { id: `d-0`, joinKey: `d` }, + ], + offset: 0, + limit: 2, + secondaryPublication: `preloaded` as const, + }, + { + name: `reconciles a secondary publication after continuation starts`, + secondaryRows: [{ id: `a-0`, joinKey: `a` }], + offset: 0, + limit: 1, + secondaryPublication: `after-primary-continuation` as const, + }, + { + name: `reconciles a secondary publication after primary exhaustion`, + secondaryRows: [{ id: `d-0`, joinKey: `d` }], + offset: 0, + limit: 2, + secondaryPublication: `after-primary-exhaustion` as const, + }, + { + name: `counts joined multiplicity before applying offset and limit`, + secondaryRows: [ + { id: `a-0`, joinKey: `a` }, + { id: `a-1`, joinKey: `a` }, ], - secondaryJoinKeys: [`c`, `d`], - targetSize: 2, + offset: 1, + limit: 1, + secondaryPublication: `preloaded` as const, + }, +] satisfies ReadonlyArray< + Pick< + MultiSourceOrderedScenario, + `secondaryRows` | `offset` | `limit` | `secondaryPublication` + > & { name: string } +>)(`$name`, async ({ name: _name, ...scenario }) => { + await runMultiSourceOrderedScenario({ + ...scenario, + primaryRows: orderedPrimaryFixture, direction: `asc`, - secondaryPublication: `preloaded`, }) }) @@ -442,12 +614,16 @@ it(`projects the minimal primary prefix needed by a joined window`, () => { { key: `c`, joinKey: `z` }, { key: `d`, joinKey: `x` }, ], - secondaryJoinKeys: new Set([`x`, `z`]), - targetSize: 2, + secondaryRows: [ + { key: `x-0`, joinKey: `x` }, + { key: `z-0`, joinKey: `z` }, + ], + offset: 0, + limit: 2, }) expect(projection).toEqual({ - visibleKeys: [`a`, `c`], + visiblePairKeys: [`a:x-0`, `c:z-0`], scannedPrimaryKeys: [`a`, `b`, `c`], primaryCursorKeys: [undefined, `a`, `b`], demandedJoinKeys: [`x`, `y`, `z`], @@ -463,8 +639,12 @@ it(`erases join-key spelling and ignores unreachable secondary rows`, () => { { key: `b`, joinKey: `y` }, { key: `c`, joinKey: `x` }, ], - secondaryJoinKeys: new Set([`x`, `unused`]), - targetSize: 2, + secondaryRows: [ + { key: `match-0`, joinKey: `x` }, + { key: `unreachable`, joinKey: `unused` }, + ], + offset: 0, + limit: 2, }) const renamed = projectMultiSourceOrderedWindow({ primaryOrder: [ @@ -472,18 +652,19 @@ it(`erases join-key spelling and ignores unreachable secondary rows`, () => { { key: `b`, joinKey: `renamed-y` }, { key: `c`, joinKey: `renamed-x` }, ], - secondaryJoinKeys: new Set([`renamed-x`]), - targetSize: 2, + secondaryRows: [{ key: `match-0`, joinKey: `renamed-x` }], + offset: 0, + limit: 2, }) expect({ - visibleKeys: original.visibleKeys, + visiblePairKeys: original.visiblePairKeys, scannedPrimaryKeys: original.scannedPrimaryKeys, primaryCursorKeys: original.primaryCursorKeys, rowsNeeded: original.rowsNeeded, sourceExhausted: original.sourceExhausted, }).toEqual({ - visibleKeys: renamed.visibleKeys, + visiblePairKeys: renamed.visiblePairKeys, scannedPrimaryKeys: renamed.scannedPrimaryKeys, primaryCursorKeys: renamed.primaryCursorKeys, rowsNeeded: renamed.rowsNeeded, @@ -499,35 +680,54 @@ it(`exhausts the bounded multi-source ordered-window model`, () => { ] const joinKeys = [`x`, `y`, `z`] as const - for (let mask = 0; mask < 1 << joinKeys.length; mask++) { - const secondaryJoinKeys = new Set( - joinKeys.filter((_, index) => (mask & (1 << index)) !== 0), - ) - for (const targetSize of [1, 2, 3]) { - const projection = projectMultiSourceOrderedWindow({ - primaryOrder: rows, - secondaryJoinKeys, - targetSize, - }) - const direct = rows - .filter(({ joinKey }) => secondaryJoinKeys.has(joinKey)) - .slice(0, targetSize) - .map(({ key }) => key) - - expect(projection.visibleKeys).toEqual(direct) - expect(projection.rowsNeeded).toBe( - Math.max(0, targetSize - direct.length), - ) - if (projection.scannedPrimaryKeys.length < rows.length) { - const shorterPrefix = rows.slice( - 0, - projection.scannedPrimaryKeys.length - 1, + for (const xCount of [0, 1, 2]) { + for (const yCount of [0, 1, 2]) { + for (const zCount of [0, 1, 2]) { + const counts = [xCount, yCount, zCount] + const secondaryRows = joinKeys.flatMap((joinKey, index) => + Array.from({ length: counts[index]! }, (_, matchIndex) => ({ + key: `${joinKey}-${matchIndex}`, + joinKey, + })), ) - expect( - shorterPrefix.filter(({ joinKey }) => secondaryJoinKeys.has(joinKey)), - ).toHaveLength(targetSize - 1) - } else { - expect(projection.sourceExhausted).toBe(true) + for (const offset of [0, 1, 2]) { + for (const limit of [1, 2]) { + const projection = projectMultiSourceOrderedWindow({ + primaryOrder: rows, + secondaryRows, + offset, + limit, + }) + const direct = rows + .flatMap((row) => + secondaryRows + .filter(({ joinKey }) => joinKey === row.joinKey) + .map((secondaryRow) => `${row.key}:${secondaryRow.key}`), + ) + .slice(offset, offset + limit) + + expect(projection.visiblePairKeys).toEqual(direct) + expect(projection.rowsNeeded).toBe( + Math.max(0, limit - direct.length), + ) + if (projection.scannedPrimaryKeys.length < rows.length) { + const shorterPrefix = rows.slice( + 0, + projection.scannedPrimaryKeys.length - 1, + ) + const shorterPairCount = shorterPrefix.reduce( + (count, row) => + count + + secondaryRows.filter(({ joinKey }) => joinKey === row.joinKey) + .length, + 0, + ) + expect(shorterPairCount).toBeLessThan(offset + limit) + } else { + expect(projection.sourceExhausted).toBe(true) + } + } + } } } } From e3d464c530073ec5bfced07278040ca6935afda7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 16:28:37 -0600 Subject: [PATCH 083/327] test(db): harden multi-source load oracle --- packages/db/src/query/live/ARCHITECTURE.md | 4 +- .../db/tests/load-subset-full-flow-model.ts | 2 +- ...d-subset-full-flow-oracle.property.test.ts | 548 ++++++++++++++++-- 3 files changed, 488 insertions(+), 66 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 3bfa950c2..81ad439c7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -411,7 +411,9 @@ reverse join demand can instead make a later matching row readable without claiming reusable ordered-prefix coverage for skipped rows. A second source may settle after a continuation is already in flight, so safe extra primary rows may become readable. None of these paths may change the direct result or let an -exhaustible source leave a provable window under-filled. +unsettled source region leave a provable window under-filled. A short result is +valid only when authoritative completeness across every involved source region +proves that no remaining row can contribute before the window boundary. Test adapters must obey the same boundary contract as production adapters. A mock that reports exhaustion must have made every matching source row readable diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 5a2ce3c2f..8de3a988a 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -77,7 +77,7 @@ export function projectMultiSourceOrderedWindow(options: { const joinedPairKeys: Array = [] const demandedJoinKeys: Array = [] const seenJoinKeys = new Set() - const targetSize = options.offset + options.limit + const targetSize = options.limit === 0 ? 0 : options.offset + options.limit const secondaryRows = [...options.secondaryRows].sort((left, right) => left.key.localeCompare(right.key), ) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 8d8af7155..b8bf18add 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -6,6 +6,7 @@ import { BTreeIndex, ReverseIndex } from '../../src/index.js' import { Func, PropRef, Value } from '../../src/query/ir.js' import { createEffect } from '../../src/query/effect.js' import { createLiveQueryCollection, eq, gte } from '../../src/query/index.js' +import { getLoadSubsetDemandKey } from '../../src/query/ir-stable-identity.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { normalizeValue } from '../../src/utils/comparison.js' import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' @@ -162,11 +163,22 @@ type MultiSourceOrderedScenario = { direction: `asc` | `desc` secondaryPublication: | `preloaded` + | `preloaded-delayed-receipt` | `after-primary-continuation` | `after-primary-exhaustion` + secondaryPageSize: 1 | 2 + secondaryCommitOrder: `insertion` | `reverse` } const multiSourceJoinKeyArbitrary = fc.constantFrom(`x`, `y`, `z`) +const secondaryJoinKeyOrders = [ + [`x`, `y`, `z`], + [`x`, `z`, `y`], + [`y`, `x`, `z`], + [`y`, `z`, `x`], + [`z`, `x`, `y`], + [`z`, `y`, `x`], +] as const const multiSourceOrderedScenarioArbitrary: fc.Arbitrary = fc .record({ @@ -187,32 +199,48 @@ const multiSourceOrderedScenarioArbitrary: fc.Arbitrary ({ - ...scenario, - primaryRows: [`a`, `b`, `c`, `d`].map((id, index) => ({ - id, - rank: ranks[index]!, - joinKey: joinKeys[index]!, - })), - secondaryRows: [`x`, `y`, `z`].flatMap((joinKey, index) => - Array.from( - { length: secondaryMatchCounts[index]! }, - (_, matchIndex) => ({ + .map( + ({ + ranks, + joinKeys, + secondaryMatchCounts, + secondaryJoinKeyOrder, + reverseSecondaryMatches, + ...scenario + }) => ({ + ...scenario, + primaryRows: [`a`, `b`, `c`, `d`].map((id, index) => ({ + id, + rank: ranks[index]!, + joinKey: joinKeys[index]!, + })), + secondaryRows: secondaryJoinKeyOrder.flatMap((joinKey) => { + const count = secondaryMatchCounts[[`x`, `y`, `z`].indexOf(joinKey)]! + const rows = Array.from({ length: count }, (_, matchIndex) => ({ id: `${joinKey}-${matchIndex}`, joinKey, - }), - ), - ), - })) + })) + return reverseSecondaryMatches ? rows.reverse() : rows + }), + }), + ) if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { fc.statistics( @@ -224,11 +252,18 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { limit, direction, secondaryPublication, + secondaryPageSize, + secondaryCommitOrder, }) => [ `direction=${direction}`, `offset=${offset}`, `limit=${limit}`, `secondary=${secondaryPublication}`, + `secondary-page-size=${secondaryPageSize}`, + `secondary-commit-order=${secondaryCommitOrder}`, + `secondary-insertion-order=${secondaryRows + .map(({ id }) => id) + .join(`,`)}`, `exhaustion=${ primaryRows.reduce( (count, { joinKey }) => @@ -248,6 +283,8 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { limit, direction, secondaryPublication, + secondaryPageSize, + secondaryCommitOrder, })[0]!.joinKey, )}`, `multiplicity=${new Set(secondaryRows.map(({ joinKey }) => joinKey)).size < secondaryRows.length}`, @@ -269,6 +306,29 @@ function orderedPrimaryRows( }) } +function hasPreloadedSecondary(scenario: MultiSourceOrderedScenario): boolean { + return ( + scenario.secondaryPublication === `preloaded` || + scenario.secondaryPublication === `preloaded-delayed-receipt` + ) +} + +function collectStringLiterals( + expression: Func | PropRef | Value, +): Array { + if (expression instanceof Func) { + return expression.args.flatMap((argument) => + collectStringLiterals(argument), + ) + } + if (!(expression instanceof Value)) return [] + if (typeof expression.value === `string`) return [expression.value] + if (!Array.isArray(expression.value)) return [] + return expression.value.filter( + (value): value is string => typeof value === `string`, + ) +} + let multiSourceOrderedHarnessId = 0 async function runMultiSourceOrderedScenario( @@ -291,13 +351,29 @@ async function runMultiSourceOrderedScenario( limit: scenario.limit, }) const primaryCalls: Array = [] + const primaryCallProgress: Array<{ + demandKey: string + establishedPrimaryCount: number + establishedSecondaryCount: number + }> = [] + const primaryReceipts: Array> = [] const primaryOrderedVisitedKeys: Array = [] const secondaryCalls: Array = [] const secondaryReceipts: Array> = [] + const secondaryLoadCommitSizes: Array = [] + const delayedSecondaryReceiptWaiters: Array<{ + index: number + gate: ReturnType> + }> = [] + const delayedSecondaryReceiptCompletionOrder: Array = [] + let releaseDelayedSecondaryReceipts = false const secondaryPublicationGate = createDeferred() const establishedPrimaryKeys = new Set() + const primaryKeysEstablishedByLoads = new Set() const establishedSecondaryKeys = new Set() let primaryOrderedCallCount = 0 + let primaryOrderedCallCountAtSecondaryRelease: number | undefined + let primaryKeysAtSecondaryRelease: ReadonlyArray | undefined let primaryKeysBeforeSecondaryPublication: ReadonlyArray | undefined let primaryBegin!: () => void let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void @@ -311,6 +387,7 @@ async function runMultiSourceOrderedScenario( primaryBegin() for (const row of freshRows) { establishedPrimaryKeys.add(row.id) + primaryKeysEstablishedByLoads.add(row.id) primaryWrite({ type: `insert`, value: row }) } const applied = primaryCommit() @@ -318,6 +395,22 @@ async function runMultiSourceOrderedScenario( return freshRows.map(({ id }) => id) } + const releaseSecondaryPublication = (): void => { + primaryOrderedCallCountAtSecondaryRelease ??= primaryOrderedCallCount + primaryKeysAtSecondaryRelease ??= [...new Set(primaryOrderedVisitedKeys)] + secondaryPublicationGate.resolve() + } + if (scenario.limit === 0) secondaryPublicationGate.resolve() + + const recordPrimaryCall = (options: LoadSubsetOptions): void => { + primaryCalls.push(options) + primaryCallProgress.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + establishedPrimaryCount: establishedPrimaryKeys.size, + establishedSecondaryCount: establishedSecondaryKeys.size, + }) + } + const primary = createCollection({ id: `multi-source-ordered-primary-${multiSourceOrderedHarnessId}`, getKey: (row) => row.id, @@ -333,16 +426,18 @@ async function runMultiSourceOrderedScenario( params.markReady() return { loadSubset: async (options) => { - primaryCalls.push(options) + recordPrimaryCall(options) if (!options.orderBy) { const rows = primaryOrder.filter( (row) => options.where === undefined || evaluateReferenceExpression(options.where, row), ) + const appliedRowKeys = await applyPrimaryRows(rows) + primaryReceipts.push(appliedRowKeys) return { hasMore: false, - appliedRowKeys: await applyPrimaryRows(rows), + appliedRowKeys, } } @@ -366,14 +461,15 @@ async function runMultiSourceOrderedScenario( scenario.secondaryPublication === `after-primary-continuation` && primaryOrderedCallCount >= 2 ) { - secondaryPublicationGate.resolve() + releaseSecondaryPublication() } if ( scenario.secondaryPublication === `after-primary-exhaustion` && !hasMore ) { - secondaryPublicationGate.resolve() + releaseSecondaryPublication() } + primaryReceipts.push(appliedRowKeys) return { hasMore, appliedRowKeys, @@ -397,6 +493,7 @@ async function runMultiSourceOrderedScenario( ): Promise> => { const freshRows = rows.filter(({ id }) => !establishedSecondaryKeys.has(id)) if (freshRows.length === 0) return [] + secondaryLoadCommitSizes.push(freshRows.length) secondaryBegin() for (const row of freshRows) { establishedSecondaryKeys.add(row.id) @@ -410,7 +507,7 @@ async function runMultiSourceOrderedScenario( id: `multi-source-ordered-secondary-${multiSourceOrderedHarnessId}`, getKey: (row) => row.id, syncMode: `on-demand`, - startSync: true, + startSync: hasPreloadedSecondary(scenario), autoIndex: `eager`, defaultIndexType: BTreeIndex, sync: { @@ -418,10 +515,7 @@ async function runMultiSourceOrderedScenario( secondaryBegin = params.begin secondaryWrite = params.write secondaryCommit = params.commit - if ( - scenario.secondaryPublication === `preloaded` && - secondaryRows.length > 0 - ) { + if (hasPreloadedSecondary(scenario) && secondaryRows.length > 0) { secondaryBegin() for (const row of secondaryRows) { establishedSecondaryKeys.add(row.id) @@ -436,18 +530,48 @@ async function runMultiSourceOrderedScenario( return { loadSubset: async (options) => { secondaryCalls.push(options) - if (scenario.secondaryPublication !== `preloaded`) { + if (!hasPreloadedSecondary(scenario)) { await secondaryPublicationGate.promise primaryKeysBeforeSecondaryPublication ??= [ ...new Set(primaryOrderedVisitedKeys), ] } + if ( + scenario.secondaryPublication === `preloaded-delayed-receipt` && + !releaseDelayedSecondaryReceipts + ) { + const waiter = { + index: delayedSecondaryReceiptWaiters.length, + gate: createDeferred(), + } + delayedSecondaryReceiptWaiters.push(waiter) + await waiter.gate.promise + delayedSecondaryReceiptCompletionOrder.push(waiter.index) + } const matchingRows = secondaryRows.filter( (row) => options.where === undefined || evaluateReferenceExpression(options.where, row), ) - const appliedRowKeys = await applySecondaryRows(matchingRows) + const rowsInCommitOrder = + scenario.secondaryCommitOrder === `reverse` + ? [...matchingRows].reverse() + : matchingRows + const appliedRowKeys: Array = [] + for ( + let index = 0; + index < rowsInCommitOrder.length; + index += scenario.secondaryPageSize + ) { + appliedRowKeys.push( + ...(await applySecondaryRows( + rowsInCommitOrder.slice( + index, + index + scenario.secondaryPageSize, + ), + )), + ) + } secondaryReceipts.push(appliedRowKeys) return { hasMore: false, @@ -476,7 +600,19 @@ async function runMultiSourceOrderedScenario( }) try { - await live.preload() + const preload = live.preload() + if (scenario.secondaryPublication === `preloaded-delayed-receipt`) { + await flushPromises() + if (scenario.secondaryRows.length > 0 && scenario.limit > 0) { + expect(delayedSecondaryReceiptWaiters.length).toBeGreaterThan(0) + } + releaseDelayedSecondaryReceipts = true + for (const waiter of [...delayedSecondaryReceiptWaiters].reverse()) { + waiter.gate.resolve() + await flushPromises() + } + } + await preload await flushPromises() expect( @@ -484,41 +620,108 @@ async function runMultiSourceOrderedScenario( ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, ), ).toEqual(projection.visiblePairKeys) - expect(primaryCalls.some(({ orderBy }) => orderBy !== undefined)).toBe(true) - expect(primaryCalls.length).toBeLessThanOrEqual(32) - expect(secondaryCalls.length).toBeGreaterThan(0) + + const refinedOffset = scenario.offset === 0 ? 1 : 0 + const refinedLimit = scenario.limit === 0 ? 1 : scenario.limit + 1 + const refinedProjection = projectMultiSourceOrderedWindow({ + primaryOrder: primaryOrder.map(({ id, joinKey }) => ({ + key: id, + joinKey, + })), + secondaryRows: scenario.secondaryRows.map(({ id, joinKey }) => ({ + key: id, + joinKey, + })), + offset: refinedOffset, + limit: refinedLimit, + }) + await live.utils.setWindow({ + offset: refinedOffset, + limit: refinedLimit, + }) + await flushPromises() + expect( + live.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual(refinedProjection.visiblePairKeys) + + if (scenario.limit > 0) { + expect(primaryCalls.some(({ orderBy }) => orderBy !== undefined)).toBe( + true, + ) + expect(secondaryCalls.length).toBeGreaterThan(0) + } + + expect(primaryCallProgress).toHaveLength(primaryCalls.length) + const previousProgressByDemand = new Map< + string, + (typeof primaryCallProgress)[number] + >() + for (const progress of primaryCallProgress) { + const previous = previousProgressByDemand.get(progress.demandKey) + if (previous) { + expect( + progress.establishedPrimaryCount > previous.establishedPrimaryCount || + progress.establishedSecondaryCount > + previous.establishedSecondaryCount, + ).toBe(true) + } + previousProgressByDemand.set(progress.demandKey, progress) + } + const claimedPrimaryKeys = primaryReceipts.flat() + expect(new Set(claimedPrimaryKeys).size).toBe(claimedPrimaryKeys.length) + expect([...claimedPrimaryKeys].sort()).toEqual( + [...primaryKeysEstablishedByLoads].sort(), + ) + expect( + claimedPrimaryKeys.every((key) => + scenario.primaryRows.some(({ id }) => id === key), + ), + ).toBe(true) const claimedSecondaryKeys = secondaryReceipts.flat() expect(new Set(claimedSecondaryKeys).size).toBe(claimedSecondaryKeys.length) - const expectedClaimedSecondaryKeys = - scenario.secondaryPublication === `preloaded` - ? [] - : scenario.secondaryRows - .filter((row) => - secondaryCalls.some( - ({ where }) => - where === undefined || - evaluateReferenceExpression(where, row), - ), - ) - .map(({ id }) => id) - .sort() + const expectedClaimedSecondaryKeys = hasPreloadedSecondary(scenario) + ? [] + : scenario.secondaryRows + .filter((row) => + secondaryCalls.some( + ({ where }) => + where === undefined || evaluateReferenceExpression(where, row), + ), + ) + .map(({ id }) => id) + .sort() expect([...claimedSecondaryKeys].sort()).toEqual( expectedClaimedSecondaryKeys, ) + expect( + secondaryLoadCommitSizes.every( + (commitSize) => commitSize <= scenario.secondaryPageSize, + ), + ).toBe(true) const primaryJoinKeys = new Set( scenario.primaryRows.map(({ joinKey }) => joinKey), ) - const probedJoinKeys = [...primaryJoinKeys, `never-demanded`] const joinCalls = secondaryCalls.filter(({ where }) => where !== undefined) - if (scenario.secondaryPublication === `preloaded`) { + if ( + hasPreloadedSecondary(scenario) && + scenario.secondaryRows.length > 0 && + scenario.limit > 0 + ) { expect(joinCalls.length).toBeGreaterThan(0) } + if (scenario.secondaryPublication === `preloaded-delayed-receipt`) { + expect(delayedSecondaryReceiptCompletionOrder).toEqual( + delayedSecondaryReceiptWaiters.map(({ index }) => index).reverse(), + ) + } if (joinCalls.length > 0) { const requestedJoinKeys = new Set( joinCalls.flatMap(({ where }) => - probedJoinKeys.filter((joinKey) => + [...primaryJoinKeys].filter((joinKey) => evaluateReferenceExpression(where!, { id: `probe-${joinKey}`, joinKey, @@ -526,7 +729,12 @@ async function runMultiSourceOrderedScenario( ), ), ) - expect(requestedJoinKeys.has(`never-demanded`)).toBe(false) + const literalJoinKeys = joinCalls.flatMap(({ where }) => + collectStringLiterals(where!), + ) + expect( + literalJoinKeys.every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) for (const joinKey of projection.demandedJoinKeys) { expect(requestedJoinKeys.has(joinKey)).toBe(true) } @@ -536,17 +744,23 @@ async function runMultiSourceOrderedScenario( } if (scenario.secondaryPublication === `after-primary-continuation`) { - expect( - primaryKeysBeforeSecondaryPublication?.length, - ).toBeGreaterThanOrEqual(1) - expect(primaryOrderedCallCount).toBeGreaterThanOrEqual(2) + if (scenario.limit > 0) { + expect(primaryOrderedCallCountAtSecondaryRelease).toBe(2) + expect(primaryKeysBeforeSecondaryPublication?.length).toBe(2) + } } if (scenario.secondaryPublication === `after-primary-exhaustion`) { - expect(primaryKeysBeforeSecondaryPublication).toEqual( - primaryOrder.map(({ id }) => id), - ) + if (scenario.limit > 0) { + expect(primaryKeysAtSecondaryRelease).toEqual( + primaryOrder.map(({ id }) => id), + ) + expect(primaryKeysBeforeSecondaryPublication).toEqual( + primaryOrder.map(({ id }) => id), + ) + } } } finally { + for (const waiter of delayedSecondaryReceiptWaiters) waiter.gate.resolve() await Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]) } } @@ -560,7 +774,7 @@ const orderedPrimaryFixture = [ it.each([ { - name: `continues across rejected rows when the secondary is preloaded`, + name: `preloaded rejection continuation`, secondaryRows: [ { id: `c-0`, joinKey: `c` }, { id: `d-0`, joinKey: `d` }, @@ -568,35 +782,72 @@ it.each([ offset: 0, limit: 2, secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, }, { - name: `reconciles a secondary publication after continuation starts`, - secondaryRows: [{ id: `a-0`, joinKey: `a` }], + name: `late secondary after continuation`, + secondaryRows: [ + { id: `b-0`, joinKey: `b` }, + { id: `a-0`, joinKey: `a` }, + ], offset: 0, - limit: 1, + limit: 2, secondaryPublication: `after-primary-continuation` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `reverse` as const, }, { - name: `reconciles a secondary publication after primary exhaustion`, + name: `delayed filtered secondary receipt`, + secondaryRows: [ + { id: `c-0`, joinKey: `c` }, + { id: `d-0`, joinKey: `d` }, + ], + offset: 0, + limit: 2, + secondaryPublication: `preloaded-delayed-receipt` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `reverse` as const, + }, + { + name: `late secondary after primary exhaustion`, secondaryRows: [{ id: `d-0`, joinKey: `d` }], offset: 0, limit: 2, secondaryPublication: `after-primary-exhaustion` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, }, { - name: `counts joined multiplicity before applying offset and limit`, + name: `joined multiplicity before offset`, secondaryRows: [ - { id: `a-0`, joinKey: `a` }, { id: `a-1`, joinKey: `a` }, + { id: `a-0`, joinKey: `a` }, ], offset: 1, limit: 1, secondaryPublication: `preloaded` as const, + secondaryPageSize: 2 as const, + secondaryCommitOrder: `insertion` as const, + }, + { + name: `zero-limit window`, + secondaryRows: [], + offset: 2, + limit: 0, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, }, ] satisfies ReadonlyArray< Pick< MultiSourceOrderedScenario, - `secondaryRows` | `offset` | `limit` | `secondaryPublication` + | `secondaryRows` + | `offset` + | `limit` + | `secondaryPublication` + | `secondaryPageSize` + | `secondaryCommitOrder` > & { name: string } >)(`$name`, async ({ name: _name, ...scenario }) => { await runMultiSourceOrderedScenario({ @@ -606,6 +857,171 @@ it.each([ }) }) +it(`settles concurrent secondary loads out of order across paged commits`, async () => { + type PrimaryRow = { id: string; rank: number; joinKey: string } + type SecondaryRow = { id: string; joinKey: string } + type PendingSecondaryLoad = { + requestIndex: number + options: LoadSubsetOptions + gate: ReturnType> + joinKeys: ReadonlyArray + } + + const primaryOptions = mockSyncCollectionOptions({ + id: `multi-source-filtered-primary`, + initialData: [ + { id: `a`, rank: 1, joinKey: `a` }, + { id: `b`, rank: 2, joinKey: `b` }, + ], + getKey: (row) => row.id, + syncMode: `eager`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }) + const primary = createCollection(primaryOptions) + const secondaryRows = [ + { id: `a-1`, joinKey: `a` }, + { id: `a-0`, joinKey: `a` }, + { id: `b-1`, joinKey: `b` }, + { id: `b-0`, joinKey: `b` }, + { id: `c-0`, joinKey: `c` }, + ] + const pendingSecondaryLoads: Array = [] + const secondaryCompletionOrder: Array = [] + const secondaryReceipts: Array> = [] + const secondaryLoadCommitSizes: Array = [] + let secondaryBegin!: () => void + let secondaryWrite!: (message: { + type: `insert` + value: SecondaryRow + }) => void + let secondaryCommit!: () => true | Promise + const establishedSecondaryKeys = new Set() + const secondary = createCollection({ + id: `multi-source-filtered-secondary`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + secondaryBegin = params.begin + secondaryWrite = params.write + secondaryCommit = params.commit + secondaryBegin() + secondaryWrite({ + type: `insert`, + value: { id: `unrelated`, joinKey: `unrelated` }, + }) + establishedSecondaryKeys.add(`unrelated`) + const seeded = secondaryCommit() + if (seeded !== true) { + throw new Error(`Expected synchronous secondary seed`) + } + params.markReady() + return { + loadSubset: async (options) => { + const matchingRows = secondaryRows.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + const joinKeys = [ + ...new Set(matchingRows.map(({ joinKey }) => joinKey)), + ] + const pending = { + requestIndex: pendingSecondaryLoads.length, + options, + gate: createDeferred(), + joinKeys, + } + pendingSecondaryLoads.push(pending) + await pending.gate.promise + + const appliedRowKeys: Array = [] + for (const row of [...matchingRows].reverse()) { + if (establishedSecondaryKeys.has(row.id)) continue + establishedSecondaryKeys.add(row.id) + secondaryLoadCommitSizes.push(1) + secondaryBegin() + secondaryWrite({ type: `insert`, value: row }) + const applied = secondaryCommit() + if (applied !== true) await applied + appliedRowKeys.push(row.id) + } + secondaryCompletionOrder.push(pending.requestIndex) + secondaryReceipts.push(appliedRowKeys) + return { hasMore: false, appliedRowKeys } + }, + unloadSubset: () => {}, + } + }, + }, + }) + const createFilteredLive = (id: string, primaryId: string) => + createLiveQueryCollection({ + id, + query: (q) => + q + .from({ primaryRow: primary }) + .where(({ primaryRow }) => eq(primaryRow.id, primaryId)) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank) + .limit(2), + startSync: true, + }) + const liveA = createFilteredLive(`multi-source-filtered-live-a`, `a`) + const liveB = createFilteredLive(`multi-source-filtered-live-b`, `b`) + + try { + const preload = Promise.all([liveA.preload(), liveB.preload()]) + await flushPromises() + expect(pendingSecondaryLoads).toHaveLength(2) + expect(pendingSecondaryLoads.every(({ options }) => !options.where)).toBe( + true, + ) + expect(pendingSecondaryLoads.map(({ joinKeys }) => joinKeys)).toEqual([ + [`a`, `b`, `c`], + [`a`, `b`, `c`], + ]) + + pendingSecondaryLoads[1]!.gate.resolve() + await flushPromises() + pendingSecondaryLoads[0]!.gate.resolve() + await preload + await flushPromises() + + expect(secondaryCompletionOrder).toEqual([1, 0]) + expect(secondaryLoadCommitSizes).toEqual([1, 1, 1, 1, 1]) + expect(new Set(secondaryReceipts.flat())).toEqual( + new Set(secondaryRows.map(({ id }) => id)), + ) + expect( + liveA.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual([`a:a-0`, `a:a-1`]) + expect( + liveB.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual([`b:b-0`, `b:b-1`]) + } finally { + for (const pending of pendingSecondaryLoads) pending.gate.resolve() + await Promise.all([ + liveA.cleanup(), + liveB.cleanup(), + primary.cleanup(), + secondary.cleanup(), + ]) + } +}) + it(`projects the minimal primary prefix needed by a joined window`, () => { const projection = projectMultiSourceOrderedWindow({ primaryOrder: [ @@ -691,7 +1107,7 @@ it(`exhausts the bounded multi-source ordered-window model`, () => { })), ) for (const offset of [0, 1, 2]) { - for (const limit of [1, 2]) { + for (const limit of [0, 1, 2]) { const projection = projectMultiSourceOrderedWindow({ primaryOrder: rows, secondaryRows, @@ -710,6 +1126,10 @@ it(`exhausts the bounded multi-source ordered-window model`, () => { expect(projection.rowsNeeded).toBe( Math.max(0, limit - direct.length), ) + if (limit === 0) { + expect(projection.scannedPrimaryKeys).toEqual([]) + continue + } if (projection.scannedPrimaryKeys.length < rows.length) { const shorterPrefix = rows.slice( 0, From 1c6f92e6de0ea256b6abb4314a278e7211d7a755 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 17:03:00 -0600 Subject: [PATCH 084/327] fix(db): keep zero-width loads transport-free --- packages/db/src/query/effect.ts | 4 +- packages/db/src/query/live/ARCHITECTURE.md | 7 + .../src/query/live/collection-subscriber.ts | 6 +- packages/db/src/query/subset-dedupe.ts | 22 ++- ...d-subset-full-flow-oracle.property.test.ts | 126 +++++++++++++++++- .../query/load-subset-oracle.property.test.ts | 37 ++++- 6 files changed, 186 insertions(+), 16 deletions(-) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 4400d74b8..5f50ac74b 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -961,7 +961,7 @@ class EffectPipelineRunner { if (index) { subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ - limit: offset + limit, + limit: limit === 0 ? 0 : offset + limit, orderBy: normalizedOrderBy, trackLoadSubsetPromise: false, onLoadSubsetResult: (result) => @@ -997,7 +997,7 @@ class EffectPipelineRunner { this.optimizableOrderByCollections, )) { if (!orderByInfo.dataNeeded || !orderByInfo.index) continue - + if (orderByInfo.limit === 0) continue const subscription = this.subscriptions[orderByInfo.sourceId] if (!subscription) continue subscription.ensureOrderedWindowSize( diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 81ad439c7..16e040ec9 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -686,6 +686,13 @@ continuing page establishes neither fact, core leaves the window uncovered, does not repeat the same request, and records a nonfatal no-progress diagnostic in `lastSubsetError`. +An ordered window with an active limit of zero creates no ordered transport +demand. Its coordinator remains alive so a later window change can load from +the same order, but neither the initial offset nor a result deficit may turn +the empty window into a positive request. The dedupe helper applies the same +law when adapters call it directly: a zero-width request establishes no +coverage and owns no physical acquisition. + Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the no-progress guard until its demanded prefix or total-order boundary changes; diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index fd0439825..fd1461da3 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -442,7 +442,7 @@ export class CollectionSubscriber< subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ - limit: offset + limit, + limit: limit === 0 ? 0 : offset + limit, orderBy: normalizedOrderBy, trackLoadSubsetPromise: false, onLoadSubsetResult: handleLoadSubsetResult, @@ -475,6 +475,10 @@ export class CollectionSubscriber< const { dataNeeded, index, offset, limit, refillFromResultDeficit } = orderByInfo + // The ordered subscription keeps its coordinator for later window changes, + // but an empty active window must not start continuation work. + if (limit === 0) return true + if (!dataNeeded || !index) { // dataNeeded is not set when there's no index (e.g., non-ref expression // or auto-indexing is disabled). Without an index, lazy loading can't work — diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 67c79beed..85c3ab74f 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -26,6 +26,7 @@ type SharedAbortLease = { type LogicalLoadReservation = { generation: number + invalidatesCoverage: boolean inflight?: InflightCall } @@ -119,8 +120,16 @@ export class DeduplicatedLoadSubset { loadSubset = ( options: LoadSubsetOptions, ): true | Promise => { - const reservation = this.reserveOwner(options) + const reservation = this.reserveOwner(options, options.limit !== 0) try { + // A zero-width window has no rows to acquire and establishes no coverage. + // Keep only its logical reservation so reused option objects still + // release in invocation order without invalidating another request. + if (options.limit === 0) { + this.onDeduplicate?.(options) + return true + } + return this.loadSubsetRequest(options, reservation) } catch (error) { this.removeOwnerReservation(options, reservation) @@ -309,6 +318,7 @@ export class DeduplicatedLoadSubset { // still release that logical demand later, but it must not invalidate a // newer request that happens to use equivalent options. if (!reservation || reservation.generation !== this.generation) return + if (!reservation.invalidatesCoverage) return this.clearLoadedTracking() const inflight = reservation.inflight @@ -339,8 +349,14 @@ export class DeduplicatedLoadSubset { this.generation++ } - private reserveOwner(options: LoadSubsetOptions): LogicalLoadReservation { - const reservation = { generation: this.generation } + private reserveOwner( + options: LoadSubsetOptions, + invalidatesCoverage: boolean, + ): LogicalLoadReservation { + const reservation = { + generation: this.generation, + invalidatesCoverage, + } const reservations = this.ownerReservations.get(options) if (reservations) reservations.push(reservation) else this.ownerReservations.set(options, [reservation]) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index b8bf18add..f8a25c729 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -369,11 +369,13 @@ async function runMultiSourceOrderedScenario( let releaseDelayedSecondaryReceipts = false const secondaryPublicationGate = createDeferred() const establishedPrimaryKeys = new Set() + const committedPrimaryKeys = new Set() const primaryKeysEstablishedByLoads = new Set() const establishedSecondaryKeys = new Set() let primaryOrderedCallCount = 0 let primaryOrderedCallCountAtSecondaryRelease: number | undefined let primaryKeysAtSecondaryRelease: ReadonlyArray | undefined + let primaryCommittedKeysAtSecondaryRelease: ReadonlyArray | undefined let primaryKeysBeforeSecondaryPublication: ReadonlyArray | undefined let primaryBegin!: () => void let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void @@ -392,12 +394,14 @@ async function runMultiSourceOrderedScenario( } const applied = primaryCommit() if (applied !== true) await applied + for (const row of freshRows) committedPrimaryKeys.add(row.id) return freshRows.map(({ id }) => id) } const releaseSecondaryPublication = (): void => { primaryOrderedCallCountAtSecondaryRelease ??= primaryOrderedCallCount primaryKeysAtSecondaryRelease ??= [...new Set(primaryOrderedVisitedKeys)] + primaryCommittedKeysAtSecondaryRelease ??= [...committedPrimaryKeys] secondaryPublicationGate.resolve() } if (scenario.limit === 0) secondaryPublicationGate.resolve() @@ -601,10 +605,21 @@ async function runMultiSourceOrderedScenario( try { const preload = live.preload() + let preloadSettled = false + void preload.then( + () => { + preloadSettled = true + }, + () => { + preloadSettled = true + }, + ) if (scenario.secondaryPublication === `preloaded-delayed-receipt`) { await flushPromises() if (scenario.secondaryRows.length > 0 && scenario.limit > 0) { expect(delayedSecondaryReceiptWaiters.length).toBeGreaterThan(0) + expect(preloadSettled).toBe(false) + expect(live.isReady()).toBe(false) } releaseDelayedSecondaryReceipts = true for (const waiter of [...delayedSecondaryReceiptWaiters].reverse()) { @@ -614,6 +629,7 @@ async function runMultiSourceOrderedScenario( } await preload await flushPromises() + expect(preloadSettled).toBe(true) expect( live.toArray.map( @@ -621,6 +637,15 @@ async function runMultiSourceOrderedScenario( ), ).toEqual(projection.visiblePairKeys) + const initialPrimaryCallCount = primaryCalls.length + if (scenario.limit === 0) { + expect( + primaryCalls + .slice(0, initialPrimaryCallCount) + .filter(({ orderBy }) => orderBy !== undefined), + ).toEqual([]) + } + const refinedOffset = scenario.offset === 0 ? 1 : 0 const refinedLimit = scenario.limit === 0 ? 1 : scenario.limit + 1 const refinedProjection = projectMultiSourceOrderedWindow({ @@ -646,6 +671,16 @@ async function runMultiSourceOrderedScenario( ), ).toEqual(refinedProjection.visiblePairKeys) + const primaryCallsBeforeZeroShrink = primaryCalls.length + await live.utils.setWindow({ offset: 2, limit: 0 }) + await flushPromises() + expect(live.toArray).toEqual([]) + expect( + primaryCalls + .slice(primaryCallsBeforeZeroShrink) + .filter(({ orderBy }) => orderBy !== undefined), + ).toEqual([]) + if (scenario.limit > 0) { expect(primaryCalls.some(({ orderBy }) => orderBy !== undefined)).toBe( true, @@ -746,6 +781,9 @@ async function runMultiSourceOrderedScenario( if (scenario.secondaryPublication === `after-primary-continuation`) { if (scenario.limit > 0) { expect(primaryOrderedCallCountAtSecondaryRelease).toBe(2) + expect(primaryCommittedKeysAtSecondaryRelease).toEqual( + expect.arrayContaining(primaryOrderedVisitedKeys.slice(0, 2)), + ) expect(primaryKeysBeforeSecondaryPublication?.length).toBe(2) } } @@ -857,6 +895,71 @@ it.each([ }) }) +it(`keeps an Effect zero-limit join free of ordered transport work`, async () => { + type PrimaryRow = { id: string; rank: number; joinKey: string } + type SecondaryRow = { id: string; joinKey: string } + const primaryLoads: Array = [] + const primary = createCollection({ + id: `multi-source-zero-limit-effect-primary`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + primaryLoads.push(options) + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const secondary = createCollection({ + id: `multi-source-zero-limit-effect-secondary`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => {}, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank) + .offset(2) + .limit(0), + onBatch: () => {}, + }) + + try { + await flushPromises() + expect(primaryLoads).toEqual([]) + } finally { + await effect.dispose() + await Promise.all([primary.cleanup(), secondary.cleanup()]) + } +}) + it(`settles concurrent secondary loads out of order across paged commits`, async () => { type PrimaryRow = { id: string; rank: number; joinKey: string } type SecondaryRow = { id: string; joinKey: string } @@ -888,7 +991,10 @@ it(`settles concurrent secondary loads out of order across paged commits`, async ] const pendingSecondaryLoads: Array = [] const secondaryCompletionOrder: Array = [] - const secondaryReceipts: Array> = [] + const secondaryReceipts: Array<{ + requestIndex: number + appliedRowKeys: ReadonlyArray + }> = [] const secondaryLoadCommitSizes: Array = [] let secondaryBegin!: () => void let secondaryWrite!: (message: { @@ -951,7 +1057,10 @@ it(`settles concurrent secondary loads out of order across paged commits`, async appliedRowKeys.push(row.id) } secondaryCompletionOrder.push(pending.requestIndex) - secondaryReceipts.push(appliedRowKeys) + secondaryReceipts.push({ + requestIndex: pending.requestIndex, + appliedRowKeys, + }) return { hasMore: false, appliedRowKeys } }, unloadSubset: () => {}, @@ -998,9 +1107,20 @@ it(`settles concurrent secondary loads out of order across paged commits`, async expect(secondaryCompletionOrder).toEqual([1, 0]) expect(secondaryLoadCommitSizes).toEqual([1, 1, 1, 1, 1]) - expect(new Set(secondaryReceipts.flat())).toEqual( + expect(secondaryReceipts.map(({ requestIndex }) => requestIndex)).toEqual([ + 1, 0, + ]) + const claimedSecondaryKeys = secondaryReceipts.flatMap( + ({ appliedRowKeys }) => appliedRowKeys, + ) + expect(new Set(claimedSecondaryKeys).size).toBe(claimedSecondaryKeys.length) + expect(new Set(claimedSecondaryKeys)).toEqual( new Set(secondaryRows.map(({ id }) => id)), ) + expect(secondaryReceipts[0]?.appliedRowKeys).toEqual( + [...secondaryRows].reverse().map(({ id }) => id), + ) + expect(secondaryReceipts[1]?.appliedRowKeys).toEqual([]) expect( liveA.toArray.map( ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index fed44b28b..8009fb8e2 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -2164,14 +2164,37 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: an empty ordered window issues no transport work`, - expectExactCountFailure( - () => countWindowLoads([{ direction: `asc`, offset: 0, limit: 0 }]), - 1, + it(`an empty ordered window issues no transport work`, () => { + expect(countWindowLoads([{ direction: `asc`, offset: 0, limit: 0 }])).toBe( 0, - ), - ) + ) + }) + + it(`releases a reused zero-window owner without invalidating later coverage`, () => { + let loads = 0 + const dedupe = new DeduplicatedLoadSubset({ + loadSubset: () => { + loads++ + return true + }, + }) + const reusedOptions = toWindowOptions({ + direction: `asc`, + offset: 0, + limit: 0, + }) + + expect(dedupe.loadSubset(reusedOptions)).toBe(true) + reusedOptions.limit = 1 + expect(dedupe.loadSubset(reusedOptions)).toBe(true) + dedupe.unloadSubset(reusedOptions) + expect( + dedupe.loadSubset( + toWindowOptions({ direction: `asc`, offset: 0, limit: 1 }), + ), + ).toBe(true) + expect(loads).toBe(1) + }) it( `discovered trace: an empty filtered window issues no transport work`, From 77f8a8fd5c5dd93a7bbba5556f304cb08c46a922 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 17:32:11 -0600 Subject: [PATCH 085/327] fix(db): defer empty unindexed snapshots --- packages/db/src/query/effect.ts | 4 +- packages/db/src/query/live/ARCHITECTURE.md | 4 +- .../src/query/live/collection-subscriber.ts | 60 ++++- ...d-subset-full-flow-oracle.property.test.ts | 223 ++++++++++++++---- 4 files changed, 229 insertions(+), 62 deletions(-) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 5f50ac74b..40d58a50c 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -958,10 +958,12 @@ class EffectPipelineRunner { const { orderBy, offset, limit, index } = orderByInfo const normalizedOrderBy = normalizeOrderByPaths(orderBy, alias) + if (limit === 0) return + if (index) { subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ - limit: limit === 0 ? 0 : offset + limit, + limit: offset + limit, orderBy: normalizedOrderBy, trackLoadSubsetPromise: false, onLoadSubsetResult: (result) => diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 16e040ec9..def987edf 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -691,7 +691,9 @@ demand. Its coordinator remains alive so a later window change can load from the same order, but neither the initial offset nor a result deficit may turn the empty window into a positive request. The dedupe helper applies the same law when adapters call it directly: a zero-width request establishes no -coverage and owns no physical acquisition. +coverage and owns no physical acquisition. This also holds when no usable +order index exists: core defers the full-snapshot fallback until the window +first becomes positive, then requests it once for that subscription session. Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index fd1461da3..071dec373 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -61,6 +61,7 @@ export class CollectionSubscriber< private pendingOrderedLoadPromise: | Promise | undefined + private unindexedSnapshotSubscription: CollectionSubscription | undefined private readonly demand = new SubsetDemandController() constructor( @@ -161,6 +162,7 @@ export class CollectionSubscriber< trackLoadResult, onLoadSubsetError, ) + if (orderByInfo.limit === 0) initialSubsetPending = false } else { // Lazy sources load only the subsets demanded by the compiled graph. const includeInitialState = !this.collectionConfigBuilder.isLazySource( @@ -353,6 +355,7 @@ export class CollectionSubscriber< onLoadSubsetError: (event: SubscriptionLoadSubsetErrorEvent) => void, ): CollectionSubscription { const { orderBy, offset, limit, index } = orderByInfo + this.unindexedSnapshotSubscription = undefined // Store the callback so loadNextItems can also use direct tracking. // Track in-flight ordered loads to avoid issuing redundant requests while @@ -421,6 +424,9 @@ export class CollectionSubscriber< subscriptionHolder.current = undefined this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined + if (this.unindexedSnapshotSubscription === subscription) { + this.unindexedSnapshotSubscription = undefined + } // Ordered continuations belong to this subscription session. A settled // load from a cleaned session must not refill through a later session. @@ -447,14 +453,10 @@ export class CollectionSubscriber< trackLoadSubsetPromise: false, onLoadSubsetResult: handleLoadSubsetResult, }) - } else { + } else if (limit > 0) { // Without an index there is no sound cursor continuation. Load the full // ordered source so later relational operators cannot underfill top-K. - subscription.requestSnapshot({ - orderBy: normalizedOrderBy, - trackLoadSubsetPromise: false, - onLoadSubsetResult: handleLoadSubsetResult, - }) + this.requestUnindexedSnapshot(subscription, normalizedOrderBy) } return subscription @@ -479,10 +481,18 @@ export class CollectionSubscriber< // but an empty active window must not start continuation work. if (limit === 0) return true - if (!dataNeeded || !index) { - // dataNeeded is not set when there's no index (e.g., non-ref expression - // or auto-indexing is disabled). Without an index, lazy loading can't work — - // all data was already loaded eagerly via requestSnapshot. + if (!index) { + // A zero-width subscription defers this full fallback until the window + // first becomes positive. Once requested, the snapshot covers every + // later window because cursor continuation is unavailable. + this.requestUnindexedSnapshot( + subscription, + normalizeOrderByPaths(orderByInfo.orderBy, this.alias), + ) + return true + } + + if (!dataNeeded) { return true } @@ -539,6 +549,36 @@ export class CollectionSubscriber< return true } + private requestUnindexedSnapshot( + subscription: CollectionSubscription, + orderBy: LoadSubsetOptions[`orderBy`], + ): void { + if (this.unindexedSnapshotSubscription === subscription) return + + this.unindexedSnapshotSubscription = subscription + try { + subscription.requestSnapshot({ + orderBy, + trackLoadSubsetPromise: false, + onLoadSubsetResult: (result, demand) => { + if (result instanceof Promise) { + void result.catch(() => { + if (this.unindexedSnapshotSubscription === subscription) { + this.unindexedSnapshotSubscription = undefined + } + }) + } + this.orderedLoadSubsetResult?.(result, demand) + }, + }) + } catch (error) { + if (this.unindexedSnapshotSubscription === subscription) { + this.unindexedSnapshotSubscription = undefined + } + throw error + } + } + private sendChangesToPipelineWithTracking( changes: Iterable>, subscription: CollectionSubscription, diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index f8a25c729..895970577 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -161,6 +161,7 @@ type MultiSourceOrderedScenario = { offset: number limit: number direction: `asc` | `desc` + primaryAutoIndex: `eager` | `off` secondaryPublication: | `preloaded` | `preloaded-delayed-receipt` @@ -204,6 +205,7 @@ const multiSourceOrderedScenarioArbitrary: fc.Arbitrary [ `direction=${direction}`, + `primary-auto-index=${primaryAutoIndex}`, `offset=${offset}`, `limit=${limit}`, `secondary=${secondaryPublication}`, @@ -282,6 +286,7 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { offset, limit, direction, + primaryAutoIndex, secondaryPublication, secondaryPageSize, secondaryCommitOrder, @@ -420,7 +425,7 @@ async function runMultiSourceOrderedScenario( getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, - autoIndex: `eager`, + autoIndex: scenario.primaryAutoIndex, defaultIndexType: BTreeIndex, sync: { sync: (params) => { @@ -446,6 +451,24 @@ async function runMultiSourceOrderedScenario( } primaryOrderedCallCount++ + if (options.limit === undefined) { + primaryOrderedVisitedKeys.push( + ...primaryOrder.map(({ id }) => id), + ) + const appliedRowKeys = await applyPrimaryRows(primaryOrder) + if ( + scenario.secondaryPublication === + `after-primary-continuation` || + scenario.secondaryPublication === `after-primary-exhaustion` + ) { + releaseSecondaryPublication() + } + primaryReceipts.push(appliedRowKeys) + return { + hasMore: false, + appliedRowKeys, + } + } const lastKey = options.cursor?.lastKey const previousIndex = lastKey === undefined @@ -670,6 +693,16 @@ async function runMultiSourceOrderedScenario( ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, ), ).toEqual(refinedProjection.visiblePairKeys) + if (scenario.limit === 0) { + const refinementCalls = primaryCalls + .slice(initialPrimaryCallCount) + .filter(({ orderBy }) => orderBy !== undefined) + expect(refinementCalls.length).toBeGreaterThan(0) + if (scenario.primaryAutoIndex === `off`) { + expect(refinementCalls).toHaveLength(1) + expect(refinementCalls[0]?.limit).toBeUndefined() + } + } const primaryCallsBeforeZeroShrink = primaryCalls.length await live.utils.setWindow({ offset: 2, limit: 0 }) @@ -780,11 +813,21 @@ async function runMultiSourceOrderedScenario( if (scenario.secondaryPublication === `after-primary-continuation`) { if (scenario.limit > 0) { - expect(primaryOrderedCallCountAtSecondaryRelease).toBe(2) - expect(primaryCommittedKeysAtSecondaryRelease).toEqual( - expect.arrayContaining(primaryOrderedVisitedKeys.slice(0, 2)), - ) - expect(primaryKeysBeforeSecondaryPublication?.length).toBe(2) + if (scenario.primaryAutoIndex === `eager`) { + expect(primaryOrderedCallCountAtSecondaryRelease).toBe(2) + expect(primaryCommittedKeysAtSecondaryRelease).toEqual( + expect.arrayContaining(primaryOrderedVisitedKeys.slice(0, 2)), + ) + expect(primaryKeysBeforeSecondaryPublication?.length).toBe(2) + } else { + expect(primaryOrderedCallCountAtSecondaryRelease).toBe(1) + expect(primaryCommittedKeysAtSecondaryRelease).toEqual( + expect.arrayContaining(primaryOrder.map(({ id }) => id)), + ) + expect(primaryKeysBeforeSecondaryPublication).toEqual( + primaryOrder.map(({ id }) => id), + ) + } } } if (scenario.secondaryPublication === `after-primary-exhaustion`) { @@ -822,6 +865,7 @@ it.each([ secondaryPublication: `preloaded` as const, secondaryPageSize: 1 as const, secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, }, { name: `late secondary after continuation`, @@ -834,6 +878,7 @@ it.each([ secondaryPublication: `after-primary-continuation` as const, secondaryPageSize: 1 as const, secondaryCommitOrder: `reverse` as const, + primaryAutoIndex: `eager` as const, }, { name: `delayed filtered secondary receipt`, @@ -846,6 +891,7 @@ it.each([ secondaryPublication: `preloaded-delayed-receipt` as const, secondaryPageSize: 1 as const, secondaryCommitOrder: `reverse` as const, + primaryAutoIndex: `eager` as const, }, { name: `late secondary after primary exhaustion`, @@ -855,6 +901,7 @@ it.each([ secondaryPublication: `after-primary-exhaustion` as const, secondaryPageSize: 1 as const, secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, }, { name: `joined multiplicity before offset`, @@ -867,15 +914,27 @@ it.each([ secondaryPublication: `preloaded` as const, secondaryPageSize: 2 as const, secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, }, { - name: `zero-limit window`, - secondaryRows: [], + name: `indexed zero-limit window`, + secondaryRows: [{ id: `a-0`, joinKey: `a` }], offset: 2, limit: 0, secondaryPublication: `preloaded` as const, secondaryPageSize: 1 as const, secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `unindexed zero-limit window`, + secondaryRows: [{ id: `a-0`, joinKey: `a` }], + offset: 2, + limit: 0, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `off` as const, }, ] satisfies ReadonlyArray< Pick< @@ -886,6 +945,7 @@ it.each([ | `secondaryPublication` | `secondaryPageSize` | `secondaryCommitOrder` + | `primaryAutoIndex` > & { name: string } >)(`$name`, async ({ name: _name, ...scenario }) => { await runMultiSourceOrderedScenario({ @@ -895,71 +955,134 @@ it.each([ }) }) -it(`keeps an Effect zero-limit join free of ordered transport work`, async () => { - type PrimaryRow = { id: string; rank: number; joinKey: string } - type SecondaryRow = { id: string; joinKey: string } - const primaryLoads: Array = [] - const primary = createCollection({ - id: `multi-source-zero-limit-effect-primary`, +it(`retries unindexed transport after a rejected zero-to-positive refinement`, async () => { + type Row = { id: string; rank: number } + let attempts = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `unindexed-zero-refinement-retry`, getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, - autoIndex: `eager`, + autoIndex: `off`, defaultIndexType: BTreeIndex, sync: { - sync: ({ markReady }) => { - markReady() + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() return { - loadSubset: (options) => { - primaryLoads.push(options) - return true + loadSubset: async () => { + attempts++ + if (attempts === 1) throw new Error(`fallback failed`) + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + await commit() + return { hasMore: false, appliedRowKeys: [`a`] } }, unloadSubset: () => {}, } }, }, }) - const secondary = createCollection({ - id: `multi-source-zero-limit-effect-secondary`, - getKey: (row) => row.id, - syncMode: `on-demand`, - startSync: true, - autoIndex: `eager`, - defaultIndexType: BTreeIndex, - sync: { - sync: ({ markReady }) => { - markReady() - return { - loadSubset: () => true, - unloadSubset: () => {}, - } - }, - }, - }) - const effect = createEffect({ + const live = createLiveQueryCollection({ + id: `unindexed-zero-refinement-retry-live`, query: (q) => q - .from({ primaryRow: primary }) - .innerJoin( - { secondaryRow: secondary }, - ({ primaryRow, secondaryRow }) => - eq(primaryRow.joinKey, secondaryRow.joinKey), - ) - .orderBy(({ primaryRow }) => primaryRow.rank) - .offset(2) + .from({ row: source }) + .orderBy(({ row }) => row.rank) .limit(0), - onBatch: () => {}, + startSync: true, }) try { + await live.preload() + expect(attempts).toBe(0) + + await expect(live.utils.setWindow({ offset: 0, limit: 1 })).rejects.toThrow( + `fallback failed`, + ) + await flushPromises() + await live.utils.setWindow({ offset: 0, limit: 1 }) await flushPromises() - expect(primaryLoads).toEqual([]) + + expect(attempts).toBe(2) } finally { - await effect.dispose() - await Promise.all([primary.cleanup(), secondary.cleanup()]) + await Promise.all([live.cleanup(), source.cleanup()]) } }) +it.each([`eager`, `off`] as const)( + `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, + async (autoIndex) => { + type PrimaryRow = { id: string; rank: number; joinKey: string } + type SecondaryRow = { id: string; joinKey: string } + const primaryLoads: Array = [] + const primary = createCollection({ + id: `multi-source-zero-limit-effect-primary-${autoIndex}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + primaryLoads.push(options) + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const secondary = createCollection({ + id: `multi-source-zero-limit-effect-secondary-${autoIndex}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => {}, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank) + .offset(2) + .limit(0), + onBatch: () => {}, + }) + + try { + await flushPromises() + expect(primaryLoads).toEqual([]) + } finally { + await effect.dispose() + await Promise.all([primary.cleanup(), secondary.cleanup()]) + } + }, +) + it(`settles concurrent secondary loads out of order across paged commits`, async () => { type PrimaryRow = { id: string; rank: number; joinKey: string } type SecondaryRow = { id: string; joinKey: string } From 6caacc308d8d116bc576f457bfc7ed1f9ca17eea Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 17:59:48 -0600 Subject: [PATCH 086/327] fix(db): publish unindexed fallback retries --- packages/db/src/query/live/ARCHITECTURE.md | 14 +- .../query/live/collection-config-builder.ts | 16 +- ...d-subset-full-flow-oracle.property.test.ts | 239 ++++++++++++++++-- 3 files changed, 241 insertions(+), 28 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index def987edf..4bd5667ad 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -693,7 +693,19 @@ the empty window into a positive request. The dedupe helper applies the same law when adapters call it directly: a zero-width request establishes no coverage and owns no physical acquisition. This also holds when no usable order index exists: core defers the full-snapshot fallback until the window -first becomes positive, then requests it once for that subscription session. +first becomes positive. One successful or pending fallback covers that +subscription session. A synchronous throw or rejected fallback clears only +that subscription's guard, so the same live query can retry. Cleanup creates a +new subscription and a late settlement from the old one cannot clear the new +guard. Truncate replay belongs to the subscription's retained demand; the live +coordinator must not add a second fallback while that replay is in flight. + +The graph loader is part of the same quiescence pass as source processing. If a +window change reaches the pass with no graph work, core calls the loader first. +It then drains every graph step created by a synchronous adapter commit before +publishing. Async settlement schedules another pass under the same rule. A +successful retry therefore cannot commit source rows while leaving the live +result stale until an unrelated later window change. Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 870e370aa..2f2a04f62 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -569,11 +569,16 @@ export class CollectionConfigBuilder< // Always run the graph if subscribed (eager execution) if (syncState.subscribedToAllCollections) { - let callbackCalled = false + // A window change can reach this point with no pending graph work. + // Let the loader run first so any synchronous source commit it starts + // becomes part of this same quiescence pass. + if (!syncState.graph.pendingWork()) { + callback?.() + } + while (syncState.graph.pendingWork()) { syncState.graph.run() callback?.() - callbackCalled = true } // Publish only after every operator has reached quiescence. A source @@ -581,13 +586,6 @@ export class CollectionConfigBuilder< // flushing between those steps would expose a mixed root snapshot. syncState.flushPendingChanges?.() - // Ensure the callback runs at least once even when the graph has no pending work. - // This handles lazy loading scenarios where setWindow() increases the limit or - // an async loadSubset completes and we need to re-check if more data is needed. - if (!callbackCalled) { - callback?.() - } - // On the initial run, we may need to do an empty commit to ensure that // the collection is initialized if (syncState.messagesCount === 0) { diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 895970577..eca0bac03 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -955,14 +955,95 @@ it.each([ }) }) -it(`retries unindexed transport after a rejected zero-to-positive refinement`, async () => { +it.each([`sync throw`, `async reject`] as const)( + `retries unindexed transport after a %s during zero-to-positive refinement`, + async (failureMode) => { + type Row = { id: string; rank: number } + let attempts = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `unindexed-zero-refinement-retry`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + attempts++ + if (attempts === 1) { + const error = new Error(`fallback failed`) + if (failureMode === `sync throw`) throw error + return Promise.reject(error) + } + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + const outcome = { hasMore: false, appliedRowKeys: [`a`] } + return applied === true + ? Promise.resolve(outcome) + : applied.then(() => outcome) + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-zero-refinement-retry-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + + try { + await live.preload() + expect(attempts).toBe(0) + + if (failureMode === `sync throw`) { + expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow( + `fallback failed`, + ) + } else { + await expect( + live.utils.setWindow({ offset: 0, limit: 1 }), + ).rejects.toThrow(`fallback failed`) + } + await flushPromises() + await live.utils.setWindow({ offset: 0, limit: 1 }) + await flushPromises() + + expect(attempts).toBe(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + await Promise.all([live.cleanup(), source.cleanup()]) + } + }, +) + +it(`fences an unindexed fallback settlement from a cleaned query session`, async () => { type Row = { id: string; rank: number } - let attempts = 0 + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void let commit!: () => true | Promise const source = createCollection({ - id: `unindexed-zero-refinement-retry`, + id: `unindexed-fallback-session-fence`, getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, @@ -975,13 +1056,10 @@ it(`retries unindexed transport after a rejected zero-to-positive refinement`, a commit = params.commit params.markReady() return { - loadSubset: async () => { - attempts++ - if (attempts === 1) throw new Error(`fallback failed`) - begin() - write({ type: `insert`, value: { id: `a`, rank: 1 } }) - await commit() - return { hasMore: false, appliedRowKeys: [`a`] } + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise }, unloadSubset: () => {}, } @@ -989,7 +1067,7 @@ it(`retries unindexed transport after a rejected zero-to-positive refinement`, a }, }) const live = createLiveQueryCollection({ - id: `unindexed-zero-refinement-retry-live`, + id: `unindexed-fallback-session-fence-live`, query: (q) => q .from({ row: source }) @@ -997,21 +1075,146 @@ it(`retries unindexed transport after a rejected zero-to-positive refinement`, a .limit(0), startSync: true, }) + let firstWindow: true | Promise | undefined + let secondWindow: true | Promise | undefined try { await live.preload() - expect(attempts).toBe(0) + firstWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + void Promise.resolve(firstWindow).catch(() => {}) + expect(pending).toHaveLength(1) - await expect(live.utils.setWindow({ offset: 0, limit: 1 })).rejects.toThrow( - `fallback failed`, - ) + await live.cleanup() + await live.preload() + secondWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + expect(pending).toHaveLength(2) + + pending[0]!.reject(new Error(`stale fallback failed`)) + await flushPromises() + const repeatedWindow = live.utils.setWindow({ offset: 0, limit: 2 }) + void Promise.resolve(repeatedWindow).catch(() => {}) + expect(pending).toHaveLength(2) + + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + if (applied !== true) await applied + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await secondWindow + await flushPromises() + + expect(pending).toHaveLength(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await Promise.all([ + Promise.resolve(firstWindow).catch(() => undefined), + Promise.resolve(secondWindow).catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) + } +}) + +it(`replays one unindexed fallback and publishes one replacement after truncate`, async () => { + type Row = { id: string; rank: number } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] + const batches: Array> = [] + const callbackReads: Array> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `unindexed-fallback-truncate-replay`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-fallback-truncate-replay-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + const subscription = live.subscribeChanges( + (changes) => { + batches.push(changes.map(({ key }) => String(key)).sort()) + callbackReads.push(live.toArray.map(({ id }) => id).sort()) + }, + { includeInitialState: false }, + ) + const preload = live.preload() + + try { + expect(pending).toHaveLength(1) + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const initialApplied = commit() + if (initialApplied !== true) await initialApplied + pending[0]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await preload await flushPromises() - await live.utils.setWindow({ offset: 0, limit: 1 }) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + + batches.length = 0 + callbackReads.length = 0 + begin() + truncate() + const replacement = commit() + await flushPromises() + expect(pending).toHaveLength(2) + + begin() + write({ type: `insert`, value: { id: `b`, rank: 2 } }) + const replacementApplied = commit() + if (replacementApplied !== true) await replacementApplied + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`b`] }) + if (replacement !== true) await replacement await flushPromises() - expect(attempts).toBe(2) + expect(pending).toHaveLength(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`b`]) + expect(batches).toHaveLength(1) + expect(callbackReads).toEqual([[`b`]]) } finally { - await Promise.all([live.cleanup(), source.cleanup()]) + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + subscription.unsubscribe() + await Promise.all([ + preload.catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) } }) From 971241887d93240e423a4c94f67580599a52bd01 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 18:17:12 -0600 Subject: [PATCH 087/327] test(db): prove loader quiescent publication --- ...d-subset-full-flow-oracle.property.test.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index eca0bac03..8020cd669 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1032,6 +1032,81 @@ it.each([`sync throw`, `async reject`] as const)( }, ) +it(`publishes once after a loader fills an indexed window across graph turns`, async () => { + type Row = { id: string; rank: number } + const remoteRows: ReadonlyArray = [ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ] + const batches: Array> = [] + const callbackReads: Array> = [] + let loads = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `indexed-loader-quiescent-publication`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + const row = remoteRows[loads++] + if (!row) return true + begin() + write({ type: `insert`, value: row }) + const applied = commit() + if (applied !== true) { + throw new Error(`Expected synchronous source application`) + } + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `indexed-loader-quiescent-publication-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + let subscription: ReturnType | undefined + + try { + await live.preload() + subscription = live.subscribeChanges( + (changes) => { + batches.push(changes.map(({ key }) => String(key)).sort()) + callbackReads.push(live.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + await live.utils.setWindow({ offset: 0, limit: 2 }) + await flushPromises() + + expect(loads).toBe(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`, `b`]) + expect(batches).toEqual([[`a`, `b`]]) + expect(callbackReads).toEqual([[`a`, `b`]]) + } finally { + subscription?.unsubscribe() + await Promise.all([live.cleanup(), source.cleanup()]) + } +}) + it(`fences an unindexed fallback settlement from a cleaned query session`, async () => { type Row = { id: string; rank: number } type Result = { @@ -1091,6 +1166,7 @@ it(`fences an unindexed fallback settlement from a cleaned query session`, async pending[0]!.reject(new Error(`stale fallback failed`)) await flushPromises() + expect(live.utils.lastSubsetError).toBeUndefined() const repeatedWindow = live.utils.setWindow({ offset: 0, limit: 2 }) void Promise.resolve(repeatedWindow).catch(() => {}) expect(pending).toHaveLength(2) @@ -1192,11 +1268,17 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` const replacement = commit() await flushPromises() expect(pending).toHaveLength(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) begin() write({ type: `insert`, value: { id: `b`, rank: 2 } }) const replacementApplied = commit() if (replacementApplied !== true) await replacementApplied + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`b`] }) if (replacement !== true) await replacement await flushPromises() From 69a84763ba11a24c5511a72bb2f6c783e374fa04 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 18:36:50 -0600 Subject: [PATCH 088/327] test(db): strengthen unindexed session observations --- packages/db/src/query/live/ARCHITECTURE.md | 5 + ...d-subset-full-flow-oracle.property.test.ts | 168 ++++++++++++++++-- 2 files changed, 162 insertions(+), 11 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 4bd5667ad..c652f846b 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -699,6 +699,11 @@ that subscription's guard, so the same live query can retry. Cleanup creates a new subscription and a late settlement from the old one cannot clear the new guard. Truncate replay belongs to the subscription's retained demand; the live coordinator must not add a second fallback while that replay is in flight. +Cleanup also aborts and settles the subscription-visible acquisition before a +raw adapter promise can affect a replacement session. The live coordinator's +subscription-identity check is a second fence, not a substitute for that lower +abort boundary. Session tests must prove the public loading, readiness, error, +and row history rather than depend on reaching either private fence alone. The graph loader is part of the same quiescence pass as source processing. If a window change reaches the pass with no graph work, core calls the loader first. diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 8020cd669..055974246 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1089,7 +1089,14 @@ it(`publishes once after a loader fills an indexed window across graph turns`, a await live.preload() subscription = live.subscribeChanges( (changes) => { - batches.push(changes.map(({ key }) => String(key)).sort()) + batches.push( + changes + .map( + ({ type, key, value }) => + `${type}:${String(key)}:${String(value.id)}`, + ) + .sort(), + ) callbackReads.push(live.toArray.map(({ id }) => id)) }, { includeInitialState: false }, @@ -1099,7 +1106,7 @@ it(`publishes once after a loader fills an indexed window across graph turns`, a expect(loads).toBe(2) expect(live.toArray.map(({ id }) => id)).toEqual([`a`, `b`]) - expect(batches).toEqual([[`a`, `b`]]) + expect(batches).toEqual([[`insert:a:a`, `insert:b:b`]]) expect(callbackReads).toEqual([[`a`, `b`]]) } finally { subscription?.unsubscribe() @@ -1150,44 +1157,164 @@ it(`fences an unindexed fallback settlement from a cleaned query session`, async .limit(0), startSync: true, }) + let failedWindow: true | Promise | undefined let firstWindow: true | Promise | undefined let secondWindow: true | Promise | undefined + let repeatedWindow: true | Promise | undefined try { await live.preload() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + + const visibleFailure = new Error(`visible fallback failed`) + failedWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + expect(pending).toHaveLength(1) + expect(live.isLoadingSubset).toBe(true) + pending[0]!.reject(visibleFailure) + await expect(Promise.resolve(failedWindow)).rejects.toBe(visibleFailure) + await flushPromises() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBe(visibleFailure) + firstWindow = live.utils.setWindow({ offset: 0, limit: 1 }) void Promise.resolve(firstWindow).catch(() => {}) - expect(pending).toHaveLength(1) + expect(pending).toHaveLength(2) + expect(live.isLoadingSubset).toBe(true) await live.cleanup() await live.preload() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() secondWindow = live.utils.setWindow({ offset: 0, limit: 1 }) - expect(pending).toHaveLength(2) + expect(pending).toHaveLength(3) + expect(live.isLoadingSubset).toBe(true) - pending[0]!.reject(new Error(`stale fallback failed`)) + pending[1]!.reject(new Error(`stale fallback failed`)) await flushPromises() expect(live.utils.lastSubsetError).toBeUndefined() - const repeatedWindow = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(true) + repeatedWindow = live.utils.setWindow({ offset: 0, limit: 2 }) void Promise.resolve(repeatedWindow).catch(() => {}) + expect(pending).toHaveLength(3) + + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + if (applied !== true) await applied + pending[2]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await Promise.all([secondWindow, repeatedWindow]) + await flushPromises() + + expect(pending).toHaveLength(3) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await Promise.all([ + Promise.resolve(failedWindow).catch(() => undefined), + Promise.resolve(firstWindow).catch(() => undefined), + Promise.resolve(secondWindow).catch(() => undefined), + Promise.resolve(repeatedWindow).catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) + } +}) + +it(`keeps an initial unindexed load scoped to its query session`, async () => { + type Row = { id: string; rank: number } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `unindexed-initial-session-fence`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-initial-session-fence-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + let firstPreload: Promise | undefined + let secondPreload: Promise | undefined + + try { + firstPreload = live.preload() + void firstPreload.catch(() => {}) + expect(pending).toHaveLength(1) + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + + await live.cleanup() + secondPreload = live.preload() expect(pending).toHaveLength(2) + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() + + pending[0]!.reject(new Error(`stale initial fallback failed`)) + await flushPromises() + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() begin() write({ type: `insert`, value: { id: `a`, rank: 1 } }) const applied = commit() if (applied !== true) await applied pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) - await secondWindow + await secondPreload await flushPromises() expect(pending).toHaveLength(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) } finally { for (const request of pending) { request.reject(new Error(`test cleanup`)) } await Promise.all([ - Promise.resolve(firstWindow).catch(() => undefined), - Promise.resolve(secondWindow).catch(() => undefined), + firstPreload?.catch(() => undefined), + secondPreload?.catch(() => undefined), live.cleanup(), source.cleanup(), ]) @@ -1243,7 +1370,14 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` }) const subscription = live.subscribeChanges( (changes) => { - batches.push(changes.map(({ key }) => String(key)).sort()) + batches.push( + changes + .map( + ({ type, key, value }) => + `${type}:${String(key)}:${String(value.id)}`, + ) + .sort(), + ) callbackReads.push(live.toArray.map(({ id }) => id).sort()) }, { includeInitialState: false }, @@ -1265,7 +1399,13 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` callbackReads.length = 0 begin() truncate() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) const replacement = commit() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) await flushPromises() expect(pending).toHaveLength(2) expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) @@ -1274,7 +1414,13 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` begin() write({ type: `insert`, value: { id: `b`, rank: 2 } }) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) const replacementApplied = commit() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) if (replacementApplied !== true) await replacementApplied expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) expect(batches).toHaveLength(0) @@ -1285,7 +1431,7 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` expect(pending).toHaveLength(2) expect(live.toArray.map(({ id }) => id)).toEqual([`b`]) - expect(batches).toHaveLength(1) + expect(batches).toEqual([[`delete:a:a`, `insert:b:b`]]) expect(callbackReads).toEqual([[`b`]]) } finally { for (const request of pending) { From 7cd45b18824e3ad6ac969400b25b4c68aa324f51 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 18:50:30 -0600 Subject: [PATCH 089/327] test(db): preserve full publication payloads --- ...d-subset-full-flow-oracle.property.test.ts | 90 +++++++++++++------ 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 055974246..4cd432906 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1034,12 +1034,17 @@ it.each([`sync throw`, `async reject`] as const)( it(`publishes once after a loader fills an indexed window across graph turns`, async () => { type Row = { id: string; rank: number } + type ObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: Row + } const remoteRows: ReadonlyArray = [ { id: `a`, rank: 1 }, { id: `b`, rank: 2 }, ] - const batches: Array> = [] - const callbackReads: Array> = [] + const batches: Array> = [] + const callbackReads: Array> = [] let loads = 0 let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void @@ -1083,6 +1088,7 @@ it(`publishes once after a loader fills an indexed window across graph turns`, a .limit(0), startSync: true, }) + const readRows = () => live.toArray.map(({ id, rank }) => ({ id, rank })) let subscription: ReturnType | undefined try { @@ -1091,13 +1097,14 @@ it(`publishes once after a loader fills an indexed window across graph turns`, a (changes) => { batches.push( changes - .map( - ({ type, key, value }) => - `${type}:${String(key)}:${String(value.id)}`, - ) - .sort(), + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), ) - callbackReads.push(live.toArray.map(({ id }) => id)) + callbackReads.push(readRows()) }, { includeInitialState: false }, ) @@ -1105,9 +1112,22 @@ it(`publishes once after a loader fills an indexed window across graph turns`, a await flushPromises() expect(loads).toBe(2) - expect(live.toArray.map(({ id }) => id)).toEqual([`a`, `b`]) - expect(batches).toEqual([[`insert:a:a`, `insert:b:b`]]) - expect(callbackReads).toEqual([[`a`, `b`]]) + expect(readRows()).toEqual([ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ]) + expect(batches).toEqual([ + [ + { type: `insert`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(callbackReads).toEqual([ + [ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ], + ]) } finally { subscription?.unsubscribe() await Promise.all([live.cleanup(), source.cleanup()]) @@ -1323,13 +1343,18 @@ it(`keeps an initial unindexed load scoped to its query session`, async () => { it(`replays one unindexed fallback and publishes one replacement after truncate`, async () => { type Row = { id: string; rank: number } + type ObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: Row + } type Result = { hasMore: boolean appliedRowKeys: ReadonlyArray } const pending: Array>> = [] - const batches: Array> = [] - const callbackReads: Array> = [] + const batches: Array> = [] + const callbackReads: Array> = [] let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void let commit!: () => true | Promise @@ -1368,17 +1393,19 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` .limit(1), startSync: true, }) + const readRows = () => live.toArray.map(({ id, rank }) => ({ id, rank })) const subscription = live.subscribeChanges( (changes) => { batches.push( changes - .map( - ({ type, key, value }) => - `${type}:${String(key)}:${String(value.id)}`, - ) - .sort(), + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), ) - callbackReads.push(live.toArray.map(({ id }) => id).sort()) + callbackReads.push(readRows()) }, { includeInitialState: false }, ) @@ -1393,36 +1420,36 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` pending[0]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) await preload await flushPromises() - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) batches.length = 0 callbackReads.length = 0 begin() truncate() - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) const replacement = commit() - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) await flushPromises() expect(pending).toHaveLength(2) - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) begin() write({ type: `insert`, value: { id: `b`, rank: 2 } }) - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) const replacementApplied = commit() - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) if (replacementApplied !== true) await replacementApplied - expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) expect(batches).toHaveLength(0) expect(callbackReads).toHaveLength(0) pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`b`] }) @@ -1430,9 +1457,14 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` await flushPromises() expect(pending).toHaveLength(2) - expect(live.toArray.map(({ id }) => id)).toEqual([`b`]) - expect(batches).toEqual([[`delete:a:a`, `insert:b:b`]]) - expect(callbackReads).toEqual([[`b`]]) + expect(readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) } finally { for (const request of pending) { request.reject(new Error(`test cleanup`)) From c3eac1e13ea59aa3c36010aa660c8bc2b27261ff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 19:02:07 -0600 Subject: [PATCH 090/327] fix(db): retain unindexed replay guard after retry --- packages/db/src/query/live/ARCHITECTURE.md | 5 +- .../src/query/live/collection-subscriber.ts | 6 + ...d-subset-full-flow-oracle.property.test.ts | 184 ++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index c652f846b..a28bc78e9 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -698,7 +698,10 @@ subscription session. A synchronous throw or rejected fallback clears only that subscription's guard, so the same live query can retry. Cleanup creates a new subscription and a late settlement from the old one cannot clear the new guard. Truncate replay belongs to the subscription's retained demand; the live -coordinator must not add a second fallback while that replay is in flight. +coordinator must not add a second fallback while that replay is in flight. A +replay that starts after an earlier rejection reclaims the same subscription +guard. Success keeps it claimed, so replacement publication cannot schedule a +duplicate full-source fallback; rejection releases it for a later retry. Cleanup also aborts and settles the subscription-visible acquisition before a raw adapter promise can affect a replacement session. The live coordinator's subscription-identity check is a second fence, not a substitute for that lower diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 071dec373..c46c190cb 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -561,6 +561,12 @@ export class CollectionSubscriber< orderBy, trackLoadSubsetPromise: false, onLoadSubsetResult: (result, demand) => { + if ( + this.unindexedSnapshotSubscription === undefined || + this.unindexedSnapshotSubscription === subscription + ) { + this.unindexedSnapshotSubscription = subscription + } if (result instanceof Promise) { void result.catch(() => { if (this.unindexedSnapshotSubscription === subscription) { diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 4cd432906..786db79e0 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1478,6 +1478,190 @@ it(`replays one unindexed fallback and publishes one replacement after truncate` } }) +type UnindexedReplayRow = { id: string; rank: number } +type UnindexedReplayResult = { + hasMore: boolean + appliedRowKeys: ReadonlyArray +} +type UnindexedReplayObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: UnindexedReplayRow +} + +function createUnindexedReplayHarness(id: string) { + const pending: Array<{ + options: LoadSubsetOptions + request: ReturnType> + }> = [] + const batches: Array> = [] + const callbackReads: Array> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: UnindexedReplayRow }) => void + let commit!: () => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `${id}-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: (options) => { + const request = createDeferred() + pending.push({ options, request }) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `${id}-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + const readRows = () => + live.toArray.map(({ id: rowId, rank }) => ({ id: rowId, rank })) + let observer: ReturnType | undefined + const startObserving = () => { + observer = live.subscribeChanges( + (changes) => { + batches.push( + changes + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), + ) + callbackReads.push(readRows()) + }, + { includeInitialState: false }, + ) + } + const stopObserving = () => { + observer?.unsubscribe() + observer = undefined + } + const clearObservations = () => { + batches.length = 0 + callbackReads.length = 0 + } + const applyRows = async (rows: ReadonlyArray) => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + const receipt = commit() + if (receipt !== true) await receipt + } + const startTruncate = () => { + begin() + truncate() + return commit() + } + const cleanup = async () => { + for (const { request } of pending) { + request.reject(new Error(`test cleanup`)) + } + stopObserving() + await Promise.all([live.cleanup(), source.cleanup()]) + } + + startObserving() + return { + source, + live, + pending, + batches, + callbackReads, + readRows, + startObserving, + stopObserving, + clearObservations, + applyRows, + startTruncate, + cleanup, + } +} + +it(`retries one unindexed fallback after a rejected truncate replay`, async () => { + const harness = createUnindexedReplayHarness( + `unindexed-rejected-truncate-retry`, + ) + const preload = harness.live.preload() + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await preload + await flushPromises() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + const replayFailure = new Error(`truncate replay failed`) + const failedReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + harness.pending[1]!.request.reject(replayFailure) + await Promise.resolve(failedReplacement).catch(() => undefined) + await flushPromises() + + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const successfulReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.isLoadingSubset).toBe(true) + await harness.applyRows([{ id: `b`, rank: 2 }]) + harness.pending[2]!.request.resolve({ + hasMore: false, + appliedRowKeys: [`b`], + }) + if (successfulReplacement !== true) await successfulReplacement + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) + } finally { + await Promise.all([preload.catch(() => undefined), harness.cleanup()]) + } +}) + it.each([`eager`, `off`] as const)( `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, async (autoIndex) => { From 7a2639da44f7b61842cf54f5d6de2d5c3d99d70a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 19:16:39 -0600 Subject: [PATCH 091/327] test(db): observe unindexed replay ownership --- ...d-subset-full-flow-oracle.property.test.ts | 201 ++++++++++++------ 1 file changed, 139 insertions(+), 62 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 786db79e0..54419bfab 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1492,8 +1492,10 @@ type UnindexedReplayObservedChange = { function createUnindexedReplayHarness(id: string) { const pending: Array<{ options: LoadSubsetOptions - request: ReturnType> + request?: ReturnType> }> = [] + const unloads: Array = [] + const synchronousLoads = new Map>() const batches: Array> = [] const callbackReads: Array> = [] let begin!: () => void @@ -1516,11 +1518,24 @@ function createUnindexedReplayHarness(id: string) { params.markReady() return { loadSubset: (options) => { + const loadIndex = pending.length + const synchronousRows = synchronousLoads.get(loadIndex) + if (synchronousRows) { + pending.push({ options }) + begin() + for (const row of synchronousRows) { + write({ type: `insert`, value: row }) + } + commit() + return true + } const request = createDeferred() pending.push({ options, request }) return request.promise }, - unloadSubset: () => {}, + unloadSubset: (options) => { + unloads.push(options) + }, } }, }, @@ -1575,7 +1590,7 @@ function createUnindexedReplayHarness(id: string) { } const cleanup = async () => { for (const { request } of pending) { - request.reject(new Error(`test cleanup`)) + request?.reject(new Error(`test cleanup`)) } stopObserving() await Promise.all([live.cleanup(), source.cleanup()]) @@ -1586,6 +1601,8 @@ function createUnindexedReplayHarness(id: string) { source, live, pending, + unloads, + synchronousLoads, batches, callbackReads, readRows, @@ -1598,69 +1615,129 @@ function createUnindexedReplayHarness(id: string) { } } -it(`retries one unindexed fallback after a rejected truncate replay`, async () => { - const harness = createUnindexedReplayHarness( - `unindexed-rejected-truncate-retry`, - ) - const preload = harness.live.preload() - - try { - expect(harness.pending).toHaveLength(1) - await harness.applyRows([{ id: `a`, rank: 1 }]) - harness.pending[0]!.request.resolve({ - hasMore: false, - appliedRowKeys: [`a`], - }) - await preload - await flushPromises() - expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) +it.each([`async`, `sync`] as const)( + `retries one unindexed fallback after a rejected truncate replay with %s success`, + async (successMode) => { + const harness = createUnindexedReplayHarness( + `unindexed-rejected-truncate-retry-${successMode}`, + ) + const preload = harness.live.preload() + let cleaned = false - harness.clearObservations() - const replayFailure = new Error(`truncate replay failed`) - const failedReplacement = harness.startTruncate() - await flushPromises() - expect(harness.pending).toHaveLength(2) - expect(harness.live.status).toBe(`ready`) - expect(harness.live.isLoadingSubset).toBe(true) - harness.pending[1]!.request.reject(replayFailure) - await Promise.resolve(failedReplacement).catch(() => undefined) - await flushPromises() + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await preload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + const replayFailure = new Error(`truncate replay failed`) + const failedReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + harness.pending[1]!.request!.reject(replayFailure) + await Promise.resolve(failedReplacement).catch(() => undefined) + await flushPromises() - expect(harness.pending).toHaveLength(2) - expect(harness.live.status).toBe(`ready`) - expect(harness.live.isLoadingSubset).toBe(false) - expect(harness.live.utils.lastSubsetError).toBe(replayFailure) - expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) - expect(harness.batches).toEqual([]) - expect(harness.callbackReads).toEqual([]) + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) - const successfulReplacement = harness.startTruncate() - await flushPromises() - expect(harness.pending).toHaveLength(3) - expect(harness.live.isLoadingSubset).toBe(true) - await harness.applyRows([{ id: `b`, rank: 2 }]) - harness.pending[2]!.request.resolve({ - hasMore: false, - appliedRowKeys: [`b`], - }) - if (successfulReplacement !== true) await successfulReplacement - await flushPromises() + if (successMode === `sync`) { + harness.synchronousLoads.set(2, [{ id: `b`, rank: 2 }]) + } + const successfulReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.isLoadingSubset).toBe(successMode === `async`) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + if (successMode === `async`) { + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + await harness.applyRows([{ id: `b`, rank: 2 }]) + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`b`], + }) + } else { + expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + } + if (successfulReplacement !== true) await successfulReplacement + await flushPromises() - expect(harness.pending).toHaveLength(3) - expect(harness.live.status).toBe(`ready`) - expect(harness.live.isLoadingSubset).toBe(false) - expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) - expect(harness.batches).toEqual([ - [ - { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, - { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, - ], - ]) - expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) - } finally { - await Promise.all([preload.catch(() => undefined), harness.cleanup()]) - } -}) + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) + + for (const { options } of harness.pending) { + expect(options).toMatchObject({ + orderBy: [ + { + expression: { type: `ref`, path: [`rank`] }, + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ], + }) + expect(options.limit).toBeUndefined() + expect(options.offset).toBeUndefined() + expect(options.cursor).toBeUndefined() + expect(options.signal).toBeInstanceOf(AbortSignal) + } + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect(harness.unloads).toHaveLength(2) + expect(harness.unloads[0]).toEqual(harness.pending[0]!.options) + expect(harness.unloads[1]).toEqual(harness.pending[1]!.options) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(harness.unloads).toHaveLength(3) + for (const [index, options] of harness.unloads.entries()) { + expect(options).toEqual(harness.pending[index]!.options) + } + } finally { + await Promise.all([ + preload.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) it.each([`eager`, `off`] as const)( `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, From bfa4b78cf7fbdad0dbf7764650e886c353aa5f34 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 19:25:19 -0600 Subject: [PATCH 092/327] test(db): prove unindexed replay result paths --- ...load-subset-full-flow-oracle.property.test.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 54419bfab..ca24c0548 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1494,6 +1494,7 @@ function createUnindexedReplayHarness(id: string) { options: LoadSubsetOptions request?: ReturnType> }> = [] + const loadResults: Array> = [] const unloads: Array = [] const synchronousLoads = new Map>() const batches: Array> = [] @@ -1527,11 +1528,15 @@ function createUnindexedReplayHarness(id: string) { write({ type: `insert`, value: row }) } commit() - return true + const result = true as const + loadResults.push(result) + return result } const request = createDeferred() pending.push({ options, request }) - return request.promise + const result = request.promise + loadResults.push(result) + return result }, unloadSubset: (options) => { unloads.push(options) @@ -1601,6 +1606,7 @@ function createUnindexedReplayHarness(id: string) { source, live, pending, + loadResults, unloads, synchronousLoads, batches, @@ -1710,8 +1716,14 @@ it.each([`async`, `sync`] as const)( expect(options.limit).toBeUndefined() expect(options.offset).toBeUndefined() expect(options.cursor).toBeUndefined() + expect(options.where).toBeUndefined() expect(options.signal).toBeInstanceOf(AbortSignal) } + expect( + harness.loadResults.map((result) => + result === true ? `sync` : `async`, + ), + ).toEqual([`async`, `async`, successMode === `sync` ? `sync` : `async`]) const signals = harness.pending.map(({ options }) => options.signal!) expect(new Set(signals)).toHaveLength(3) expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) From bed68561a1ad3dc3ac0c5898dd4f17924b447ace Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 19:37:52 -0600 Subject: [PATCH 093/327] test(db): preserve replay acquisition identity --- ...d-subset-full-flow-oracle.property.test.ts | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index ca24c0548..6bb6ca1ad 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1705,19 +1705,36 @@ it.each([`async`, `sync`] as const)( expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) for (const { options } of harness.pending) { - expect(options).toMatchObject({ - orderBy: [ - { - expression: { type: `ref`, path: [`rank`] }, - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - }) + expect(Object.keys(options).sort()).toEqual([ + `cursor`, + `limit`, + `orderBy`, + `signal`, + `subscription`, + `where`, + ]) + expect(options.where).toBeUndefined() expect(options.limit).toBeUndefined() expect(options.offset).toBeUndefined() expect(options.cursor).toBeUndefined() - expect(options.where).toBeUndefined() + expect(options.orderBy).toHaveLength(1) + const ordering = options.orderBy![0]! + expect(Object.keys(ordering).sort()).toEqual([ + `compareOptions`, + `expression`, + ]) + expect(Object.keys(ordering.expression).sort()).toEqual([ + `path`, + `type`, + ]) + expect(ordering.expression).toEqual({ type: `ref`, path: [`rank`] }) + expect(ordering.compareOptions).toStrictEqual({ + direction: `asc`, + nulls: `first`, + stringSort: `locale`, + }) expect(options.signal).toBeInstanceOf(AbortSignal) + expect(options.subscription).toBeDefined() } expect( harness.loadResults.map((result) => @@ -1728,8 +1745,11 @@ it.each([`async`, `sync`] as const)( expect(new Set(signals)).toHaveLength(3) expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) expect(harness.unloads).toHaveLength(2) - expect(harness.unloads[0]).toEqual(harness.pending[0]!.options) - expect(harness.unloads[1]).toEqual(harness.pending[1]!.options) + expect( + harness.unloads.map((options) => + harness.pending.findIndex((pending) => pending.options === options), + ), + ).toEqual([1, 0]) await harness.cleanup() cleaned = true @@ -1739,9 +1759,11 @@ it.each([`async`, `sync`] as const)( expect(harness.readRows()).toEqual([]) expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) expect(harness.unloads).toHaveLength(3) - for (const [index, options] of harness.unloads.entries()) { - expect(options).toEqual(harness.pending[index]!.options) - } + expect( + harness.unloads.map((options) => + harness.pending.findIndex((pending) => pending.options === options), + ), + ).toEqual([1, 0, 2]) } finally { await Promise.all([ preload.catch(() => undefined), From ae273f49fa7f7f1d35dd83036a9f6ded87731ae3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 19:47:21 -0600 Subject: [PATCH 094/327] test(db): retain replay ownership key --- .../tests/query/load-subset-full-flow-oracle.property.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 6bb6ca1ad..d94b3788d 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1744,6 +1744,9 @@ it.each([`async`, `sync`] as const)( const signals = harness.pending.map(({ options }) => options.signal!) expect(new Set(signals)).toHaveLength(3) expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect( + new Set(harness.pending.map(({ options }) => options.subscription)), + ).toHaveLength(1) expect(harness.unloads).toHaveLength(2) expect( harness.unloads.map((options) => From 02c5cd84306d1db1d7115aa7aa7e00bae3327895 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 20:04:42 -0600 Subject: [PATCH 095/327] test(db): fence cleaned truncate replays --- ...d-subset-full-flow-oracle.property.test.ts | 228 +++++++++++++++--- 1 file changed, 188 insertions(+), 40 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index d94b3788d..57cedd3cb 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1621,6 +1621,42 @@ function createUnindexedReplayHarness(id: string) { } } +function expectUnindexedFullSnapshotRequest(options: LoadSubsetOptions): void { + expect(Object.keys(options).sort()).toEqual([ + `cursor`, + `limit`, + `orderBy`, + `signal`, + `subscription`, + `where`, + ]) + expect(options.where).toBeUndefined() + expect(options.limit).toBeUndefined() + expect(options.offset).toBeUndefined() + expect(options.cursor).toBeUndefined() + expect(options.orderBy).toHaveLength(1) + const ordering = options.orderBy![0]! + expect(Object.keys(ordering).sort()).toEqual([`compareOptions`, `expression`]) + expect(Object.keys(ordering.expression).sort()).toEqual([`path`, `type`]) + expect(ordering.expression).toEqual({ type: `ref`, path: [`rank`] }) + expect(ordering.compareOptions).toStrictEqual({ + direction: `asc`, + nulls: `first`, + stringSort: `locale`, + }) + expect(options.signal).toBeInstanceOf(AbortSignal) + expect(options.subscription).toBeDefined() +} + +function acquisitionIndices( + acquisitions: ReadonlyArray<{ options: LoadSubsetOptions }>, + releases: ReadonlyArray, +): ReadonlyArray { + return releases.map((options) => + acquisitions.findIndex((acquisition) => acquisition.options === options), + ) +} + it.each([`async`, `sync`] as const)( `retries one unindexed fallback after a rejected truncate replay with %s success`, async (successMode) => { @@ -1705,36 +1741,7 @@ it.each([`async`, `sync`] as const)( expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) for (const { options } of harness.pending) { - expect(Object.keys(options).sort()).toEqual([ - `cursor`, - `limit`, - `orderBy`, - `signal`, - `subscription`, - `where`, - ]) - expect(options.where).toBeUndefined() - expect(options.limit).toBeUndefined() - expect(options.offset).toBeUndefined() - expect(options.cursor).toBeUndefined() - expect(options.orderBy).toHaveLength(1) - const ordering = options.orderBy![0]! - expect(Object.keys(ordering).sort()).toEqual([ - `compareOptions`, - `expression`, - ]) - expect(Object.keys(ordering.expression).sort()).toEqual([ - `path`, - `type`, - ]) - expect(ordering.expression).toEqual({ type: `ref`, path: [`rank`] }) - expect(ordering.compareOptions).toStrictEqual({ - direction: `asc`, - nulls: `first`, - stringSort: `locale`, - }) - expect(options.signal).toBeInstanceOf(AbortSignal) - expect(options.subscription).toBeDefined() + expectUnindexedFullSnapshotRequest(options) } expect( harness.loadResults.map((result) => @@ -1748,11 +1755,9 @@ it.each([`async`, `sync`] as const)( new Set(harness.pending.map(({ options }) => options.subscription)), ).toHaveLength(1) expect(harness.unloads).toHaveLength(2) - expect( - harness.unloads.map((options) => - harness.pending.findIndex((pending) => pending.options === options), - ), - ).toEqual([1, 0]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, + ]) await harness.cleanup() cleaned = true @@ -1762,11 +1767,9 @@ it.each([`async`, `sync`] as const)( expect(harness.readRows()).toEqual([]) expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) expect(harness.unloads).toHaveLength(3) - expect( - harness.unloads.map((options) => - harness.pending.findIndex((pending) => pending.options === options), - ), - ).toEqual([1, 0, 2]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, 2, + ]) } finally { await Promise.all([ preload.catch(() => undefined), @@ -1776,6 +1779,151 @@ it.each([`async`, `sync`] as const)( }, ) +it.each([`resolve`, `reject`] as const)( + `fences a %s settlement from a truncate replay cleaned before completion`, + async (lateSettlement) => { + const harness = createUnindexedReplayHarness( + `unindexed-pending-replay-cleanup-${lateSettlement}`, + ) + const firstPreload = harness.live.preload() + let restartPreload: Promise | undefined + let replacement: true | Promise | undefined + let cleaned = false + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await firstPreload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + replacement = harness.startTruncate() + void Promise.resolve(replacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const firstSessionSubscription = harness.pending[0]!.options.subscription + harness.stopObserving() + await harness.live.cleanup() + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + expect(harness.unloads).toHaveLength(2) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, + ]) + + restartPreload = harness.live.preload() + harness.startObserving() + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`loading`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + + const staleError = new Error(`stale replay failed`) + if (lateSettlement === `resolve`) { + harness.pending[1]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [], + }) + } else { + harness.pending[1]!.request!.reject(staleError) + } + await Promise.resolve(replacement).catch(() => undefined) + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`loading`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + await harness.applyRows([{ id: `c`, rank: 3 }]) + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [{ type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }], + ]) + expect(harness.callbackReads).toEqual([[{ id: `c`, rank: 3 }]]) + const appliedStateRevision = harness.live._stateRevision + const appliedLayoutRevision = harness.live._layoutRevision + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`c`], + }) + await restartPreload + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [{ type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }], + [], + ]) + expect(harness.callbackReads).toEqual([ + [{ id: `c`, rank: 3 }], + [{ id: `c`, rank: 3 }], + ]) + expect(harness.live._stateRevision).toBe(appliedStateRevision) + expect(harness.live._layoutRevision).toBe(appliedLayoutRevision) + + for (const { options } of harness.pending) { + expectUnindexedFullSnapshotRequest(options) + } + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect(harness.pending[1]!.options.subscription).toBe( + firstSessionSubscription, + ) + expect(harness.pending[2]!.options.subscription).not.toBe( + firstSessionSubscription, + ) + expect(harness.loadResults.every((result) => result !== true)).toBe(true) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(harness.unloads).toHaveLength(3) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, 2, + ]) + } finally { + await Promise.all([ + firstPreload.catch(() => undefined), + restartPreload?.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) + it.each([`eager`, `off`] as const)( `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, async (autoIndex) => { From 3a13375a9bcec8a30cb0fe3c1470c00a07ad8d4d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 20:22:17 -0600 Subject: [PATCH 096/327] test(db): observe stale replay settlements --- ...d-subset-full-flow-oracle.property.test.ts | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 57cedd3cb..8b568ce66 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -2,6 +2,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' +import { SyncTransactionAbortedError } from '../../src/errors.js' import { BTreeIndex, ReverseIndex } from '../../src/index.js' import { Func, PropRef, Value } from '../../src/query/ir.js' import { createEffect } from '../../src/query/effect.js' @@ -1501,7 +1502,7 @@ function createUnindexedReplayHarness(id: string) { const callbackReads: Array> = [] let begin!: () => void let write!: (message: { type: `insert`; value: UnindexedReplayRow }) => void - let commit!: () => true | Promise + let commit!: (signal?: AbortSignal) => true | Promise let truncate!: () => void const source = createCollection({ id: `${id}-source`, @@ -1588,6 +1589,14 @@ function createUnindexedReplayHarness(id: string) { const receipt = commit() if (receipt !== true) await receipt } + const applyRowsForRequest = ( + requestIndex: number, + rows: ReadonlyArray, + ): Promise => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + return Promise.resolve(commit(pending[requestIndex]!.options.signal)) + } const startTruncate = () => { begin() truncate() @@ -1616,6 +1625,7 @@ function createUnindexedReplayHarness(id: string) { stopObserving, clearObservations, applyRows, + applyRowsForRequest, startTruncate, cleanup, } @@ -1840,10 +1850,13 @@ it.each([`resolve`, `reject`] as const)( expect(harness.readRows()).toEqual([]) const staleError = new Error(`stale replay failed`) + await expect( + harness.applyRowsForRequest(1, [{ id: `stale`, rank: -1 }]), + ).rejects.toBeInstanceOf(SyncTransactionAbortedError) if (lateSettlement === `resolve`) { harness.pending[1]!.request!.resolve({ hasMore: false, - appliedRowKeys: [], + appliedRowKeys: [`stale`], }) } else { harness.pending[1]!.request!.reject(staleError) @@ -1901,7 +1914,28 @@ it.each([`resolve`, `reject`] as const)( expect(harness.pending[2]!.options.subscription).not.toBe( firstSessionSubscription, ) - expect(harness.loadResults.every((result) => result !== true)).toBe(true) + const loadResults = await Promise.allSettled( + harness.loadResults.map((result) => Promise.resolve(result)), + ) + expect(loadResults[0]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`a`] }, + }) + if (lateSettlement === `resolve`) { + expect(loadResults[1]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`stale`] }, + }) + } else { + expect(loadResults[1]!.status).toBe(`rejected`) + if (loadResults[1]!.status === `rejected`) { + expect(loadResults[1]!.reason).toBe(staleError) + } + } + expect(loadResults[2]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`c`] }, + }) await harness.cleanup() cleaned = true From c4e238361fb0b6ddd82166615ae3806e90afd621 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 20:34:59 -0600 Subject: [PATCH 097/327] fix(db): fence superseded unindexed replays --- .../src/query/live/collection-subscriber.ts | 47 +++-- ...d-subset-full-flow-oracle.property.test.ts | 171 ++++++++++++++++++ 2 files changed, 202 insertions(+), 16 deletions(-) diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index c46c190cb..1f8bd74d2 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -61,7 +61,14 @@ export class CollectionSubscriber< private pendingOrderedLoadPromise: | Promise | undefined - private unindexedSnapshotSubscription: CollectionSubscription | undefined + // Overlapping replays share one subscription, so only the latest result + // token may clear the full-source acquisition guard. + private unindexedSnapshot: + | { + subscription: CollectionSubscription + token: symbol + } + | undefined private readonly demand = new SubsetDemandController() constructor( @@ -355,7 +362,7 @@ export class CollectionSubscriber< onLoadSubsetError: (event: SubscriptionLoadSubsetErrorEvent) => void, ): CollectionSubscription { const { orderBy, offset, limit, index } = orderByInfo - this.unindexedSnapshotSubscription = undefined + this.unindexedSnapshot = undefined // Store the callback so loadNextItems can also use direct tracking. // Track in-flight ordered loads to avoid issuing redundant requests while @@ -424,8 +431,8 @@ export class CollectionSubscriber< subscriptionHolder.current = undefined this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined - if (this.unindexedSnapshotSubscription === subscription) { - this.unindexedSnapshotSubscription = undefined + if (this.unindexedSnapshot?.subscription === subscription) { + this.unindexedSnapshot = undefined } // Ordered continuations belong to this subscription session. A settled @@ -553,24 +560,28 @@ export class CollectionSubscriber< subscription: CollectionSubscription, orderBy: LoadSubsetOptions[`orderBy`], ): void { - if (this.unindexedSnapshotSubscription === subscription) return + if (this.unindexedSnapshot?.subscription === subscription) return - this.unindexedSnapshotSubscription = subscription + const requestToken = Symbol() + this.unindexedSnapshot = { + subscription, + token: requestToken, + } try { subscription.requestSnapshot({ orderBy, trackLoadSubsetPromise: false, onLoadSubsetResult: (result, demand) => { - if ( - this.unindexedSnapshotSubscription === undefined || - this.unindexedSnapshotSubscription === subscription - ) { - this.unindexedSnapshotSubscription = subscription - } + const token = Symbol() + this.unindexedSnapshot = { subscription, token } if (result instanceof Promise) { void result.catch(() => { - if (this.unindexedSnapshotSubscription === subscription) { - this.unindexedSnapshotSubscription = undefined + const current = this.unindexedSnapshot + if ( + current?.subscription === subscription && + current.token === token + ) { + this.unindexedSnapshot = undefined } }) } @@ -578,8 +589,12 @@ export class CollectionSubscriber< }, }) } catch (error) { - if (this.unindexedSnapshotSubscription === subscription) { - this.unindexedSnapshotSubscription = undefined + const current = this.unindexedSnapshot + if ( + current.subscription === subscription && + current.token === requestToken + ) { + this.unindexedSnapshot = undefined } throw error } diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 8b568ce66..ea9807a46 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1958,6 +1958,177 @@ it.each([`resolve`, `reject`] as const)( }, ) +it.each( + ([`resolve`, `reject`] as const).flatMap((supersededSettlement) => + ([`superseded-first`, `current-first`] as const).map((settlementOrder) => ({ + supersededSettlement, + settlementOrder, + })), + ), +)( + `publishes only the current replay when an overlapping replay settles $settlementOrder with $supersededSettlement`, + async ({ supersededSettlement, settlementOrder }) => { + const harness = createUnindexedReplayHarness( + `unindexed-overlapping-replays-${supersededSettlement}-${settlementOrder}`, + ) + const preload = harness.live.preload() + let firstReplacement: true | Promise | undefined + let currentReplacement: true | Promise | undefined + let cleaned = false + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await preload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + firstReplacement = harness.startTruncate() + void Promise.resolve(firstReplacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(2) + + currentReplacement = harness.startTruncate() + void Promise.resolve(currentReplacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect( + new Set(harness.pending.map(({ options }) => options.subscription)), + ).toHaveLength(1) + expect(harness.unloads).toEqual([]) + + await expect( + harness.applyRowsForRequest(1, [{ id: `stale`, rank: -1 }]), + ).rejects.toBeInstanceOf(SyncTransactionAbortedError) + await harness.applyRowsForRequest(2, [{ id: `c`, rank: 3 }]) + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const supersededError = new Error(`superseded replay failed`) + const settleSuperseded = () => { + if (supersededSettlement === `resolve`) { + harness.pending[1]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`stale`], + }) + } else { + harness.pending[1]!.request!.reject(supersededError) + } + } + const settleCurrent = () => { + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`c`], + }) + } + const settleFirst = + settlementOrder === `superseded-first` + ? settleSuperseded + : settleCurrent + const settleLast = + settlementOrder === `superseded-first` + ? settleCurrent + : settleSuperseded + + settleFirst() + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1] : [0], + ) + + settleLast() + await Promise.all([ + Promise.resolve(firstReplacement).catch(() => undefined), + Promise.resolve(currentReplacement).catch(() => undefined), + ]) + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }, + ], + ]) + expect(harness.callbackReads).toEqual([[{ id: `c`, rank: 3 }]]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1, 0] : [0, 1], + ) + + for (const { options } of harness.pending) { + expectUnindexedFullSnapshotRequest(options) + } + const loadResults = await Promise.allSettled( + harness.loadResults.map((result) => Promise.resolve(result)), + ) + expect(loadResults[0]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`a`] }, + }) + if (supersededSettlement === `resolve`) { + expect(loadResults[1]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`stale`] }, + }) + } else { + expect(loadResults[1]!.status).toBe(`rejected`) + if (loadResults[1]!.status === `rejected`) { + expect(loadResults[1]!.reason).toBe(supersededError) + } + } + expect(loadResults[2]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`c`] }, + }) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1, 0, 2] : [0, 1, 2], + ) + } finally { + await Promise.all([ + preload.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) + it.each([`eager`, `off`] as const)( `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, async (autoIndex) => { From 421ceb9d86fe7a2e98d13d3dda82a1852e97f6ff Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 20:51:02 -0600 Subject: [PATCH 098/327] test(db): observe replay commit receipts --- .../tests/query/load-subset-full-flow-oracle.property.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index ea9807a46..77a61a165 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -2063,8 +2063,8 @@ it.each( settleLast() await Promise.all([ - Promise.resolve(firstReplacement).catch(() => undefined), - Promise.resolve(currentReplacement).catch(() => undefined), + Promise.resolve(firstReplacement), + Promise.resolve(currentReplacement), ]) await flushPromises() From 87406ab9ac54e8d8b71ca5e76ae19df52f3be806 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:05:38 -0600 Subject: [PATCH 099/327] test(db): prove refinement identity erasure --- ...d-subset-refinement-model.property.test.ts | 713 +++++++++++++++++- 1 file changed, 689 insertions(+), 24 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 3584ea499..f0c9e3203 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -7,7 +7,9 @@ import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { projectAcquisitionSettlement, projectAdapterLifecycle, + projectAtomicOrderedPublicationState, projectAuthorizedContinuationStarts, + projectOrderedPublicationBoundary, projectReplayPublication, projectRetainedRowKeys, projectReusableDemands, @@ -566,6 +568,8 @@ function renameHistoryIds( demandId: `${event.demandId}-${suffix}`, attemptId: `${event.attemptId}-${suffix}`, } + case `truncateSource`: + return { ...event, sessionId: `${event.sessionId}-${suffix}` } case `settleDemandWithoutEvidence`: return { ...event, @@ -628,12 +632,62 @@ function renameHistoryIds( ...event, acquisitionId: `${event.acquisitionId}-${suffix}`, } + case `stagePublicationRows`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `commitPublication`: + case `establishReplacementCoverage`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + } + case `beginReplacement`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + demandIds: event.demandIds.map((demandId) => `${demandId}-${suffix}`), + } + case `settleReplacement`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } default: return event } }) } +function expectObservationPreservedAfterEveryPrefix( + history: ReadonlyArray, + suffix: string, + project: ( + prefix: ReadonlyArray, + suffix: string, + ) => T, + normalize: (observation: T, suffix: string) => unknown = (observation) => + observation, +): void { + for (let prefixLength = 0; prefixLength <= history.length; prefixLength++) { + const prefix = history.slice(0, prefixLength) + expect( + normalize(project(renameHistoryIds(prefix, suffix), suffix), suffix), + JSON.stringify({ prefixLength, prefix }), + ).toEqual(normalize(project(prefix, ``), ``)) + } +} + +function removeRenamingSuffix(value: string, suffix: string): string { + const marker = `-${suffix}` + return suffix !== `` && value.endsWith(marker) + ? value.slice(0, -marker.length) + : value +} + for (const campaign of refinementCampaigns(1_779_002)) { fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( `source demand names are observationally erased (${campaign.label})`, @@ -660,8 +714,10 @@ for (const campaign of refinementCampaigns(1_779_002)) { }, ] - expect(projectSourceReadiness(renameHistoryIds(history, suffix))).toEqual( - projectSourceReadiness(history), + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSourceReadiness, ) }, ) @@ -763,6 +819,47 @@ for (const campaign of refinementCampaigns(1_779_003)) { renameHistoryIds(continuationHistory, suffix), ), ).toBe(projectAuthorizedContinuationStarts(continuationHistory)) + + for (const history of [ + demandHistory, + evidenceFreeHistory, + continuationHistory, + ]) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectTransportLoads, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedRowKeys, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableDemands, + (demandIds, renamingSuffix) => + demandIds.map((demandId) => + removeRenamingSuffix(demandId, renamingSuffix), + ), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAdapterLifecycle, + (events, renamingSuffix) => + events.map(({ type, ownerId }) => ({ + type, + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAuthorizedContinuationStarts, + ) + } }, ) } @@ -786,6 +883,19 @@ for (const campaign of refinementCampaigns(1_779_004)) { callbackReads: original.callbackReads, receiptStates: original.receipts.map(({ state }) => state), }) + + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSyncTransactions, + (observation, renamingSuffix) => ({ + ...observation, + receipts: observation.receipts.map(({ transactionId, state }) => ({ + transactionId: removeRenamingSuffix(transactionId, renamingSuffix), + state, + })), + }), + ) }, ) } @@ -819,6 +929,24 @@ for (const campaign of refinementCampaigns(1_779_005)) { expect(normalizeOwners(projectAcquisitionSettlement(renamed))).toEqual( normalizeOwners(projectAcquisitionSettlement(history)), ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAcquisitionSettlement, + (observation, renamingSuffix) => ({ + physicalStarts: observation.physicalStarts.map((acquisitionId) => + removeRenamingSuffix(acquisitionId, renamingSuffix), + ), + owners: observation.owners.map( + ({ ownerId, state, rowKeys: keys }) => ({ + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + state, + rowKeys: keys, + }), + ), + visibleRowKeys: observation.visibleRowKeys, + }), + ) }, ) } @@ -926,9 +1054,15 @@ for (const campaign of refinementCampaigns(1_779_006)) { } for (const campaign of refinementCampaigns(1_779_007)) { - fcTest.prop([fc.integer({ min: -10, max: 10 })], campaign.options)( + fcTest.prop( + [ + fc.integer({ min: -10, max: 10 }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( `replay attempt names are observationally erased (${campaign.label})`, - (replacementVersion) => { + (replacementVersion, suffix) => { const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } const replacement = { sourceId: `source`, @@ -936,26 +1070,17 @@ for (const campaign of refinementCampaigns(1_779_007)) { version: replacementVersion, } - expect( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `attempt-a`, - `attempt-b`, - `new-first`, - ), - ), - ).toEqual( - projectReplayPublication( - overlappingReplayHistory( - baseline, - replacement, - `renamed-old`, - `renamed-new`, - `new-first`, - ), - ), + const history = overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReplayPublication, ) }, ) @@ -1005,6 +1130,546 @@ function acquisitionHistory( ] } +function sourceErasureHistories(): Array> { + const register = ( + sessionId: string, + sourceId: string, + demandId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `registerSourceDemand`, + sessionId, + sourceId, + demandId, + }) + const settle = ( + sessionId: string, + sourceId: string, + demandId: string, + outcome: `resolve` | `reject`, + ): LoadSubsetFullFlowEvent => ({ + type: `settleSourceDemand`, + sessionId, + sourceId, + demandId, + outcome, + }) + + return [ + [ + register(`session-a`, `source-a`, `demand-a`), + register(`session-a`, `source-b`, `demand-b`), + settle(`session-a`, `source-a`, `demand-a`, `resolve`), + settle(`session-a`, `source-b`, `demand-b`, `reject`), + ], + [ + register(`session-a`, `source-a`, `demand-a`), + { type: `cleanupSession`, sessionId: `session-a` }, + settle(`session-a`, `source-a`, `demand-a`, `resolve`), + ], + [ + register(`session-a`, `source-a`, `demand-a`), + { + type: `restartSession`, + previousSessionId: `session-a`, + nextSessionId: `session-b`, + }, + settle(`session-a`, `source-a`, `demand-a`, `reject`), + register(`session-b`, `source-b`, `demand-b`), + settle(`session-b`, `source-b`, `demand-b`, `resolve`), + ], + ] +} + +function demandErasureHistories(): Array> { + const request = ( + ownerId: string, + attemptId: string, + alreadyAborted = false, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + ownerId, + sessionId: `session-a`, + demandId: `demand-a`, + attemptId, + alreadyAborted, + }) + const release = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + ownerId, + demandId: `demand-a`, + attemptId, + rowKeys: [`row-a`], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }) + + return [ + [ + request(`owner-a`, `attempt-a`), + { + type: `applyAuthoritativeRows`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`row-a`], + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `applyUnprovenRows`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`row-a`], + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `rejectDemand`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `settleDemandWithoutEvidence`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + release(`owner-a`, `attempt-a`), + ], + [request(`owner-a`, `attempt-a`, true), release(`owner-a`, `attempt-a`)], + [ + request(`owner-a`, `attempt-a`), + { type: `truncateSource`, sessionId: `session-a` }, + request(`owner-b`, `attempt-b`), + { + type: `applyAuthoritativeRows`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`stale-row`], + }, + { + type: `applyAuthoritativeRows`, + ownerId: `owner-b`, + demandId: `demand-a`, + attemptId: `attempt-b`, + rowKeys: [`row-a`], + }, + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `scheduleContinuation`, + taskId: `task-a`, + sessionId: `session-a`, + windowRevision: 0, + }, + { + type: `advanceWindowRevision`, + sessionId: `session-a`, + revision: 1, + }, + { type: `runContinuation`, taskId: `task-a` }, + { type: `cleanupSession`, sessionId: `session-a` }, + { + type: `restartSession`, + previousSessionId: `session-a`, + nextSessionId: `session-b`, + }, + { + type: `requestDemand`, + ownerId: `owner-b`, + sessionId: `session-b`, + demandId: `demand-b`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `scheduleContinuation`, + taskId: `task-b`, + sessionId: `session-b`, + windowRevision: 0, + }, + { type: `runContinuation`, taskId: `task-b` }, + ], + ] +} + +function transactionErasureHistories(): Array> { + const stage: LoadSubsetFullFlowEvent = { + type: `stageSyncTransaction`, + transactionId: `transaction`, + sourceId: `source`, + rowKeys: [`row`], + } + const settle: LoadSubsetFullFlowEvent = { + type: `settleSyncReceipt`, + transactionId: `transaction`, + } + + return [ + successfulTransaction(`transaction`, `source`, `row`), + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: false, + signalAborted: true, + }, + settle, + ], + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: true, + signalAborted: false, + }, + { type: `abortSyncTransaction`, transactionId: `transaction` }, + settle, + ], + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: true, + signalAborted: false, + }, + { type: `enterSyncApplication`, transactionId: `transaction` }, + { type: `publishSyncTransaction`, transactionId: `transaction` }, + settle, + ], + ] +} + +function replayErasureHistories(): Array> { + const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } + const replacement = { sourceId: `source`, rowKey: `row`, version: 1 } + + return [ + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `old-first`, + ), + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ), + [ + { type: `establishPublication`, sourceId: `source`, rows: [baseline] }, + { type: `startReplay`, attemptId: `attempt-a`, sourceId: `source` }, + { + type: `writeReplayRows`, + attemptId: `attempt-a`, + rows: [replacement], + acceptedByCore: false, + }, + { type: `settleReplay`, attemptId: `attempt-a`, outcome: `resolve` }, + ], + [ + { type: `establishPublication`, sourceId: `source`, rows: [baseline] }, + { type: `startReplay`, attemptId: `attempt-a`, sourceId: `source` }, + { type: `settleReplay`, attemptId: `attempt-a`, outcome: `reject` }, + ], + ] +} + +function publicationErasureHistories(): Array> { + const orderedRows = [ + { key: `row-a`, orderValue: 1 }, + { key: `row-b`, orderValue: 2 }, + ] + const relatedRows = [{ key: `related`, orderValue: 3 }] + const requestRelated: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + ownerId: `owner-related`, + sessionId: `session`, + demandId: `related`, + attemptId: `attempt-related`, + alreadyAborted: false, + } + + return [ + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + { type: `resizeOrderedWindow`, size: 2 }, + ], + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + requestRelated, + { + type: `stagePublicationRows`, + publicationId: `publication-b`, + demandId: `ordered`, + rows: orderedRows.slice(1), + }, + { + type: `stagePublicationRows`, + publicationId: `publication-b`, + demandId: `related`, + rows: relatedRows, + }, + { + type: `beginReplacement`, + publicationId: `publication-b`, + demandIds: [`ordered`, `related`], + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + demandId: `related`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + demandId: `ordered`, + outcome: `success`, + extent: `continues`, + }, + { + type: `establishReplacementCoverage`, + publicationId: `publication-b`, + }, + { + type: `releaseDemand`, + ownerId: `owner-related`, + demandId: `related`, + attemptId: `attempt-related`, + rowKeys: [`related`], + finalRowOwner: true, + invalidatesAdapterEvidence: true, + }, + ], + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + requestRelated, + { + type: `beginReplacement`, + publicationId: `publication-b`, + demandIds: [`ordered`, `related`], + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + demandId: `related`, + outcome: `abort`, + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + demandId: `ordered`, + outcome: `failure`, + }, + { type: `cleanupSession`, sessionId: `session` }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + ], + ] +} + +for (const campaign of refinementCampaigns(1_779_009)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `erased identities preserve every bounded next-command observation (${campaign.label})`, + (suffix) => { + for (const history of sourceErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSourceReadiness, + ) + } + + for (const history of demandErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectTransportLoads, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedRowKeys, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableDemands, + (demandIds, renamingSuffix) => + demandIds.map((demandId) => + removeRenamingSuffix(demandId, renamingSuffix), + ), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAdapterLifecycle, + (events, renamingSuffix) => + events.map(({ type, ownerId }) => ({ + type, + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAuthorizedContinuationStarts, + ) + } + + for (const history of transactionErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSyncTransactions, + (observation, renamingSuffix) => ({ + ...observation, + receipts: observation.receipts.map(({ transactionId, state }) => ({ + transactionId: removeRenamingSuffix( + transactionId, + renamingSuffix, + ), + state, + })), + }), + ) + } + + for (const history of replayErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReplayPublication, + ) + } + + for (const history of publicationErasureHistories()) { + const orderedProjection = ( + prefix: ReadonlyArray, + renamingSuffix: string, + ) => + projectAtomicOrderedPublicationState(prefix, { + demandId: + renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, + direction: `asc`, + initialWindowSize: 1, + }) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + orderedProjection, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + (prefix, renamingSuffix) => + projectOrderedPublicationBoundary(prefix, { + demandId: + renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, + direction: `asc`, + prefixSize: 2, + }), + ) + } + + for (const history of [ + acquisitionHistory(`shared`, [`row-a`, `row-b`]), + acquisitionHistory(`separate`, [`row-a`, `row-b`]), + [ + { + type: `startAcquisition`, + acquisitionId: `acquisition`, + sourceId: `source`, + demandId: `demand`, + }, + { + type: `attachAcquisitionOwner`, + acquisitionId: `acquisition`, + ownerId: `owner`, + }, + { + type: `settleAcquisition`, + acquisitionId: `acquisition`, + outcome: `reject`, + rowKeys: [`ghost-row`], + }, + ] satisfies Array, + ]) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAcquisitionSettlement, + (observation, renamingSuffix) => ({ + physicalStarts: observation.physicalStarts.map((acquisitionId) => + removeRenamingSuffix(acquisitionId, renamingSuffix), + ), + owners: observation.owners.map(({ ownerId, state, rowKeys }) => ({ + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + state, + rowKeys, + })), + visibleRowKeys: observation.visibleRowKeys, + }), + ) + } + }, + ) +} + function semanticAcquisitionResult( history: ReadonlyArray, ) { From 8b6a7303b49fdd60c375bf960e3149dd5f5fe191 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:11:49 -0600 Subject: [PATCH 100/327] test(db): calibrate replay identity erasure --- ...d-subset-refinement-model.property.test.ts | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index f0c9e3203..25f3363bd 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -615,6 +615,13 @@ function renameHistoryIds( ...event, transactionId: `${event.transactionId}-${suffix}`, } + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + return { + ...event, + attemptId: `${event.attemptId}-${suffix}`, + } case `startAcquisition`: return { ...event, @@ -1395,6 +1402,18 @@ function replayErasureHistories(): Array> { ] } +function replayAttemptIds( + history: ReadonlyArray, +): Array { + return history.flatMap((event) => + event.type === `startReplay` || + event.type === `writeReplayRows` || + event.type === `settleReplay` + ? [event.attemptId] + : [], + ) +} + function publicationErasureHistories(): Array> { const orderedRows = [ { key: `row-a`, orderValue: 1 }, @@ -1590,6 +1609,11 @@ for (const campaign of refinementCampaigns(1_779_009)) { } for (const history of replayErasureHistories()) { + expect(replayAttemptIds(renameHistoryIds(history, suffix))).toEqual( + replayAttemptIds(history).map( + (attemptId) => `${attemptId}-${suffix}`, + ), + ) expectObservationPreservedAfterEveryPrefix( history, suffix, From 2c18eea0fb0eb28508eb2ac08bb2e98166c8d4c3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:20:24 -0600 Subject: [PATCH 101/327] test(db): prove every erased identity changes --- ...d-subset-refinement-model.property.test.ts | 153 ++++++++++++++++-- 1 file changed, 140 insertions(+), 13 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 25f3363bd..6b415d671 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1402,15 +1402,116 @@ function replayErasureHistories(): Array> { ] } -function replayAttemptIds( +function erasedIdentityReferences( history: ReadonlyArray, -): Array { - return history.flatMap((event) => - event.type === `startReplay` || - event.type === `writeReplayRows` || - event.type === `settleReplay` - ? [event.attemptId] - : [], +): Array<{ field: string; value: string }> { + const references: Array<{ field: string; value: string }> = [] + const add = (field: string, value: string) => { + references.push({ field, value }) + } + + for (const event of history) { + switch (event.type) { + case `requestDemand`: + add(`ownerId`, event.ownerId) + add(`sessionId`, event.sessionId) + add(`demandId`, event.demandId) + add(`attemptId`, event.attemptId) + break + case `applyAuthoritativeRows`: + case `applyUnprovenRows`: + case `rejectDemand`: + case `releaseDemand`: + add(`ownerId`, event.ownerId) + add(`demandId`, event.demandId) + add(`attemptId`, event.attemptId) + break + case `settleDemandWithoutEvidence`: + add(`demandId`, event.demandId) + add(`attemptId`, event.attemptId) + break + case `truncateSource`: + case `cleanupSession`: + case `advanceWindowRevision`: + add(`sessionId`, event.sessionId) + break + case `restartSession`: + add(`previousSessionId`, event.previousSessionId) + add(`nextSessionId`, event.nextSessionId) + break + case `scheduleContinuation`: + add(`taskId`, event.taskId) + add(`sessionId`, event.sessionId) + break + case `runContinuation`: + add(`taskId`, event.taskId) + break + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + add(`transactionId`, event.transactionId) + break + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + add(`attemptId`, event.attemptId) + break + case `registerSourceDemand`: + case `settleSourceDemand`: + add(`sessionId`, event.sessionId) + add(`demandId`, event.demandId) + break + case `startAcquisition`: + add(`acquisitionId`, event.acquisitionId) + add(`demandId`, event.demandId) + break + case `attachAcquisitionOwner`: + add(`acquisitionId`, event.acquisitionId) + add(`ownerId`, event.ownerId) + break + case `settleAcquisition`: + add(`acquisitionId`, event.acquisitionId) + break + case `stagePublicationRows`: + add(`publicationId`, event.publicationId) + add(`demandId`, event.demandId) + break + case `commitPublication`: + case `establishReplacementCoverage`: + add(`publicationId`, event.publicationId) + break + case `beginReplacement`: + add(`publicationId`, event.publicationId) + event.demandIds.forEach((demandId) => add(`demandId`, demandId)) + break + case `settleReplacement`: + add(`publicationId`, event.publicationId) + add(`demandId`, event.demandId) + break + case `establishPublication`: + case `resizeOrderedWindow`: + break + } + } + + return references +} + +function expectEveryErasedIdentityRenamed( + history: ReadonlyArray, + suffix: string, +): void { + expect( + erasedIdentityReferences(renameHistoryIds(history, suffix)), + JSON.stringify(history), + ).toEqual( + erasedIdentityReferences(history).map(({ field, value }) => ({ + field, + value: `${value}-${suffix}`, + })), ) } @@ -1545,6 +1646,37 @@ for (const campaign of refinementCampaigns(1_779_009)) { fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( `erased identities preserve every bounded next-command observation (${campaign.label})`, (suffix) => { + for (const history of [ + ...sourceErasureHistories(), + ...demandErasureHistories(), + ...transactionErasureHistories(), + ...replayErasureHistories(), + ...publicationErasureHistories(), + acquisitionHistory(`shared`, [`row-a`, `row-b`]), + acquisitionHistory(`separate`, [`row-a`, `row-b`]), + [ + { + type: `startAcquisition`, + acquisitionId: `acquisition`, + sourceId: `source`, + demandId: `demand`, + }, + { + type: `attachAcquisitionOwner`, + acquisitionId: `acquisition`, + ownerId: `owner`, + }, + { + type: `settleAcquisition`, + acquisitionId: `acquisition`, + outcome: `reject`, + rowKeys: [`ghost-row`], + }, + ] satisfies Array, + ]) { + expectEveryErasedIdentityRenamed(history, suffix) + } + for (const history of sourceErasureHistories()) { expectObservationPreservedAfterEveryPrefix( history, @@ -1609,11 +1741,6 @@ for (const campaign of refinementCampaigns(1_779_009)) { } for (const history of replayErasureHistories()) { - expect(replayAttemptIds(renameHistoryIds(history, suffix))).toEqual( - replayAttemptIds(history).map( - (attemptId) => `${attemptId}-${suffix}`, - ), - ) expectObservationPreservedAfterEveryPrefix( history, suffix, From 44088c5d04cb3eb33554e5f1120442e6ba255285 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:27:37 -0600 Subject: [PATCH 102/327] test(db): guard transaction rename collisions --- .../tests/query/load-subset-refinement-model.property.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 6b415d671..f0b7b5316 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1328,6 +1328,10 @@ function transactionErasureHistories(): Array> { return [ successfulTransaction(`transaction`, `source`, `row`), + [ + ...successfulTransaction(`transaction-a`, `source-a`, `row-a`), + ...successfulTransaction(`transaction-b`, `source-b`, `row-b`), + ], [ stage, { From 13e56c06db008ece710e7e99e3f5f34bb0123ddf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:36:47 -0600 Subject: [PATCH 103/327] test(db): preserve semantic fields during renaming --- ...d-subset-refinement-model.property.test.ts | 131 ++++++++++++------ 1 file changed, 90 insertions(+), 41 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index f0b7b5316..3a3e62750 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1408,47 +1408,52 @@ function replayErasureHistories(): Array> { function erasedIdentityReferences( history: ReadonlyArray, -): Array<{ field: string; value: string }> { - const references: Array<{ field: string; value: string }> = [] - const add = (field: string, value: string) => { - references.push({ field, value }) +): Array<{ path: string; field: string; value: string }> { + const references: Array<{ path: string; field: string; value: string }> = [] + const add = ( + eventIndex: number, + field: string, + value: string, + fieldPath = field, + ) => { + references.push({ path: `${eventIndex}.${fieldPath}`, field, value }) } - for (const event of history) { + for (const [eventIndex, event] of history.entries()) { switch (event.type) { case `requestDemand`: - add(`ownerId`, event.ownerId) - add(`sessionId`, event.sessionId) - add(`demandId`, event.demandId) - add(`attemptId`, event.attemptId) + add(eventIndex, `ownerId`, event.ownerId) + add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) break case `applyAuthoritativeRows`: case `applyUnprovenRows`: case `rejectDemand`: case `releaseDemand`: - add(`ownerId`, event.ownerId) - add(`demandId`, event.demandId) - add(`attemptId`, event.attemptId) + add(eventIndex, `ownerId`, event.ownerId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) break case `settleDemandWithoutEvidence`: - add(`demandId`, event.demandId) - add(`attemptId`, event.attemptId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) break case `truncateSource`: case `cleanupSession`: case `advanceWindowRevision`: - add(`sessionId`, event.sessionId) + add(eventIndex, `sessionId`, event.sessionId) break case `restartSession`: - add(`previousSessionId`, event.previousSessionId) - add(`nextSessionId`, event.nextSessionId) + add(eventIndex, `previousSessionId`, event.previousSessionId) + add(eventIndex, `nextSessionId`, event.nextSessionId) break case `scheduleContinuation`: - add(`taskId`, event.taskId) - add(`sessionId`, event.sessionId) + add(eventIndex, `taskId`, event.taskId) + add(eventIndex, `sessionId`, event.sessionId) break case `runContinuation`: - add(`taskId`, event.taskId) + add(eventIndex, `taskId`, event.taskId) break case `stageSyncTransaction`: case `commitSyncTransaction`: @@ -1456,44 +1461,46 @@ function erasedIdentityReferences( case `abortSyncTransaction`: case `publishSyncTransaction`: case `settleSyncReceipt`: - add(`transactionId`, event.transactionId) + add(eventIndex, `transactionId`, event.transactionId) break case `startReplay`: case `writeReplayRows`: case `settleReplay`: - add(`attemptId`, event.attemptId) + add(eventIndex, `attemptId`, event.attemptId) break case `registerSourceDemand`: case `settleSourceDemand`: - add(`sessionId`, event.sessionId) - add(`demandId`, event.demandId) + add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `demandId`, event.demandId) break case `startAcquisition`: - add(`acquisitionId`, event.acquisitionId) - add(`demandId`, event.demandId) + add(eventIndex, `acquisitionId`, event.acquisitionId) + add(eventIndex, `demandId`, event.demandId) break case `attachAcquisitionOwner`: - add(`acquisitionId`, event.acquisitionId) - add(`ownerId`, event.ownerId) + add(eventIndex, `acquisitionId`, event.acquisitionId) + add(eventIndex, `ownerId`, event.ownerId) break case `settleAcquisition`: - add(`acquisitionId`, event.acquisitionId) + add(eventIndex, `acquisitionId`, event.acquisitionId) break case `stagePublicationRows`: - add(`publicationId`, event.publicationId) - add(`demandId`, event.demandId) + add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `demandId`, event.demandId) break case `commitPublication`: case `establishReplacementCoverage`: - add(`publicationId`, event.publicationId) + add(eventIndex, `publicationId`, event.publicationId) break case `beginReplacement`: - add(`publicationId`, event.publicationId) - event.demandIds.forEach((demandId) => add(`demandId`, demandId)) + add(eventIndex, `publicationId`, event.publicationId) + event.demandIds.forEach((demandId, demandIndex) => + add(eventIndex, `demandId`, demandId, `demandIds.${demandIndex}`), + ) break case `settleReplacement`: - add(`publicationId`, event.publicationId) - add(`demandId`, event.demandId) + add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `demandId`, event.demandId) break case `establishPublication`: case `resizeOrderedWindow`: @@ -1504,19 +1511,61 @@ function erasedIdentityReferences( return references } +function changedLeafPaths( + left: unknown, + right: unknown, + path = ``, +): Array { + if (Object.is(left, right)) return [] + if (Array.isArray(left) && Array.isArray(right)) { + if (left.length !== right.length) return [path] + return left.flatMap((value, index) => + changedLeafPaths( + value, + right[index], + path === `` ? `${index}` : `${path}.${index}`, + ), + ) + } + if ( + typeof left === `object` && + left !== null && + typeof right === `object` && + right !== null + ) { + const leftRecord = left as Record + const rightRecord = right as Record + const keys = [ + ...new Set([...Object.keys(leftRecord), ...Object.keys(rightRecord)]), + ].sort() + return keys.flatMap((key) => + changedLeafPaths( + leftRecord[key], + rightRecord[key], + path === `` ? key : `${path}.${key}`, + ), + ) + } + return [path] +} + function expectEveryErasedIdentityRenamed( history: ReadonlyArray, suffix: string, ): void { - expect( - erasedIdentityReferences(renameHistoryIds(history, suffix)), - JSON.stringify(history), - ).toEqual( - erasedIdentityReferences(history).map(({ field, value }) => ({ + const renamed = renameHistoryIds(history, suffix) + const references = erasedIdentityReferences(history) + expect(erasedIdentityReferences(renamed), JSON.stringify(history)).toEqual( + references.map(({ path, field, value }) => ({ + path, field, value: `${value}-${suffix}`, })), ) + expect( + changedLeafPaths(history, renamed).sort(), + JSON.stringify(history), + ).toEqual(references.map(({ path }) => path).sort()) } function publicationErasureHistories(): Array> { From fa36261b4d1aa6c4c91ecc0f781f60d35c5d3f25 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:44:58 -0600 Subject: [PATCH 104/327] test(db): cover non-invalidating demand release --- .../load-subset-refinement-model.property.test.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 3a3e62750..2eb48c095 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1256,6 +1256,18 @@ function demandErasureHistories(): Array> { release(`owner-a`, `attempt-a`), ], [request(`owner-a`, `attempt-a`, true), release(`owner-a`, `attempt-a`)], + [ + request(`owner-a`, `attempt-a`), + { + type: `releaseDemand`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`row-a`], + finalRowOwner: false, + invalidatesAdapterEvidence: false, + }, + ], [ request(`owner-a`, `attempt-a`), { type: `truncateSource`, sessionId: `session-a` }, From 8172e622e902b8aa0755374513bbc20b4a9d2134 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 21:57:38 -0600 Subject: [PATCH 105/327] test(db): separate relational and refinement oracles --- packages/db/src/query/live/ARCHITECTURE.md | 16 ++ .../db/tests/load-subset-full-flow-model.ts | 84 +++---- ...d-subset-full-flow-oracle.property.test.ts | 214 +++++++++++------- 3 files changed, 180 insertions(+), 134 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a28bc78e9..bd2cbaca8 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1135,6 +1135,22 @@ create recursive Collection machinery. | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | +### Oracle family boundary + +The shared load-subset refinement model begins after relational evaluation. It +may vary opaque source topology, demand relationships, already-evaluated result +contributions, and public window state. It owns asynchronous demand, applied +evidence, row support, coverage, publication, source progress, and resource +work. It must not interpret query IR, weighted deltas, predicates, joins, +grouping, ordering, or nested materialization. + +DBSP operator suites own incremental relational laws. The includes suites own +compiled routes and materialized nested results. A load-subset production +harness may use an eager query from those paths as its relational control, then +compare lazy demand and source progress with a small refinement projection. It +must not copy those paths into a second relational engine inside the shared +model. + Each oracle identifies the first divergent checkpoint and compares either the whole result or one exact structural difference. Correlated-materialization scenarios use direct assertions. A boundary suite may retain an exact diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 8de3a988a..ffd46e594 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -40,77 +40,65 @@ export type OrderedContinuationEvidence = { rowsNeeded: number } -export type MultiSourceOrderedRow = { - key: string - joinKey: string -} - -export type MultiSourceSecondaryRow = { - key: string - joinKey: string +export type OrderedSourceStep = { + sourceKey: string + resultKeys: ReadonlyArray + demandKeys: ReadonlyArray } -export type MultiSourceOrderedWindow = { - visiblePairKeys: ReadonlyArray - scannedPrimaryKeys: ReadonlyArray - primaryCursorKeys: ReadonlyArray - demandedJoinKeys: ReadonlyArray +export type OrderedSourceProgress = { + visibleResultKeys: ReadonlyArray + scannedSourceKeys: ReadonlyArray + sourceCursorKeys: ReadonlyArray + demandedKeys: ReadonlyArray rowsNeeded: number sourceExhausted: boolean } /** - * Projects the smallest forward primary-source scan that fills a joined - * window. The caller supplies primary rows in total order, so this projector - * only owns the cross-source relational law: every scanned primary row advances - * source progress, while joined pair multiplicity fills offset plus limit. - * Production may prove the same result with reverse authoritative demands; the - * boundary harness compares the public result and each transport law separately. + * Projects the smallest forward source scan that fills a result window. Each + * step contains result contributions already evaluated by the owning DBSP + * oracle or an eager production control. This model owns source progress only; + * it does not interpret predicates, joins, grouping, ordering, or includes. */ -export function projectMultiSourceOrderedWindow(options: { - primaryOrder: ReadonlyArray - secondaryRows: ReadonlyArray +export function projectOrderedSourceProgress(options: { + sourceSteps: ReadonlyArray offset: number limit: number -}): MultiSourceOrderedWindow { - const scannedPrimaryKeys: Array = [] - const joinedPairKeys: Array = [] - const demandedJoinKeys: Array = [] - const seenJoinKeys = new Set() +}): OrderedSourceProgress { + const scannedSourceKeys: Array = [] + const resultKeys: Array = [] + const demandedKeys: Array = [] + const seenDemandKeys = new Set() const targetSize = options.limit === 0 ? 0 : options.offset + options.limit - const secondaryRows = [...options.secondaryRows].sort((left, right) => - left.key.localeCompare(right.key), - ) - for (const row of options.primaryOrder) { - if (joinedPairKeys.length >= targetSize) break + for (const step of options.sourceSteps) { + if (resultKeys.length >= targetSize) break - scannedPrimaryKeys.push(row.key) - if (!seenJoinKeys.has(row.joinKey)) { - seenJoinKeys.add(row.joinKey) - demandedJoinKeys.push(row.joinKey) - } - for (const secondaryRow of secondaryRows) { - if (secondaryRow.joinKey === row.joinKey) { - joinedPairKeys.push(`${row.key}:${secondaryRow.key}`) + scannedSourceKeys.push(step.sourceKey) + for (const demandKey of step.demandKeys) { + if (!seenDemandKeys.has(demandKey)) { + seenDemandKeys.add(demandKey) + demandedKeys.push(demandKey) } } + resultKeys.push(...step.resultKeys) } - const visiblePairKeys = joinedPairKeys.slice( + const visibleResultKeys = resultKeys.slice( options.offset, options.offset + options.limit, ) return { - visiblePairKeys, - scannedPrimaryKeys, - primaryCursorKeys: scannedPrimaryKeys.map((_, index) => - index === 0 ? undefined : scannedPrimaryKeys[index - 1], + visibleResultKeys, + scannedSourceKeys, + sourceCursorKeys: scannedSourceKeys.map((_, index) => + index === 0 ? undefined : scannedSourceKeys[index - 1], ), - demandedJoinKeys, - rowsNeeded: Math.max(0, options.limit - visiblePairKeys.length), - sourceExhausted: scannedPrimaryKeys.length === options.primaryOrder.length, + demandedKeys, + rowsNeeded: Math.max(0, options.limit - visibleResultKeys.length), + sourceExhausted: scannedSourceKeys.length === options.sourceSteps.length, } } diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 77a61a165..996667e89 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -4,6 +4,7 @@ import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { SyncTransactionAbortedError } from '../../src/errors.js' import { BTreeIndex, ReverseIndex } from '../../src/index.js' +import { localOnlyCollectionOptions } from '../../src/local-only.js' import { Func, PropRef, Value } from '../../src/query/ir.js' import { createEffect } from '../../src/query/effect.js' import { createLiveQueryCollection, eq, gte } from '../../src/query/index.js' @@ -19,9 +20,9 @@ import { projectAtomicOrderedPublicationState, projectAtomicOrderedPublications, projectAuthorizedContinuationStarts, - projectMultiSourceOrderedWindow, projectOrderedContinuationEvidence, projectOrderedPublicationBoundary, + projectOrderedSourceProgress, projectRetainedRowKeys, projectReusableDemands, projectTransportLoads, @@ -37,7 +38,10 @@ import { } from '../oracle-config.js' import type { InitialQueryBuilder } from '../../src/query/builder/index.js' import type { LoadSubsetOptions, WritableDeep } from '../../src/types.js' -import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { + LoadSubsetFullFlowEvent, + OrderedSourceStep, +} from '../load-subset-full-flow-model.js' type AdapterLifecycleEvent = | { type: `start`; options: LoadSubsetOptions } @@ -312,6 +316,66 @@ function orderedPrimaryRows( }) } +let multiSourceOrderedControlId = 0 + +async function observeOrderedSourceSteps( + scenario: MultiSourceOrderedScenario, +): Promise> { + const controlId = multiSourceOrderedControlId++ + const primary = createCollection( + localOnlyCollectionOptions({ + id: `multi-source-control-primary-${controlId}`, + getKey: (row) => row.id, + initialData: [...scenario.primaryRows], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + const secondary = createCollection( + localOnlyCollectionOptions({ + id: `multi-source-control-secondary-${controlId}`, + getKey: (row) => row.id, + initialData: [...scenario.secondaryRows], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + const result = createLiveQueryCollection({ + id: `multi-source-control-result-${controlId}`, + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank, scenario.direction), + startSync: true, + }) + + try { + await result.preload() + const resultKeysBySource = new Map>() + for (const { primaryRow, secondaryRow } of result.toArray) { + const keys = resultKeysBySource.get(primaryRow.id) ?? [] + keys.push(`${primaryRow.id}:${secondaryRow.id}`) + resultKeysBySource.set(primaryRow.id, keys) + } + return orderedPrimaryRows(scenario).map((row) => ({ + sourceKey: row.id, + resultKeys: resultKeysBySource.get(row.id) ?? [], + demandKeys: [row.joinKey], + })) + } finally { + await Promise.all([ + result.cleanup(), + primary.cleanup(), + secondary.cleanup(), + ]) + } +} + function hasPreloadedSecondary(scenario: MultiSourceOrderedScenario): boolean { return ( scenario.secondaryPublication === `preloaded` || @@ -344,15 +408,9 @@ async function runMultiSourceOrderedScenario( type SecondaryRow = { id: string; joinKey: string } const primaryOrder = orderedPrimaryRows(scenario) - const projection = projectMultiSourceOrderedWindow({ - primaryOrder: primaryOrder.map(({ id, joinKey }) => ({ - key: id, - joinKey, - })), - secondaryRows: scenario.secondaryRows.map(({ id, joinKey }) => ({ - key: id, - joinKey, - })), + const sourceSteps = await observeOrderedSourceSteps(scenario) + const projection = projectOrderedSourceProgress({ + sourceSteps, offset: scenario.offset, limit: scenario.limit, }) @@ -659,7 +717,7 @@ async function runMultiSourceOrderedScenario( live.toArray.map( ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, ), - ).toEqual(projection.visiblePairKeys) + ).toEqual(projection.visibleResultKeys) const initialPrimaryCallCount = primaryCalls.length if (scenario.limit === 0) { @@ -672,15 +730,8 @@ async function runMultiSourceOrderedScenario( const refinedOffset = scenario.offset === 0 ? 1 : 0 const refinedLimit = scenario.limit === 0 ? 1 : scenario.limit + 1 - const refinedProjection = projectMultiSourceOrderedWindow({ - primaryOrder: primaryOrder.map(({ id, joinKey }) => ({ - key: id, - joinKey, - })), - secondaryRows: scenario.secondaryRows.map(({ id, joinKey }) => ({ - key: id, - joinKey, - })), + const refinedProjection = projectOrderedSourceProgress({ + sourceSteps, offset: refinedOffset, limit: refinedLimit, }) @@ -693,7 +744,7 @@ async function runMultiSourceOrderedScenario( live.toArray.map( ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, ), - ).toEqual(refinedProjection.visiblePairKeys) + ).toEqual(refinedProjection.visibleResultKeys) if (scenario.limit === 0) { const refinementCalls = primaryCalls .slice(initialPrimaryCallCount) @@ -804,7 +855,7 @@ async function runMultiSourceOrderedScenario( expect( literalJoinKeys.every((joinKey) => primaryJoinKeys.has(joinKey)), ).toBe(true) - for (const joinKey of projection.demandedJoinKeys) { + for (const joinKey of projection.demandedKeys) { expect(requestedJoinKeys.has(joinKey)).toBe(true) } expect( @@ -2379,67 +2430,66 @@ it(`settles concurrent secondary loads out of order across paged commits`, async } }) -it(`projects the minimal primary prefix needed by a joined window`, () => { - const projection = projectMultiSourceOrderedWindow({ - primaryOrder: [ - { key: `a`, joinKey: `x` }, - { key: `b`, joinKey: `y` }, - { key: `c`, joinKey: `z` }, - { key: `d`, joinKey: `x` }, - ], - secondaryRows: [ - { key: `x-0`, joinKey: `x` }, - { key: `z-0`, joinKey: `z` }, +it(`projects the minimal source prefix needed by evaluated result contributions`, () => { + const projection = projectOrderedSourceProgress({ + sourceSteps: [ + { sourceKey: `a`, resultKeys: [`a:x-0`], demandKeys: [`x`] }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`y`] }, + { sourceKey: `c`, resultKeys: [`c:z-0`], demandKeys: [`z`] }, + { sourceKey: `d`, resultKeys: [`d:x-0`], demandKeys: [`x`] }, ], offset: 0, limit: 2, }) expect(projection).toEqual({ - visiblePairKeys: [`a:x-0`, `c:z-0`], - scannedPrimaryKeys: [`a`, `b`, `c`], - primaryCursorKeys: [undefined, `a`, `b`], - demandedJoinKeys: [`x`, `y`, `z`], + visibleResultKeys: [`a:x-0`, `c:z-0`], + scannedSourceKeys: [`a`, `b`, `c`], + sourceCursorKeys: [undefined, `a`, `b`], + demandedKeys: [`x`, `y`, `z`], rowsNeeded: 0, sourceExhausted: false, }) }) -it(`erases join-key spelling and ignores unreachable secondary rows`, () => { - const original = projectMultiSourceOrderedWindow({ - primaryOrder: [ - { key: `a`, joinKey: `x` }, - { key: `b`, joinKey: `y` }, - { key: `c`, joinKey: `x` }, - ], - secondaryRows: [ - { key: `match-0`, joinKey: `x` }, - { key: `unreachable`, joinKey: `unused` }, +it(`erases demand-key spelling without changing source progress`, () => { + const original = projectOrderedSourceProgress({ + sourceSteps: [ + { sourceKey: `a`, resultKeys: [`a:match-0`], demandKeys: [`x`] }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`y`] }, + { sourceKey: `c`, resultKeys: [`c:match-0`], demandKeys: [`x`] }, ], offset: 0, limit: 2, }) - const renamed = projectMultiSourceOrderedWindow({ - primaryOrder: [ - { key: `a`, joinKey: `renamed-x` }, - { key: `b`, joinKey: `renamed-y` }, - { key: `c`, joinKey: `renamed-x` }, + const renamed = projectOrderedSourceProgress({ + sourceSteps: [ + { + sourceKey: `a`, + resultKeys: [`a:match-0`], + demandKeys: [`renamed-x`], + }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`renamed-y`] }, + { + sourceKey: `c`, + resultKeys: [`c:match-0`], + demandKeys: [`renamed-x`], + }, ], - secondaryRows: [{ key: `match-0`, joinKey: `renamed-x` }], offset: 0, limit: 2, }) expect({ - visiblePairKeys: original.visiblePairKeys, - scannedPrimaryKeys: original.scannedPrimaryKeys, - primaryCursorKeys: original.primaryCursorKeys, + visibleResultKeys: original.visibleResultKeys, + scannedSourceKeys: original.scannedSourceKeys, + sourceCursorKeys: original.sourceCursorKeys, rowsNeeded: original.rowsNeeded, sourceExhausted: original.sourceExhausted, }).toEqual({ - visiblePairKeys: renamed.visiblePairKeys, - scannedPrimaryKeys: renamed.scannedPrimaryKeys, - primaryCursorKeys: renamed.primaryCursorKeys, + visibleResultKeys: renamed.visibleResultKeys, + scannedSourceKeys: renamed.scannedSourceKeys, + sourceCursorKeys: renamed.sourceCursorKeys, rowsNeeded: renamed.rowsNeeded, sourceExhausted: renamed.sourceExhausted, }) @@ -2451,52 +2501,44 @@ it(`exhausts the bounded multi-source ordered-window model`, () => { { key: `b`, joinKey: `y` }, { key: `c`, joinKey: `z` }, ] - const joinKeys = [`x`, `y`, `z`] as const - for (const xCount of [0, 1, 2]) { for (const yCount of [0, 1, 2]) { for (const zCount of [0, 1, 2]) { const counts = [xCount, yCount, zCount] - const secondaryRows = joinKeys.flatMap((joinKey, index) => - Array.from({ length: counts[index]! }, (_, matchIndex) => ({ - key: `${joinKey}-${matchIndex}`, - joinKey, - })), - ) + const sourceSteps = rows.map((row, index) => ({ + sourceKey: row.key, + resultKeys: Array.from( + { length: counts[index]! }, + (_, matchIndex) => `${row.key}:${row.joinKey}-${matchIndex}`, + ), + demandKeys: [row.joinKey], + })) for (const offset of [0, 1, 2]) { for (const limit of [0, 1, 2]) { - const projection = projectMultiSourceOrderedWindow({ - primaryOrder: rows, - secondaryRows, + const projection = projectOrderedSourceProgress({ + sourceSteps, offset, limit, }) - const direct = rows - .flatMap((row) => - secondaryRows - .filter(({ joinKey }) => joinKey === row.joinKey) - .map((secondaryRow) => `${row.key}:${secondaryRow.key}`), - ) + const direct = sourceSteps + .flatMap(({ resultKeys }) => resultKeys) .slice(offset, offset + limit) - expect(projection.visiblePairKeys).toEqual(direct) + expect(projection.visibleResultKeys).toEqual(direct) expect(projection.rowsNeeded).toBe( Math.max(0, limit - direct.length), ) if (limit === 0) { - expect(projection.scannedPrimaryKeys).toEqual([]) + expect(projection.scannedSourceKeys).toEqual([]) continue } - if (projection.scannedPrimaryKeys.length < rows.length) { - const shorterPrefix = rows.slice( + if (projection.scannedSourceKeys.length < sourceSteps.length) { + const shorterPrefix = sourceSteps.slice( 0, - projection.scannedPrimaryKeys.length - 1, + projection.scannedSourceKeys.length - 1, ) const shorterPairCount = shorterPrefix.reduce( - (count, row) => - count + - secondaryRows.filter(({ joinKey }) => joinKey === row.joinKey) - .length, + (count, step) => count + step.resultKeys.length, 0, ) expect(shorterPairCount).toBeLessThan(offset + limit) From 8cb04c9bd5aa5f0a082bf9103e54303e70a2b0d2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:03:45 -0600 Subject: [PATCH 106/327] test(db): calibrate ordered demand control --- ...d-subset-full-flow-oracle.property.test.ts | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 996667e89..dfd6b6dd4 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -409,6 +409,14 @@ async function runMultiSourceOrderedScenario( const primaryOrder = orderedPrimaryRows(scenario) const sourceSteps = await observeOrderedSourceSteps(scenario) + expect( + sourceSteps.map(({ sourceKey, demandKeys }) => ({ sourceKey, demandKeys })), + ).toEqual( + primaryOrder.map(({ id, joinKey }) => ({ + sourceKey: id, + demandKeys: [joinKey], + })), + ) const projection = projectOrderedSourceProgress({ sourceSteps, offset: scenario.offset, @@ -838,29 +846,33 @@ async function runMultiSourceOrderedScenario( delayedSecondaryReceiptWaiters.map(({ index }) => index).reverse(), ) } - if (joinCalls.length > 0) { - const requestedJoinKeys = new Set( - joinCalls.flatMap(({ where }) => - [...primaryJoinKeys].filter((joinKey) => - evaluateReferenceExpression(where!, { - id: `probe-${joinKey}`, - joinKey, - }), - ), + const requestedJoinKeys = new Set( + joinCalls.flatMap(({ where }) => + [...primaryJoinKeys].filter((joinKey) => + evaluateReferenceExpression(where!, { + id: `probe-${joinKey}`, + joinKey, + }), ), - ) - const literalJoinKeys = joinCalls.flatMap(({ where }) => - collectStringLiterals(where!), - ) - expect( - literalJoinKeys.every((joinKey) => primaryJoinKeys.has(joinKey)), - ).toBe(true) - for (const joinKey of projection.demandedKeys) { + ), + ) + const literalJoinKeys = joinCalls.flatMap(({ where }) => + collectStringLiterals(where!), + ) + expect( + literalJoinKeys.every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) + expect( + [...requestedJoinKeys].every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) + const requiredJoinKeys = new Set([ + ...projection.demandedKeys, + ...refinedProjection.demandedKeys, + ]) + if (joinCalls.length > 0) { + for (const joinKey of requiredJoinKeys) { expect(requestedJoinKeys.has(joinKey)).toBe(true) } - expect( - [...requestedJoinKeys].every((joinKey) => primaryJoinKeys.has(joinKey)), - ).toBe(true) } if (scenario.secondaryPublication === `after-primary-continuation`) { From a4aa171e503a124d2d3bb6af009d5876cc7120cc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:15:53 -0600 Subject: [PATCH 107/327] test(db): assert acquisition work invariants --- ...d-subset-refinement-model.property.test.ts | 102 +++++++++++++++--- 1 file changed, 90 insertions(+), 12 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 2eb48c095..a516bf10c 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1899,6 +1899,9 @@ async function runAcquisitionTopology( ) { const runId = ++acquisitionRunId let physicalStarts = 0 + let logicalStarts = 0 + let logicalReleases = 0 + let deduplications = 0 const delivery = createDeferred() const createSource = (suffix: string) => { type Row = { id: string } @@ -1918,6 +1921,9 @@ async function runAcquisitionTopology( appliedRowKeys: rowKeys, } satisfies LoadSubsetResult }, + onDeduplicate: () => { + deduplications++ + }, }) return createCollection({ id: `refinement-acquisition-${runId}-${suffix}`, @@ -1931,8 +1937,14 @@ async function runAcquisitionTopology( commit = params.commit params.markReady() return { - loadSubset: deduplicated.loadSubset, - unloadSubset: deduplicated.unloadSubset, + loadSubset: (options) => { + logicalStarts++ + return deduplicated.loadSubset(options) + }, + unloadSubset: (options) => { + logicalReleases++ + deduplicated.unloadSubset(options) + }, } }, }, @@ -1967,6 +1979,13 @@ async function runAcquisitionTopology( ) const preloads = liveQueries.map((live) => live.preload()) const expectedPhysicalStarts = topology === `shared` ? 1 : 2 + let owners: Array<{ + ownerId: (typeof ownerIds)[number] + state: `resolved` + rowKeys: Array + }> = [] + let settledBatches: Array>> = [[], []] + let settledCallbackReads: Array>> = [[], []] try { for ( @@ -1977,23 +1996,34 @@ async function runAcquisitionTopology( await flushPromises() } expect(physicalStarts).toBe(expectedPhysicalStarts) + expect(logicalStarts).toBe(2) + expect(liveQueries.map((live) => live.isReady())).toEqual([false, false]) + expect(liveQueries.map((live) => live.isLoadingSubset)).toEqual([ + true, + true, + ]) + expect(liveQueries.map((live) => live.toArray)).toEqual([[], []]) + expect(batches).toEqual([[], []]) + expect(callbackReads).toEqual([[], []]) delivery.resolve() await Promise.all(preloads) - const owners = liveQueries.map((live, index) => ({ + owners = liveQueries.map((live, index) => ({ ownerId: ownerIds[index]!, state: `resolved` as const, rowKeys: live.toArray.map(({ id }) => String(id)).sort(), })) - return { - physicalStarts, - owners, - visibleRowKeys: [ - ...new Set(owners.flatMap(({ rowKeys: keys }) => keys)), - ].sort(), - batches, - callbackReads, - } + expect(liveQueries.map((live) => live.isReady())).toEqual([true, true]) + expect(liveQueries.map((live) => live.isLoadingSubset)).toEqual([ + false, + false, + ]) + settledBatches = batches.map((ownerBatches) => + ownerBatches.map((batch) => [...batch]), + ) + settledCallbackReads = callbackReads.map((ownerReads) => + ownerReads.map((read) => [...read]), + ) } finally { delivery.resolve() subscriptions.forEach((subscription) => subscription.unsubscribe()) @@ -2002,6 +2032,21 @@ async function runAcquisitionTopology( ...sources.map((source) => source.cleanup()), ]) } + + return { + physicalStarts, + logicalStarts, + logicalReleases, + deduplications, + owners, + visibleRowKeys: [ + ...new Set(owners.flatMap(({ rowKeys: keys }) => keys)), + ].sort(), + batches: settledBatches, + callbackReads: settledCallbackReads, + batchesAfterUnsubscribe: batches, + callbackReadsAfterUnsubscribe: callbackReads, + } } let acquisitionRunId = 0 @@ -2041,8 +2086,41 @@ for (const campaign of refinementCampaigns(1_779_008)) { }).toEqual(separateSemantic) expect(sharedActual.batches).toEqual(separateActual.batches) expect(sharedActual.callbackReads).toEqual(separateActual.callbackReads) + const expectedKeys = [...rowKeys].sort() + const expectedBatches = [ + [expectedKeys, []], + [expectedKeys, []], + ] + const expectedCallbackReads = [ + [expectedKeys, expectedKeys], + [expectedKeys, expectedKeys], + ] + expect(sharedActual.batches).toEqual(expectedBatches) + expect(sharedActual.callbackReads).toEqual(expectedCallbackReads) + expect(sharedActual.batchesAfterUnsubscribe).toEqual(sharedActual.batches) + expect(sharedActual.callbackReadsAfterUnsubscribe).toEqual( + sharedActual.callbackReads, + ) + expect(separateActual.batchesAfterUnsubscribe).toEqual( + separateActual.batches, + ) + expect(separateActual.callbackReadsAfterUnsubscribe).toEqual( + separateActual.callbackReads, + ) + expect(sharedActual.logicalStarts).toBe(2) + expect(separateActual.logicalStarts).toBe(2) + expect(sharedActual.logicalReleases).toBe(2) + expect(separateActual.logicalReleases).toBe(2) expect(sharedActual.physicalStarts).toBe(1) expect(separateActual.physicalStarts).toBe(2) + expect(sharedActual.deduplications).toBe(1) + expect(separateActual.deduplications).toBe(0) + expect(sharedActual.physicalStarts + sharedActual.deduplications).toBe( + sharedActual.logicalStarts, + ) + expect( + separateActual.physicalStarts + separateActual.deduplications, + ).toBe(separateActual.logicalStarts) }, ) } From f9897a13b7dcb3f1f0368d41ff53f3fecc1716f6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:22:44 -0600 Subject: [PATCH 108/327] test(db): verify release invalidates acquisition evidence --- ...d-subset-refinement-model.property.test.ts | 119 +++++++++++++++--- 1 file changed, 104 insertions(+), 15 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index a516bf10c..265faef7c 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1986,6 +1986,14 @@ async function runAcquisitionTopology( }> = [] let settledBatches: Array>> = [[], []] let settledCallbackReads: Array>> = [[], []] + let initialPhysicalStarts = 0 + let initialLogicalStarts = 0 + let initialDeduplications = 0 + let remountRowKeys: Array = [] + let remountBatches: Array> = [] + let remountCallbackReads: Array> = [] + let remountBatchesAfterUnsubscribe: Array> = [] + let remountCallbackReadsAfterUnsubscribe: Array> = [] try { for ( @@ -2024,6 +2032,49 @@ async function runAcquisitionTopology( settledCallbackReads = callbackReads.map((ownerReads) => ownerReads.map((read) => [...read]), ) + + initialPhysicalStarts = physicalStarts + initialLogicalStarts = logicalStarts + initialDeduplications = deduplications + subscriptions.forEach((subscription) => subscription.unsubscribe()) + await Promise.all(liveQueries.map((live) => live.cleanup())) + expect(logicalReleases).toBe(2) + + const remount = createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-remount`, + query: (q) => q.from({ row: sharedSource }), + startSync: false, + }) + const observedRemountBatches: Array> = [] + const observedRemountCallbackReads: Array> = [] + const remountSubscription = remount.subscribeChanges( + (changes) => { + observedRemountBatches.push( + changes.map(({ key }) => String(key)).sort(), + ) + observedRemountCallbackReads.push( + remount.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ) + try { + await remount.preload() + remountRowKeys = remount.toArray.map(({ id }) => String(id)).sort() + remountBatches = observedRemountBatches.map((batch) => [...batch]) + remountCallbackReads = observedRemountCallbackReads.map((read) => [ + ...read, + ]) + } finally { + remountSubscription.unsubscribe() + await remount.cleanup() + remountBatchesAfterUnsubscribe = observedRemountBatches.map((batch) => [ + ...batch, + ]) + remountCallbackReadsAfterUnsubscribe = observedRemountCallbackReads.map( + (read) => [...read], + ) + } } finally { delivery.resolve() subscriptions.forEach((subscription) => subscription.unsubscribe()) @@ -2034,10 +2085,13 @@ async function runAcquisitionTopology( } return { - physicalStarts, - logicalStarts, + initialPhysicalStarts, + initialLogicalStarts, + initialDeduplications, + totalPhysicalStarts: physicalStarts, + totalLogicalStarts: logicalStarts, logicalReleases, - deduplications, + totalDeduplications: deduplications, owners, visibleRowKeys: [ ...new Set(owners.flatMap(({ rowKeys: keys }) => keys)), @@ -2046,6 +2100,11 @@ async function runAcquisitionTopology( callbackReads: settledCallbackReads, batchesAfterUnsubscribe: batches, callbackReadsAfterUnsubscribe: callbackReads, + remountRowKeys, + remountBatches, + remountCallbackReads, + remountBatchesAfterUnsubscribe, + remountCallbackReadsAfterUnsubscribe, } } @@ -2107,20 +2166,50 @@ for (const campaign of refinementCampaigns(1_779_008)) { expect(separateActual.callbackReadsAfterUnsubscribe).toEqual( separateActual.callbackReads, ) - expect(sharedActual.logicalStarts).toBe(2) - expect(separateActual.logicalStarts).toBe(2) - expect(sharedActual.logicalReleases).toBe(2) - expect(separateActual.logicalReleases).toBe(2) - expect(sharedActual.physicalStarts).toBe(1) - expect(separateActual.physicalStarts).toBe(2) - expect(sharedActual.deduplications).toBe(1) - expect(separateActual.deduplications).toBe(0) - expect(sharedActual.physicalStarts + sharedActual.deduplications).toBe( - sharedActual.logicalStarts, + expect(sharedActual.initialLogicalStarts).toBe(2) + expect(separateActual.initialLogicalStarts).toBe(2) + expect(sharedActual.initialPhysicalStarts).toBe(1) + expect(separateActual.initialPhysicalStarts).toBe(2) + expect(sharedActual.initialDeduplications).toBe(1) + expect(separateActual.initialDeduplications).toBe(0) + expect(sharedActual.remountRowKeys).toEqual(expectedKeys) + expect(separateActual.remountRowKeys).toEqual(expectedKeys) + expect(sharedActual.remountBatches).toEqual([expectedKeys, []]) + expect(separateActual.remountBatches).toEqual([expectedKeys, []]) + expect(sharedActual.remountCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(separateActual.remountCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(sharedActual.remountBatchesAfterUnsubscribe).toEqual( + sharedActual.remountBatches, + ) + expect(sharedActual.remountCallbackReadsAfterUnsubscribe).toEqual( + sharedActual.remountCallbackReads, + ) + expect(separateActual.remountBatchesAfterUnsubscribe).toEqual( + separateActual.remountBatches, ) + expect(separateActual.remountCallbackReadsAfterUnsubscribe).toEqual( + separateActual.remountCallbackReads, + ) + expect(sharedActual.totalLogicalStarts).toBe(3) + expect(separateActual.totalLogicalStarts).toBe(3) + expect(sharedActual.logicalReleases).toBe(3) + expect(separateActual.logicalReleases).toBe(3) + expect(sharedActual.totalPhysicalStarts).toBe(2) + expect(separateActual.totalPhysicalStarts).toBe(3) + expect(sharedActual.totalDeduplications).toBe(1) + expect(separateActual.totalDeduplications).toBe(0) + expect( + sharedActual.totalPhysicalStarts + sharedActual.totalDeduplications, + ).toBe(sharedActual.totalLogicalStarts) expect( - separateActual.physicalStarts + separateActual.deduplications, - ).toBe(separateActual.logicalStarts) + separateActual.totalPhysicalStarts + separateActual.totalDeduplications, + ).toBe(separateActual.totalLogicalStarts) }, ) } From 0a7977edb633d1af30e4a9428b0bb0d8ba8c1783 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:37:47 -0600 Subject: [PATCH 109/327] fix(db): retain exact acquisition coverage for co-owners --- packages/db/src/query/subset-dedupe.ts | 91 ++++++++++---- .../query/load-subset-oracle.property.test.ts | 29 ++--- ...d-subset-refinement-model.property.test.ts | 116 ++++++++++++++++-- packages/db/tests/query/subset-dedupe.test.ts | 11 +- 4 files changed, 192 insertions(+), 55 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 85c3ab74f..e565eb371 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -27,17 +27,26 @@ type SharedAbortLease = { type LogicalLoadReservation = { generation: number invalidatesCoverage: boolean - inflight?: InflightCall + acquisition?: AcquisitionOwnership } -type InflightCall = { +type AcquisitionOwnership = { + matchesPhysicalRequest: (options: LoadSubsetOptions) => boolean + generation: number + reservations: Set +} + +type InflightCall = AcquisitionOwnership & { options: LoadSubsetOptions promise: Promise lease: SharedAbortLease - matchesPhysicalRequest: (options: LoadSubsetOptions) => boolean - generation: number trackable: boolean - reservations: Set +} + +function isInflightCall( + acquisition: AcquisitionOwnership, +): acquisition is InflightCall { + return `lease` in acquisition } /** @@ -87,6 +96,10 @@ export class DeduplicatedLoadSubset { // Each entry also owns the shared cancellation lease for its requesters. private inflightCalls: Array = [] + // Retain exact acquisition ownership after settlement so a later exact + // owner can share its evidence until the final logical lease releases. + private exactAcquisitions: Array = [] + // Generation counter to invalidate in-flight requests after reset() // When reset() is called, this increments, and any in-flight completion handlers // check if their captured generation matches before updating tracking state @@ -141,6 +154,18 @@ export class DeduplicatedLoadSubset { options: LoadSubsetOptions, reservation: LogicalLoadReservation, ): true | Promise { + const exactAcquisition = this.exactAcquisitions.find( + (acquisition) => + acquisition.generation === this.generation && + acquisition.matchesPhysicalRequest(options), + ) + if (exactAcquisition) { + exactAcquisition.reservations.add(reservation) + reservation.acquisition = exactAcquisition + this.onDeduplicate?.(options) + return true + } + // If we've loaded all data, everything is covered if (this.hasLoadedAllData) { this.onDeduplicate?.(options) @@ -178,7 +203,7 @@ export class DeduplicatedLoadSubset { if (matchingInflight !== undefined) { matchingInflight.reservations.add(reservation) - reservation.inflight = matchingInflight + reservation.acquisition = matchingInflight matchingInflight.lease.attach(options.signal) // An in-flight call will load data that covers this request // Every requester shares the physical work and cancellation lease. A @@ -240,6 +265,13 @@ export class DeduplicatedLoadSubset { if (resultPromise === true) { if (requestGeneration === this.generation && !lease.aborted) { this.updateTracking(trackingOptions) + const acquisition: AcquisitionOwnership = { + matchesPhysicalRequest, + generation: requestGeneration, + reservations: new Set([reservation]), + } + reservation.acquisition = acquisition + this.exactAcquisitions.push(acquisition) } lease.dispose() return true @@ -263,6 +295,7 @@ export class DeduplicatedLoadSubset { !lease.aborted ) { this.updateTracking(trackingOptions) + this.exactAcquisitions.push(inflightEntry) } return recordLoadSubsetResultDemandMatcher( result, @@ -279,7 +312,7 @@ export class DeduplicatedLoadSubset { lease.dispose() }), } - reservation.inflight = inflightEntry + reservation.acquisition = inflightEntry recordLoadSubsetPromiseDemandMatcher( inflightEntry.promise, @@ -307,10 +340,11 @@ export class DeduplicatedLoadSubset { * across live-query lifetimes must return this method as their unloadSubset * callback. * - * Settled evidence is invalidated conservatively. In-flight work is tracked - * by exact logical owner, so a late release cannot retire a newer generation - * or work that another owner still needs. Core must release the same options - * object that it passed to loadSubset; unmatched releases are no-ops. + * Settled exact evidence and in-flight work are tracked by logical owner, so + * a late release cannot retire a newer generation or work that another owner + * still needs. Broader inferred coverage remains conservative. Core must + * release the same options object that it passed to loadSubset; unmatched + * releases are no-ops. */ unloadSubset = (options: LoadSubsetOptions): void => { const reservation = this.shiftOwnerReservation(options) @@ -320,16 +354,13 @@ export class DeduplicatedLoadSubset { if (!reservation || reservation.generation !== this.generation) return if (!reservation.invalidatesCoverage) return + const acquisition = reservation.acquisition + if (acquisition) { + acquisition.reservations.delete(reservation) + if (acquisition.reservations.size > 0) return + this.retireAcquisition(acquisition) + } this.clearLoadedTracking() - const inflight = reservation.inflight - if (!inflight) return - - inflight.reservations.delete(reservation) - if (inflight.reservations.size > 0) return - - inflight.trackable = false - const index = this.inflightCalls.indexOf(inflight) - if (index !== -1) this.inflightCalls.splice(index, 1) } /** @@ -381,12 +412,19 @@ export class DeduplicatedLoadSubset { if (reservationIndex !== -1) reservations!.splice(reservationIndex, 1) if (reservations?.length === 0) this.ownerReservations.delete(options) - const inflight = reservation.inflight - if (!inflight) return - inflight.reservations.delete(reservation) - if (inflight.reservations.size > 0) return - inflight.trackable = false - const inflightIndex = this.inflightCalls.indexOf(inflight) + const acquisition = reservation.acquisition + if (!acquisition) return + acquisition.reservations.delete(reservation) + if (acquisition.reservations.size > 0) return + this.retireAcquisition(acquisition) + } + + private retireAcquisition(acquisition: AcquisitionOwnership): void { + const exactIndex = this.exactAcquisitions.indexOf(acquisition) + if (exactIndex !== -1) this.exactAcquisitions.splice(exactIndex, 1) + if (!isInflightCall(acquisition)) return + acquisition.trackable = false + const inflightIndex = this.inflightCalls.indexOf(acquisition) if (inflightIndex !== -1) this.inflightCalls.splice(inflightIndex, 1) } @@ -394,6 +432,7 @@ export class DeduplicatedLoadSubset { this.unlimitedWhere = undefined this.hasLoadedAllData = false this.limitedCalls = [] + this.exactAcquisitions = [] } private updateTracking(options: LoadSubsetOptions): void { diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index 8009fb8e2..73398ae27 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -2428,19 +2428,15 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: a composed predicate state forgets one loaded region`, - expectExactCountFailure( - () => - countLoads([ - { kind: `in`, values: [0] }, - { kind: `in`, values: [2] }, - { kind: `eq`, value: 2 }, - ]), - 3, - 2, - ), - ) + it(`retains exact coverage when predicate regions compose`, () => { + expect( + countLoads([ + { kind: `in`, values: [0] }, + { kind: `in`, values: [2] }, + { kind: `eq`, value: 2 }, + ]), + ).toBe(2) + }) it(`rejects repeated transport work for one identical compound predicate`, () => { const predicate: PredicateSpec = { @@ -2689,11 +2685,8 @@ describe(`loadSubset coverage oracle`, () => { }, ) - it(`discovered trace: settled predicate regions cover their union`, async () => { - await expectAssertionFailure(runAsyncScenario, { - checkpoint: 2, - classify: ({ actual, expected }) => actual === true && expected === false, - })({ + it(`settled predicate regions cover their union`, async () => { + await runAsyncScenario({ first: [0], second: [1], firstOutcome: `resolve`, diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 265faef7c..dbfb29ed5 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1989,6 +1989,16 @@ async function runAcquisitionTopology( let initialPhysicalStarts = 0 let initialLogicalStarts = 0 let initialDeduplications = 0 + let retainedOwnerRowKeys: Array = [] + let retainedOwnerReady = false + let coOwnerPhysicalStarts = 0 + let coOwnerLogicalStarts = 0 + let coOwnerDeduplications = 0 + let coOwnerRowKeys: Array = [] + let coOwnerBatches: Array> = [] + let coOwnerCallbackReads: Array> = [] + let coOwnerBatchesAfterUnsubscribe: Array> = [] + let coOwnerCallbackReadsAfterUnsubscribe: Array> = [] let remountRowKeys: Array = [] let remountBatches: Array> = [] let remountCallbackReads: Array> = [] @@ -2036,9 +2046,56 @@ async function runAcquisitionTopology( initialPhysicalStarts = physicalStarts initialLogicalStarts = logicalStarts initialDeduplications = deduplications - subscriptions.forEach((subscription) => subscription.unsubscribe()) - await Promise.all(liveQueries.map((live) => live.cleanup())) - expect(logicalReleases).toBe(2) + subscriptions[0]!.unsubscribe() + await liveQueries[0]!.cleanup() + expect(logicalReleases).toBe(1) + + const coOwner = createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-co-owner`, + query: (q) => q.from({ row: sharedSource }), + startSync: false, + }) + const observedCoOwnerBatches: Array> = [] + const observedCoOwnerCallbackReads: Array> = [] + const coOwnerSubscription = coOwner.subscribeChanges( + (changes) => { + observedCoOwnerBatches.push( + changes.map(({ key }) => String(key)).sort(), + ) + observedCoOwnerCallbackReads.push( + coOwner.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ) + try { + await coOwner.preload() + retainedOwnerRowKeys = liveQueries[1]!.toArray + .map(({ id }) => String(id)) + .sort() + retainedOwnerReady = liveQueries[1]!.isReady() + coOwnerPhysicalStarts = physicalStarts + coOwnerLogicalStarts = logicalStarts + coOwnerDeduplications = deduplications + coOwnerRowKeys = coOwner.toArray.map(({ id }) => String(id)).sort() + coOwnerBatches = observedCoOwnerBatches.map((batch) => [...batch]) + coOwnerCallbackReads = observedCoOwnerCallbackReads.map((read) => [ + ...read, + ]) + + subscriptions[1]!.unsubscribe() + await liveQueries[1]!.cleanup() + } finally { + coOwnerSubscription.unsubscribe() + await coOwner.cleanup() + coOwnerBatchesAfterUnsubscribe = observedCoOwnerBatches.map((batch) => [ + ...batch, + ]) + coOwnerCallbackReadsAfterUnsubscribe = observedCoOwnerCallbackReads.map( + (read) => [...read], + ) + } + expect(logicalReleases).toBe(3) const remount = createLiveQueryCollection({ id: `refinement-acquisition-${runId}-remount`, @@ -2088,6 +2145,16 @@ async function runAcquisitionTopology( initialPhysicalStarts, initialLogicalStarts, initialDeduplications, + retainedOwnerRowKeys, + retainedOwnerReady, + coOwnerPhysicalStarts, + coOwnerLogicalStarts, + coOwnerDeduplications, + coOwnerRowKeys, + coOwnerBatches, + coOwnerCallbackReads, + coOwnerBatchesAfterUnsubscribe, + coOwnerCallbackReadsAfterUnsubscribe, totalPhysicalStarts: physicalStarts, totalLogicalStarts: logicalStarts, logicalReleases, @@ -2172,6 +2239,37 @@ for (const campaign of refinementCampaigns(1_779_008)) { expect(separateActual.initialPhysicalStarts).toBe(2) expect(sharedActual.initialDeduplications).toBe(1) expect(separateActual.initialDeduplications).toBe(0) + expect(sharedActual.retainedOwnerRowKeys).toEqual(expectedKeys) + expect(separateActual.retainedOwnerRowKeys).toEqual(expectedKeys) + expect(sharedActual.retainedOwnerReady).toBe(true) + expect(separateActual.retainedOwnerReady).toBe(true) + expect(sharedActual.coOwnerRowKeys).toEqual(expectedKeys) + expect(separateActual.coOwnerRowKeys).toEqual(expectedKeys) + expect(sharedActual.coOwnerBatches).toEqual([]) + expect(separateActual.coOwnerBatches).toEqual([expectedKeys, []]) + expect(sharedActual.coOwnerCallbackReads).toEqual([]) + expect(separateActual.coOwnerCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(sharedActual.coOwnerBatchesAfterUnsubscribe).toEqual( + sharedActual.coOwnerBatches, + ) + expect(sharedActual.coOwnerCallbackReadsAfterUnsubscribe).toEqual( + sharedActual.coOwnerCallbackReads, + ) + expect(separateActual.coOwnerBatchesAfterUnsubscribe).toEqual( + separateActual.coOwnerBatches, + ) + expect(separateActual.coOwnerCallbackReadsAfterUnsubscribe).toEqual( + separateActual.coOwnerCallbackReads, + ) + expect(sharedActual.coOwnerLogicalStarts).toBe(3) + expect(separateActual.coOwnerLogicalStarts).toBe(3) + expect(sharedActual.coOwnerPhysicalStarts).toBe(1) + expect(separateActual.coOwnerPhysicalStarts).toBe(3) + expect(sharedActual.coOwnerDeduplications).toBe(2) + expect(separateActual.coOwnerDeduplications).toBe(0) expect(sharedActual.remountRowKeys).toEqual(expectedKeys) expect(separateActual.remountRowKeys).toEqual(expectedKeys) expect(sharedActual.remountBatches).toEqual([expectedKeys, []]) @@ -2196,13 +2294,13 @@ for (const campaign of refinementCampaigns(1_779_008)) { expect(separateActual.remountCallbackReadsAfterUnsubscribe).toEqual( separateActual.remountCallbackReads, ) - expect(sharedActual.totalLogicalStarts).toBe(3) - expect(separateActual.totalLogicalStarts).toBe(3) - expect(sharedActual.logicalReleases).toBe(3) - expect(separateActual.logicalReleases).toBe(3) + expect(sharedActual.totalLogicalStarts).toBe(4) + expect(separateActual.totalLogicalStarts).toBe(4) + expect(sharedActual.logicalReleases).toBe(4) + expect(separateActual.logicalReleases).toBe(4) expect(sharedActual.totalPhysicalStarts).toBe(2) - expect(separateActual.totalPhysicalStarts).toBe(3) - expect(sharedActual.totalDeduplications).toBe(1) + expect(separateActual.totalPhysicalStarts).toBe(4) + expect(sharedActual.totalDeduplications).toBe(2) expect(separateActual.totalDeduplications).toBe(0) expect( sharedActual.totalPhysicalStarts + sharedActual.totalDeduplications, diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 558e90d3d..606d00f8c 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -315,19 +315,26 @@ describe(`createDeduplicatedLoadSubset`, () => { })), ), )( - `invalidates $settlement $name settled coverage on unload`, + `invalidates $settlement $name settled coverage after its final owner unloads`, async ({ createOptions, settlement }) => { const loadSubset = vi.fn(() => settlement === `sync` ? (true as const) : Promise.resolve(), ) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) const owner = createOptions() + const peer = createOptions() await deduplicated.loadSubset(owner) - expect(deduplicated.loadSubset(createOptions())).toBe(true) + expect(deduplicated.loadSubset(peer)).toBe(true) expect(loadSubset).toHaveBeenCalledTimes(1) deduplicated.unloadSubset(owner) + const coOwner = createOptions() + expect(deduplicated.loadSubset(coOwner)).toBe(true) + expect(loadSubset).toHaveBeenCalledTimes(1) + + deduplicated.unloadSubset(peer) + deduplicated.unloadSubset(coOwner) await deduplicated.loadSubset(createOptions()) expect(loadSubset).toHaveBeenCalledTimes(2) From 0cfbb3da16093658664af6568ef1ced3783221f1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:43:50 -0600 Subject: [PATCH 110/327] fix(db): preserve newer exact acquisition evidence --- packages/db/src/query/subset-dedupe.ts | 8 ++- packages/db/tests/query/subset-dedupe.test.ts | 58 ++++++++++++++----- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index e565eb371..5049025fb 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -360,7 +360,7 @@ export class DeduplicatedLoadSubset { if (acquisition.reservations.size > 0) return this.retireAcquisition(acquisition) } - this.clearLoadedTracking() + this.clearInferredTracking() } /** @@ -429,10 +429,14 @@ export class DeduplicatedLoadSubset { } private clearLoadedTracking(): void { + this.clearInferredTracking() + this.exactAcquisitions = [] + } + + private clearInferredTracking(): void { this.unlimitedWhere = undefined this.hasLoadedAllData = false this.limitedCalls = [] - this.exactAcquisitions = [] } private updateTracking(options: LoadSubsetOptions): void { diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 606d00f8c..edec95877 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -341,7 +341,7 @@ describe(`createDeduplicatedLoadSubset`, () => { }, ) - it(`bounds conservative adapter-wide invalidation to one refetch per revisited demand`, async () => { + it(`invalidates a released acquisition without erasing other exact owners`, async () => { const loadSubset = vi.fn(() => Promise.resolve()) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) const demands = Array.from({ length: 6 }, (_, id) => ({ @@ -352,24 +352,24 @@ describe(`createDeduplicatedLoadSubset`, () => { for (const demand of demands) await deduplicated.loadSubset(demand) expect(loadSubset).toHaveBeenCalledTimes(demands.length) - // Core may delete rows owned by any remembered request when one collection - // owner leaves. Without adapter row provenance, preserving the other five - // request facts would be unsafe, so one release invalidates all six. + // A release invalidates broader coverage inferred from the combined + // request history, but each other physical acquisition still has a live + // exact owner and therefore retains its own evidence. deduplicated.unloadSubset(demands[0]!) for (const demand of demands.slice(1)) { - await deduplicated.loadSubset(demand) + expect(deduplicated.loadSubset(demand)).toBe(true) } - expect(loadSubset).toHaveBeenCalledTimes( - demands.length + demands.length - 1, - ) + expect(loadSubset).toHaveBeenCalledTimes(demands.length) - // Once those demands have rebuilt the cache, revisiting them is free again. - for (const demand of demands.slice(1)) { + await deduplicated.loadSubset(demands[0]!) + expect(loadSubset).toHaveBeenCalledTimes(demands.length + 1) + + // Once the released demand has rebuilt its acquisition, every exact owner + // can be revisited without transport. + for (const demand of demands) { expect(deduplicated.loadSubset(demand)).toBe(true) } - expect(loadSubset).toHaveBeenCalledTimes( - demands.length + demands.length - 1, - ) + expect(loadSubset).toHaveBeenCalledTimes(demands.length + 1) }) it(`does not restore invalidated coverage when unloaded work settles late`, async () => { @@ -430,6 +430,38 @@ describe(`createDeduplicatedLoadSubset`, () => { }, ) + it(`keeps newer settled exact work when a rejected older owner unloads late`, async () => { + const pending: Array<{ + resolve: () => void + reject: (error: Error) => void + }> = [] + const loadSubset = vi.fn( + () => + new Promise((resolve, reject) => { + pending.push({ resolve, reject }) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + const oldLoad = deduplicated.loadSubset(reusedOptions) + const rejected = expect(oldLoad).rejects.toThrow(`old failed`) + + pending[0]!.reject(new Error(`old failed`)) + await rejected + + const freshLoad = deduplicated.loadSubset(reusedOptions) + pending[1]!.resolve() + await freshLoad + + deduplicated.unloadSubset(reusedOptions) + const peerOptions = { limit: 2 } + expect(deduplicated.loadSubset(peerOptions)).toBe(true) + expect(loadSubset).toHaveBeenCalledTimes(2) + + deduplicated.unloadSubset(reusedOptions) + deduplicated.unloadSubset(peerOptions) + }) + it(`keeps shared exact in-flight work while another logical owner remains`, async () => { let resolveLoad: (() => void) | undefined const loadSubset = vi.fn( From d0228e80d6788b7df2677005edb0352025640425 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 22:50:07 -0600 Subject: [PATCH 111/327] fix(db): recheck subset ownership after adapter entry --- packages/db/src/query/subset-dedupe.ts | 29 +++++++++++++++---- packages/db/tests/query/subset-dedupe.test.ts | 17 +++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 5049025fb..2552f0b67 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -27,6 +27,7 @@ type SharedAbortLease = { type LogicalLoadReservation = { generation: number invalidatesCoverage: boolean + active: boolean acquisition?: AcquisitionOwnership } @@ -263,7 +264,11 @@ export class DeduplicatedLoadSubset { // Handle both sync (true) and async (Promise) return values if (resultPromise === true) { - if (requestGeneration === this.generation && !lease.aborted) { + if ( + requestGeneration === this.generation && + reservation.active && + !lease.aborted + ) { this.updateTracking(trackingOptions) const acquisition: AcquisitionOwnership = { matchesPhysicalRequest, @@ -276,14 +281,18 @@ export class DeduplicatedLoadSubset { lease.dispose() return true } else { + const ownsRequestAtAdapterReturn = + requestGeneration === this.generation && reservation.active // We need to create a reference to the in-flight entry so we can remove it later const inflightEntry: InflightCall = { options: trackingOptions, lease, matchesPhysicalRequest, generation: requestGeneration, - trackable: true, - reservations: new Set([reservation]), + trackable: ownsRequestAtAdapterReturn, + reservations: ownsRequestAtAdapterReturn + ? new Set([reservation]) + : new Set(), promise: resultPromise .then((result) => { // Only update tracking if this request is still from the current generation @@ -312,7 +321,14 @@ export class DeduplicatedLoadSubset { lease.dispose() }), } - reservation.acquisition = inflightEntry + const ownsRequestAfterHandlerInstallation = + requestGeneration === this.generation && reservation.active + inflightEntry.trackable = ownsRequestAfterHandlerInstallation + if (ownsRequestAfterHandlerInstallation) { + reservation.acquisition = inflightEntry + } else { + inflightEntry.reservations.clear() + } recordLoadSubsetPromiseDemandMatcher( inflightEntry.promise, @@ -320,7 +336,7 @@ export class DeduplicatedLoadSubset { ) // Store the in-flight entry so concurrent subset calls can wait for it - if (requestGeneration === this.generation) { + if (ownsRequestAfterHandlerInstallation) { this.inflightCalls.push(inflightEntry) } return projectLoadSubsetResultForCaller( @@ -387,6 +403,7 @@ export class DeduplicatedLoadSubset { const reservation = { generation: this.generation, invalidatesCoverage, + active: true, } const reservations = this.ownerReservations.get(options) if (reservations) reservations.push(reservation) @@ -399,6 +416,7 @@ export class DeduplicatedLoadSubset { ): LogicalLoadReservation | undefined { const reservations = this.ownerReservations.get(options) const reservation = reservations?.shift() + if (reservation) reservation.active = false if (reservations?.length === 0) this.ownerReservations.delete(options) return reservation } @@ -407,6 +425,7 @@ export class DeduplicatedLoadSubset { options: LoadSubsetOptions, reservation: LogicalLoadReservation, ): void { + reservation.active = false const reservations = this.ownerReservations.get(options) const reservationIndex = reservations?.indexOf(reservation) ?? -1 if (reservationIndex !== -1) reservations!.splice(reservationIndex, 1) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index edec95877..79ecb8037 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -462,6 +462,23 @@ describe(`createDeduplicatedLoadSubset`, () => { deduplicated.unloadSubset(peerOptions) }) + it.each([`sync`, `async`] as const)( + `does not retain exact evidence when its sole owner unloads during %s adapter entry`, + async (settlement) => { + const options = { limit: 2 } + const loadSubset = vi.fn(() => { + deduplicated.unloadSubset(options) + return settlement === `sync` ? (true as const) : Promise.resolve() + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset(options) + await deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }, + ) + it(`keeps shared exact in-flight work while another logical owner remains`, async () => { let resolveLoad: (() => void) | undefined const loadSubset = vi.fn( From ed13c4f2b5123c8788a2233b89ba221940ba2704 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:00:01 -0600 Subject: [PATCH 112/327] fix(db): clean up failed promise observation --- packages/db/src/query/subset-dedupe.ts | 51 ++++++++++++------- packages/db/tests/query/subset-dedupe.test.ts | 42 +++++++++++++++ 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 2552f0b67..4d06fa1a6 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -283,28 +283,26 @@ export class DeduplicatedLoadSubset { } else { const ownsRequestAtAdapterReturn = requestGeneration === this.generation && reservation.active - // We need to create a reference to the in-flight entry so we can remove it later - const inflightEntry: InflightCall = { - options: trackingOptions, - lease, - matchesPhysicalRequest, - generation: requestGeneration, - trackable: ownsRequestAtAdapterReturn, - reservations: ownsRequestAtAdapterReturn - ? new Set([reservation]) - : new Set(), - promise: resultPromise + // Promise subclasses can run or throw from handler installation. Keep the + // entry optional until the complete observation chain exists so failed + // installation cannot leave an owner or abort lease behind. + const installation: { entry: InflightCall | undefined } = { + entry: undefined, + } + let observedPromise: Promise + try { + observedPromise = resultPromise .then((result) => { // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared if ( - inflightEntry.trackable && - inflightEntry.generation === this.generation && + installation.entry?.trackable && + installation.entry.generation === this.generation && !lease.aborted ) { this.updateTracking(trackingOptions) - this.exactAcquisitions.push(inflightEntry) + this.exactAcquisitions.push(installation.entry) } return recordLoadSubsetResultDemandMatcher( result, @@ -314,13 +312,30 @@ export class DeduplicatedLoadSubset { .finally(() => { // Always remove from in-flight array on completion OR rejection // This ensures failed requests can be retried instead of being cached forever - const index = this.inflightCalls.indexOf(inflightEntry) - if (index !== -1) { - this.inflightCalls.splice(index, 1) + if (installation.entry) { + const index = this.inflightCalls.indexOf(installation.entry) + if (index !== -1) { + this.inflightCalls.splice(index, 1) + } } lease.dispose() - }), + }) + } catch (error) { + lease.dispose() + throw error + } + const inflightEntry: InflightCall = { + options: trackingOptions, + lease, + matchesPhysicalRequest, + generation: requestGeneration, + trackable: ownsRequestAtAdapterReturn, + reservations: ownsRequestAtAdapterReturn + ? new Set([reservation]) + : new Set(), + promise: observedPromise, } + installation.entry = inflightEntry const ownsRequestAfterHandlerInstallation = requestGeneration === this.generation && reservation.active inflightEntry.trackable = ownsRequestAfterHandlerInstallation diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 79ecb8037..037caaa63 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -690,6 +690,48 @@ describe(`createDeduplicatedLoadSubset`, () => { await Promise.all([oldLoad, freshLoad]) }) + it(`releases its abort lease when Promise handler installation throws`, async () => { + class ThrowOnThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + _onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + _onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + throw new Error(`then install failed`) + } + } + const signal = { + aborted: false, + reason: undefined, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as AbortSignal + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return loadSubsetCalls === 1 + ? new ThrowOnThenPromise((resolve) => resolve()) + : Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + expect(() => deduplicated.loadSubset({ limit: 2, signal })).toThrow( + `then install failed`, + ) + expect(signal.addEventListener).toHaveBeenCalledTimes(1) + expect(signal.removeEventListener).toHaveBeenCalledTimes(1) + + await deduplicated.loadSubset({ limit: 2 }) + expect(loadSubsetCalls).toBe(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 66d0da5667e0321523efc1bddbb5026f0a45be6d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:08:59 -0600 Subject: [PATCH 113/327] fix(db): normalize subset load promises --- packages/db/src/query/subset-dedupe.ts | 14 +++++-- .../query/load-subset-oracle.property.test.ts | 26 +++++++------ packages/db/tests/query/subset-dedupe.test.ts | 39 +++++++++++++++++-- 3 files changed, 60 insertions(+), 19 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 4d06fa1a6..6864bc8c3 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -283,15 +283,21 @@ export class DeduplicatedLoadSubset { } else { const ownsRequestAtAdapterReturn = requestGeneration === this.generation && reservation.active - // Promise subclasses can run or throw from handler installation. Keep the - // entry optional until the complete observation chain exists so failed - // installation cannot leave an owner or abort lease behind. + // Assimilate foreign Promise implementations before installing stateful + // handlers. Calling their `then` now preserves reentrant adapter effects, + // while the native bridge defers fulfillment, rejection, and thrown errors + // until the entry below exists. + const normalizedResultPromise = new Promise( + (resolve, reject) => { + resultPromise.then(resolve, reject) + }, + ) const installation: { entry: InflightCall | undefined } = { entry: undefined, } let observedPromise: Promise try { - observedPromise = resultPromise + observedPromise = normalizedResultPromise .then((result) => { // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index 73398ae27..331832a3c 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -1940,14 +1940,20 @@ async function expectDerivedSyncDuringOptimisticMutation(): Promise { async function expectDeduplicatedWaiterHandlesRejection( scenario: RejectedWaiterScenario, ): Promise { - const detachedBranches: Array> = [] + let sourceRejectionObservers = 0 class LocallyTrackedPromise extends Promise { - catch( - onRejected?: ((reason: unknown) => TResult | PromiseLike) | null, - ): Promise { - const branch = super.catch(onRejected) - detachedBranches.push(branch) - return branch + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: ((value: T) => TResult1 | PromiseLike) | null, + onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + if (onrejected) sourceRejectionObservers += 1 + return super.then(onfulfilled, onrejected) } } @@ -1970,7 +1976,6 @@ async function expectDeduplicatedWaiterHandlesRejection( } const callerOutcomes = Promise.allSettled([first, second]) - const detachedOutcomes = Promise.allSettled(detachedBranches) rejectSource(new Error(`transport failed`)) expect((await callerOutcomes).map(({ status }) => status)).toEqual([ `rejected`, @@ -1978,10 +1983,7 @@ async function expectDeduplicatedWaiterHandlesRejection( ]) try { - expect({ - branchCount: detachedBranches.length, - statuses: (await detachedOutcomes).map(({ status }) => status), - }).toEqual({ branchCount: 1, statuses: [`fulfilled`] }) + expect(sourceRejectionObservers).toBe(1) } catch (error) { throw new TraceAssertionError(0, error) } diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 037caaa63..218b3bf20 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -722,9 +722,10 @@ describe(`createDeduplicatedLoadSubset`, () => { } const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) - expect(() => deduplicated.loadSubset({ limit: 2, signal })).toThrow( - `then install failed`, - ) + const failedLoad = deduplicated.loadSubset({ limit: 2, signal }) + + expect(failedLoad).toBeInstanceOf(Promise) + await expect(failedLoad).rejects.toThrow(`then install failed`) expect(signal.addEventListener).toHaveBeenCalledTimes(1) expect(signal.removeEventListener).toHaveBeenCalledTimes(1) @@ -732,6 +733,38 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubsetCalls).toBe(2) }) + it(`retains exact evidence when a Promise subclass settles during handler installation`, async () => { + class SynchronousThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + _onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + return Promise.resolve(onfulfilled?.()) as Promise + } + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return loadSubsetCalls === 1 + ? new SynchronousThenPromise((resolve) => resolve()) + : Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset({ limit: 2 }) + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(loadSubsetCalls).toBe(1) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 11c50ddd776e9af3cffafaf39976f6a1efeab27a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:17:33 -0600 Subject: [PATCH 114/327] fix(db): publish coverage after result retention --- packages/db/src/query/subset-dedupe.ts | 11 +++--- packages/db/tests/query/subset-dedupe.test.ts | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index 6864bc8c3..29d47890f 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -299,6 +299,12 @@ export class DeduplicatedLoadSubset { try { observedPromise = normalizedResultPromise .then((result) => { + // Retain every fallible adapter result field before publishing + // coverage. A rejected caller must never leave reusable evidence. + const retainedResult = recordLoadSubsetResultDemandMatcher( + result, + matchesPhysicalRequest, + ) // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared @@ -310,10 +316,7 @@ export class DeduplicatedLoadSubset { this.updateTracking(trackingOptions) this.exactAcquisitions.push(installation.entry) } - return recordLoadSubsetResultDemandMatcher( - result, - matchesPhysicalRequest, - ) + return retainedResult }) .finally(() => { // Always remove from in-flight array on completion OR rejection diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 218b3bf20..06508a6bb 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -765,6 +765,40 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubsetCalls).toBe(1) }) + it(`does not retain coverage when fulfilled result normalization throws`, async () => { + const resultError = new Error(`result read failed`) + const hostileResult = { + get hasMore(): boolean | undefined { + throw resultError + }, + } + const signal = { + aborted: false, + reason: undefined, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as AbortSignal + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 ? hostileResult : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 2, signal })).rejects.toBe( + resultError, + ) + expect(signal.addEventListener).toHaveBeenCalledTimes(1) + expect(signal.removeEventListener).toHaveBeenCalledTimes(1) + + const retry = deduplicated.loadSubset({ limit: 2 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 769fc3a97e2b20d90008f60b92f74fdb2b92ef81 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:24:59 -0600 Subject: [PATCH 115/327] fix(db): snapshot applied subset evidence --- packages/db/src/query/load-subset-outcome.ts | 11 +++++- packages/db/tests/query/subset-dedupe.test.ts | 37 ++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/load-subset-outcome.ts b/packages/db/src/query/load-subset-outcome.ts index f95d72b94..24ed1a2ac 100644 --- a/packages/db/src/query/load-subset-outcome.ts +++ b/packages/db/src/query/load-subset-outcome.ts @@ -27,8 +27,15 @@ export function recordLoadSubsetResultDemandMatcher( if (typeof result !== `object`) return result // Give each physical acquisition its own result identity. A source may reuse - // one result object across calls with different demands. - const retainedResult = { ...result } + // one result object across calls with different demands. Snapshot nested + // source evidence here too, before the caller publishes coverage from it. + const appliedRowKeys = result.appliedRowKeys + const retainedResult: LoadSubsetResult = { + hasMore: result.hasMore, + ...(appliedRowKeys === undefined + ? {} + : { appliedRowKeys: Object.freeze([...appliedRowKeys]) }), + } loadSubsetResultDemandMatchers.set(retainedResult, matches) return retainedResult } diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 06508a6bb..ee9f3861f 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -6,7 +6,11 @@ import { import { Func, PropRef, Value } from '../../src/query/ir' import { createCrossRealmUint8Array } from '../utils' import type { BasicExpression, OrderBy } from '../../src/query/ir' -import type { LoadSubsetFn, LoadSubsetOptions } from '../../src/types' +import type { + LoadSubsetFn, + LoadSubsetOptions, + LoadSubsetResult, +} from '../../src/types' // Helper functions to build expressions more easily function ref(path: string | Array): PropRef { @@ -799,6 +803,37 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubsetCalls).toBe(2) }) + it(`does not retain coverage when row-key snapshotting throws`, async () => { + const resultError = new Error(`row-key snapshot failed`) + const hostileRowKeys = new Proxy>([1], { + get: (target, property, receiver) => { + if (property === Symbol.iterator) throw resultError + return Reflect.get(target, property, receiver) + }, + }) + const hostileResult: LoadSubsetResult = { + hasMore: false, + appliedRowKeys: hostileRowKeys, + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 ? hostileResult : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 2 })).rejects.toBe( + resultError, + ) + + const retry = deduplicated.loadSubset({ limit: 2 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From e15f1e204c6303efe7c0370057d50a4c3f80d1c4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:39:47 -0600 Subject: [PATCH 116/327] fix(db): validate applied subset evidence --- packages/db/src/query/load-subset-outcome.ts | 29 ++++++++++++++----- packages/db/tests/query/subset-dedupe.test.ts | 23 +++++++++++++++ 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/packages/db/src/query/load-subset-outcome.ts b/packages/db/src/query/load-subset-outcome.ts index 24ed1a2ac..fb5fc1600 100644 --- a/packages/db/src/query/load-subset-outcome.ts +++ b/packages/db/src/query/load-subset-outcome.ts @@ -13,6 +13,23 @@ const loadSubsetResultDemandMatchers = new WeakMap< (options: LoadSubsetOptions) => boolean >() +function snapshotAppliedRowKeys( + appliedRowKeys: ReadonlyArray | undefined, +): ReadonlyArray | undefined { + if (appliedRowKeys === undefined) return undefined + + const snapshot: Array = [] + for (const key of appliedRowKeys) { + if (typeof key !== `string` && typeof key !== `number`) { + throw new TypeError( + `loadSubset appliedRowKeys must contain only string or number keys`, + ) + } + snapshot.push(key) + } + return Object.freeze(snapshot) +} + export function recordLoadSubsetPromiseDemandMatcher( promise: Promise, matches: (options: LoadSubsetOptions) => boolean, @@ -29,12 +46,10 @@ export function recordLoadSubsetResultDemandMatcher( // Give each physical acquisition its own result identity. A source may reuse // one result object across calls with different demands. Snapshot nested // source evidence here too, before the caller publishes coverage from it. - const appliedRowKeys = result.appliedRowKeys + const appliedRowKeys = snapshotAppliedRowKeys(result.appliedRowKeys) const retainedResult: LoadSubsetResult = { hasMore: result.hasMore, - ...(appliedRowKeys === undefined - ? {} - : { appliedRowKeys: Object.freeze([...appliedRowKeys]) }), + ...(appliedRowKeys === undefined ? {} : { appliedRowKeys }), } loadSubsetResultDemandMatchers.set(retainedResult, matches) return retainedResult @@ -61,7 +76,7 @@ export function createAppliedLoadSubsetOutcome( generation: number, sourceResult: void | LoadSubsetResult, ): AppliedLoadSubsetOutcome { - const appliedRowKeys = sourceResult?.appliedRowKeys + const appliedRowKeys = snapshotAppliedRowKeys(sourceResult?.appliedRowKeys) return { collectionId, demand, @@ -72,9 +87,7 @@ export function createAppliedLoadSubsetOutcome( : sourceResult?.hasMore === false ? `exhausted` : `unknown`, - ...(appliedRowKeys === undefined - ? {} - : { appliedRowKeys: Object.freeze([...appliedRowKeys]) }), + ...(appliedRowKeys === undefined ? {} : { appliedRowKeys }), } } diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index ee9f3861f..b0c79ef2c 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -834,6 +834,29 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubsetCalls).toBe(2) }) + it(`rejects sparse applied-row evidence without retaining coverage`, async () => { + const sparseRowKeys = new Array(1) + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 + ? { hasMore: true, appliedRowKeys: sparseRowKeys } + : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 1 })).rejects.toThrow( + `appliedRowKeys must contain only string or number keys`, + ) + + const retry = deduplicated.loadSubset({ limit: 1 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined From 33dab005f9ffb090e959ded3274f989f36754f15 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:44:42 -0600 Subject: [PATCH 117/327] test(db): name deterministic oracle suites accurately --- packages/db/package.json | 2 +- ...rty.test.ts => load-subset-replay-refinement-oracle.test.ts} | 0 ...s => load-subset-source-readiness-refinement-oracle.test.ts} | 0 ...est.ts => load-subset-transaction-refinement-oracle.test.ts} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename packages/db/tests/query/{load-subset-replay-refinement-oracle.property.test.ts => load-subset-replay-refinement-oracle.test.ts} (100%) rename packages/db/tests/query/{load-subset-source-readiness-refinement-oracle.property.test.ts => load-subset-source-readiness-refinement-oracle.test.ts} (100%) rename packages/db/tests/query/{load-subset-transaction-refinement-oracle.property.test.ts => load-subset-transaction-refinement-oracle.test.ts} (100%) diff --git a/packages/db/package.json b/packages/db/package.json index e5b6fe1f9..5db484027 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.property.test.ts tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts tests/query/load-subset-transaction-refinement-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts similarity index 100% rename from packages/db/tests/query/load-subset-replay-refinement-oracle.property.test.ts rename to packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts similarity index 100% rename from packages/db/tests/query/load-subset-source-readiness-refinement-oracle.property.test.ts rename to packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts diff --git a/packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts b/packages/db/tests/query/load-subset-transaction-refinement-oracle.test.ts similarity index 100% rename from packages/db/tests/query/load-subset-transaction-refinement-oracle.property.test.ts rename to packages/db/tests/query/load-subset-transaction-refinement-oracle.test.ts From da5f6a3c36e2f68acb4fbc4a07da8117bdf3b410 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sat, 29 Aug 2026 23:57:32 -0600 Subject: [PATCH 118/327] test(db): replay oracle shrink paths --- ...ubscription-replay-oracle.property.test.ts | 14 ++--- .../tests/collection-sync-reentrancy.test.ts | 4 +- packages/db/tests/oracle-config.ts | 41 ++++++++++---- ...d-subset-full-flow-oracle.property.test.ts | 53 +++++++++++++++---- .../query/load-subset-oracle.property.test.ts | 3 +- .../query/pagination-oracle.property.test.ts | 15 +++--- packages/db/tests/utils.test.ts | 17 ++++-- packages/db/tests/utils.ts | 30 ----------- 8 files changed, 104 insertions(+), 73 deletions(-) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 04ab87945..fd9ac74d2 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -1809,7 +1809,7 @@ async function runOptimisticReplayScenario( } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, replaySeed, replayPath } = readOracleRunConfig() const generatedRuns = 30 * multiplier const generatedTimeout = 5_000 * multiplier @@ -9387,7 +9387,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `matches replay and ownership laws for a random or replayed seed`, runReplayScenario, @@ -9396,7 +9396,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sequentialReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `matches synchronous, asynchronous, and partial-failure replay laws`, runSequentialReplayScenario, @@ -9419,7 +9419,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayCompletionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `preserves replay completion authority for a random or replayed seed`, runReplayCompletionScenario, @@ -9436,7 +9436,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [cleanupRestartScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `isolates cleanup and restart sessions for a random or replayed seed`, runCleanupRestartScenario, @@ -9454,7 +9454,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sharedSubscriptionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `keeps shared transport and logical ownership distinct for a random or replayed seed`, runSharedSubscriptionScenario, @@ -9472,7 +9472,7 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [optimisticReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `preserves optimistic overlays across replay outcomes for a random or replayed seed`, runOptimisticReplayScenario, diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index a0239bf77..daec8cbfa 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -190,7 +190,7 @@ async function runListenerScenario(scenario: ListenerScenario): Promise { } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, replaySeed, replayPath } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { @@ -609,7 +609,7 @@ describe(`sync publication reentrancy`, () => { fcTest.prop( [listenerScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters(generatedRuns, replaySeed, replayPath), )( `matches the reentrant drain laws for a random or replayed seed`, runListenerScenario, diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 2a0375432..0dbf76f52 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -2,7 +2,11 @@ type OracleEnvironment = Record export function readOracleRunConfig( environment: OracleEnvironment = process.env, -): { multiplier: number; replaySeed: number | undefined } { +): { + multiplier: number + replaySeed: number | undefined + replayPath: string | undefined +} { const multiplierValue = environment.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER ?? `1` const multiplier = Number(multiplierValue) if ( @@ -16,23 +20,42 @@ export function readOracleRunConfig( } const seedValue = environment.TANSTACK_DB_ORACLE_SEED - if (seedValue === undefined) return { multiplier, replaySeed: undefined } + const replayPath = environment.TANSTACK_DB_ORACLE_PATH + if (seedValue === undefined) { + if (replayPath !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_SEED`, + ) + } + return { multiplier, replaySeed: undefined, replayPath: undefined } + } const replaySeed = Number(seedValue) if (seedValue.trim() === `` || !Number.isSafeInteger(replaySeed)) { throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) } - return { multiplier, replaySeed } + return { multiplier, replaySeed, replayPath } } export function oracleRandomParameters( numRuns: number, replaySeed: number | undefined, -): { numRuns: number; seed?: number } { - return replaySeed === undefined ? { numRuns } : { numRuns, seed: replaySeed } + replayPath?: string, +): { numRuns: number; seed?: number; path?: string } { + if (replaySeed === undefined) { + if (replayPath !== undefined) { + throw new Error(`A FastCheck replay path requires a replay seed`) + } + return { numRuns } + } + return { + numRuns, + seed: replaySeed, + ...(replayPath === undefined ? {} : { path: replayPath }), + } } -const { multiplier, replaySeed: seed } = readOracleRunConfig() +const { multiplier, replaySeed: seed, replayPath: path } = readOracleRunConfig() /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ export function oracleRuns(baseRuns: number): number { @@ -43,9 +66,7 @@ export function oracleRuns(baseRuns: number): number { export function oraclePropertyOptions(baseRuns: number): { numRuns: number seed?: number + path?: string } { - return { - numRuns: oracleRuns(baseRuns), - ...(seed === undefined ? {} : { seed }), - } + return oracleRandomParameters(oracleRuns(baseRuns), seed, path) } diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index dfd6b6dd4..c52a53d8d 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -153,8 +153,11 @@ it(`loads each side of a filtered inner join once`, async () => { } }) -const { multiplier: fullFlowMultiplier, replaySeed: fullFlowReplaySeed } = - readOracleRunConfig() +const { + multiplier: fullFlowMultiplier, + replaySeed: fullFlowReplaySeed, + replayPath: fullFlowReplayPath, +} = readOracleRunConfig() type MultiSourceOrderedScenario = { primaryRows: ReadonlyArray<{ @@ -300,7 +303,7 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { `multiplicity=${new Set(secondaryRows.map(({ joinKey }) => joinKey)).size < secondaryRows.length}`, `tied=${new Set(primaryRows.map(({ rank }) => rank)).size < primaryRows.length}`, ], - oracleRandomParameters(1_000, fullFlowReplaySeed), + oracleRandomParameters(1_000, fullFlowReplaySeed, fullFlowReplayPath), ) } @@ -2574,7 +2577,11 @@ fcTest.prop([multiSourceOrderedScenarioArbitrary], { fcTest.prop( [multiSourceOrderedScenarioArbitrary], - oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `fills joined ordered windows for a random or replayed seed`, runMultiSourceOrderedScenario, @@ -4153,7 +4160,11 @@ fcTest.prop([orderedConsumerParityScenarioArbitrary], { fcTest.prop( [orderedConsumerParityScenarioArbitrary], - oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `keeps ordered collection consumers equal for a random or replayed seed`, assertOrderedConsumerParity, @@ -4809,7 +4820,7 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { )}`, `exhaustion=${pages.some((page) => page.extent === `exhausted`)}`, ], - oracleRandomParameters(1_000, fullFlowReplaySeed), + oracleRandomParameters(1_000, fullFlowReplaySeed, fullFlowReplayPath), ) } @@ -5080,7 +5091,11 @@ fcTest.prop( maxLength: 8, }), ], - oracleRandomParameters(128 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 128 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `starts automatic continuation only for new semantic progress with a random or replayed seed`, assertAutomaticOrderedProgress, @@ -5096,7 +5111,11 @@ fcTest.prop([orderedContinuationEvidenceScenarioArbitrary], { fcTest.prop( [orderedContinuationEvidenceScenarioArbitrary], - oracleRandomParameters(64 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 64 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `derives ordered progress from applied eligible evidence for a random or replayed seed`, runOrderedContinuationEvidenceScenario, @@ -5427,7 +5446,11 @@ fcTest.prop([orderedBoundaryProvenanceArbitrary], { fcTest.prop( [orderedBoundaryProvenanceArbitrary], - oracleRandomParameters(32 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 32 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `keeps ordered boundary provenance for a random or replayed seed`, runOrderedBoundaryProvenanceScenario, @@ -6343,7 +6366,11 @@ fcTest.prop([atomicOrderedReplayArbitrary], { fcTest.prop( [atomicOrderedReplayArbitrary], - oracleRandomParameters(32 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 32 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `keeps ordered replacement publication atomic for a random or replayed seed`, runAtomicOrderedReplayScenario, @@ -6362,7 +6389,11 @@ fcTest.prop([truncateCoverageScenarioArbitrary], { fcTest.prop( [truncateCoverageScenarioArbitrary], - oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplaySeed, + fullFlowReplayPath, + ), )( `fences pre-truncate evidence for a random or replayed seed`, runTruncateCoverageScenario, diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index 331832a3c..ac978a0af 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -1119,11 +1119,12 @@ async function runAsyncScenarioWithKnownFailures( } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, replaySeed, replayPath } = readOracleRunConfig() const coverageScenarioRuns = 40 * multiplier const coverageRandomParameters = oracleRandomParameters( coverageScenarioRuns, replaySeed, + replayPath, ) let collectionSequence = 0 diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index 3342cdeef..c86a94660 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -340,12 +340,13 @@ async function cleanupAll( if (rejection) throw rejection.reason } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, replaySeed, replayPath } = readOracleRunConfig() const orderedScenarioRuns = 12 * multiplier const transitionScenarioRuns = 8 * multiplier const orderedScenarioRandomParameters = oracleRandomParameters( orderedScenarioRuns, replaySeed, + replayPath, ) let collectionSequence = 0 @@ -2521,7 +2522,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [multiOrderScenarioArbitrary], - oracleRandomParameters(orderedScenarioRuns, replaySeed), + oracleRandomParameters(orderedScenarioRuns, replaySeed, replayPath), )( `matches multi-column nullable ordering for a random or replayed seed`, runMultiOrderScenario, @@ -2537,7 +2538,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [nullableCursorScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), )( `matches nullable cursor ordering while an async response is pending for a random or replayed seed`, runNullableCursorScenario, @@ -2971,7 +2972,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingMutationScenarioArbitrary, responseTimingArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), )( `matches recomputation when source mutations cross a pending cursor response for a random or replayed seed`, runPendingMutationScenario, @@ -2992,7 +2993,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingHistoryScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), )( `matches recomputation across multi-action pending histories for a random or replayed seed`, runPendingHistoryScenario, @@ -3127,7 +3128,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [stateScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), )( `matches full recomputation across source and window transitions for a random or replayed seed`, runPaginationStateScenario, @@ -3472,7 +3473,7 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [scenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), )( `matches full recomputation when exact async cursor loads widen ordered coverage for a random or replayed seed`, runOnDemandPaginationScenario, diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index 0cfc2b27a..4e4b76289 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -5,19 +5,21 @@ import { isPromiseLike } from '../src/utils/type-guards' import { oracleRandomParameters, readOracleRunConfig } from './oracle-config' describe(`oracle run configuration`, () => { - it(`reads the multiplier and replay seed from an explicit environment`, () => { + it(`reads the multiplier and replay coordinates from an explicit environment`, () => { expect( readOracleRunConfig({ TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: `100`, TANSTACK_DB_ORACLE_SEED: `-42`, + TANSTACK_DB_ORACLE_PATH: `1:0:2`, }), - ).toEqual({ multiplier: 100, replaySeed: -42 }) + ).toEqual({ multiplier: 100, replaySeed: -42, replayPath: `1:0:2` }) }) - it(`uses one run multiplier and no replay seed by default`, () => { + it(`uses one run multiplier and no replay coordinates by default`, () => { expect(readOracleRunConfig({})).toEqual({ multiplier: 1, replaySeed: undefined, + replayPath: undefined, }) }) @@ -27,6 +29,7 @@ describe(`oracle run configuration`, () => { [{ TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: ` ` }, `positive integer`], [{ TANSTACK_DB_ORACLE_SEED: `1.5` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_SEED: ` ` }, `must be an integer`], + [{ TANSTACK_DB_ORACLE_PATH: `1:0` }, `requires TANSTACK_DB_ORACLE_SEED`], ] satisfies ReadonlyArray, string]>)( `rejects invalid environment values`, (environment, message) => { @@ -34,11 +37,15 @@ describe(`oracle run configuration`, () => { }, ) - it(`adds a seed only for replay runs`, () => { + it(`adds replay coordinates only for replay runs`, () => { expect(oracleRandomParameters(40, undefined)).toEqual({ numRuns: 40 }) - expect(oracleRandomParameters(40, -42)).toEqual({ + expect(() => oracleRandomParameters(40, undefined, `1:0:2`)).toThrow( + `requires a replay seed`, + ) + expect(oracleRandomParameters(40, -42, `1:0:2`)).toEqual({ numRuns: 40, seed: -42, + path: `1:0:2`, }) }) }) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index fdb124f04..c5cd7fa0b 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -11,8 +11,6 @@ import type { import type { IndexConstructor } from '../src/indexes/base-index' import type { WithVirtualProps } from '../src/virtual-props.js' -type OracleEnvironment = Record - export function createCrossRealmUint8Array( values: ReadonlyArray, ): Uint8Array { @@ -21,34 +19,6 @@ export function createCrossRealmUint8Array( }) as Uint8Array } -export function readOracleRunConfig( - environment: OracleEnvironment = process.env, -): { multiplier: number; replaySeed: number | undefined } { - const multiplierValue = environment.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER ?? `1` - const multiplier = Number(multiplierValue) - if (!Number.isSafeInteger(multiplier) || multiplier < 1) { - throw new Error( - `TANSTACK_DB_ORACLE_RUNS_MULTIPLIER must be a positive integer`, - ) - } - - const seedValue = environment.TANSTACK_DB_ORACLE_SEED - if (seedValue === undefined) return { multiplier, replaySeed: undefined } - - const replaySeed = Number(seedValue) - if (!Number.isSafeInteger(replaySeed)) { - throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) - } - return { multiplier, replaySeed } -} - -export function oracleRandomParameters( - numRuns: number, - replaySeed: number | undefined, -): { numRuns: number; seed?: number } { - return replaySeed === undefined ? { numRuns } : { numRuns, seed: replaySeed } -} - export type OutputWithVirtual< T extends object, TKey extends string | number = string | number, From 9d56dac97a3f961fd15f581edb1dcc7fdf32eafd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 00:10:47 -0600 Subject: [PATCH 119/327] test(db): scope oracle replay paths --- ...ubscription-replay-oracle.property.test.ts | 38 +++- .../tests/collection-sync-reentrancy.test.ts | 8 +- packages/db/tests/oracle-config.ts | 68 ++++-- .../coverage-registry-oracle.property.test.ts | 4 +- ...ncludes-collection-oracle.property.test.ts | 9 +- ...-cross-formulation-oracle.property.test.ts | 10 +- ...ncludes-optimistic-oracle.property.test.ts | 202 ++++++++++-------- .../query/includes-oracle.property.test.ts | 13 +- .../query/includes-publication-oracle.test.ts | 69 ++++-- .../query/includes-temporal-oracle.test.ts | 5 +- ...d-subset-full-flow-oracle.property.test.ts | 47 ++-- ...d-subset-lifecycle-oracle.property.test.ts | 2 +- .../query/load-subset-oracle.property.test.ts | 41 ++-- ...-subset-projection-oracle.property.test.ts | 5 +- ...d-subset-refinement-model.property.test.ts | 2 +- .../query/pagination-oracle.property.test.ts | 52 +++-- packages/db/tests/utils.test.ts | 60 +++++- 17 files changed, 425 insertions(+), 210 deletions(-) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index fd9ac74d2..ff2363c77 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -1809,7 +1809,7 @@ async function runOptimisticReplayScenario( } } -const { multiplier, replaySeed, replayPath } = readOracleRunConfig() +const { multiplier, ...oracleReplay } = readOracleRunConfig() const generatedRuns = 30 * multiplier const generatedTimeout = 5_000 * multiplier @@ -9387,7 +9387,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.ownership`, + ), )( `matches replay and ownership laws for a random or replayed seed`, runReplayScenario, @@ -9396,7 +9400,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sequentialReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.sequential`, + ), )( `matches synchronous, asynchronous, and partial-failure replay laws`, runSequentialReplayScenario, @@ -9419,7 +9427,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayCompletionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.completion`, + ), )( `preserves replay completion authority for a random or replayed seed`, runReplayCompletionScenario, @@ -9436,7 +9448,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [cleanupRestartScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.restart`, + ), )( `isolates cleanup and restart sessions for a random or replayed seed`, runCleanupRestartScenario, @@ -9454,7 +9470,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sharedSubscriptionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.shared`, + ), )( `keeps shared transport and logical ownership distinct for a random or replayed seed`, runSharedSubscriptionScenario, @@ -9472,7 +9492,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [optimisticReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.optimistic`, + ), )( `preserves optimistic overlays across replay outcomes for a random or replayed seed`, runOptimisticReplayScenario, diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index daec8cbfa..521bba7e8 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -190,7 +190,7 @@ async function runListenerScenario(scenario: ListenerScenario): Promise { } } -const { multiplier, replaySeed, replayPath } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { @@ -609,7 +609,11 @@ describe(`sync publication reentrancy`, () => { fcTest.prop( [listenerScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed, replayPath), + oracleRandomParameters( + generatedRuns, + replay, + `collection-sync.reentrant-drain`, + ), )( `matches the reentrant drain laws for a random or replayed seed`, runListenerScenario, diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 0dbf76f52..a541e6f9e 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -1,12 +1,14 @@ type OracleEnvironment = Record -export function readOracleRunConfig( - environment: OracleEnvironment = process.env, -): { - multiplier: number +export type OracleReplayConfig = { replaySeed: number | undefined replayPath: string | undefined -} { + replayProperty: string | undefined +} + +export function readOracleRunConfig( + environment: OracleEnvironment = process.env, +): OracleReplayConfig & { multiplier: number } { const multiplierValue = environment.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER ?? `1` const multiplier = Number(multiplierValue) if ( @@ -21,41 +23,66 @@ export function readOracleRunConfig( const seedValue = environment.TANSTACK_DB_ORACLE_SEED const replayPath = environment.TANSTACK_DB_ORACLE_PATH + const replayProperty = environment.TANSTACK_DB_ORACLE_PROPERTY if (seedValue === undefined) { if (replayPath !== undefined) { throw new Error( `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_SEED`, ) } - return { multiplier, replaySeed: undefined, replayPath: undefined } + return { + multiplier, + replaySeed: undefined, + replayPath: undefined, + replayProperty: undefined, + } } const replaySeed = Number(seedValue) if (seedValue.trim() === `` || !Number.isSafeInteger(replaySeed)) { throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) } - return { multiplier, replaySeed, replayPath } + if (replayPath === undefined) { + return { + multiplier, + replaySeed, + replayPath: undefined, + replayProperty: undefined, + } + } + if (replayPath.trim() === ``) { + throw new Error(`TANSTACK_DB_ORACLE_PATH must be non-empty`) + } + if (!/^\d+(?::\d+)*$/.test(replayPath)) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH must contain colon-separated nonnegative integers`, + ) + } + if (replayProperty === undefined || replayProperty.trim() === ``) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_PROPERTY`, + ) + } + return { multiplier, replaySeed, replayPath, replayProperty } } export function oracleRandomParameters( numRuns: number, - replaySeed: number | undefined, - replayPath?: string, + replay: OracleReplayConfig, + property: string, ): { numRuns: number; seed?: number; path?: string } { - if (replaySeed === undefined) { - if (replayPath !== undefined) { - throw new Error(`A FastCheck replay path requires a replay seed`) - } - return { numRuns } - } + const { replaySeed, replayPath, replayProperty } = replay + if (replaySeed === undefined) return { numRuns } return { numRuns, seed: replaySeed, - ...(replayPath === undefined ? {} : { path: replayPath }), + ...(replayPath !== undefined && replayProperty === property + ? { path: replayPath } + : {}), } } -const { multiplier, replaySeed: seed, replayPath: path } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ export function oracleRuns(baseRuns: number): number { @@ -63,10 +90,13 @@ export function oracleRuns(baseRuns: number): number { } /** Replays broad randomized properties when a campaign seed is supplied. */ -export function oraclePropertyOptions(baseRuns: number): { +export function oraclePropertyOptions( + baseRuns: number, + property: string, +): { numRuns: number seed?: number path?: string } { - return oracleRandomParameters(oracleRuns(baseRuns), seed, path) + return oracleRandomParameters(oracleRuns(baseRuns), replay, property) } diff --git a/packages/db/tests/query/coverage-registry-oracle.property.test.ts b/packages/db/tests/query/coverage-registry-oracle.property.test.ts index 8c2e8fc25..72d931255 100644 --- a/packages/db/tests/query/coverage-registry-oracle.property.test.ts +++ b/packages/db/tests/query/coverage-registry-oracle.property.test.ts @@ -1329,7 +1329,7 @@ describe(`coverage registry oracle`, () => { fcTest.prop( [claimChurnArbitrary, fc.integer({ min: 1, max: 8 })], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `coverage-registry.claim-churn`), )(`bounds long claim churn for a random or replayed seed`, runClaimChurn) it(`restores a compacted narrower fact when the wider acquisition retires`, () => { @@ -1985,7 +1985,7 @@ describe(`coverage registry oracle`, () => { maxCommands: 40, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions(100, `coverage-registry.state-machine`), )( `matches the lease, retry, settlement, publication, ownership, and disposal state machine`, (commands) => { diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 5d8c2f45a..6cf697bd9 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -326,7 +326,10 @@ const exhaustiveActions: ReadonlyArray = [ ] describe(`Collection-valued includes oracle`, () => { - fcTest.prop([collectionScenarioArbitrary], oraclePropertyOptions(30))( + fcTest.prop( + [collectionScenarioArbitrary], + oraclePropertyOptions(30, `includes-collection.relationship-history`), + )( `keeps Collection, toArray, and materialize equivalent across generated relationship histories`, ({ parentGroup, childValue, actions }) => runTrace({ @@ -1107,7 +1110,7 @@ describe(`Collection-valued includes oracle`, () => { wideId: fc.integer({ min: 10, max: 19 }), }), ], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `includes-collection.public-key-order`), )( `uses one raw public-key order across Collection and inline materializations`, async ({ smallId, wideId }) => { @@ -1770,7 +1773,7 @@ describe(`Collection-valued includes oracle`, () => { value: fc.integer({ min: -10, max: 10 }), }), ], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `includes-collection.optimistic-child-history`), )( `matches recomputation through optimistic child insert and delete confirmation and rollback`, async ({ group, insertedId, confirmedId, value }) => { diff --git a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts index 850aeb02c..7770956fa 100644 --- a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts +++ b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts @@ -497,12 +497,18 @@ describe(`includes cross-formulation oracle`, () => { }), ) - fcTest.prop([scenarioArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(8, `includes-cross-formulation.equivalence`), + )( `agrees across nested includes, flat joins, per-parent queries, and TLP partitions`, expectFormulationsEquivalent, ) - fcTest.prop([windowedScenarioArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [windowedScenarioArbitrary], + oraclePropertyOptions(12, `includes-cross-formulation.ordered-window`), + )( `matches recomputation for ordered offset and limit child windows`, ({ scenario, offset, limit }) => expectWindowedIncludeMatches(scenario, offset, limit), diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 0e125cdfc..23b4bf762 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -499,7 +499,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.rekey-detach`), + )( `an optimistic rekey detaches its old descendants immediately`, async (routes) => { await expectHistoryMatches(routes, [ @@ -514,7 +517,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.rekey-rollback`), + )( `restores the authoritative relationship after an optimistic rekey rolls back`, async (routes) => { await expectHistoryMatches(routes, [ @@ -528,7 +534,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.descendant-rollback`), + )( `rolls back a descendant update made while its ancestor is reparented`, async (routes) => { await expectHistoryMatches(routes, [ @@ -552,7 +561,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.ancestor-rollback`), + )( `rolls back a reparented ancestor while its descendant update remains pending`, async (routes) => { await expectHistoryMatches(routes, [ @@ -576,7 +588,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.confirm-same-route`), + )( `settles a confirmed optimistic reparent on the same authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -614,7 +629,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.confirm-different-route`), + )( `settles a confirmed optimistic reparent on a different authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -654,100 +672,100 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( - `restores a rekey after a sibling enters its old route`, - async (routes) => { - await expectHistoryMatches(routes, [ - { - type: `optimisticRollback`, + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.sibling-route-rollback`), + )(`restores a rekey after a sibling enters its old route`, async (routes) => { + await expectHistoryMatches(routes, [ + { + type: `optimisticRollback`, + level: 1, + id: 11, + patch: { group: routes.optimistic }, + beforeRollback: { level: 1, - id: 11, - patch: { group: routes.optimistic }, - beforeRollback: { - level: 1, - changes: [ - { - type: `insert`, - value: { - id: 12, - parentGroup: routes.rootA, - group: routes.original, - value: 120, - position: 1, - }, - }, - ], - }, - }, - { - type: `sync`, - level: 2, changes: [ { - type: `update`, + type: `insert`, value: { - id: 21, - parentGroup: routes.original, - group: routes.original + 1000, - value: 211, - position: 0, + id: 12, + parentGroup: routes.rootA, + group: routes.original, + value: 120, + position: 1, }, }, ], }, - ]) - }, - ) + }, + { + type: `sync`, + level: 2, + changes: [ + { + type: `update`, + value: { + id: 21, + parentGroup: routes.original, + group: routes.original + 1000, + value: 211, + position: 0, + }, + }, + ], + }, + ]) + }) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( - `supports repeated rollback and confirmation histories`, - async (routes) => { - await expectHistoryMatches(routes, [ - { - type: `optimistic`, - handle: `first`, - level: 1, - id: 11, - patch: { parentGroup: routes.rootB }, - }, - { type: `rollback`, handle: `first` }, - { - type: `optimistic`, - handle: `second`, - level: 1, - id: 11, - patch: { parentGroup: routes.rootB }, - }, - { - type: `confirm`, - handle: `second`, - authoritative: firstChild(routes, { - parentGroup: routes.rootB, - }), - }, - { - type: `optimisticRollback`, - level: 1, - id: 11, - patch: { group: routes.optimistic }, - }, - { - type: `sync`, - level: 2, - changes: [ - { - type: `update`, - value: { - id: 21, - parentGroup: routes.original, - group: routes.original + 1000, - value: 212, - position: 0, - }, + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.repeated-history`), + )(`supports repeated rollback and confirmation histories`, async (routes) => { + await expectHistoryMatches(routes, [ + { + type: `optimistic`, + handle: `first`, + level: 1, + id: 11, + patch: { parentGroup: routes.rootB }, + }, + { type: `rollback`, handle: `first` }, + { + type: `optimistic`, + handle: `second`, + level: 1, + id: 11, + patch: { parentGroup: routes.rootB }, + }, + { + type: `confirm`, + handle: `second`, + authoritative: firstChild(routes, { + parentGroup: routes.rootB, + }), + }, + { + type: `optimisticRollback`, + level: 1, + id: 11, + patch: { group: routes.optimistic }, + }, + { + type: `sync`, + level: 2, + changes: [ + { + type: `update`, + value: { + id: 21, + parentGroup: routes.original, + group: routes.original + 1000, + value: 212, + position: 0, }, - ], - }, - ]) - }, - ) + }, + ], + }, + ]) + }) }) diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index 8fb762587..a1e4ec201 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -281,7 +281,7 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { fc.statistics( scenarioArbitrary, classifyScenarioCoverage, - oraclePropertyOptions(1_000), + oraclePropertyOptions(1_000, `includes.scenario-statistics`), ) } @@ -4295,7 +4295,10 @@ describe(`includes recompute oracle`, () => { }) }) - fcTest.prop([scenarioArbitrary], oraclePropertyOptions(40))( + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(40, `includes.incremental-history`), + )( `matches naive recomputation after every incremental change`, expectScenarioMatches, ) @@ -4306,7 +4309,7 @@ describe(`includes recompute oracle`, () => { ({ sharedIntermediate }) => !sharedIntermediate, ), ], - oraclePropertyOptions(30), + oraclePropertyOptions(30, `includes.nested-scalar-materialization`), )( `matches recomputation for nested scalar materialization`, expectMaterializeScenarioMatches, @@ -4334,7 +4337,7 @@ describe(`includes recompute oracle`, () => { { selector: (row) => row.id, maxLength: 7 }, ), ], - oraclePropertyOptions(25), + oraclePropertyOptions(25, `includes.alpha-renaming`), )( `is unchanged by alpha-renaming, sibling declaration order, or an unrelated sibling`, async (rootRows, childRows) => { @@ -4446,7 +4449,7 @@ describe(`includes recompute oracle`, () => { fcTest.prop( [fc.integer({ min: -5, max: 5 }).filter((value) => value !== 0)], - oraclePropertyOptions(15), + oraclePropertyOptions(15, `includes.optimistic-convergence`), )( `optimistic updates converge to confirmed-only state`, async (confirmedValue) => { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index e78656d70..2e59db03e 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -413,7 +413,13 @@ describe(`layered-query publication oracle`, () => { for (const q1Shape of q1Shapes) { for (const q2Shape of q2Shapes) { - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 12, + `includes-publication.parent-scalar.${q1Shape}.${q2Shape}`, + ), + )( `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -427,7 +433,10 @@ describe(`layered-query publication oracle`, () => { fcTest.prop( [changedValueArbitrary, changedChildValueArbitrary], - oraclePropertyOptions(12), + oraclePropertyOptions( + 12, + `includes-publication.parent-then-child.${q1Shape}.${q2Shape}`, + ), )( `recovers a ${q1Shape} Q1 and ${q2Shape} Q2 after a child update`, async (parentValue, childValue) => { @@ -440,7 +449,13 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 8, + `includes-publication.optimistic-before-confirm.${q1Shape}.${q2Shape}`, + ), + )( `publishes optimistic state before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -452,7 +467,13 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 8, + `includes-publication.optimistic-after-confirm.${q1Shape}.${q2Shape}`, + ), + )( `publishes state after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -466,19 +487,22 @@ describe(`layered-query publication oracle`, () => { } } - fcTest.prop([changedChildValueArbitrary], oraclePropertyOptions(100))( + fcTest.prop( + [changedChildValueArbitrary], + oraclePropertyOptions(100, `includes-publication.child-scalar`), + )( `publishes child-only scalar updates through both layers`, async (value) => { await expectPublicationMatches({ type: `childScalar`, value }) }, ) - fcTest.prop([fc.constantFrom(20, 30)], oraclePropertyOptions(100))( - `compares route transitions at both query layers`, - async (group) => { - await expectPublicationMatches({ type: `parentRoute`, group }) - }, - ) + fcTest.prop( + [fc.constantFrom(20, 30)], + oraclePropertyOptions(100, `includes-publication.parent-route`), + )(`compares route transitions at both query layers`, async (group) => { + await expectPublicationMatches({ type: `parentRoute`, group }) + }) fcTest.prop( [ @@ -487,18 +511,21 @@ describe(`layered-query publication oracle`, () => { value: changedValueArbitrary, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions( + 100, + `includes-publication.atomic-parent-replacement`, + ), )(`compares atomic parent replacements at both query layers`, async (row) => { await expectPublicationMatches({ type: `atomicReplace`, ...row }) }) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(100))( - `publishes restored state after optimistic rollback`, - async (value) => { - await expectPublicationMatches({ - type: `optimisticRollback`, - value, - }) - }, - ) + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions(100, `includes-publication.optimistic-rollback`), + )(`publishes restored state after optimistic rollback`, async (value) => { + await expectPublicationMatches({ + type: `optimisticRollback`, + value, + }) + }) }) diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 649dccd49..87ec2ffa4 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -1196,7 +1196,10 @@ describe(`includes temporal oracle`, () => { expectObsoleteDemandCannotPublishAfterReactivation, ) - fcTest.prop([fc.scheduler()], oraclePropertyOptions(20))( + fcTest.prop( + [fc.scheduler()], + oraclePropertyOptions(20, `includes-temporal.demand-scheduling`), + )( `obsolete and current demand completions are generation-safe in either order`, expectScheduledDemandCompletionsStayGenerationSafe, ) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index c52a53d8d..2c73610a1 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -153,11 +153,8 @@ it(`loads each side of a filtered inner join once`, async () => { } }) -const { - multiplier: fullFlowMultiplier, - replaySeed: fullFlowReplaySeed, - replayPath: fullFlowReplayPath, -} = readOracleRunConfig() +const { multiplier: fullFlowMultiplier, ...fullFlowReplay } = + readOracleRunConfig() type MultiSourceOrderedScenario = { primaryRows: ReadonlyArray<{ @@ -303,7 +300,11 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { `multiplicity=${new Set(secondaryRows.map(({ joinKey }) => joinKey)).size < secondaryRows.length}`, `tied=${new Set(primaryRows.map(({ rank }) => rank)).size < primaryRows.length}`, ], - oracleRandomParameters(1_000, fullFlowReplaySeed, fullFlowReplayPath), + oracleRandomParameters( + 1_000, + fullFlowReplay, + `load-subset-full-flow.multi-source-statistics`, + ), ) } @@ -2579,8 +2580,8 @@ fcTest.prop( [multiSourceOrderedScenarioArbitrary], oracleRandomParameters( 12 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.multi-source-ordered`, ), )( `fills joined ordered windows for a random or replayed seed`, @@ -4162,8 +4163,8 @@ fcTest.prop( [orderedConsumerParityScenarioArbitrary], oracleRandomParameters( 12 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.consumer-parity`, ), )( `keeps ordered collection consumers equal for a random or replayed seed`, @@ -4820,7 +4821,11 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { )}`, `exhaustion=${pages.some((page) => page.extent === `exhausted`)}`, ], - oracleRandomParameters(1_000, fullFlowReplaySeed, fullFlowReplayPath), + oracleRandomParameters( + 1_000, + fullFlowReplay, + `load-subset-full-flow.continuation-statistics`, + ), ) } @@ -5093,8 +5098,8 @@ fcTest.prop( ], oracleRandomParameters( 128 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.automatic-progress`, ), )( `starts automatic continuation only for new semantic progress with a random or replayed seed`, @@ -5113,8 +5118,8 @@ fcTest.prop( [orderedContinuationEvidenceScenarioArbitrary], oracleRandomParameters( 64 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.continuation-evidence`, ), )( `derives ordered progress from applied eligible evidence for a random or replayed seed`, @@ -5448,8 +5453,8 @@ fcTest.prop( [orderedBoundaryProvenanceArbitrary], oracleRandomParameters( 32 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.boundary-provenance`, ), )( `keeps ordered boundary provenance for a random or replayed seed`, @@ -6368,8 +6373,8 @@ fcTest.prop( [atomicOrderedReplayArbitrary], oracleRandomParameters( 32 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.atomic-replacement`, ), )( `keeps ordered replacement publication atomic for a random or replayed seed`, @@ -6391,8 +6396,8 @@ fcTest.prop( [truncateCoverageScenarioArbitrary], oracleRandomParameters( 12 * fullFlowMultiplier, - fullFlowReplaySeed, - fullFlowReplayPath, + fullFlowReplay, + `load-subset-full-flow.truncate-evidence`, ), )( `fences pre-truncate evidence for a random or replayed seed`, diff --git a/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts b/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts index 433881dde..8dc4e634b 100644 --- a/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts @@ -348,7 +348,7 @@ fcTest.prop( maxCommands: 20, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions(100, `load-subset-lifecycle.state-machine`), )( `matches the scheduled acquisition, coverage, release, teardown, and stale-settlement lifecycle`, (commands) => { diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index ac978a0af..5b5dada62 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -1119,13 +1119,10 @@ async function runAsyncScenarioWithKnownFailures( } } -const { multiplier, replaySeed, replayPath } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() const coverageScenarioRuns = 40 * multiplier -const coverageRandomParameters = oracleRandomParameters( - coverageScenarioRuns, - replaySeed, - replayPath, -) +const coverageRandomParameters = (property: string) => + oracleRandomParameters(coverageScenarioRuns, replay, property) let collectionSequence = 0 @@ -2707,7 +2704,10 @@ describe(`loadSubset coverage oracle`, () => { runCoverageTraceWithKnownFailures, ) - fcTest.prop([requestTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [requestTraceArbitrary], + coverageRandomParameters(`load-subset.coverage`), + )( `matches finite-domain coverage for a random or replayed seed`, runCoverageTraceWithKnownFailures, ) @@ -2720,7 +2720,10 @@ describe(`loadSubset coverage oracle`, () => { runAsyncScenarioWithKnownFailures, ) - fcTest.prop([asyncScenarioArbitrary], coverageRandomParameters)( + fcTest.prop( + [asyncScenarioArbitrary], + coverageRandomParameters(`load-subset.async-settlement`), + )( `settles, retries, and resets in-flight set requests for a random or replayed seed`, runAsyncScenarioWithKnownFailures, ) @@ -2735,7 +2738,7 @@ describe(`loadSubset coverage oracle`, () => { fcTest.prop( [concurrentAsyncScenarioArbitrary, resultWrapperModeArbitrary], - coverageRandomParameters, + coverageRandomParameters(`load-subset.concurrent-dedupe`), )( `deduplicates three or more concurrent requests for a random or replayed seed`, runConcurrentAsyncScenario, @@ -2749,7 +2752,10 @@ describe(`loadSubset coverage oracle`, () => { expectDeduplicatedWaiterHandlesRejection, ) - fcTest.prop([rejectedWaiterScenarioArbitrary], coverageRandomParameters)( + fcTest.prop( + [rejectedWaiterScenarioArbitrary], + coverageRandomParameters(`load-subset.rejected-waiter`), + )( `checks rejected requests observed by an in-flight waiter for a random or replayed seed`, expectDeduplicatedWaiterHandlesRejection, ) @@ -2762,7 +2768,10 @@ describe(`loadSubset coverage oracle`, () => { runWindowCoverageTraceWithKnownFailures, ) - fcTest.prop([windowTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [windowTraceArbitrary], + coverageRandomParameters(`load-subset.ordered-window`), + )( `never treats uncovered ordered windows as loaded for a random or replayed seed`, runWindowCoverageTraceWithKnownFailures, ) @@ -2775,7 +2784,10 @@ describe(`loadSubset coverage oracle`, () => { runWindowCoverageTraceWithKnownFailures, ) - fcTest.prop([changingWhereWindowTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [changingWhereWindowTraceArbitrary], + coverageRandomParameters(`load-subset.changing-predicate`), + )( `keeps changing predicates distinct across window histories for a random or replayed seed`, runWindowCoverageTraceWithKnownFailures, ) @@ -2788,7 +2800,10 @@ describe(`loadSubset coverage oracle`, () => { expectDistinctWhereStartsDistinctLimitedWindowLoads, ) - fcTest.prop([distinctWindowWherePairArbitrary], coverageRandomParameters)( + fcTest.prop( + [distinctWindowWherePairArbitrary], + coverageRandomParameters(`load-subset.distinct-window-predicate`), + )( `keeps distinct limited-window predicates separate for a random or replayed seed`, expectDistinctWhereStartsDistinctLimitedWindowLoads, ) diff --git a/packages/db/tests/query/load-subset-projection-oracle.property.test.ts b/packages/db/tests/query/load-subset-projection-oracle.property.test.ts index d17a396b4..ca721343c 100644 --- a/packages/db/tests/query/load-subset-projection-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-projection-oracle.property.test.ts @@ -256,7 +256,10 @@ const projectionScenarioArbitrary = fc return { sourceSize, callerOffset, callerLimit } }) -fcTest.prop([projectionScenarioArbitrary], oraclePropertyOptions(50))( +fcTest.prop( + [projectionScenarioArbitrary], + oraclePropertyOptions(50, `load-subset-projection.state-equivalence`), +)( `projects covering exhaustion relative to a finite source world`, async ({ sourceSize, callerOffset, callerLimit }) => { const rows = Array.from({ length: sourceSize }, (_, id) => ({ id })) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index dbfb29ed5..4135d272f 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -33,7 +33,7 @@ function refinementCampaigns(fixedSeed: number) { }, { label: `random or replayed seed`, - options: oraclePropertyOptions(50), + options: oraclePropertyOptions(50, `load-subset-refinement.${fixedSeed}`), }, ] as const } diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index c86a94660..7338da376 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -340,14 +340,9 @@ async function cleanupAll( if (rejection) throw rejection.reason } -const { multiplier, replaySeed, replayPath } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() const orderedScenarioRuns = 12 * multiplier const transitionScenarioRuns = 8 * multiplier -const orderedScenarioRandomParameters = oracleRandomParameters( - orderedScenarioRuns, - replaySeed, - replayPath, -) let collectionSequence = 0 @@ -2522,7 +2517,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [multiOrderScenarioArbitrary], - oracleRandomParameters(orderedScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + orderedScenarioRuns, + replay, + `pagination.multi-order`, + ), )( `matches multi-column nullable ordering for a random or replayed seed`, runMultiOrderScenario, @@ -2538,7 +2537,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [nullableCursorScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.nullable-cursor`, + ), )( `matches nullable cursor ordering while an async response is pending for a random or replayed seed`, runNullableCursorScenario, @@ -2972,7 +2975,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingMutationScenarioArbitrary, responseTimingArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.pending-mutation`, + ), )( `matches recomputation when source mutations cross a pending cursor response for a random or replayed seed`, runPendingMutationScenario, @@ -2993,7 +3000,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingHistoryScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.pending-history`, + ), )( `matches recomputation across multi-action pending histories for a random or replayed seed`, runPendingHistoryScenario, @@ -3113,7 +3124,14 @@ describe(`pagination recomputation oracle`, () => { runPaginationScenario, ) - fcTest.prop([scenarioArbitrary], orderedScenarioRandomParameters)( + fcTest.prop( + [scenarioArbitrary], + oracleRandomParameters( + orderedScenarioRuns, + replay, + `pagination.ordered-window`, + ), + )( `matches full recomputation across ordered windows for a random or replayed seed`, runPaginationScenario, ) @@ -3128,7 +3146,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [stateScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.window-transition`, + ), )( `matches full recomputation across source and window transitions for a random or replayed seed`, runPaginationStateScenario, @@ -3473,7 +3495,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [scenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed, replayPath), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.async-cursor`, + ), )( `matches full recomputation when exact async cursor loads widen ordered coverage for a random or replayed seed`, runOnDemandPaginationScenario, diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index 4e4b76289..e20ff5c00 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -11,8 +11,14 @@ describe(`oracle run configuration`, () => { TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: `100`, TANSTACK_DB_ORACLE_SEED: `-42`, TANSTACK_DB_ORACLE_PATH: `1:0:2`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, }), - ).toEqual({ multiplier: 100, replaySeed: -42, replayPath: `1:0:2` }) + ).toEqual({ + multiplier: 100, + replaySeed: -42, + replayPath: `1:0:2`, + replayProperty: `coverage.claim-churn`, + }) }) it(`uses one run multiplier and no replay coordinates by default`, () => { @@ -20,6 +26,7 @@ describe(`oracle run configuration`, () => { multiplier: 1, replaySeed: undefined, replayPath: undefined, + replayProperty: undefined, }) }) @@ -30,6 +37,29 @@ describe(`oracle run configuration`, () => { [{ TANSTACK_DB_ORACLE_SEED: `1.5` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_SEED: ` ` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_PATH: `1:0` }, `requires TANSTACK_DB_ORACLE_SEED`], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: ` `, + TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, + }, + `must be non-empty`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:-1`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, + }, + `colon-separated nonnegative integers`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:0`, + }, + `requires TANSTACK_DB_ORACLE_PROPERTY`, + ], ] satisfies ReadonlyArray, string]>)( `rejects invalid environment values`, (environment, message) => { @@ -37,12 +67,30 @@ describe(`oracle run configuration`, () => { }, ) - it(`adds replay coordinates only for replay runs`, () => { - expect(oracleRandomParameters(40, undefined)).toEqual({ numRuns: 40 }) - expect(() => oracleRandomParameters(40, undefined, `1:0:2`)).toThrow( - `requires a replay seed`, + it(`adds a shrink path only to its named property`, () => { + const ordinaryRun = { + replaySeed: undefined, + replayPath: undefined, + replayProperty: undefined, + } + const replayRun = { + replaySeed: -42, + replayPath: `1:0:2`, + replayProperty: `coverage.claim-churn`, + } + + expect( + oracleRandomParameters(40, ordinaryRun, `coverage.claim-churn`), + ).toEqual({ numRuns: 40 }) + expect(oracleRandomParameters(40, replayRun, `coverage.other-law`)).toEqual( + { + numRuns: 40, + seed: -42, + }, ) - expect(oracleRandomParameters(40, -42, `1:0:2`)).toEqual({ + expect( + oracleRandomParameters(40, replayRun, `coverage.claim-churn`), + ).toEqual({ numRuns: 40, seed: -42, path: `1:0:2`, From 8a8fa424e9dd5e757932dffffd7c5e68c0ae7c38 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 00:19:20 -0600 Subject: [PATCH 120/327] test(db): reject invalid oracle replay targets --- packages/db/tests/oracle-config.ts | 115 +++++++++++++++++++++++++++++ packages/db/tests/utils.test.ts | 59 +++++++++++---- 2 files changed, 160 insertions(+), 14 deletions(-) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index a541e6f9e..2faa04245 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -1,5 +1,108 @@ type OracleEnvironment = Record +const staticOracleProperties = [ + `collection-sync.reentrant-drain`, + `coverage-registry.claim-churn`, + `coverage-registry.state-machine`, + `includes-collection.optimistic-child-history`, + `includes-collection.public-key-order`, + `includes-collection.relationship-history`, + `includes-cross-formulation.equivalence`, + `includes-cross-formulation.ordered-window`, + `includes-optimistic.ancestor-rollback`, + `includes-optimistic.confirm-different-route`, + `includes-optimistic.confirm-same-route`, + `includes-optimistic.descendant-rollback`, + `includes-optimistic.rekey-detach`, + `includes-optimistic.rekey-rollback`, + `includes-optimistic.repeated-history`, + `includes-optimistic.sibling-route-rollback`, + `includes-publication.atomic-parent-replacement`, + `includes-publication.child-scalar`, + `includes-publication.optimistic-rollback`, + `includes-publication.parent-route`, + `includes-temporal.demand-scheduling`, + `includes.alpha-renaming`, + `includes.incremental-history`, + `includes.nested-scalar-materialization`, + `includes.optimistic-convergence`, + `includes.scenario-statistics`, + `load-subset-full-flow.atomic-replacement`, + `load-subset-full-flow.automatic-progress`, + `load-subset-full-flow.boundary-provenance`, + `load-subset-full-flow.consumer-parity`, + `load-subset-full-flow.continuation-evidence`, + `load-subset-full-flow.continuation-statistics`, + `load-subset-full-flow.multi-source-ordered`, + `load-subset-full-flow.multi-source-statistics`, + `load-subset-full-flow.truncate-evidence`, + `load-subset-lifecycle.state-machine`, + `load-subset-projection.state-equivalence`, + `load-subset.async-settlement`, + `load-subset.changing-predicate`, + `load-subset.concurrent-dedupe`, + `load-subset.coverage`, + `load-subset.distinct-window-predicate`, + `load-subset.ordered-window`, + `load-subset.rejected-waiter`, + `pagination.async-cursor`, + `pagination.multi-order`, + `pagination.nullable-cursor`, + `pagination.ordered-window`, + `pagination.pending-history`, + `pagination.pending-mutation`, + `pagination.window-transition`, + `subscription-replay.completion`, + `subscription-replay.optimistic`, + `subscription-replay.ownership`, + `subscription-replay.restart`, + `subscription-replay.sequential`, + `subscription-replay.shared`, +] as const + +const publicationProperties = [ + `parent-scalar`, + `parent-then-child`, + `optimistic-before-confirm`, + `optimistic-after-confirm`, +].flatMap((law) => + [`direct`, `joined`].flatMap((q1Shape) => + [`passThrough`, `where`, `orderBy`, `select`].map( + (q2Shape) => `includes-publication.${law}.${q1Shape}.${q2Shape}`, + ), + ), +) + +const refinementProperties = Array.from( + { length: 9 }, + (_, index) => `load-subset-refinement.${1_779_001 + index}`, +) + +export function validateOraclePropertyRegistry( + properties: ReadonlyArray, +): ReadonlySet { + const registry = new Set() + for (const property of properties) { + if (registry.has(property)) { + throw new Error(`duplicate oracle property: ${property}`) + } + registry.add(property) + } + return registry +} + +const registeredOracleProperties = validateOraclePropertyRegistry([ + ...staticOracleProperties, + ...publicationProperties, + ...refinementProperties, +]) + +function assertRegisteredOracleProperty(property: string): void { + if (!registeredOracleProperties.has(property)) { + throw new Error(`unknown oracle property: ${property}`) + } +} + export type OracleReplayConfig = { replaySeed: number | undefined replayPath: string | undefined @@ -30,6 +133,11 @@ export function readOracleRunConfig( `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_SEED`, ) } + if (replayProperty !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PROPERTY requires TANSTACK_DB_ORACLE_PATH`, + ) + } return { multiplier, replaySeed: undefined, @@ -43,6 +151,11 @@ export function readOracleRunConfig( throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) } if (replayPath === undefined) { + if (replayProperty !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PROPERTY requires TANSTACK_DB_ORACLE_PATH`, + ) + } return { multiplier, replaySeed, @@ -63,6 +176,7 @@ export function readOracleRunConfig( `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_PROPERTY`, ) } + assertRegisteredOracleProperty(replayProperty) return { multiplier, replaySeed, replayPath, replayProperty } } @@ -71,6 +185,7 @@ export function oracleRandomParameters( replay: OracleReplayConfig, property: string, ): { numRuns: number; seed?: number; path?: string } { + assertRegisteredOracleProperty(property) const { replaySeed, replayPath, replayProperty } = replay if (replaySeed === undefined) return { numRuns } return { diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index e20ff5c00..ae46c102d 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -2,7 +2,11 @@ import { describe, expect, it } from 'vitest' import { Temporal } from 'temporal-polyfill' import { deepEquals } from '../src/utils' import { isPromiseLike } from '../src/utils/type-guards' -import { oracleRandomParameters, readOracleRunConfig } from './oracle-config' +import { + oracleRandomParameters, + readOracleRunConfig, + validateOraclePropertyRegistry, +} from './oracle-config' describe(`oracle run configuration`, () => { it(`reads the multiplier and replay coordinates from an explicit environment`, () => { @@ -11,13 +15,13 @@ describe(`oracle run configuration`, () => { TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: `100`, TANSTACK_DB_ORACLE_SEED: `-42`, TANSTACK_DB_ORACLE_PATH: `1:0:2`, - TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, }), ).toEqual({ multiplier: 100, replaySeed: -42, replayPath: `1:0:2`, - replayProperty: `coverage.claim-churn`, + replayProperty: `coverage-registry.claim-churn`, }) }) @@ -37,11 +41,17 @@ describe(`oracle run configuration`, () => { [{ TANSTACK_DB_ORACLE_SEED: `1.5` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_SEED: ` ` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_PATH: `1:0` }, `requires TANSTACK_DB_ORACLE_SEED`], + [ + { + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `requires TANSTACK_DB_ORACLE_PATH`, + ], [ { TANSTACK_DB_ORACLE_SEED: `42`, TANSTACK_DB_ORACLE_PATH: ` `, - TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, }, `must be non-empty`, ], @@ -49,7 +59,7 @@ describe(`oracle run configuration`, () => { { TANSTACK_DB_ORACLE_SEED: `42`, TANSTACK_DB_ORACLE_PATH: `1:-1`, - TANSTACK_DB_ORACLE_PROPERTY: `coverage.claim-churn`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, }, `colon-separated nonnegative integers`, ], @@ -60,6 +70,21 @@ describe(`oracle run configuration`, () => { }, `requires TANSTACK_DB_ORACLE_PROPERTY`, ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:0`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.typo`, + }, + `unknown oracle property`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `requires TANSTACK_DB_ORACLE_PATH`, + ], ] satisfies ReadonlyArray, string]>)( `rejects invalid environment values`, (environment, message) => { @@ -67,6 +92,12 @@ describe(`oracle run configuration`, () => { }, ) + it(`rejects duplicate registered property names`, () => { + expect(() => + validateOraclePropertyRegistry([`one.property`, `one.property`]), + ).toThrow(`duplicate oracle property`) + }) + it(`adds a shrink path only to its named property`, () => { const ordinaryRun = { replaySeed: undefined, @@ -76,20 +107,20 @@ describe(`oracle run configuration`, () => { const replayRun = { replaySeed: -42, replayPath: `1:0:2`, - replayProperty: `coverage.claim-churn`, + replayProperty: `coverage-registry.claim-churn`, } expect( - oracleRandomParameters(40, ordinaryRun, `coverage.claim-churn`), + oracleRandomParameters(40, ordinaryRun, `coverage-registry.claim-churn`), ).toEqual({ numRuns: 40 }) - expect(oracleRandomParameters(40, replayRun, `coverage.other-law`)).toEqual( - { - numRuns: 40, - seed: -42, - }, - ) expect( - oracleRandomParameters(40, replayRun, `coverage.claim-churn`), + oracleRandomParameters(40, replayRun, `coverage-registry.state-machine`), + ).toEqual({ + numRuns: 40, + seed: -42, + }) + expect( + oracleRandomParameters(40, replayRun, `coverage-registry.claim-churn`), ).toEqual({ numRuns: 40, seed: -42, From c06a6dac29c55215ad4c0e8ebc2fb99e2f3bba8b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 00:24:05 -0600 Subject: [PATCH 121/327] docs(db): define demand evidence boundaries --- packages/db/src/query/live/ARCHITECTURE.md | 105 +++++++++++++++++---- 1 file changed, 89 insertions(+), 16 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index bd2cbaca8..80cac2d4c 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -794,6 +794,24 @@ type DemandSet = readonly [ ] ``` +### Demand facts + +The demand plane keeps these facts separate. One fact may justify creating the +next, but none is an alias for another. + +| Fact | Meaning | What it does not prove | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Demand snapshot | Immutable semantic work requested by one caller | That any source work started or any row arrived | +| Logical lease | One active owner of that demand | That it owns a distinct physical request | +| Physical acquisition | One exact adapter attempt, signal, options snapshot, and settlement | That its requested region was applied or is reusable | +| Applied outcome | Extent and row keys established by that acquisition after its writes became visible | Coverage for a different demand or generation | +| Coverage fact | Caller-relative proof that an applied outcome satisfies a demand | Row lifetime or consumer publication | +| Row ownership | Acquisition support for applied row keys | Ordered-prefix membership or public visibility | +| Publication snapshot | The last complete reader-visible rows and ordered boundary | Current private progress, request extent, or source ownership | + +Consumer-local scheduling, loading state, and error state are observations over +these facts. They are not extra coverage or ownership facts. + One request may cover many buckets, and the adapter may coalesce or reuse requests according to the compiled demand plan. A coalesced request has one shared abort lease. If one owner releases its lease, the source request remains @@ -863,6 +881,8 @@ Those buckets no longer participate in readiness and cannot receive rows through routes that no longer exist. Sharing source work never merges the route rows themselves. +### Adapter obligations + The source contract stays abstract: a demand request eventually establishes one coherent baseline and identifies when that baseline is complete. Each request receives an `AbortSignal`. Cancellation is cooperative at this source @@ -873,6 +893,17 @@ adapter from writing after it ignores that signal. Buffering, snapshot tokens, shape offsets, Collection transactions, and local indexes are source-specific ways to satisfy that contract; they are not materializer state. +A conforming adapter must: + +- treat the received options snapshot and signal as one exact acquisition; +- honor cancellation immediately before publishing request-scoped rows; +- await or return every applied receipt which establishes its result; +- report only row keys established by that acquisition and report source extent + only when it knows it authoritatively; +- make every supplied release callback idempotent and non-throwing, and return + the paired `unloadSubset` callback when it keeps dedupe state across + lifetimes. + Every sync `commit()` returns an applied receipt: `true` when that transaction's writes and events are already visible, or a promise when the transaction is parked in the causal queue. The promise resolves only after the @@ -979,6 +1010,26 @@ waiting on the preload. Use an adapter's documented mutation acknowledgement helper instead; it must confirm the optimistic write without starting new collection demand. +### Conservative fallbacks + +When evidence is missing, core chooses less reuse or more source work instead +of guessing: + +- an omitted outcome or unknown extent proves no reusable coverage; +- requested limits and current Collection rows never stand in for applied row + evidence; +- an order that lacks an expressible total boundary, exact collation, or usable + range index loads the full filtered source region and lets D2 refine it; +- an unsupported demand value fails before retention instead of receiving a + lossy snapshot or identity; +- a throwing release keeps its lease, acquisition, coverage, and row ownership + as retryable cleanup debt; +- automatic continuation stops when it makes no semantic progress and resumes + only after demand or authoritative evidence changes. + +These fallbacks may cost work or delay reclamation. They must not change the +query result, invent coverage, or expose a private replacement publication. + This project uses a single graph-run order rather than multi-dimensional timely-dataflow frontiers. Do not introduce a general timestamp or frontier framework unless a source contract proves that the generation and up-to-date @@ -1104,6 +1155,15 @@ create recursive Collection machinery. - **Hydration:** establishing an initial snapshot before forwarding later changes. - **Generation:** a token that rejects obsolete asynchronous work. +- **Demand snapshot:** an immutable description of work requested by one + logical caller. +- **Logical lease:** one active owner of a demand. +- **Physical acquisition:** one exact adapter attempt and its settlement. +- **Applied outcome:** the source extent and row keys established by one + acquisition after its writes become visible. +- **Coverage fact:** caller-relative proof that applied evidence satisfies a + demand. +- **Row ownership:** the acquisition support that keeps applied row keys alive. - **Source extent:** an authoritative source fact that more rows continue past an exact demand, that the source is exhausted there, or that neither is known. - **Collection facade:** a stable public Collection view shared by the parents @@ -1129,20 +1189,22 @@ create recursive Collection machinery. | Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | | Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | | End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | -| Subset acquisition, readiness, receipt, and replay refinement laws | `packages/db/tests/query/load-subset-refinement-*.property.test.ts` | -| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.property.test.ts` | +| Shared subset acquisition, readiness, receipt, and replay interpreter | `packages/db/tests/query/load-subset-refinement-model.property.test.ts` | +| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.test.ts` | | Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | | Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | ### Oracle family boundary -The shared load-subset refinement model begins after relational evaluation. It -may vary opaque source topology, demand relationships, already-evaluated result -contributions, and public window state. It owns asynchronous demand, applied -evidence, row support, coverage, publication, source progress, and resource -work. It must not interpret query IR, weighted deltas, predicates, joins, -grouping, ordering, or nested materialization. +The shared load-subset refinement model begins after relational evaluation. Its +closed event grammar varies opaque source topology, demand relationships, +already-evaluated result contributions, public window state, settlement, +release, and teardown. It owns asynchronous demand, applied evidence, row +support, coverage, publication, source progress, and resource work. It must not +interpret query IR, weighted deltas, predicates, joins, grouping, ordering, or +nested materialization. A new regression must reduce to this grammar or justify +a grammar change; it must not add a one-off event named after the bug. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production @@ -1151,19 +1213,30 @@ compare lazy demand and source progress with a small refinement projection. It must not copy those paths into a second relational engine inside the shared model. -Each oracle identifies the first divergent checkpoint and compares either the -whole result or one exact structural difference. Correlated-materialization -scenarios use direct assertions. A boundary suite may retain an exact -expected-failure guard for a planner or ownership defect that this graph does -not own. +Each model law has its own projection instead of one monolithic expected-state +reducer. Each oracle identifies the first divergent checkpoint and compares +either the whole result or one exact structural difference. +Correlated-materialization scenarios use direct assertions. A boundary suite +may retain an exact expected-failure guard for a planner or ownership defect +that this graph does not own. Run the DB oracle set with `pnpm test:oracles` from `packages/db`. Broad properties use FastCheck's random seed, while structural matrices keep fixed seeds so each run covers the same named cells. Increase both corpora with `TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve FastCheck's -reported seed and shrink path while reducing a failure. Replay a broad -campaign with `TANSTACK_DB_ORACLE_SEED= pnpm test:oracles`, then add the -smallest case as a deterministic regression trace. +reported property key, seed, and shrink path while reducing a failure. Replay +one exact property with: + +```sh +TANSTACK_DB_ORACLE_PROPERTY= \ +TANSTACK_DB_ORACLE_SEED= \ +TANSTACK_DB_ORACLE_PATH= \ +pnpm test:oracles +``` + +The replay registry rejects partial, unknown, stale, and duplicate property +coordinates. A seed without a property and path still runs the broad campaign. +After shrinking, add the smallest case as a deterministic regression trace. The broad relationship history changes correlation keys rather than freezing them. Set `TANSTACK_DB_ORACLE_STATISTICS=1` to print its generated depth, From e5b97080198d665115ddc0c248d34e4ea7bcbf2b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 00:29:08 -0600 Subject: [PATCH 122/327] docs(db): correct oracle architecture contracts --- packages/db/src/query/live/ARCHITECTURE.md | 47 +++++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 80cac2d4c..829a933e6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -873,10 +873,10 @@ options. Its semantic contract is: -> Every active, satisfiable bucket must be covered by a settled current demand -> request before initial preload completes. +> Every active, satisfiable bucket must have its current demand load settle +> before initial preload completes. -A request may remain in flight after some covered buckets become inactive. +A request may remain in flight after some buckets it targeted become inactive. Those buckets no longer participate in readiness and cannot receive rows through routes that no longer exist. Sharing source work never merges the route rows themselves. @@ -1035,12 +1035,13 @@ timely-dataflow frontiers. Do not introduce a general timestamp or frontier framework unless a source contract proves that the generation and up-to-date protocol cannot express its ordering. -**Initial readiness:** preload is complete when every demand currently -reachable from the initial query graph is covered by a settled request. Demand -that is no longer reachable does not block completion. An empty outer relation -has no child demand, but its root demand must still settle. Later readiness -transitions follow the existing Collection contract until an executable test -defines another public behavior. +**Initial readiness:** preload is complete when the current load for every +demand reachable from the initial query graph has settled. An outcome-free load +may settle readiness without proving reusable coverage. Demand that is no +longer reachable does not block completion. An empty outer relation has no +child demand, but its root demand must still settle. Later readiness transitions +follow the existing Collection contract until an executable test defines +another public behavior. Pending demand does not hide the parent row. An active empty bucket gives it the current canonical bucket value, and available partial source rows produce @@ -1125,8 +1126,8 @@ create recursive Collection machinery. materialized output relation of its children. 9. **Publication:** reads, events, and downstream queries observe the same complete graph result. -10. **Initial demand:** preload completes when every initially reachable demand - is covered; obsolete demand does not block it. +10. **Initial demand:** preload completes when the current load for every + initially reachable demand settles; obsolete demand does not block it. 11. **Ownership:** a query-db row exists exactly while an explicit owner remains. 12. **Work:** irrelevant rows do not cause unrelated scans or activate unrelated @@ -1164,6 +1165,8 @@ create recursive Collection machinery. - **Coverage fact:** caller-relative proof that applied evidence satisfies a demand. - **Row ownership:** the acquisition support that keeps applied row keys alive. +- **Publication snapshot:** the last complete reader-visible rows and ordered + boundary. - **Source extent:** an authoritative source fact that more rows continue past an exact demand, that the source is exhausted there, or that neither is known. - **Collection facade:** a stable public Collection view shared by the parents @@ -1202,9 +1205,11 @@ closed event grammar varies opaque source topology, demand relationships, already-evaluated result contributions, public window state, settlement, release, and teardown. It owns asynchronous demand, applied evidence, row support, coverage, publication, source progress, and resource work. It must not -interpret query IR, weighted deltas, predicates, joins, grouping, ordering, or -nested materialization. A new regression must reduce to this grammar or justify -a grammar change; it must not add a one-off event named after the bug. +interpret query IR, weighted deltas, predicates, joins, grouping, query-level +ordering, or nested materialization. It may project already-evaluated total-order +coordinates and public window state. A new regression must reduce to this +grammar or justify a grammar change; it must not add a one-off event named after +the bug. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production @@ -1223,9 +1228,9 @@ that this graph does not own. Run the DB oracle set with `pnpm test:oracles` from `packages/db`. Broad properties use FastCheck's random seed, while structural matrices keep fixed seeds so each run covers the same named cells. Increase both corpora with -`TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve FastCheck's -reported property key, seed, and shrink path while reducing a failure. Replay -one exact property with: +`TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve the oracle +property key beside FastCheck's reported seed and shrink path while reducing a +failure. Replay one exact property with: ```sh TANSTACK_DB_ORACLE_PROPERTY= \ @@ -1234,9 +1239,11 @@ TANSTACK_DB_ORACLE_PATH= \ pnpm test:oracles ``` -The replay registry rejects partial, unknown, stale, and duplicate property -coordinates. A seed without a property and path still runs the broad campaign. -After shrinking, add the smallest case as a deterministic regression trace. +The replay registry rejects partial or unknown coordinates and duplicate +registered names. Its static inventory must stay equal to the property helper +call sites; a missing registration fails at the helper boundary. A seed without +a property and path still runs the broad campaign. After shrinking, add the +smallest case as a deterministic regression trace. The broad relationship history changes correlation keys rather than freezing them. Set `TANSTACK_DB_ORACLE_STATISTICS=1` to print its generated depth, From 3477f24ba3b73544b6108ac64572ef57d06ba039 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 01:45:27 -0600 Subject: [PATCH 123/327] fix(db): terminate nested predicate differences --- packages/db/src/query/predicate-utils.ts | 30 +++++-------------- .../db/tests/query/predicate-utils.test.ts | 18 +++++++++++ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index 3241f9e55..bff11e345 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -1043,29 +1043,15 @@ function removeConditions( predicate: BasicExpression, conditionsToRemove: Array>, ): BasicExpression | undefined { - if (predicate.type === `func` && predicate.name === `and`) { - const remainingArgs = predicate.args.filter( - (arg) => - !conditionsToRemove.some((cond) => - areExpressionsEqual(arg as BasicExpression, cond), - ), - ) - - if (remainingArgs.length === 0) { - return undefined - } else if (remainingArgs.length === 1) { - return remainingArgs[0]! - } else { - return { - type: `func`, - name: `and`, - args: remainingArgs, - } as BasicExpression - } - } + const remaining = extractAllConditions(predicate).filter( + (candidate) => + !conditionsToRemove.some((condition) => + areExpressionsEqual(candidate, condition), + ), + ) - // For non-AND predicates, don't remove anything - return predicate + if (remaining.length === 0) return undefined + return combineConditions(remaining) } /** diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 6471950de..cdd1c785a 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -1431,6 +1431,24 @@ describe(`minusWherePredicates`, () => { }) describe(`common conditions`, () => { + it(`removes a reordered IN condition from a nested conjunction`, () => { + const requested = inOp(ref(`score`), [2, -2, 0, -3, -1, 3]) + const alreadyLoaded = and( + inOp(ref(`score`), [0]), + and( + lt(ref(`score`), val(1)), + inOp(ref(`score`), [2, -3, -2, 3, 0, -1]), + ), + ) + + expect(minusWherePredicates(requested, alreadyLoaded)).toEqual( + and( + requested, + func(`not`, and(inOp(ref(`score`), [0]), lt(ref(`score`), val(1)))), + ), + ) + }) + it(`should handle common conditions: (age > 10 AND status = 'active') - (age > 20 AND status = 'active') = (age > 10 AND age <= 20 AND status = 'active')`, () => { const from = and( gt(ref(`age`), val(10)), From 5ceed21ffe70ba6bf98a9c661679340aceaeed82 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 02:09:03 -0600 Subject: [PATCH 124/327] test(db): bound coverage claims by acquisition --- .../coverage-registry-oracle.property.test.ts | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/packages/db/tests/query/coverage-registry-oracle.property.test.ts b/packages/db/tests/query/coverage-registry-oracle.property.test.ts index 72d931255..7ca042ce8 100644 --- a/packages/db/tests/query/coverage-registry-oracle.property.test.ts +++ b/packages/db/tests/query/coverage-registry-oracle.property.test.ts @@ -6,6 +6,7 @@ import { createLoadSubsetCoverageRegistry, } from '../../src/query/coverage-registry.js' import { oraclePropertyOptions } from '../oracle-config.js' +import type { CoverageRegistryResourceCounts } from '../../src/query/coverage-registry.js' import type { AppliedLoadSubsetOutcome } from '../../src/types.js' import type { Command } from 'fast-check' @@ -468,6 +469,22 @@ function createReleaseProbe(failFirst: boolean): ReleaseProbe { return probe } +function expectRegistryResourceBounds( + resourceCounts: CoverageRegistryResourceCounts, +): void { + // One logical lease may own several physical attempts. Bound each retained + // slot by claims, not by the number of unique lease tokens. + expect(resourceCounts.claims).toBeLessThanOrEqual( + resourceCounts.retainedDemands + resourceCounts.unsettledClaims, + ) + expect(resourceCounts.retainedDemands).toBeLessThanOrEqual( + resourceCounts.claims, + ) + expect(resourceCounts.retainedOutcomes).toBeLessThanOrEqual( + resourceCounts.claims, + ) +} + function expectReleaseFailure(release: () => unknown): void { let threw = false try { @@ -573,15 +590,7 @@ function assertRegistryModel(model: RegistryModel, real: RegistryReal): void { } const resourceCounts = real.registry.resourceCounts() expect(resourceCounts).toEqual(expectedResourceCounts) - expect(resourceCounts.claims).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) - expect(resourceCounts.retainedDemands).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) - expect(resourceCounts.retainedOutcomes).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) + expectRegistryResourceBounds(resourceCounts) for (const row of modelRows) { expect(real.registry.rowOwnerCount(row)).toBe( model.acquisitions.filter( @@ -1090,6 +1099,38 @@ class DisposeCommand implements Command { } describe(`coverage registry oracle`, () => { + it(`bounds evidence when one lease owns parallel physical acquisitions`, () => { + const registry = createPrefixRegistry() + const lease = registry.addLease(1) + const acquisitions = [ + addPrefixAcquisition(registry, { + generation: 1, + leases: [lease], + release: vi.fn(), + prefix: 1, + }), + addPrefixAcquisition(registry, { + generation: 1, + leases: [lease], + release: vi.fn(), + prefix: 1, + }), + ] + + acquisitions.forEach((acquisition) => + registry.settleLease(acquisition, lease), + ) + + expect(registry.resourceCounts()).toMatchObject({ + liveLeases: 1, + acquisitions: 2, + claims: 2, + unsettledClaims: 0, + retainedDemands: 2, + }) + expectRegistryResourceBounds(registry.resourceCounts()) + }) + it(`fences old evidence while retaining its physical release obligation`, () => { const registry = createPrefixRegistry() const oldRelease = vi.fn() From 9209e8f1c0b05faf29d302fdb2c865bedcbc5c4d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 03:19:00 -0600 Subject: [PATCH 125/327] fix(db): guard ordered refill adapter entry --- packages/db/src/query/live/ARCHITECTURE.md | 8 + .../src/query/live/collection-subscriber.ts | 43 +++-- ...d-subset-full-flow-oracle.property.test.ts | 174 +++++++++++++++--- 3 files changed, 186 insertions(+), 39 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 829a933e6..54adafc16 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -422,6 +422,10 @@ offset, cursor, and boundary-class refinement requests. Every applied row key must name a row established by that acquisition. Tests that withhold rows while claiming exhaustion, or ignore a refinement request, do not model a valid adapter and cannot establish a runtime defect. +An acquisition may establish a row that is already readable by applying the +same authoritative value again under its own request signal and awaiting that +receipt. Merely observing a row installed by another demand does not transfer +ownership or make it an applied row of the new acquisition. Every continuation boundary comes from rows established by the same ordered demand. Rows retained for another query, join, or window cannot move it. During @@ -685,6 +689,10 @@ start another request only when that prefix grows or that boundary moves. If a continuing page establishes neither fact, core leaves the window uncovered, does not repeat the same request, and records a nonfatal no-progress diagnostic in `lastSubsetError`. +Adapter entry is itself pending work. A request-scoped commit can publish rows +before an async `loadSubset` call returns its Promise, so graph callbacks during +that entry cannot start another ordered request. Once entry returns, the normal +in-flight Promise guard owns the request until settlement. An ordered window with an active limit of zero creates no ordered transport demand. Its coordinator remains alive so a later window change can load from diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 1f8bd74d2..c988bdc8c 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -61,6 +61,10 @@ export class CollectionSubscriber< private pendingOrderedLoadPromise: | Promise | undefined + // A sync commit can publish rows before an async loadSubset call returns its + // Promise. Block graph callbacks in that entry window; the Promise guard + // takes over as soon as requestLimitedSnapshot returns. + private orderedLoadStartInProgress = false // Overlapping replays share one subscription, so only the latest result // token may clear the full-source acquisition guard. private unindexedSnapshot: @@ -512,6 +516,8 @@ export class CollectionSubscriber< return true } + if (this.orderedLoadStartInProgress) return true + if (this.pendingOrderedLoadPromise) { // The current window still needs the in-flight coverage. Attach it to // this operation without making an unrelated or superseded request a @@ -666,22 +672,27 @@ export class CollectionSubscriber< // Omit offset so requestLimitedSnapshot can advance based on // the number of rows already loaded (supports offset-based backends). try { - subscription.requestLimitedSnapshot({ - orderBy: cursor.normalizedOrderBy, - limit: n, - minValues: cursor.minValues, - trackLoadSubsetPromise: false, - onLoadSubsetResult: (result, demand) => { - if (result instanceof Promise) { - void result.catch(() => { - if (this.lastLoadRequestKey === loadRequestKey) { - this.lastLoadRequestKey = undefined - } - }) - } - this.orderedLoadSubsetResult?.(result, demand) - }, - }) + this.orderedLoadStartInProgress = true + try { + subscription.requestLimitedSnapshot({ + orderBy: cursor.normalizedOrderBy, + limit: n, + minValues: cursor.minValues, + trackLoadSubsetPromise: false, + onLoadSubsetResult: (result, demand) => { + if (result instanceof Promise) { + void result.catch(() => { + if (this.lastLoadRequestKey === loadRequestKey) { + this.lastLoadRequestKey = undefined + } + }) + } + this.orderedLoadSubsetResult?.(result, demand) + }, + }) + } finally { + this.orderedLoadStartInProgress = false + } } catch (error) { if (this.lastLoadRequestKey === loadRequestKey) { this.lastLoadRequestKey = undefined diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 2c73610a1..2c2cdeb9e 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -405,6 +405,28 @@ function collectStringLiterals( let multiSourceOrderedHarnessId = 0 +async function expectMultiSourceStepToSettle( + scenario: MultiSourceOrderedScenario, + step: string, + promise: Promise, +): Promise { + let timeout: ReturnType | undefined + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error(`${step} did not settle for ${JSON.stringify(scenario)}`), + ) + }, 5_000) + }), + ]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } +} + async function runMultiSourceOrderedScenario( scenario: MultiSourceOrderedScenario, ): Promise { @@ -412,7 +434,11 @@ async function runMultiSourceOrderedScenario( type SecondaryRow = { id: string; joinKey: string } const primaryOrder = orderedPrimaryRows(scenario) - const sourceSteps = await observeOrderedSourceSteps(scenario) + const sourceSteps = await expectMultiSourceStepToSettle( + scenario, + `control projection`, + observeOrderedSourceSteps(scenario), + ) expect( sourceSteps.map(({ sourceKey, demandKeys }) => ({ sourceKey, demandKeys })), ).toEqual( @@ -455,23 +481,23 @@ async function runMultiSourceOrderedScenario( let primaryKeysBeforeSecondaryPublication: ReadonlyArray | undefined let primaryBegin!: () => void let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void - let primaryCommit!: () => true | Promise + let primaryCommit!: (signal?: AbortSignal) => true | Promise const applyPrimaryRows = async ( rows: ReadonlyArray, + signal: AbortSignal | undefined, ): Promise> => { - const freshRows = rows.filter(({ id }) => !establishedPrimaryKeys.has(id)) - if (freshRows.length === 0) return [] + if (rows.length === 0) return [] primaryBegin() - for (const row of freshRows) { + for (const row of rows) { establishedPrimaryKeys.add(row.id) primaryKeysEstablishedByLoads.add(row.id) primaryWrite({ type: `insert`, value: row }) } - const applied = primaryCommit() + const applied = primaryCommit(signal) if (applied !== true) await applied - for (const row of freshRows) committedPrimaryKeys.add(row.id) - return freshRows.map(({ id }) => id) + for (const row of rows) committedPrimaryKeys.add(row.id) + return rows.map(({ id }) => id) } const releaseSecondaryPublication = (): void => { @@ -484,6 +510,16 @@ async function runMultiSourceOrderedScenario( const recordPrimaryCall = (options: LoadSubsetOptions): void => { primaryCalls.push(options) + // Four source rows, one initial window, and one positive refinement cannot + // require an unbounded number of physical acquisitions. Keep a generous + // ceiling so a microtask refill loop becomes a shrinkable oracle failure. + if (primaryCalls.length > 32) { + throw new Error( + `primary loadSubset exceeded the bounded source grammar at call ${primaryCalls.length}: ${JSON.stringify( + { limit: options.limit, cursor: options.cursor }, + )}`, + ) + } primaryCallProgress.push({ demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, establishedPrimaryCount: establishedPrimaryKeys.size, @@ -513,7 +549,10 @@ async function runMultiSourceOrderedScenario( options.where === undefined || evaluateReferenceExpression(options.where, row), ) - const appliedRowKeys = await applyPrimaryRows(rows) + const appliedRowKeys = await applyPrimaryRows( + rows, + options.signal, + ) primaryReceipts.push(appliedRowKeys) return { hasMore: false, @@ -526,7 +565,10 @@ async function runMultiSourceOrderedScenario( primaryOrderedVisitedKeys.push( ...primaryOrder.map(({ id }) => id), ) - const appliedRowKeys = await applyPrimaryRows(primaryOrder) + const appliedRowKeys = await applyPrimaryRows( + primaryOrder, + options.signal, + ) if ( scenario.secondaryPublication === `after-primary-continuation` || @@ -552,7 +594,7 @@ async function runMultiSourceOrderedScenario( let appliedRowKeys: Array = [] if (row) { primaryOrderedVisitedKeys.push(row.id) - appliedRowKeys = await applyPrimaryRows([row]) + appliedRowKeys = await applyPrimaryRows([row], options.signal) } const hasMore = previousIndex + 1 < primaryOrder.length - 1 if ( @@ -584,10 +626,11 @@ async function runMultiSourceOrderedScenario( type: `insert` value: SecondaryRow }) => void - let secondaryCommit!: () => true | Promise + let secondaryCommit!: (signal?: AbortSignal) => true | Promise const secondaryRows = scenario.secondaryRows const applySecondaryRows = async ( rows: ReadonlyArray, + signal: AbortSignal | undefined, ): Promise> => { const freshRows = rows.filter(({ id }) => !establishedSecondaryKeys.has(id)) if (freshRows.length === 0) return [] @@ -597,7 +640,7 @@ async function runMultiSourceOrderedScenario( establishedSecondaryKeys.add(row.id) secondaryWrite({ type: `insert`, value: row }) } - const applied = secondaryCommit() + const applied = secondaryCommit(signal) if (applied !== true) await applied return freshRows.map(({ id }) => id) } @@ -628,6 +671,11 @@ async function runMultiSourceOrderedScenario( return { loadSubset: async (options) => { secondaryCalls.push(options) + if (secondaryCalls.length > 32) { + throw new Error( + `secondary loadSubset exceeded the bounded source grammar`, + ) + } if (!hasPreloadedSecondary(scenario)) { await secondaryPublicationGate.promise primaryKeysBeforeSecondaryPublication ??= [ @@ -667,6 +715,7 @@ async function runMultiSourceOrderedScenario( index, index + scenario.secondaryPageSize, ), + options.signal, )), ) } @@ -721,7 +770,7 @@ async function runMultiSourceOrderedScenario( await flushPromises() } } - await preload + await expectMultiSourceStepToSettle(scenario, `preload`, preload) await flushPromises() expect(preloadSettled).toBe(true) @@ -747,10 +796,14 @@ async function runMultiSourceOrderedScenario( offset: refinedOffset, limit: refinedLimit, }) - await live.utils.setWindow({ - offset: refinedOffset, - limit: refinedLimit, - }) + await expectMultiSourceStepToSettle( + scenario, + `positive window refinement`, + live.utils.setWindow({ + offset: refinedOffset, + limit: refinedLimit, + }), + ) await flushPromises() expect( live.toArray.map( @@ -769,7 +822,11 @@ async function runMultiSourceOrderedScenario( } const primaryCallsBeforeZeroShrink = primaryCalls.length - await live.utils.setWindow({ offset: 2, limit: 0 }) + await expectMultiSourceStepToSettle( + scenario, + `zero window refinement`, + live.utils.setWindow({ offset: 2, limit: 0 }), + ) await flushPromises() expect(live.toArray).toEqual([]) expect( @@ -801,13 +858,15 @@ async function runMultiSourceOrderedScenario( } previousProgressByDemand.set(progress.demandKey, progress) } - const claimedPrimaryKeys = primaryReceipts.flat() - expect(new Set(claimedPrimaryKeys).size).toBe(claimedPrimaryKeys.length) + for (const receipt of primaryReceipts) { + expect(new Set(receipt).size).toBe(receipt.length) + } + const claimedPrimaryKeys = new Set(primaryReceipts.flat()) expect([...claimedPrimaryKeys].sort()).toEqual( [...primaryKeysEstablishedByLoads].sort(), ) expect( - claimedPrimaryKeys.every((key) => + [...claimedPrimaryKeys].every((key) => scenario.primaryRows.some(({ id }) => id === key), ), ).toBe(true) @@ -909,8 +968,13 @@ async function runMultiSourceOrderedScenario( } } } finally { + secondaryPublicationGate.resolve() for (const waiter of delayedSecondaryReceiptWaiters) waiter.gate.resolve() - await Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]) + await expectMultiSourceStepToSettle( + scenario, + `cleanup`, + Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]), + ) } } @@ -1023,6 +1087,70 @@ it.each([ }) }) +it(`settles a late secondary load after tied primary continuations`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 1, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-continuation`, + secondaryPageSize: 1, + secondaryCommitOrder: `reverse`, + primaryRows: [ + { id: `a`, rank: 2, joinKey: `x` }, + { id: `b`, rank: 0, joinKey: `z` }, + { id: `c`, rank: 0, joinKey: `y` }, + { id: `d`, rank: 2, joinKey: `y` }, + ], + secondaryRows: [ + { id: `x-0`, joinKey: `x` }, + { id: `z-0`, joinKey: `z` }, + ], + }) +}) + +it(`settles an empty join after exhausting tied primary rows`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 1, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-exhaustion`, + secondaryPageSize: 1, + secondaryCommitOrder: `insertion`, + primaryRows: [ + { id: `a`, rank: 0, joinKey: `x` }, + { id: `b`, rank: 0, joinKey: `x` }, + { id: `c`, rank: 0, joinKey: `x` }, + { id: `d`, rank: 0, joinKey: `x` }, + ], + secondaryRows: [], + }) +}) + +it(`does not start duplicate ordered work from an applying receipt`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 2, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-continuation`, + secondaryPageSize: 1, + secondaryCommitOrder: `insertion`, + primaryRows: [ + { id: `a`, rank: 0, joinKey: `y` }, + { id: `b`, rank: 1, joinKey: `x` }, + { id: `c`, rank: 1, joinKey: `y` }, + { id: `d`, rank: 0, joinKey: `z` }, + ], + secondaryRows: [ + { id: `z-0`, joinKey: `z` }, + { id: `z-1`, joinKey: `z` }, + { id: `y-0`, joinKey: `y` }, + ], + }) +}) + it.each([`sync throw`, `async reject`] as const)( `retries unindexed transport after a %s during zero-to-positive refinement`, async (failureMode) => { From afe5cf2ee80e33349ba28b5cc306767649884e82 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 03:25:55 -0600 Subject: [PATCH 126/327] test(db): preserve secondary acquisition ownership --- ...d-subset-full-flow-oracle.property.test.ts | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 2c2cdeb9e..7c547b3d8 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -632,17 +632,16 @@ async function runMultiSourceOrderedScenario( rows: ReadonlyArray, signal: AbortSignal | undefined, ): Promise> => { - const freshRows = rows.filter(({ id }) => !establishedSecondaryKeys.has(id)) - if (freshRows.length === 0) return [] - secondaryLoadCommitSizes.push(freshRows.length) + if (rows.length === 0) return [] + secondaryLoadCommitSizes.push(rows.length) secondaryBegin() - for (const row of freshRows) { + for (const row of rows) { establishedSecondaryKeys.add(row.id) secondaryWrite({ type: `insert`, value: row }) } const applied = secondaryCommit(signal) if (applied !== true) await applied - return freshRows.map(({ id }) => id) + return rows.map(({ id }) => id) } const secondary = createCollection({ id: `multi-source-ordered-secondary-${multiSourceOrderedHarnessId}`, @@ -871,19 +870,19 @@ async function runMultiSourceOrderedScenario( ), ).toBe(true) - const claimedSecondaryKeys = secondaryReceipts.flat() - expect(new Set(claimedSecondaryKeys).size).toBe(claimedSecondaryKeys.length) - const expectedClaimedSecondaryKeys = hasPreloadedSecondary(scenario) - ? [] - : scenario.secondaryRows - .filter((row) => - secondaryCalls.some( - ({ where }) => - where === undefined || evaluateReferenceExpression(where, row), - ), - ) - .map(({ id }) => id) - .sort() + for (const receipt of secondaryReceipts) { + expect(new Set(receipt).size).toBe(receipt.length) + } + const claimedSecondaryKeys = new Set(secondaryReceipts.flat()) + const expectedClaimedSecondaryKeys = scenario.secondaryRows + .filter((row) => + secondaryCalls.some( + ({ where }) => + where === undefined || evaluateReferenceExpression(where, row), + ), + ) + .map(({ id }) => id) + .sort() expect([...claimedSecondaryKeys].sort()).toEqual( expectedClaimedSecondaryKeys, ) From 1d794ea3f90ca2ac35118ad5d644fd377b9ee353 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 03:32:15 -0600 Subject: [PATCH 127/327] test(db): bind receipts to acquisitions --- ...d-subset-full-flow-oracle.property.test.ts | 76 +++++++++++-------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 7c547b3d8..14f0cff3e 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -458,10 +458,18 @@ async function runMultiSourceOrderedScenario( establishedPrimaryCount: number establishedSecondaryCount: number }> = [] - const primaryReceipts: Array> = [] + const primaryReceipts: Array<{ + demandKey: string + expectedRowKeys: ReadonlyArray + appliedRowKeys: ReadonlyArray + }> = [] const primaryOrderedVisitedKeys: Array = [] const secondaryCalls: Array = [] - const secondaryReceipts: Array> = [] + const secondaryReceipts: Array<{ + demandKey: string + expectedRowKeys: ReadonlyArray + appliedRowKeys: ReadonlyArray + }> = [] const secondaryLoadCommitSizes: Array = [] const delayedSecondaryReceiptWaiters: Array<{ index: number @@ -472,7 +480,6 @@ async function runMultiSourceOrderedScenario( const secondaryPublicationGate = createDeferred() const establishedPrimaryKeys = new Set() const committedPrimaryKeys = new Set() - const primaryKeysEstablishedByLoads = new Set() const establishedSecondaryKeys = new Set() let primaryOrderedCallCount = 0 let primaryOrderedCallCountAtSecondaryRelease: number | undefined @@ -491,7 +498,6 @@ async function runMultiSourceOrderedScenario( primaryBegin() for (const row of rows) { establishedPrimaryKeys.add(row.id) - primaryKeysEstablishedByLoads.add(row.id) primaryWrite({ type: `insert`, value: row }) } const applied = primaryCommit(signal) @@ -553,7 +559,11 @@ async function runMultiSourceOrderedScenario( rows, options.signal, ) - primaryReceipts.push(appliedRowKeys) + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: rows.map(({ id }) => id), + appliedRowKeys, + }) return { hasMore: false, appliedRowKeys, @@ -576,7 +586,11 @@ async function runMultiSourceOrderedScenario( ) { releaseSecondaryPublication() } - primaryReceipts.push(appliedRowKeys) + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: primaryOrder.map(({ id }) => id), + appliedRowKeys, + }) return { hasMore: false, appliedRowKeys, @@ -609,7 +623,11 @@ async function runMultiSourceOrderedScenario( ) { releaseSecondaryPublication() } - primaryReceipts.push(appliedRowKeys) + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: row ? [row.id] : [], + appliedRowKeys, + }) return { hasMore, appliedRowKeys, @@ -718,7 +736,11 @@ async function runMultiSourceOrderedScenario( )), ) } - secondaryReceipts.push(appliedRowKeys) + secondaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: rowsInCommitOrder.map(({ id }) => id), + appliedRowKeys, + }) return { hasMore: false, appliedRowKeys, @@ -857,35 +879,25 @@ async function runMultiSourceOrderedScenario( } previousProgressByDemand.set(progress.demandKey, progress) } + expect(primaryReceipts).toHaveLength(primaryCalls.length) for (const receipt of primaryReceipts) { - expect(new Set(receipt).size).toBe(receipt.length) + expect(new Set(receipt.appliedRowKeys).size).toBe( + receipt.appliedRowKeys.length, + ) + expect([...receipt.appliedRowKeys].sort(), receipt.demandKey).toEqual( + [...receipt.expectedRowKeys].sort(), + ) } - const claimedPrimaryKeys = new Set(primaryReceipts.flat()) - expect([...claimedPrimaryKeys].sort()).toEqual( - [...primaryKeysEstablishedByLoads].sort(), - ) - expect( - [...claimedPrimaryKeys].every((key) => - scenario.primaryRows.some(({ id }) => id === key), - ), - ).toBe(true) + expect(secondaryReceipts).toHaveLength(secondaryCalls.length) for (const receipt of secondaryReceipts) { - expect(new Set(receipt).size).toBe(receipt.length) - } - const claimedSecondaryKeys = new Set(secondaryReceipts.flat()) - const expectedClaimedSecondaryKeys = scenario.secondaryRows - .filter((row) => - secondaryCalls.some( - ({ where }) => - where === undefined || evaluateReferenceExpression(where, row), - ), + expect(new Set(receipt.appliedRowKeys).size).toBe( + receipt.appliedRowKeys.length, ) - .map(({ id }) => id) - .sort() - expect([...claimedSecondaryKeys].sort()).toEqual( - expectedClaimedSecondaryKeys, - ) + expect([...receipt.appliedRowKeys].sort(), receipt.demandKey).toEqual( + [...receipt.expectedRowKeys].sort(), + ) + } expect( secondaryLoadCommitSizes.every( (commitSize) => commitSize <= scenario.secondaryPageSize, From becdd05ec95c4c125eb128a526d8470d622e42d8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 03:39:14 -0600 Subject: [PATCH 128/327] docs(db): define refinement oracle grammar --- packages/db/src/query/live/ARCHITECTURE.md | 59 ++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 54adafc16..8fd720ded 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1219,6 +1219,65 @@ coordinates and public window state. A new regression must reduce to this grammar or justify a grammar change; it must not add a one-off event named after the bug. +The grammar composes these independent axes: + +| Axis | Values owned by this oracle family | +| --------------- | ------------------------------------------------------------------------------------------------------------- | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, released | +| Identity | Owner, session, demand, attempt, acquisition, source, transaction, publication | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Result rows, readiness, error, ordered boundary, receipt, ownership, and physical-work counts | + +An executable history chooses values on these axes, then combines them through +the demand facts above. A logical request installs its owner before adapter +entry. It either attaches to an acquisition or starts one. Request-scoped sync +transactions make row versions visible and settle their receipts before the +acquisition can publish an outcome. Applied evidence may then establish +caller-relative coverage and row ownership. Ordered evidence may update private +window progress; only a complete publication snapshot reaches readers. +Release, truncate, replacement, restart, and cleanup change the relevant +identity or generation without changing this sequence. + +Adapter entry and every result, cleanup, and listener callback are reentrancy +boundaries. Any otherwise legal event may occur before that boundary returns. +Work which has entered an adapter but has not yet returned a promise is already +pending work. A production-boundary driver must include this synchronous phase; +promise-only overlap does not reconstruct the source. + +Each projection may erase axes it does not own. It must preserve the identity +and cardinality of the fact it claims to check. In particular: + +- receipt laws compare each acquisition with its own applied keys; +- coverage laws keep caller demand separate from acquisition outcome; +- ownership laws keep logical leases separate from physical row support; +- publication laws keep demand origin, row version, and generation separate; +- work laws count physical starts separately from logical owners; +- renaming laws erase names only after every allowed next-command observation + remains equal. + +Set unions, final-state equality, and settled promises are therefore supporting +views, not universal oracles. Each can hide a wrong acquisition, transient +publication, duplicate start, stale generation, or lost owner. + +The reconstruction control for a new finding is: + +1. express its source topology as already-evaluated contributions; +2. name every logical and physical identity involved; +3. place each action at its exact boundary phase and source origin; +4. derive evidence, ownership, coverage, and publication independently; +5. compare the first public or resource observation that can differ; and +6. verify the same grammar admits the nearest marginal case but rejects a raw + relational or materialization problem. + +Zero-sized windows, empty sources, unknown extent, and synchronous adapter +results are marginal cases of this grammar, not separate families. Predicate +evaluation, join multiplicity, aggregate deltas, and nested materialization are +outside it and remain negative controls. + DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production harness may use an eager query from those paths as its relational control, then From 542eb2c14233f4441608afbbc8d070eecd59bc70 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 03:48:52 -0600 Subject: [PATCH 129/327] docs(db): preserve refinement grammar distinctions --- packages/db/src/query/live/ARCHITECTURE.md | 49 +++++++++++++++------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 8fd720ded..89216aeff 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,16 +1221,17 @@ the bug. The grammar composes these independent axes: -| Axis | Values owned by this oracle family | -| --------------- | ------------------------------------------------------------------------------------------------------------- | -| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | -| Demand relation | Exact, shared, covered, uncovered, ordered, additional, released | -| Identity | Owner, session, demand, attempt, acquisition, source, transaction, publication | -| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | -| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | -| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | -| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | -| Observation | Result rows, readiness, error, ordered boundary, receipt, ownership, and physical-work counts | +| Axis | Values owned by this oracle family | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Identity | Owner, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Result rows, readiness, ordered boundary, exact error occurrences, receipts, ownership, physical starts, and retained-space counts | An executable history chooses values on these axes, then combines them through the demand facts above. A logical request installs its owner before adapter @@ -1242,12 +1243,26 @@ window progress; only a complete publication snapshot reaches readers. Release, truncate, replacement, restart, and cleanup change the relevant identity or generation without changing this sequence. +Logical release and durable physical release are separate transitions. A +throwing cleanup leaves a release-pending acquisition, its coverage, and row +support as retryable debt. Only accepted cleanup retires those physical facts. +Resource observations therefore count leases, acquisitions, coverage claims, +unsettled claims, retained demands, outcomes, and row-key slots separately from +transport starts. + Adapter entry and every result, cleanup, and listener callback are reentrancy boundaries. Any otherwise legal event may occur before that boundary returns. Work which has entered an adapter but has not yet returned a promise is already pending work. A production-boundary driver must include this synchronous phase; promise-only overlap does not reconstruct the source. +Failures also carry boundary identity. One occurrence names its originating +options, creation order, and containing callback or acquisition frames. A +private propagation token may carry that occurrence through an authorized +nested frame, but payload equality never merges two boundaries. `undefined`, +`NaN`, primitives, and the same `Error` object can each be the payload of a +distinct occurrence. + Each projection may erase axes it does not own. It must preserve the identity and cardinality of the fact it claims to check. In particular: @@ -1256,6 +1271,10 @@ and cardinality of the fact it claims to check. In particular: - ownership laws keep logical leases separate from physical row support; - publication laws keep demand origin, row version, and generation separate; - work laws count physical starts separately from logical owners; +- space laws count each retained resource category separately; +- release laws distinguish requested, retryable, accepted, and disposed work; +- error laws preserve occurrence, originating options, and report order rather + than deduplicating by payload; - renaming laws erase names only after every allowed next-command observation remains equal. @@ -1267,10 +1286,12 @@ The reconstruction control for a new finding is: 1. express its source topology as already-evaluated contributions; 2. name every logical and physical identity involved; -3. place each action at its exact boundary phase and source origin; -4. derive evidence, ownership, coverage, and publication independently; -5. compare the first public or resource observation that can differ; and -6. verify the same grammar admits the nearest marginal case but rejects a raw +3. state the adapter capability which makes each transport transition legal; +4. place each action at its exact boundary phase and source origin; +5. derive evidence, ownership, coverage, publication, failure occurrences, and + retained resources independently; +6. compare the first public or resource observation that can differ; and +7. verify the same grammar admits the nearest marginal case but rejects a raw relational or materialization problem. Zero-sized windows, empty sources, unknown extent, and synchronous adapter From 84b28bba19ed89482bf3def6e5888886fb94f25a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 04:00:36 -0600 Subject: [PATCH 130/327] docs(db): preserve work and release order laws --- packages/db/src/query/live/ARCHITECTURE.md | 48 +++++++++++++--------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 89216aeff..00ac11775 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,17 +1221,18 @@ the bug. The grammar composes these independent axes: -| Axis | Values owned by this oracle family | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | -| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | -| Identity | Owner, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | -| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | -| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | -| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | -| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | -| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | -| Observation | Result rows, readiness, ordered boundary, exact error occurrences, receipts, ownership, physical starts, and retained-space counts | +| Axis | Values owned by this oracle family | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Identity | Owner, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Result rows, readiness, ordered boundary, canonically ordered removals, exact error occurrences, receipts, ownership, physical starts, evidence-path work, and retained-space counts | An executable history chooses values on these axes, then combines them through the demand facts above. A logical request installs its owner before adapter @@ -1248,7 +1249,9 @@ throwing cleanup leaves a release-pending acquisition, its coverage, and row support as retryable debt. Only accepted cleanup retires those physical facts. Resource observations therefore count leases, acquisitions, coverage claims, unsettled claims, retained demands, outcomes, and row-key slots separately from -transport starts. +transport starts. Algorithmic work is another independent observation: count +row-key copies, demand snapshots, and demand-key derivations rather than using +transport starts or retained space as a proxy for evidence computation. Adapter entry and every result, cleanup, and listener callback are reentrancy boundaries. Any otherwise legal event may occur before that boundary returns. @@ -1271,8 +1274,12 @@ and cardinality of the fact it claims to check. In particular: - ownership laws keep logical leases separate from physical row support; - publication laws keep demand origin, row version, and generation separate; - work laws count physical starts separately from logical owners; +- evidence-work laws count row-key copies, demand snapshots, and demand-key + derivations separately and bound them independently of candidate count; - space laws count each retained resource category separately; - release laws distinguish requested, retryable, accepted, and disposed work; +- removal laws preserve the shared `compareKeys` sequence across mixed string, + number, ASCII, and non-ASCII keys rather than comparing only a set; - error laws preserve occurrence, originating options, and report order rather than deduplicating by payload; - renaming laws erase names only after every allowed next-command observation @@ -1288,16 +1295,19 @@ The reconstruction control for a new finding is: 2. name every logical and physical identity involved; 3. state the adapter capability which makes each transport transition legal; 4. place each action at its exact boundary phase and source origin; -5. derive evidence, ownership, coverage, publication, failure occurrences, and - retained resources independently; -6. compare the first public or resource observation that can differ; and +5. derive evidence, ownership, coverage, publication, failure occurrences, + canonical removal order, evidence-path work, and retained resources + independently; +6. compare the first public, algorithmic-work, or retained-resource observation + that can differ; and 7. verify the same grammar admits the nearest marginal case but rejects a raw relational or materialization problem. -Zero-sized windows, empty sources, unknown extent, and synchronous adapter -results are marginal cases of this grammar, not separate families. Predicate -evaluation, join multiplicity, aggregate deltas, and nested materialization are -outside it and remain negative controls. +Zero-sized windows, empty sources, unknown extent, synchronous adapter results, +and mixed string/number or non-ASCII row keys are marginal cases of this +grammar, not separate families. Predicate evaluation, join multiplicity, +aggregate deltas, and nested materialization are outside it and remain negative +controls. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production From 14ec043f84bf7192a7f40fdbfa6d4edc963e31e7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 04:13:30 -0600 Subject: [PATCH 131/327] docs(db): preserve operation and trace laws --- packages/db/src/query/live/ARCHITECTURE.md | 62 +++++++++++++++------- 1 file changed, 42 insertions(+), 20 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 00ac11775..5d6c90b99 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,18 +1221,19 @@ the bug. The grammar composes these independent axes: -| Axis | Values owned by this oracle family | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | -| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | -| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | -| Identity | Owner, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | -| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, cleanup | -| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | -| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | -| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | -| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | -| Observation | Result rows, readiness, ordered boundary, canonically ordered removals, exact error occurrences, receipts, ownership, physical starts, evidence-path work, and retained-space counts | +| Axis | Values owned by this oracle family | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | +| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, and retained space | An executable history chooses values on these axes, then combines them through the demand facts above. A logical request installs its owner before adapter @@ -1244,6 +1245,14 @@ window progress; only a complete publication snapshot reaches readers. Release, truncate, replacement, restart, and cleanup change the relevant identity or generation without changing this sequence. +An imperative load operation is a separate caller boundary around this flow. +It owns the future requests caused while it is current, retains the promises it +already acquired after a newer operation supersedes it, and settles only after +synchronous follow-up requests have had a chance to join. Its outcome includes +caller-relative retained evidence even when coverage reuse starts no transport. +Physical acquisition count, readiness, and an operation's pending set or result +are therefore different projections. + Logical release and durable physical release are separate transitions. A throwing cleanup leaves a release-pending acquisition, its coverage, and row support as retryable debt. Only accepted cleanup retires those physical facts. @@ -1266,10 +1275,19 @@ nested frame, but payload equality never merges two boundaries. `undefined`, `NaN`, primitives, and the same `Error` object can each be the payload of a distinct occurrence. +Publication is an ordered observation, not only a final state. Transaction and +replay laws preserve each emitted change batch, adapter invocation and release, +and the rows synchronously visible inside its callback. Terminal teardown first +reports retained failures, then emits `unsubscribed` exactly once, then clears +listeners. Reentrant teardown and cleanup retries must not duplicate or reorder +that lifecycle edge. + Each projection may erase axes it does not own. It must preserve the identity and cardinality of the fact it claims to check. In particular: - receipt laws compare each acquisition with its own applied keys; +- operation laws keep caller identity, acquired promises, first error, and + per-source/collection/generation outcomes separate from acquisition state; - coverage laws keep caller demand separate from acquisition outcome; - ownership laws keep logical leases separate from physical row support; - publication laws keep demand origin, row version, and generation separate; @@ -1280,6 +1298,9 @@ and cardinality of the fact it claims to check. In particular: - release laws distinguish requested, retryable, accepted, and disposed work; - removal laws preserve the shared `compareKeys` sequence across mixed string, number, ASCII, and non-ASCII keys rather than comparing only a set; +- trace laws preserve adapter calls, releases, emitted change batches, + callback-time reads, and terminal lifecycle events in order rather than + comparing only final state; - error laws preserve occurrence, originating options, and report order rather than deduplicating by payload; - renaming laws erase names only after every allowed next-command observation @@ -1292,22 +1313,23 @@ publication, duplicate start, stale generation, or lost owner. The reconstruction control for a new finding is: 1. express its source topology as already-evaluated contributions; -2. name every logical and physical identity involved; +2. name every logical, imperative-operation, and physical identity involved; 3. state the adapter capability which makes each transport transition legal; 4. place each action at its exact boundary phase and source origin; -5. derive evidence, ownership, coverage, publication, failure occurrences, - canonical removal order, evidence-path work, and retained resources - independently; +5. derive operation settlement, evidence, ownership, coverage, publication, + ordered observation traces, failure occurrences, canonical removal order, + evidence-path work, and retained resources independently; 6. compare the first public, algorithmic-work, or retained-resource observation that can differ; and 7. verify the same grammar admits the nearest marginal case but rejects a raw relational or materialization problem. Zero-sized windows, empty sources, unknown extent, synchronous adapter results, -and mixed string/number or non-ASCII row keys are marginal cases of this -grammar, not separate families. Predicate evaluation, join multiplicity, -aggregate deltas, and nested materialization are outside it and remain negative -controls. +coverage-reused operations with no transport, superseded overlapping +operations, reentrant terminal cleanup, and mixed string/number or non-ASCII +row keys are marginal cases of this grammar, not separate families. Predicate +evaluation, join multiplicity, aggregate deltas, and nested materialization are +outside it and remain negative controls. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production From 05dbfb7c69860af4cfa391663d16a46853d8ecb6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 04:15:59 -0600 Subject: [PATCH 132/327] test(db): accept synchronous oracle steps --- .../query/load-subset-full-flow-oracle.property.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 14f0cff3e..b6cc85c55 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -408,13 +408,13 @@ let multiSourceOrderedHarnessId = 0 async function expectMultiSourceStepToSettle( scenario: MultiSourceOrderedScenario, step: string, - promise: Promise, -): Promise { + result: T, +): Promise> { let timeout: ReturnType | undefined try { return await Promise.race([ - promise, - new Promise((_, reject) => { + Promise.resolve(result), + new Promise((_, reject) => { timeout = setTimeout(() => { reject( new Error(`${step} did not settle for ${JSON.stringify(scenario)}`), From 2af40609bf6d4da8739dae7f7c59c91c7dfd7269 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 04:24:16 -0600 Subject: [PATCH 133/327] docs(db): preserve ordered work laws --- packages/db/src/query/live/ARCHITECTURE.md | 46 +++++++++++++--------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 5d6c90b99..1c3508f0f 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,19 +1221,19 @@ the bug. The grammar composes these independent axes: -| Axis | Values owned by this oracle family | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | -| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | -| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | -| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | -| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | -| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | -| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | -| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | -| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | -| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | -| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, and retained space | +| Axis | Values owned by this oracle family | +| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | +| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, ordered-path work, and retained space | An executable history chooses values on these axes, then combines them through the demand facts above. A logical request installs its owner before adapter @@ -1260,7 +1260,11 @@ Resource observations therefore count leases, acquisitions, coverage claims, unsettled claims, retained demands, outcomes, and row-key slots separately from transport starts. Algorithmic work is another independent observation: count row-key copies, demand snapshots, and demand-key derivations rather than using -transport starts or retained space as a proxy for evidence computation. +transport starts or retained space as a proxy for evidence computation. Ordered +source work is separate again: preserve the exact scan and cursor sequence, and +count source reads or snapshots, sorts or total-order refinements, and predicate +compilations independently. A stable result and request trace can still hide +repeated local work. Adapter entry and every result, cleanup, and listener callback are reentrancy boundaries. Any otherwise legal event may occur before that boundary returns. @@ -1294,6 +1298,9 @@ and cardinality of the fact it claims to check. In particular: - work laws count physical starts separately from logical owners; - evidence-work laws count row-key copies, demand snapshots, and demand-key derivations separately and bound them independently of candidate count; +- ordered-work laws preserve the exact source-read and cursor sequence and + count source snapshots, sorts or total-order refinements, and predicate + compilations separately from transport and coverage-evidence work; - space laws count each retained resource category separately; - release laws distinguish requested, retryable, accepted, and disposed work; - removal laws preserve the shared `compareKeys` sequence across mixed string, @@ -1318,7 +1325,7 @@ The reconstruction control for a new finding is: 4. place each action at its exact boundary phase and source origin; 5. derive operation settlement, evidence, ownership, coverage, publication, ordered observation traces, failure occurrences, canonical removal order, - evidence-path work, and retained resources independently; + evidence-path work, ordered-path work, and retained resources independently; 6. compare the first public, algorithmic-work, or retained-resource observation that can differ; and 7. verify the same grammar admits the nearest marginal case but rejects a raw @@ -1326,10 +1333,11 @@ The reconstruction control for a new finding is: Zero-sized windows, empty sources, unknown extent, synchronous adapter results, coverage-reused operations with no transport, superseded overlapping -operations, reentrant terminal cleanup, and mixed string/number or non-ASCII -row keys are marginal cases of this grammar, not separate families. Predicate -evaluation, join multiplicity, aggregate deltas, and nested materialization are -outside it and remain negative controls. +operations, reentrant terminal cleanup, zero-contribution source steps, +all-tied boundaries, and mixed string/number or non-ASCII row keys are marginal +cases of this grammar, not separate families. Predicate evaluation, join +multiplicity, aggregate deltas, and nested materialization are outside it and +remain negative controls. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production From cc984e00ee5fa46077998d2607ac7b91c35c87fc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 04:31:26 -0600 Subject: [PATCH 134/327] docs(db): preserve runtime identity space cost --- packages/db/src/query/live/ARCHITECTURE.md | 43 ++++++++++++++-------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 1c3508f0f..b1f082bd7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,19 +1221,19 @@ the bug. The grammar composes these independent axes: -| Axis | Values owned by this oracle family | -| --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | -| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | -| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | -| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | -| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence | -| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | -| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | -| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | -| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | -| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | -| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, ordered-path work, and retained space | +| Axis | Values owned by this oracle family | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | +| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence, runtime reference slot | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, ordered-path work, transient retained space, and lifetime symbol-identity entries | An executable history chooses values on these axes, then combines them through the demand facts above. A logical request installs its owner before adapter @@ -1266,6 +1266,14 @@ count source reads or snapshots, sorts or total-order refinements, and predicate compilations independently. A stable result and request trace can still hide repeated local work. +Runtime reference identity has a different lifetime again. Objects use weak +identity, but JavaScript symbols cannot be weak keys. Stable equality for the +same live symbol therefore retains one strong entry per distinct symbol for the +runtime identity factory's lifetime. This monotonic, usage-proportional cost is +not part of the live-demand resource bound. Eviction is not valid unless the +platform supplies weak symbol identity or another scheme proves that one live +symbol can never receive a different identity. + Adapter entry and every result, cleanup, and listener callback are reentrancy boundaries. Any otherwise legal event may occur before that boundary returns. Work which has entered an adapter but has not yet returned a promise is already @@ -1302,6 +1310,8 @@ and cardinality of the fact it claims to check. In particular: count source snapshots, sorts or total-order refinements, and predicate compilations separately from transport and coverage-evidence work; - space laws count each retained resource category separately; +- identity-space laws count process-lifetime symbol entries separately from + transient demand resources and preserve stable same-symbol identity; - release laws distinguish requested, retryable, accepted, and disposed work; - removal laws preserve the shared `compareKeys` sequence across mixed string, number, ASCII, and non-ASCII keys rather than comparing only a set; @@ -1325,7 +1335,8 @@ The reconstruction control for a new finding is: 4. place each action at its exact boundary phase and source origin; 5. derive operation settlement, evidence, ownership, coverage, publication, ordered observation traces, failure occurrences, canonical removal order, - evidence-path work, ordered-path work, and retained resources independently; + evidence-path work, ordered-path work, transient retained resources, and + lifetime identity entries independently; 6. compare the first public, algorithmic-work, or retained-resource observation that can differ; and 7. verify the same grammar admits the nearest marginal case but rejects a raw @@ -1337,7 +1348,9 @@ operations, reentrant terminal cleanup, zero-contribution source steps, all-tied boundaries, and mixed string/number or non-ASCII row keys are marginal cases of this grammar, not separate families. Predicate evaluation, join multiplicity, aggregate deltas, and nested materialization are outside it and -remain negative controls. +remain negative controls. Reclaiming live symbol-identity entries is also +outside the current platform contract; their accepted factory-lifetime cost +must remain visible. DBSP operator suites own incremental relational laws. The includes suites own compiled routes and materialized nested results. A load-subset production From 4db17cf12a36d52aee6f2cd475b63c8c7801671f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:03:33 -0600 Subject: [PATCH 135/327] fix(db): fence reentrant window cleanup --- .../query/live/collection-config-builder.ts | 15 ++- .../src/query/live/collection-subscriber.ts | 4 +- ...d-subset-full-flow-oracle.property.test.ts | 116 +++++++++++++++++- 3 files changed, 126 insertions(+), 9 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 2f2a04f62..813851834 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -323,6 +323,8 @@ export class CollectionConfigBuilder< } catch (error) { if ( previousWindow && + syncSession === this.syncSession && + this.currentSyncConfig !== undefined && windowOperationGeneration === this.windowOperationGeneration ) { try { @@ -559,8 +561,11 @@ export class CollectionConfigBuilder< this.isGraphRunning = true try { - const { begin, commit } = this.currentSyncConfig + const config = this.currentSyncConfig + const { begin, commit } = config const syncState = this.currentSyncState + const sessionIsActive = () => + this.currentSyncConfig === config && this.currentSyncState === syncState // Don't run if the live query is in an error state if (this.isInErrorState) { @@ -574,23 +579,29 @@ export class CollectionConfigBuilder< // becomes part of this same quiescence pass. if (!syncState.graph.pendingWork()) { callback?.() + if (!sessionIsActive()) return } while (syncState.graph.pendingWork()) { syncState.graph.run() + if (!sessionIsActive()) return callback?.() + if (!sessionIsActive()) return } // Publish only after every operator has reached quiescence. A source // change can reach sibling materializations in different graph steps; // flushing between those steps would expose a mixed root snapshot. syncState.flushPendingChanges?.() + if (!sessionIsActive()) return // On the initial run, we may need to do an empty commit to ensure that // the collection is initialized if (syncState.messagesCount === 0) { begin() + if (!sessionIsActive()) return commit() + if (!sessionIsActive()) return } // After graph processing completes, check if we should mark ready. @@ -598,7 +609,7 @@ export class CollectionConfigBuilder< // 1. All data has been processed through the graph // 2. All source collections have had a chance to send their initial data // This prevents marking ready before data is processed (fixes isReady=true with empty data) - this.updateLiveQueryStatus(this.currentSyncConfig) + this.updateLiveQueryStatus(config) } } finally { this.isGraphRunning = false diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index c988bdc8c..05921fc69 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -597,7 +597,9 @@ export class CollectionSubscriber< } catch (error) { const current = this.unindexedSnapshot if ( - current.subscription === subscription && + // requestSnapshot can reentrantly unsubscribe and clear this field. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + current?.subscription === subscription && current.token === requestToken ) { this.unindexedSnapshot = undefined diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index b6cc85c55..18489307a 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1162,20 +1162,114 @@ it(`does not start duplicate ordered work from an applying receipt`, async () => }) }) -it.each([`sync throw`, `async reject`] as const)( - `retries unindexed transport after a %s during zero-to-positive refinement`, - async (failureMode) => { +it(`preserves a synchronous unindexed load error after reentrant cleanup`, async () => { + type Row = { id: string; rank: number } + const failure = new Error(`unindexed load failed after cleanup`) + let cleanupLive: () => Promise = () => Promise.resolve() + const source = createCollection({ + id: `unindexed-reentrant-cleanup-error`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + void cleanupLive() + throw failure + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-reentrant-cleanup-error-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + cleanupLive = () => live.cleanup() + + try { + await live.preload() + + expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow(failure) + } finally { + await Promise.all([live.cleanup(), source.cleanup()]) + } +}) + +it.each([ + { + name: `indexed sync throw without cleanup`, + autoIndex: `eager` as const, + failureMode: `sync throw` as const, + reentrantCleanup: false, + }, + { + name: `indexed async reject without cleanup`, + autoIndex: `eager` as const, + failureMode: `async reject` as const, + reentrantCleanup: false, + }, + { + name: `unindexed sync throw without cleanup`, + autoIndex: `off` as const, + failureMode: `sync throw` as const, + reentrantCleanup: false, + }, + { + name: `unindexed async reject without cleanup`, + autoIndex: `off` as const, + failureMode: `async reject` as const, + reentrantCleanup: false, + }, + { + name: `indexed sync throw with cleanup`, + autoIndex: `eager` as const, + failureMode: `sync throw` as const, + reentrantCleanup: true, + }, + { + name: `indexed async reject with cleanup`, + autoIndex: `eager` as const, + failureMode: `async reject` as const, + reentrantCleanup: true, + }, + { + name: `unindexed sync throw with cleanup`, + autoIndex: `off` as const, + failureMode: `sync throw` as const, + reentrantCleanup: true, + }, + { + name: `unindexed async reject with cleanup`, + autoIndex: `off` as const, + failureMode: `async reject` as const, + reentrantCleanup: true, + }, +])( + `preserves refinement failure and retry state for $name`, + async ({ autoIndex, failureMode, reentrantCleanup }) => { type Row = { id: string; rank: number } let attempts = 0 let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void let commit!: () => true | Promise + let cleanupLive: () => Promise = () => Promise.resolve() const source = createCollection({ - id: `unindexed-zero-refinement-retry`, + id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}`, getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, - autoIndex: `off`, + autoIndex, defaultIndexType: BTreeIndex, sync: { sync: (params) => { @@ -1188,6 +1282,7 @@ it.each([`sync throw`, `async reject`] as const)( attempts++ if (attempts === 1) { const error = new Error(`fallback failed`) + if (reentrantCleanup) void cleanupLive() if (failureMode === `sync throw`) throw error return Promise.reject(error) } @@ -1205,7 +1300,7 @@ it.each([`sync throw`, `async reject`] as const)( }, }) const live = createLiveQueryCollection({ - id: `unindexed-zero-refinement-retry-live`, + id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}-live`, query: (q) => q .from({ row: source }) @@ -1213,6 +1308,7 @@ it.each([`sync throw`, `async reject`] as const)( .limit(0), startSync: true, }) + cleanupLive = () => live.cleanup() try { await live.preload() @@ -1222,12 +1318,20 @@ it.each([`sync throw`, `async reject`] as const)( expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow( `fallback failed`, ) + } else if (reentrantCleanup) { + expect(live.utils.setWindow({ offset: 0, limit: 1 })).toBe(true) } else { await expect( live.utils.setWindow({ offset: 0, limit: 1 }), ).rejects.toThrow(`fallback failed`) } await flushPromises() + + if (reentrantCleanup) { + expect(attempts).toBe(1) + return + } + await live.utils.setWindow({ offset: 0, limit: 1 }) await flushPromises() From c8dd3045812f0b3694dcd55c4cfd6bc183d1ab36 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:14:29 -0600 Subject: [PATCH 136/327] test(db): cover reentrant cleanup restart --- ...d-subset-full-flow-oracle.property.test.ts | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 18489307a..a491b1d9d 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1259,11 +1259,13 @@ it.each([ `preserves refinement failure and retry state for $name`, async ({ autoIndex, failureMode, reentrantCleanup }) => { type Row = { id: string; rank: number } + const failure = new Error(`fallback failed`) let attempts = 0 let begin!: () => void let write!: (message: { type: `insert`; value: Row }) => void let commit!: () => true | Promise let cleanupLive: () => Promise = () => Promise.resolve() + let staleFailure: ReturnType> | undefined const source = createCollection({ id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}`, getKey: (row) => row.id, @@ -1281,10 +1283,13 @@ it.each([ loadSubset: () => { attempts++ if (attempts === 1) { - const error = new Error(`fallback failed`) if (reentrantCleanup) void cleanupLive() - if (failureMode === `sync throw`) throw error - return Promise.reject(error) + if (failureMode === `sync throw`) throw failure + if (reentrantCleanup) { + staleFailure = createDeferred() + return staleFailure.promise + } + return Promise.reject(failure) } begin() write({ type: `insert`, value: { id: `a`, rank: 1 } }) @@ -1316,19 +1321,50 @@ it.each([ if (failureMode === `sync throw`) { expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow( - `fallback failed`, + failure, ) } else if (reentrantCleanup) { expect(live.utils.setWindow({ offset: 0, limit: 1 })).toBe(true) } else { await expect( live.utils.setWindow({ offset: 0, limit: 1 }), - ).rejects.toThrow(`fallback failed`) + ).rejects.toBe(failure) } await flushPromises() if (reentrantCleanup) { expect(attempts).toBe(1) + expect(live.status).toBe(`cleaned-up`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray).toEqual([]) + expect(live.utils.getWindow()).toEqual({ + offset: 0, + limit: failureMode === `sync throw` ? 0 : 1, + }) + + await live.preload() + if (failureMode === `sync throw`) { + expect(attempts).toBe(1) + await live.utils.setWindow({ offset: 0, limit: 1 }) + } + await flushPromises() + + expect(attempts).toBe(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) + + staleFailure?.reject(failure) + await flushPromises() + + expect(attempts).toBe(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) return } @@ -1338,6 +1374,7 @@ it.each([ expect(attempts).toBe(2) expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) } finally { + staleFailure?.reject(failure) await Promise.all([live.cleanup(), source.cleanup()]) } }, From 09624d2e0594faeae70615b0c6bbbce954b41806 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:23:12 -0600 Subject: [PATCH 137/327] test(db): observe refinement rollback before retry --- ...d-subset-full-flow-oracle.property.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index a491b1d9d..199ccc349 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1365,9 +1365,29 @@ it.each([ expect(live.isLoadingSubset).toBe(false) expect(live.utils.lastSubsetError).toBeUndefined() expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) return } + expect(attempts).toBe(1) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBe(failure) + expect(live.toArray).toEqual([]) + expect(live.utils.getWindow()).toEqual({ + offset: 0, + limit: failureMode === `sync throw` ? 0 : 1, + }) + + if (failureMode === `sync throw`) { + begin() + write({ type: `insert`, value: { id: `b`, rank: 2 } }) + await commit() + await flushPromises() + expect(live.toArray).toEqual([]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 0 }) + } + await live.utils.setWindow({ offset: 0, limit: 1 }) await flushPromises() From 68b804482d53a82a049420427490f091517a3afd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:35:41 -0600 Subject: [PATCH 138/327] test(db): observe refinement cleanup ownership --- ...d-subset-full-flow-oracle.property.test.ts | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 199ccc349..a1e3ea1bf 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1200,7 +1200,13 @@ it(`preserves a synchronous unindexed load error after reentrant cleanup`, async try { await live.preload() - expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow(failure) + let thrown: unknown + try { + live.utils.setWindow({ offset: 0, limit: 1 }) + } catch (error) { + thrown = error + } + expect(thrown).toBe(failure) } finally { await Promise.all([live.cleanup(), source.cleanup()]) } @@ -1266,6 +1272,8 @@ it.each([ let commit!: () => true | Promise let cleanupLive: () => Promise = () => Promise.resolve() let staleFailure: ReturnType> | undefined + const signals: Array = [] + let unloads = 0 const source = createCollection({ id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}`, getKey: (row) => row.id, @@ -1280,8 +1288,9 @@ it.each([ commit = params.commit params.markReady() return { - loadSubset: () => { + loadSubset: ({ signal }) => { attempts++ + signals.push(signal) if (attempts === 1) { if (reentrantCleanup) void cleanupLive() if (failureMode === `sync throw`) throw failure @@ -1299,7 +1308,9 @@ it.each([ ? Promise.resolve(outcome) : applied.then(() => outcome) }, - unloadSubset: () => {}, + unloadSubset: () => { + unloads++ + }, } }, }, @@ -1320,9 +1331,13 @@ it.each([ expect(attempts).toBe(0) if (failureMode === `sync throw`) { - expect(() => live.utils.setWindow({ offset: 0, limit: 1 })).toThrow( - failure, - ) + let thrown: unknown + try { + live.utils.setWindow({ offset: 0, limit: 1 }) + } catch (error) { + thrown = error + } + expect(thrown).toBe(failure) } else if (reentrantCleanup) { expect(live.utils.setWindow({ offset: 0, limit: 1 })).toBe(true) } else { @@ -1338,6 +1353,9 @@ it.each([ expect(live.isLoadingSubset).toBe(false) expect(live.utils.lastSubsetError).toBeUndefined() expect(live.toArray).toEqual([]) + expect(signals).toHaveLength(1) + expect(signals[0]!.aborted).toBe(true) + expect(unloads).toBe(1) expect(live.utils.getWindow()).toEqual({ offset: 0, limit: failureMode === `sync throw` ? 0 : 1, From f77d455f34279e90e242c0a8f1c4978538cda452 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:38:26 -0600 Subject: [PATCH 139/327] test(db): type optional cleanup signal --- .../query/load-subset-full-flow-oracle.property.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index a1e3ea1bf..83978598a 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1272,7 +1272,7 @@ it.each([ let commit!: () => true | Promise let cleanupLive: () => Promise = () => Promise.resolve() let staleFailure: ReturnType> | undefined - const signals: Array = [] + const signals: Array = [] let unloads = 0 const source = createCollection({ id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}`, @@ -1354,7 +1354,8 @@ it.each([ expect(live.utils.lastSubsetError).toBeUndefined() expect(live.toArray).toEqual([]) expect(signals).toHaveLength(1) - expect(signals[0]!.aborted).toBe(true) + expect(signals[0]).toBeInstanceOf(AbortSignal) + expect(signals[0]?.aborted).toBe(true) expect(unloads).toBe(1) expect(live.utils.getWindow()).toEqual({ offset: 0, From 933e63a6b8440fed0e4785d53997812bd7208446 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:39:06 -0600 Subject: [PATCH 140/327] test(db): derive shared demand ownership --- .../db/tests/load-subset-full-flow-model.ts | 182 ++++++++++--- ...d-subset-full-flow-oracle.property.test.ts | 171 ++++++++++-- ...d-subset-refinement-model.property.test.ts | 246 ++++++++++++++---- 3 files changed, 501 insertions(+), 98 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index ffd46e594..d320b7676 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -224,9 +224,6 @@ export type LoadSubsetFullFlowEvent = ownerId: FullFlowOwnerId demandId: FullFlowDemandId attemptId: FullFlowAttemptId - rowKeys: ReadonlyArray - finalRowOwner: boolean - invalidatesAdapterEvidence: boolean } | { type: `restartSession` @@ -371,6 +368,34 @@ export type LoadSubsetFullFlowEvent = export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` ownerId: FullFlowOwnerId + attemptId: FullFlowAttemptId +} + +type ActiveDemandAttempts = Map> + +function addActiveDemandAttempt( + activeAttempts: ActiveDemandAttempts, + demandId: FullFlowDemandId, + attemptId: FullFlowAttemptId, +): void { + let attempts = activeAttempts.get(demandId) + if (!attempts) { + attempts = new Set() + activeAttempts.set(demandId, attempts) + } + attempts.add(attemptId) +} + +function releaseActiveDemandAttempt( + activeAttempts: ActiveDemandAttempts, + demandId: FullFlowDemandId, + attemptId: FullFlowAttemptId, +): boolean { + const attempts = activeAttempts.get(demandId) + if (!attempts?.delete(attemptId)) return false + if (attempts.size > 0) return false + activeAttempts.delete(demandId) + return true } type DemandAttemptRecord = { @@ -455,16 +480,28 @@ function assertWellFormedDemandAttempts( export function projectAdapterLifecycle( history: ReadonlyArray, ): Array { - const invokedOwners = new Set() + assertWellFormedDemandAttempts(history) + const invokedAttempts = new Set() const projected: Array = [] for (const event of history) { if (event.type === `requestDemand` && !event.alreadyAborted) { - invokedOwners.add(event.ownerId) - projected.push({ type: `invoke`, ownerId: event.ownerId }) + invokedAttempts.add(event.attemptId) + projected.push({ + type: `invoke`, + ownerId: event.ownerId, + attemptId: event.attemptId, + }) } - if (event.type === `releaseDemand` && invokedOwners.delete(event.ownerId)) { - projected.push({ type: `release`, ownerId: event.ownerId }) + if ( + event.type === `releaseDemand` && + invokedAttempts.delete(event.attemptId) + ) { + projected.push({ + type: `release`, + ownerId: event.ownerId, + attemptId: event.attemptId, + }) } } @@ -485,13 +522,15 @@ export function projectTransportLoads( assertWellFormedDemandAttempts(history) const reusableDemands = new Map() const inFlightDemands = new Map() + const activeAttempts: ActiveDemandAttempts = new Map() let loads = 0 for (const event of history) { switch (event.type) { case `requestDemand`: + if (event.alreadyAborted) break + addActiveDemandAttempt(activeAttempts, event.demandId, event.attemptId) if ( - !event.alreadyAborted && !reusableDemands.has(event.demandId) && !inFlightDemands.has(event.demandId) ) { @@ -517,13 +556,15 @@ export function projectTransportLoads( } break case `releaseDemand`: - if (event.invalidatesAdapterEvidence) { - if (reusableDemands.get(event.demandId) === event.attemptId) { - reusableDemands.delete(event.demandId) - } - if (inFlightDemands.get(event.demandId) === event.attemptId) { - inFlightDemands.delete(event.demandId) - } + if ( + releaseActiveDemandAttempt( + activeAttempts, + event.demandId, + event.attemptId, + ) + ) { + reusableDemands.delete(event.demandId) + inFlightDemands.delete(event.demandId) } break case `restartSession`: @@ -654,6 +695,7 @@ export function projectReusableDemands( assertWellFormedDemandAttempts(history) const reusableDemands = new Map() const attemptEpochs = new Map() + const activeAttempts: ActiveDemandAttempts = new Map() let sourceEpoch = 0 for (const event of history) { @@ -661,6 +703,11 @@ export function projectReusableDemands( case `requestDemand`: if (!event.alreadyAborted) { attemptEpochs.set(event.attemptId, sourceEpoch) + addActiveDemandAttempt( + activeAttempts, + event.demandId, + event.attemptId, + ) } break case `applyAuthoritativeRows`: @@ -673,11 +720,15 @@ export function projectReusableDemands( reusableDemands.clear() break case `releaseDemand`: - if (event.invalidatesAdapterEvidence) { - attemptEpochs.delete(event.attemptId) - if (reusableDemands.get(event.demandId) === event.attemptId) { - reusableDemands.delete(event.demandId) - } + attemptEpochs.delete(event.attemptId) + if ( + releaseActiveDemandAttempt( + activeAttempts, + event.demandId, + event.attemptId, + ) + ) { + reusableDemands.delete(event.demandId) } break case `applyUnprovenRows`: @@ -798,6 +849,7 @@ export function projectAtomicOrderedPublicationState( initialWindowSize: number }, ): AtomicOrderedPublicationProjection { + assertWellFormedDemandAttempts(history) const staged = new Map< FullFlowPublicationId, Map> @@ -811,7 +863,7 @@ export function projectAtomicOrderedPublicationState( | undefined > >() - const activeAdditionalDemands = new Set() + const activeAdditionalDemands: ActiveDemandAttempts = new Map() const publications: Array> = [] let currentPublication: AtomicOrderedPublicationState | undefined let retainsPreviousPublication = false @@ -839,7 +891,7 @@ export function projectAtomicOrderedPublicationState( const orderedPrefix = sortRows(orderedRows).slice(0, retainedSize) const desired = new Map(orderedPrefix.map((row) => [row.key, row] as const)) - for (const demandId of activeAdditionalDemands) { + for (const demandId of activeAdditionalDemands.keys()) { for (const row of publication.get(demandId) ?? []) { desired.set(row.key, row) } @@ -886,7 +938,7 @@ export function projectAtomicOrderedPublicationState( const current = attempts.get(currentReplacement) const ordered = current?.get(options.demandId) - const activeDemandFailed = [...activeAdditionalDemands].some( + const activeDemandFailed = [...activeAdditionalDemands.keys()].some( (demandId) => current?.get(demandId)?.outcome !== `success`, ) if (ordered?.outcome !== `success` || activeDemandFailed) { @@ -958,7 +1010,11 @@ export function projectAtomicOrderedPublicationState( } case `requestDemand`: if (!event.alreadyAborted && event.demandId !== options.demandId) { - activeAdditionalDemands.add(event.demandId) + addActiveDemandAttempt( + activeAdditionalDemands, + event.demandId, + event.attemptId, + ) } break case `applyAuthoritativeRows`: @@ -966,7 +1022,11 @@ export function projectAtomicOrderedPublicationState( case `rejectDemand`: break case `releaseDemand`: - activeAdditionalDemands.delete(event.demandId) + releaseActiveDemandAttempt( + activeAdditionalDemands, + event.demandId, + event.attemptId, + ) break case `truncateSource`: case `restartSession`: @@ -995,22 +1055,80 @@ export function projectAtomicOrderedPublicationState( export function projectRetainedRowKeys( history: ReadonlyArray, ): Array { - const retainedRows = new Set() + assertWellFormedDemandAttempts(history) + const activeAttempts: ActiveDemandAttempts = new Map() + const reusableRows = new Map>() + const rowClaims = new Map>() + const attemptRows = new Map>() + + const claimRows = ( + attemptId: FullFlowAttemptId, + rowKeys: Iterable, + ) => { + let claimed = attemptRows.get(attemptId) + if (!claimed) { + claimed = new Set() + attemptRows.set(attemptId, claimed) + } + for (const rowKey of rowKeys) { + claimed.add(rowKey) + let claims = rowClaims.get(rowKey) + if (!claims) { + claims = new Set() + rowClaims.set(rowKey, claims) + } + claims.add(attemptId) + } + } + + const releaseRows = (attemptId: FullFlowAttemptId) => { + for (const rowKey of attemptRows.get(attemptId) ?? []) { + const claims = rowClaims.get(rowKey) + claims?.delete(attemptId) + if (claims?.size === 0) rowClaims.delete(rowKey) + } + attemptRows.delete(attemptId) + } for (const event of history) { + if (event.type === `requestDemand` && !event.alreadyAborted) { + addActiveDemandAttempt(activeAttempts, event.demandId, event.attemptId) + const retained = reusableRows.get(event.demandId) + if (retained) claimRows(event.attemptId, retained) + } + if (event.type === `applyAuthoritativeRows`) { + let retained = reusableRows.get(event.demandId) + if (!retained) { + retained = new Set() + reusableRows.set(event.demandId, retained) + } + event.rowKeys.forEach((rowKey) => retained.add(rowKey)) + } if ( event.type === `applyAuthoritativeRows` || event.type === `applyUnprovenRows` ) { - event.rowKeys.forEach((key) => retainedRows.add(key)) + for (const attemptId of activeAttempts.get(event.demandId) ?? []) { + claimRows(attemptId, event.rowKeys) + } + } + if (event.type === `truncateSource`) { + reusableRows.clear() + rowClaims.clear() + attemptRows.clear() } - if (event.type === `truncateSource`) retainedRows.clear() - if (event.type === `releaseDemand` && event.finalRowOwner) { - event.rowKeys.forEach((key) => retainedRows.delete(key)) + if (event.type === `releaseDemand`) { + const releasedFinalAttempt = releaseActiveDemandAttempt( + activeAttempts, + event.demandId, + event.attemptId, + ) + releaseRows(event.attemptId) + if (releasedFinalAttempt) reusableRows.delete(event.demandId) } } - return [...retainedRows].sort() + return [...rowClaims.keys()].sort() } export type ExpectedSyncReceiptState = `pending` | `resolved` | `rejected` diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 83978598a..70921983b 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -3117,16 +3117,6 @@ async function runTruncateCoverageScenario( ? `old` : `fresh` }-attempt`, - rowKeys: - options === initialOptions - ? [`initial`] - : options === oldOptions - ? [`old`] - : scenario.freshResult === `reject` - ? [] - : [`fresh`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }) } expect(unloadSubset.mock.calls.map(([options]) => options)).toEqual( @@ -3275,6 +3265,155 @@ it.each([ }, ) +it(`keeps adapter release obligations distinct across attempts by one owner`, () => { + const history: ReadonlyArray = [ + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt-1`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt-2`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt-1`, + }, + { + type: `releaseDemand`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt-2`, + }, + ] + + expect(projectAdapterLifecycle(history)).toEqual([ + { type: `invoke`, ownerId: `owner`, attemptId: `attempt-1` }, + { type: `invoke`, ownerId: `owner`, attemptId: `attempt-2` }, + { type: `release`, ownerId: `owner`, attemptId: `attempt-1` }, + { type: `release`, ownerId: `owner`, attemptId: `attempt-2` }, + ]) +}) + +it(`derives shared row and evidence lifetime from active attempts`, () => { + const sharedHistory: ReadonlyArray = [ + { + type: `requestDemand`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + ownerId: `owner-a`, + demandId: `shared`, + attemptId: `attempt-a`, + rowKeys: [`x`], + }, + { + type: `releaseDemand`, + ownerId: `owner-a`, + demandId: `shared`, + attemptId: `attempt-a`, + }, + ] + + expect(projectRetainedRowKeys(sharedHistory)).toEqual([`x`]) + expect( + projectTransportLoads([ + ...sharedHistory, + { + type: `requestDemand`, + ownerId: `owner-c`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-c`, + alreadyAborted: false, + }, + ]), + ).toBe(1) + + expect( + projectRetainedRowKeys([ + ...sharedHistory, + { + type: `releaseDemand`, + ownerId: `owner-b`, + demandId: `shared`, + attemptId: `attempt-b`, + }, + ]), + ).toEqual([]) +}) + +it(`keeps an additional demand active until its final attempt releases`, () => { + const history: ReadonlyArray = [ + { + type: `requestDemand`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `other`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `other`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `next`, + demandId: `ordered`, + rows: [{ key: `o`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `next`, + demandId: `other`, + rows: [{ key: `x`, orderValue: 1 }], + }, + { + type: `releaseDemand`, + ownerId: `owner-a`, + demandId: `other`, + attemptId: `attempt-a`, + }, + { type: `commitPublication`, publicationId: `next` }, + ] + + expect( + projectAtomicOrderedPublicationState(history, { + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`o`, `x`]) +}) + it(`does not release physical work when an already-aborted demand skips adapter start`, async () => { const ownerId = `aborted-owner` const requestEvent: LoadSubsetFullFlowEvent = { @@ -3292,9 +3431,6 @@ it(`does not release physical work when an already-aborted demand skips adapter ownerId, demandId: `all-rows`, attemptId: `aborted-attempt`, - rowKeys: [], - finalRowOwner: false, - invalidatesAdapterEvidence: false, }, ] const adapterEvents: Array = [] @@ -3841,9 +3977,6 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad ownerId: `owner-1`, demandId: `all-rows`, attemptId: `attempt-1`, - rowKeys: [row.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, { type: `restartSession`, @@ -6188,9 +6321,6 @@ async function runAtomicOrderedReplayScenario( ownerId: `other-owner`, demandId: `other`, attemptId: `other-attempt`, - rowKeys: [replacementOtherRow.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }) expectPublicationHistory() } @@ -6277,9 +6407,6 @@ async function runAtomicOrderedReplayScenario( ownerId: `other-owner`, demandId: `other`, attemptId: `other-attempt`, - rowKeys: [replacementOtherRow.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }) expectPublicationHistory() } diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 4135d272f..4dcfbb549 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -87,7 +87,11 @@ for (const campaign of refinementCampaigns(1_779_001)) { type DemandLifecycleCase = { history: Array - expected: Array<{ type: `invoke` | `release`; ownerId: string }> + expected: Array<{ + type: `invoke` | `release` + ownerId: string + attemptId: string + }> } function enumerateDemandLifecycles(): Array { @@ -117,7 +121,14 @@ function enumerateDemandLifecycles(): Array { ], alreadyAborted ? expected - : [...expected, { type: `invoke`, ownerId }], + : [ + ...expected, + { + type: `invoke`, + ownerId, + attemptId: `${ownerId}-attempt`, + }, + ], unseenOwners.filter((owner) => owner !== ownerId), alreadyAborted ? activeOwners : [...activeOwners, ownerId], ) @@ -132,12 +143,16 @@ function enumerateDemandLifecycles(): Array { ownerId, demandId: `demand`, attemptId: `${ownerId}-attempt`, - rowKeys: [], - finalRowOwner: false, - invalidatesAdapterEvidence: false, }, ], - [...expected, { type: `release`, ownerId }], + [ + ...expected, + { + type: `release`, + ownerId, + attemptId: `${ownerId}-attempt`, + }, + ], unseenOwners, activeOwners.filter((owner) => owner !== ownerId), ) @@ -152,17 +167,18 @@ it(`exhaustively projects exact adapter starts and releases for two owners`, () for (const { history, expected } of enumerateDemandLifecycles()) { const lifecycle = projectAdapterLifecycle(history) expect(lifecycle, JSON.stringify(history)).toEqual(expected) - const activeOwners = new Set() + const activeAttempts = new Set() for (const event of lifecycle) { if (event.type === `invoke`) { - expect(activeOwners.has(event.ownerId), JSON.stringify(history)).toBe( - false, - ) - activeOwners.add(event.ownerId) + expect( + activeAttempts.has(event.attemptId), + JSON.stringify(history), + ).toBe(false) + activeAttempts.add(event.attemptId) } else { expect( - activeOwners.delete(event.ownerId), + activeAttempts.delete(event.attemptId), JSON.stringify(history), ).toBe(true) } @@ -196,9 +212,24 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ownerId: `owner-a`, demandId: `exact-demand`, attemptId: `owner-a-attempt`, - rowKeys: [], - finalRowOwner: true, - invalidatesAdapterEvidence: true, + }, + request(`owner-c`), + ]), + ).toBe(1) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `releaseDemand`, + ownerId: `owner-a`, + demandId: `exact-demand`, + attemptId: `owner-a-attempt`, + }, + { + type: `releaseDemand`, + ownerId: `owner-b`, + demandId: `exact-demand`, + attemptId: `owner-b-attempt`, }, request(`owner-c`), ]), @@ -229,6 +260,158 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ).toBe(1) }) +it(`retains a row until its last independent demand claim releases`, () => { + const request = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + ownerId: attemptId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const apply = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `applyUnprovenRows`, + ownerId: attemptId, + demandId, + attemptId, + rowKeys: [`x`], + }) + const release = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + ownerId: attemptId, + demandId, + attemptId, + }) + const sharedClaims = [ + request(`left`, `left-attempt`), + request(`right`, `right-attempt`), + apply(`left`, `left-attempt`), + apply(`right`, `right-attempt`), + ] + + expect( + projectRetainedRowKeys([...sharedClaims, release(`left`, `left-attempt`)]), + ).toEqual([`x`]) + expect( + projectRetainedRowKeys([ + ...sharedClaims, + release(`left`, `left-attempt`), + release(`right`, `right-attempt`), + ]), + ).toEqual([]) +}) + +it.each([ + { + name: `one owner releases the first attempt first`, + owners: [`owner`, `owner`] as const, + releaseOrder: [0, 1] as const, + }, + { + name: `one owner releases the second attempt first`, + owners: [`owner`, `owner`] as const, + releaseOrder: [1, 0] as const, + }, + { + name: `two owners release the first attempt first`, + owners: [`owner-a`, `owner-b`] as const, + releaseOrder: [0, 1] as const, + }, + { + name: `two owners release the second attempt first`, + owners: [`owner-a`, `owner-b`] as const, + releaseOrder: [1, 0] as const, + }, +])(`derives shared ownership for $name`, ({ owners, releaseOrder }) => { + const demandId = `shared` + const attempts = owners.map((ownerId, index) => ({ + ownerId, + attemptId: `attempt-${index}`, + })) + const requests = attempts.map( + ({ ownerId, attemptId }) => ({ + type: `requestDemand`, + ownerId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }), + ) + const settlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + ownerId: attempts[0]!.ownerId, + demandId, + attemptId: attempts[0]!.attemptId, + rowKeys: [`x`], + } + const releases = releaseOrder.map((index) => ({ + type: `releaseDemand`, + ownerId: attempts[index]!.ownerId, + demandId, + attemptId: attempts[index]!.attemptId, + })) + + for (let released = 0; released <= releases.length; released++) { + const active = released < releases.length + const history = [...requests, settlement, ...releases.slice(0, released)] + const lifecycle = projectAdapterLifecycle(history) + + expect(lifecycle.filter(({ type }) => type === `invoke`)).toHaveLength(2) + expect(lifecycle.filter(({ type }) => type === `release`)).toHaveLength( + released, + ) + expect(projectRetainedRowKeys(history)).toEqual(active ? [`x`] : []) + expect(projectReusableDemands(history)).toEqual(active ? [demandId] : []) + const peerRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + ownerId: `peer`, + sessionId: `session`, + demandId, + attemptId: `peer-after-${released}`, + alreadyAborted: false, + } + expect(projectRetainedRowKeys([...history, peerRequest])).toEqual( + active ? [`x`] : [], + ) + expect(projectTransportLoads([...history, peerRequest])).toBe( + active ? 1 : 2, + ) + + const publication = projectAtomicOrderedPublicationState( + [ + ...history, + { + type: `stagePublicationRows`, + publicationId: `publication`, + demandId: `ordered`, + rows: [{ key: `o`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + demandId, + rows: [{ key: `x`, orderValue: 1 }], + }, + { type: `commitPublication`, publicationId: `publication` }, + ], + { demandId: `ordered`, direction: `asc`, initialWindowSize: 1 }, + ) + expect(publication.currentPublication?.rows.map(({ key }) => key)).toEqual( + active ? [`o`, `x`] : [`o`], + ) + } +}) + it.each([ { name: `authoritative`, @@ -274,9 +457,6 @@ it.each([ ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, - rowKeys: [], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, }, ] satisfies ReadonlyArray<{ @@ -348,9 +528,6 @@ it(`scopes reusable evidence to the physical attempt when an owner is reused`, ( ownerId: `stable-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, - rowKeys: [`stale-row`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, } const beforeFreshSettlement = [ oldRequest, @@ -400,9 +577,6 @@ it(`does not rebuild coverage when a released attempt settles after its replacem ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, - rowKeys: [], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, { type: `requestDemand`, @@ -456,9 +630,6 @@ it(`keeps fresh same-epoch work shared after an older rejected attempt releases` ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, - rowKeys: [], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, { ...freshRequest, @@ -484,9 +655,6 @@ it(`rejects histories that reuse one demand attempt identity`, () => { ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `reused-attempt`, - rowKeys: [], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, { type: `requestDemand`, @@ -755,9 +923,6 @@ for (const campaign of refinementCampaigns(1_779_003)) { ownerId: `owner`, demandId: `demand`, attemptId: `attempt`, - rowKeys: [`row`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, ] const continuationHistory: Array = [ @@ -856,9 +1021,10 @@ for (const campaign of refinementCampaigns(1_779_003)) { suffix, projectAdapterLifecycle, (events, renamingSuffix) => - events.map(({ type, ownerId }) => ({ + events.map(({ type, ownerId, attemptId }) => ({ type, ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + attemptId: removeRenamingSuffix(attemptId, renamingSuffix), })), ) expectObservationPreservedAfterEveryPrefix( @@ -1208,9 +1374,6 @@ function demandErasureHistories(): Array> { ownerId, demandId: `demand-a`, attemptId, - rowKeys: [`row-a`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }) return [ @@ -1263,9 +1426,6 @@ function demandErasureHistories(): Array> { ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, - rowKeys: [`row-a`], - finalRowOwner: false, - invalidatesAdapterEvidence: false, }, ], [ @@ -1661,9 +1821,6 @@ function publicationErasureHistories(): Array> { ownerId: `owner-related`, demandId: `related`, attemptId: `attempt-related`, - rowKeys: [`related`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, }, ], [ @@ -1775,9 +1932,10 @@ for (const campaign of refinementCampaigns(1_779_009)) { suffix, projectAdapterLifecycle, (events, renamingSuffix) => - events.map(({ type, ownerId }) => ({ + events.map(({ type, ownerId, attemptId }) => ({ type, ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + attemptId: removeRenamingSuffix(attemptId, renamingSuffix), })), ) expectObservationPreservedAfterEveryPrefix( From db898056705bcb4ff1c77ed240637108f40e11f2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 05:56:12 -0600 Subject: [PATCH 141/327] test(db): scope late rows to shared acquisition --- .../db/tests/load-subset-full-flow-model.ts | 122 ++++++++++++++---- ...d-subset-refinement-model.property.test.ts | 76 +++++++++++ 2 files changed, 174 insertions(+), 24 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index d320b7676..775f405c9 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -693,34 +693,44 @@ export function projectReusableDemands( history: ReadonlyArray, ): Array { assertWellFormedDemandAttempts(history) - const reusableDemands = new Map() - const attemptEpochs = new Map() const activeAttempts: ActiveDemandAttempts = new Map() - let sourceEpoch = 0 + const currentAcquisitions = new Map() + const reusableAcquisitions = new Map() + const attemptAcquisitions = new Map() for (const event of history) { switch (event.type) { case `requestDemand`: if (!event.alreadyAborted) { - attemptEpochs.set(event.attemptId, sourceEpoch) addActiveDemandAttempt( activeAttempts, event.demandId, event.attemptId, ) + const acquisitionId = + currentAcquisitions.get(event.demandId) ?? + reusableAcquisitions.get(event.demandId) ?? + event.attemptId + currentAcquisitions.set(event.demandId, acquisitionId) + attemptAcquisitions.set(event.attemptId, acquisitionId) } break - case `applyAuthoritativeRows`: - if (attemptEpochs.get(event.attemptId) === sourceEpoch) { - reusableDemands.set(event.demandId, event.attemptId) + case `applyAuthoritativeRows`: { + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(event.demandId) === acquisitionId + ) { + reusableAcquisitions.set(event.demandId, acquisitionId) + currentAcquisitions.delete(event.demandId) } break + } case `truncateSource`: - sourceEpoch++ - reusableDemands.clear() + currentAcquisitions.clear() + reusableAcquisitions.clear() break case `releaseDemand`: - attemptEpochs.delete(event.attemptId) if ( releaseActiveDemandAttempt( activeAttempts, @@ -728,7 +738,8 @@ export function projectReusableDemands( event.attemptId, ) ) { - reusableDemands.delete(event.demandId) + currentAcquisitions.delete(event.demandId) + reusableAcquisitions.delete(event.demandId) } break case `applyUnprovenRows`: @@ -748,7 +759,7 @@ export function projectReusableDemands( } } - return [...reusableDemands.keys()].sort() + return [...reusableAcquisitions.keys()].sort() } /** @@ -1057,7 +1068,17 @@ export function projectRetainedRowKeys( ): Array { assertWellFormedDemandAttempts(history) const activeAttempts: ActiveDemandAttempts = new Map() - const reusableRows = new Map>() + const activeAttemptIds = new Set() + const currentAcquisitions = new Map() + const reusableRows = new Map< + FullFlowDemandId, + { acquisitionId: FullFlowAttemptId; rows: Set } + >() + const attemptAcquisitions = new Map() + const acquisitionAttempts = new Map< + FullFlowAttemptId, + Set + >() const rowClaims = new Map>() const attemptRows = new Map>() @@ -1093,38 +1114,91 @@ export function projectRetainedRowKeys( for (const event of history) { if (event.type === `requestDemand` && !event.alreadyAborted) { addActiveDemandAttempt(activeAttempts, event.demandId, event.attemptId) + activeAttemptIds.add(event.attemptId) const retained = reusableRows.get(event.demandId) - if (retained) claimRows(event.attemptId, retained) + const acquisitionId = + currentAcquisitions.get(event.demandId) ?? + retained?.acquisitionId ?? + event.attemptId + currentAcquisitions.set(event.demandId, acquisitionId) + attemptAcquisitions.set(event.attemptId, acquisitionId) + let participants = acquisitionAttempts.get(acquisitionId) + if (!participants) { + participants = new Set() + acquisitionAttempts.set(acquisitionId, participants) + } + participants.add(event.attemptId) + if (retained) claimRows(event.attemptId, retained.rows) } if (event.type === `applyAuthoritativeRows`) { - let retained = reusableRows.get(event.demandId) - if (!retained) { - retained = new Set() - reusableRows.set(event.demandId, retained) + const acquisitionId = attemptAcquisitions.get(event.attemptId) + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(event.demandId) === acquisitionId + ) { + const rows = new Set(event.rowKeys) + reusableRows.set(event.demandId, { acquisitionId, rows }) + currentAcquisitions.delete(event.demandId) + } + for (const attemptId of participants) { + if (activeAttemptIds.has(attemptId)) { + claimRows(attemptId, event.rowKeys) + } + } + } + if (event.type === `applyUnprovenRows`) { + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(event.demandId) === acquisitionId + ) { + currentAcquisitions.delete(event.demandId) + } + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + for (const attemptId of participants) { + if (activeAttemptIds.has(attemptId)) { + claimRows(attemptId, event.rowKeys) + } } - event.rowKeys.forEach((rowKey) => retained.add(rowKey)) } if ( - event.type === `applyAuthoritativeRows` || - event.type === `applyUnprovenRows` + event.type === `rejectDemand` || + event.type === `settleDemandWithoutEvidence` ) { - for (const attemptId of activeAttempts.get(event.demandId) ?? []) { - claimRows(attemptId, event.rowKeys) + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if (currentAcquisitions.get(event.demandId) === acquisitionId) { + currentAcquisitions.delete(event.demandId) } } if (event.type === `truncateSource`) { + currentAcquisitions.clear() reusableRows.clear() rowClaims.clear() attemptRows.clear() } if (event.type === `releaseDemand`) { + activeAttemptIds.delete(event.attemptId) + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if (acquisitionId) { + acquisitionAttempts.get(acquisitionId)?.delete(event.attemptId) + } const releasedFinalAttempt = releaseActiveDemandAttempt( activeAttempts, event.demandId, event.attemptId, ) releaseRows(event.attemptId) - if (releasedFinalAttempt) reusableRows.delete(event.demandId) + if (releasedFinalAttempt) { + currentAcquisitions.delete(event.demandId) + reusableRows.delete(event.demandId) + } } } diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 4dcfbb549..a903cad4c 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -310,6 +310,56 @@ it(`retains a row until its last independent demand claim releases`, () => { ).toEqual([]) }) +it(`attaches late rows only to attempts that shared the settling acquisition`, () => { + const request = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + ownerId, + sessionId: `session`, + demandId: `shared`, + attemptId, + alreadyAborted: false, + }) + const release = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + ownerId, + demandId: `shared`, + attemptId, + }) + const lateSettlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + ownerId: `old-owner`, + demandId: `shared`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + } + const oldRequest = request(`old-owner`, `old-attempt`) + const oldRelease = release(`old-owner`, `old-attempt`) + + const freshCohort = [ + oldRequest, + oldRelease, + request(`fresh-owner`, `fresh-attempt`), + lateSettlement, + ] + expect(projectRetainedRowKeys(freshCohort)).toEqual([]) + expect(projectReusableDemands(freshCohort)).toEqual([]) + + const attachedPeer = [ + oldRequest, + request(`peer-owner`, `peer-attempt`), + oldRelease, + lateSettlement, + ] + expect(projectRetainedRowKeys(attachedPeer)).toEqual([`stale-row`]) + expect(projectReusableDemands(attachedPeer)).toEqual([`shared`]) +}) + it.each([ { name: `one owner releases the first attempt first`, @@ -410,6 +460,32 @@ it.each([ active ? [`o`, `x`] : [`o`], ) } + + const lateSharedSettlement = [ + requests[0]!, + requests[1]!, + releases[0]!, + settlement, + ] + expect(projectRetainedRowKeys(lateSharedSettlement)).toEqual([`x`]) + expect(projectReusableDemands(lateSharedSettlement)).toEqual([demandId]) + + const fullyReleasedBeforeSettlement = [ + ...requests, + ...releases, + { + type: `requestDemand`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId, + attemptId: `fresh-attempt`, + alreadyAborted: false, + } satisfies LoadSubsetFullFlowEvent, + settlement, + ] + expect(projectRetainedRowKeys(fullyReleasedBeforeSettlement)).toEqual([]) + expect(projectReusableDemands(fullyReleasedBeforeSettlement)).toEqual([]) + expect(projectTransportLoads(fullyReleasedBeforeSettlement)).toBe(2) }) it.each([ From 005d3b755ee38ac101f6512a7ddf3d5bf11d4402 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 06:06:04 -0600 Subject: [PATCH 142/327] test(db): retire demand acquisitions independently --- .../db/tests/load-subset-full-flow-model.ts | 168 +++++++++++++----- ...d-subset-refinement-model.property.test.ts | 35 ++++ 2 files changed, 155 insertions(+), 48 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 775f405c9..086a49b91 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -372,6 +372,7 @@ export type ExpectedAdapterLifecycleEvent = { } type ActiveDemandAttempts = Map> +type AcquisitionAttempts = Map> function addActiveDemandAttempt( activeAttempts: ActiveDemandAttempts, @@ -398,6 +399,30 @@ function releaseActiveDemandAttempt( return true } +function addAcquisitionAttempt( + acquisitionAttempts: AcquisitionAttempts, + acquisitionId: FullFlowAttemptId, + attemptId: FullFlowAttemptId, +): void { + let attempts = acquisitionAttempts.get(acquisitionId) + if (!attempts) { + attempts = new Set() + acquisitionAttempts.set(acquisitionId, attempts) + } + attempts.add(attemptId) +} + +function releaseAcquisitionAttempt( + acquisitionAttempts: AcquisitionAttempts, + acquisitionId: FullFlowAttemptId, + attemptId: FullFlowAttemptId, +): boolean { + const attempts = acquisitionAttempts.get(acquisitionId) + if (!attempts?.delete(attemptId) || attempts.size > 0) return false + acquisitionAttempts.delete(acquisitionId) + return true +} + type DemandAttemptRecord = { ownerId: FullFlowOwnerId demandId: FullFlowDemandId @@ -520,53 +545,79 @@ export function projectTransportLoads( history: ReadonlyArray, ): number { assertWellFormedDemandAttempts(history) - const reusableDemands = new Map() - const inFlightDemands = new Map() - const activeAttempts: ActiveDemandAttempts = new Map() + const reusableAcquisitions = new Map() + const inFlightAcquisitions = new Map() + const attemptAcquisitions = new Map() + const acquisitionAttempts: AcquisitionAttempts = new Map() let loads = 0 for (const event of history) { switch (event.type) { - case `requestDemand`: + case `requestDemand`: { if (event.alreadyAborted) break - addActiveDemandAttempt(activeAttempts, event.demandId, event.attemptId) - if ( - !reusableDemands.has(event.demandId) && - !inFlightDemands.has(event.demandId) - ) { + let acquisitionId = + inFlightAcquisitions.get(event.demandId) ?? + reusableAcquisitions.get(event.demandId) + if (acquisitionId === undefined) { loads++ - inFlightDemands.set(event.demandId, event.attemptId) + acquisitionId = event.attemptId + inFlightAcquisitions.set(event.demandId, acquisitionId) } + attemptAcquisitions.set(event.attemptId, acquisitionId) + addAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + event.attemptId, + ) break + } case `applyAuthoritativeRows`: { - if (inFlightDemands.get(event.demandId) !== event.attemptId) break - inFlightDemands.delete(event.demandId) - reusableDemands.set(event.demandId, event.attemptId) + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if ( + acquisitionId === undefined || + inFlightAcquisitions.get(event.demandId) !== acquisitionId + ) { + break + } + inFlightAcquisitions.delete(event.demandId) + reusableAcquisitions.set(event.demandId, acquisitionId) break } case `truncateSource`: - reusableDemands.clear() - inFlightDemands.clear() + reusableAcquisitions.clear() + inFlightAcquisitions.clear() break case `applyUnprovenRows`: case `rejectDemand`: - case `settleDemandWithoutEvidence`: - if (inFlightDemands.get(event.demandId) === event.attemptId) { - inFlightDemands.delete(event.demandId) + case `settleDemandWithoutEvidence`: { + const acquisitionId = attemptAcquisitions.get(event.attemptId) + if ( + acquisitionId !== undefined && + inFlightAcquisitions.get(event.demandId) === acquisitionId + ) { + inFlightAcquisitions.delete(event.demandId) } break - case `releaseDemand`: + } + case `releaseDemand`: { + const acquisitionId = attemptAcquisitions.get(event.attemptId) if ( - releaseActiveDemandAttempt( - activeAttempts, - event.demandId, + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, event.attemptId, ) ) { - reusableDemands.delete(event.demandId) - inFlightDemands.delete(event.demandId) + if (reusableAcquisitions.get(event.demandId) === acquisitionId) { + reusableAcquisitions.delete(event.demandId) + } + if (inFlightAcquisitions.get(event.demandId) === acquisitionId) { + inFlightAcquisitions.delete(event.demandId) + } } break + } case `restartSession`: case `cleanupSession`: case `advanceWindowRevision`: @@ -697,6 +748,7 @@ export function projectReusableDemands( const currentAcquisitions = new Map() const reusableAcquisitions = new Map() const attemptAcquisitions = new Map() + const acquisitionAttempts: AcquisitionAttempts = new Map() for (const event of history) { switch (event.type) { @@ -713,6 +765,11 @@ export function projectReusableDemands( event.attemptId currentAcquisitions.set(event.demandId, acquisitionId) attemptAcquisitions.set(event.attemptId, acquisitionId) + addAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + event.attemptId, + ) } break case `applyAuthoritativeRows`: { @@ -730,18 +787,30 @@ export function projectReusableDemands( currentAcquisitions.clear() reusableAcquisitions.clear() break - case `releaseDemand`: + case `releaseDemand`: { + releaseActiveDemandAttempt( + activeAttempts, + event.demandId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(event.attemptId) if ( - releaseActiveDemandAttempt( - activeAttempts, - event.demandId, + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, event.attemptId, ) ) { - currentAcquisitions.delete(event.demandId) - reusableAcquisitions.delete(event.demandId) + if (currentAcquisitions.get(event.demandId) === acquisitionId) { + currentAcquisitions.delete(event.demandId) + } + if (reusableAcquisitions.get(event.demandId) === acquisitionId) { + reusableAcquisitions.delete(event.demandId) + } } break + } case `applyUnprovenRows`: case `rejectDemand`: case `restartSession`: @@ -1075,10 +1144,7 @@ export function projectRetainedRowKeys( { acquisitionId: FullFlowAttemptId; rows: Set } >() const attemptAcquisitions = new Map() - const acquisitionAttempts = new Map< - FullFlowAttemptId, - Set - >() + const acquisitionAttempts: AcquisitionAttempts = new Map() const rowClaims = new Map>() const attemptRows = new Map>() @@ -1122,12 +1188,7 @@ export function projectRetainedRowKeys( event.attemptId currentAcquisitions.set(event.demandId, acquisitionId) attemptAcquisitions.set(event.attemptId, acquisitionId) - let participants = acquisitionAttempts.get(acquisitionId) - if (!participants) { - participants = new Set() - acquisitionAttempts.set(acquisitionId, participants) - } - participants.add(event.attemptId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, event.attemptId) if (retained) claimRows(event.attemptId, retained.rows) } if (event.type === `applyAuthoritativeRows`) { @@ -1173,7 +1234,10 @@ export function projectRetainedRowKeys( event.type === `settleDemandWithoutEvidence` ) { const acquisitionId = attemptAcquisitions.get(event.attemptId) - if (currentAcquisitions.get(event.demandId) === acquisitionId) { + if ( + acquisitionId !== undefined && + currentAcquisitions.get(event.demandId) === acquisitionId + ) { currentAcquisitions.delete(event.demandId) } } @@ -1186,18 +1250,26 @@ export function projectRetainedRowKeys( if (event.type === `releaseDemand`) { activeAttemptIds.delete(event.attemptId) const acquisitionId = attemptAcquisitions.get(event.attemptId) - if (acquisitionId) { - acquisitionAttempts.get(acquisitionId)?.delete(event.attemptId) - } - const releasedFinalAttempt = releaseActiveDemandAttempt( + releaseActiveDemandAttempt( activeAttempts, event.demandId, event.attemptId, ) releaseRows(event.attemptId) - if (releasedFinalAttempt) { - currentAcquisitions.delete(event.demandId) - reusableRows.delete(event.demandId) + if ( + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + event.attemptId, + ) + ) { + if (currentAcquisitions.get(event.demandId) === acquisitionId) { + currentAcquisitions.delete(event.demandId) + } + if (reusableRows.get(event.demandId)?.acquisitionId === acquisitionId) { + reusableRows.delete(event.demandId) + } } } } diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index a903cad4c..20e65b92b 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -360,6 +360,41 @@ it(`attaches late rows only to attempts that shared the settling acquisition`, ( expect(projectReusableDemands(attachedPeer)).toEqual([`shared`]) }) +it(`retires an ownerless acquisition without disturbing another cohort for the same demand`, () => { + const demandId = `shared` + const request = (attemptId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + ownerId: attemptId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const history: ReadonlyArray = [ + request(`attempt-a`), + { type: `truncateSource`, sessionId: `session` }, + request(`attempt-b`), + { + type: `releaseDemand`, + ownerId: `attempt-b`, + demandId, + attemptId: `attempt-b`, + }, + request(`attempt-c`), + { + type: `applyAuthoritativeRows`, + ownerId: `attempt-b`, + demandId, + attemptId: `attempt-b`, + rowKeys: [`stale-b`], + }, + ] + + expect(projectTransportLoads(history)).toBe(3) + expect(projectRetainedRowKeys(history)).toEqual([]) + expect(projectReusableDemands(history)).toEqual([]) +}) + it.each([ { name: `one owner releases the first attempt first`, From 075790026df040e88028c08e613abd68bf4f7f27 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 06:14:46 -0600 Subject: [PATCH 143/327] test(db): preserve sibling acquisition ownership --- ...load-subset-refinement-model.property.test.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 20e65b92b..d91f30e59 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -393,6 +393,22 @@ it(`retires an ownerless acquisition without disturbing another cohort for the s expect(projectTransportLoads(history)).toBe(3) expect(projectRetainedRowKeys(history)).toEqual([]) expect(projectReusableDemands(history)).toEqual([]) + + const survivingAcquisitionSettles = [ + ...history, + { + type: `applyAuthoritativeRows`, + ownerId: `attempt-a`, + demandId, + attemptId: `attempt-a`, + rowKeys: [`live-a`], + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectTransportLoads(survivingAcquisitionSettles)).toBe(3) + expect(projectRetainedRowKeys(survivingAcquisitionSettles)).toEqual([ + `live-a`, + ]) + expect(projectReusableDemands(survivingAcquisitionSettles)).toEqual([]) }) it.each([ From 58f83c32ef56463f6a7ea0bbe206902c15e55c3f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 06:44:31 -0600 Subject: [PATCH 144/327] test(db): qualify refinement grammar by source --- ...ubscription-replay-oracle.property.test.ts | 1 + .../db/tests/load-subset-full-flow-model.ts | 617 ++++++++++++------ packages/db/tests/oracle-config.ts | 2 +- ...d-subset-full-flow-oracle.property.test.ts | 172 ++++- ...d-subset-refinement-model.property.test.ts | 570 +++++++++++++++- ...source-readiness-refinement-oracle.test.ts | 5 + 6 files changed, 1151 insertions(+), 216 deletions(-) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index ff2363c77..cb28169a3 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -8951,6 +8951,7 @@ describe(`CollectionSubscription replay oracle`, () => { { type: `commitPublication`, publicationId: `initial` }, { type: `requestDemand`, + sourceId: `source`, ownerId: `other-owner`, sessionId: `session`, demandId: `other`, diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 086a49b91..1cfbf3031 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -186,6 +186,7 @@ export type LoadSubsetFullFlowEvent = type: `requestDemand` ownerId: FullFlowOwnerId sessionId: FullFlowSessionId + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId alreadyAborted: boolean @@ -193,18 +194,21 @@ export type LoadSubsetFullFlowEvent = | { type: `applyAuthoritativeRows` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId rowKeys: ReadonlyArray } | { type: `settleDemandWithoutEvidence` + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId } | { type: `applyUnprovenRows` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId rowKeys: ReadonlyArray @@ -212,16 +216,19 @@ export type LoadSubsetFullFlowEvent = | { type: `rejectDemand` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId } | { type: `truncateSource` sessionId: FullFlowSessionId + sourceId: FullFlowSourceId } | { type: `releaseDemand` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId attemptId: FullFlowAttemptId } @@ -303,14 +310,23 @@ export type LoadSubsetFullFlowEvent = sessionId: FullFlowSessionId sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId } | { type: `settleSourceDemand` sessionId: FullFlowSessionId sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId outcome: `resolve` | `reject` } + | { + type: `retireSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + } | { type: `startAcquisition` acquisitionId: FullFlowAcquisitionId @@ -368,16 +384,58 @@ export type LoadSubsetFullFlowEvent = export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId attemptId: FullFlowAttemptId } -type ActiveDemandAttempts = Map> -type AcquisitionAttempts = Map> +type ScopedIdentity = string +type ActiveDemandAttempts = Map> +type AcquisitionAttempts = Map> -function addActiveDemandAttempt( - activeAttempts: ActiveDemandAttempts, +function scopedIdentity(...parts: ReadonlyArray): ScopedIdentity { + return parts.map((part) => `${part.length}:${part}`).join(`|`) +} + +function sourceDemandIdentity( + sourceId: FullFlowSourceId, demandId: FullFlowDemandId, +): ScopedIdentity { + return scopedIdentity(sourceId, demandId) +} + +function sourceAttemptIdentity( + sourceId: FullFlowSourceId, attemptId: FullFlowAttemptId, +): ScopedIdentity { + return scopedIdentity(sourceId, attemptId) +} + +function sourceDemandAttemptIdentity( + sourceId: FullFlowSourceId, + demandId: FullFlowDemandId, + attemptId: FullFlowAttemptId, +): ScopedIdentity { + return scopedIdentity(sourceId, demandId, attemptId) +} + +function sourceRowIdentity( + sourceId: FullFlowSourceId, + rowKey: string, +): ScopedIdentity { + return scopedIdentity(sourceId, rowKey) +} + +function belongsToSource( + identity: ScopedIdentity, + sourceId: FullFlowSourceId, +): boolean { + return identity.startsWith(`${sourceId.length}:${sourceId}|`) +} + +function addActiveDemandAttempt( + activeAttempts: ActiveDemandAttempts, + demandId: ScopedIdentity, + attemptId: ScopedIdentity, ): void { let attempts = activeAttempts.get(demandId) if (!attempts) { @@ -389,8 +447,8 @@ function addActiveDemandAttempt( function releaseActiveDemandAttempt( activeAttempts: ActiveDemandAttempts, - demandId: FullFlowDemandId, - attemptId: FullFlowAttemptId, + demandId: ScopedIdentity, + attemptId: ScopedIdentity, ): boolean { const attempts = activeAttempts.get(demandId) if (!attempts?.delete(attemptId)) return false @@ -401,8 +459,8 @@ function releaseActiveDemandAttempt( function addAcquisitionAttempt( acquisitionAttempts: AcquisitionAttempts, - acquisitionId: FullFlowAttemptId, - attemptId: FullFlowAttemptId, + acquisitionId: ScopedIdentity, + attemptId: ScopedIdentity, ): void { let attempts = acquisitionAttempts.get(acquisitionId) if (!attempts) { @@ -414,8 +472,8 @@ function addAcquisitionAttempt( function releaseAcquisitionAttempt( acquisitionAttempts: AcquisitionAttempts, - acquisitionId: FullFlowAttemptId, - attemptId: FullFlowAttemptId, + acquisitionId: ScopedIdentity, + attemptId: ScopedIdentity, ): boolean { const attempts = acquisitionAttempts.get(acquisitionId) if (!attempts?.delete(attemptId) || attempts.size > 0) return false @@ -434,16 +492,17 @@ type DemandAttemptRecord = { function assertWellFormedDemandAttempts( history: ReadonlyArray, ): void { - const attempts = new Map() + const attempts = new Map() for (const event of history) { if (event.type === `requestDemand`) { - if (attempts.has(event.attemptId)) { + const attemptKey = sourceAttemptIdentity(event.sourceId, event.attemptId) + if (attempts.has(attemptKey)) { throw new Error( `Demand attempt "${event.attemptId}" was requested more than once`, ) } - attempts.set(event.attemptId, { + attempts.set(attemptKey, { ownerId: event.ownerId, demandId: event.demandId, settled: false, @@ -460,7 +519,9 @@ function assertWellFormedDemandAttempts( event.type === `releaseDemand` if (!usesDemandAttempt) continue - const attempt = attempts.get(event.attemptId) + const attempt = attempts.get( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) if (!attempt) { throw new Error( `Demand attempt "${event.attemptId}" was used before it was requested`, @@ -506,25 +567,31 @@ export function projectAdapterLifecycle( history: ReadonlyArray, ): Array { assertWellFormedDemandAttempts(history) - const invokedAttempts = new Set() + const invokedAttempts = new Set() const projected: Array = [] for (const event of history) { if (event.type === `requestDemand` && !event.alreadyAborted) { - invokedAttempts.add(event.attemptId) + invokedAttempts.add( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) projected.push({ type: `invoke`, ownerId: event.ownerId, + sourceId: event.sourceId, attemptId: event.attemptId, }) } if ( event.type === `releaseDemand` && - invokedAttempts.delete(event.attemptId) + invokedAttempts.delete( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) ) { projected.push({ type: `release`, ownerId: event.ownerId, + sourceId: event.sourceId, attemptId: event.attemptId, }) } @@ -545,9 +612,9 @@ export function projectTransportLoads( history: ReadonlyArray, ): number { assertWellFormedDemandAttempts(history) - const reusableAcquisitions = new Map() - const inFlightAcquisitions = new Map() - const attemptAcquisitions = new Map() + const reusableAcquisitions = new Map() + const inFlightAcquisitions = new Map() + const attemptAcquisitions = new Map() const acquisitionAttempts: AcquisitionAttempts = new Map() let loads = 0 @@ -555,65 +622,89 @@ export function projectTransportLoads( switch (event.type) { case `requestDemand`: { if (event.alreadyAborted) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) let acquisitionId = - inFlightAcquisitions.get(event.demandId) ?? - reusableAcquisitions.get(event.demandId) + inFlightAcquisitions.get(demandKey) ?? + reusableAcquisitions.get(demandKey) if (acquisitionId === undefined) { loads++ - acquisitionId = event.attemptId - inFlightAcquisitions.set(event.demandId, acquisitionId) + acquisitionId = attemptKey + inFlightAcquisitions.set(demandKey, acquisitionId) } - attemptAcquisitions.set(event.attemptId, acquisitionId) - addAcquisitionAttempt( - acquisitionAttempts, - acquisitionId, - event.attemptId, - ) + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) break } case `applyAuthoritativeRows`: { - const acquisitionId = attemptAcquisitions.get(event.attemptId) + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) if ( acquisitionId === undefined || - inFlightAcquisitions.get(event.demandId) !== acquisitionId + inFlightAcquisitions.get(demandKey) !== acquisitionId ) { break } - inFlightAcquisitions.delete(event.demandId) - reusableAcquisitions.set(event.demandId, acquisitionId) + inFlightAcquisitions.delete(demandKey) + reusableAcquisitions.set(demandKey, acquisitionId) break } case `truncateSource`: - reusableAcquisitions.clear() - inFlightAcquisitions.clear() + for (const demandKey of reusableAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + reusableAcquisitions.delete(demandKey) + } + } + for (const demandKey of inFlightAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + inFlightAcquisitions.delete(demandKey) + } + } break case `applyUnprovenRows`: case `rejectDemand`: case `settleDemandWithoutEvidence`: { - const acquisitionId = attemptAcquisitions.get(event.attemptId) + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) if ( acquisitionId !== undefined && - inFlightAcquisitions.get(event.demandId) === acquisitionId + inFlightAcquisitions.get(demandKey) === acquisitionId ) { - inFlightAcquisitions.delete(event.demandId) + inFlightAcquisitions.delete(demandKey) } break } case `releaseDemand`: { - const acquisitionId = attemptAcquisitions.get(event.attemptId) + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) if ( acquisitionId !== undefined && releaseAcquisitionAttempt( acquisitionAttempts, acquisitionId, - event.attemptId, + attemptKey, ) ) { - if (reusableAcquisitions.get(event.demandId) === acquisitionId) { - reusableAcquisitions.delete(event.demandId) + if (reusableAcquisitions.get(demandKey) === acquisitionId) { + reusableAcquisitions.delete(demandKey) } - if (inFlightAcquisitions.get(event.demandId) === acquisitionId) { - inFlightAcquisitions.delete(event.demandId) + if (inFlightAcquisitions.get(demandKey) === acquisitionId) { + inFlightAcquisitions.delete(demandKey) } } break @@ -635,6 +726,7 @@ export function projectTransportLoads( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: case `startAcquisition`: case `attachAcquisitionOwner`: case `settleAcquisition`: @@ -723,6 +815,7 @@ export function projectAuthorizedContinuationStarts( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: case `startAcquisition`: case `attachAcquisitionOwner`: case `settleAcquisition`: @@ -739,85 +832,126 @@ export function projectAuthorizedContinuationStarts( return starts } -/** Projects reusable demand evidence without using registry state. */ -export function projectReusableDemands( +export type ExpectedReusableDemand = { + sourceId: FullFlowSourceId + demandId: FullFlowDemandId +} + +/** Projects source-qualified reusable demand evidence without registry state. */ +export function projectReusableSourceDemands( history: ReadonlyArray, -): Array { +): Array { assertWellFormedDemandAttempts(history) const activeAttempts: ActiveDemandAttempts = new Map() - const currentAcquisitions = new Map() - const reusableAcquisitions = new Map() - const attemptAcquisitions = new Map() + const currentAcquisitions = new Map() + const reusableAcquisitions = new Map< + ScopedIdentity, + { acquisitionId: ScopedIdentity; demand: ExpectedReusableDemand } + >() + const attemptAcquisitions = new Map() const acquisitionAttempts: AcquisitionAttempts = new Map() for (const event of history) { switch (event.type) { case `requestDemand`: if (!event.alreadyAborted) { - addActiveDemandAttempt( - activeAttempts, - event.demandId, + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, event.attemptId, ) + addActiveDemandAttempt(activeAttempts, demandKey, attemptKey) const acquisitionId = - currentAcquisitions.get(event.demandId) ?? - reusableAcquisitions.get(event.demandId) ?? - event.attemptId - currentAcquisitions.set(event.demandId, acquisitionId) - attemptAcquisitions.set(event.attemptId, acquisitionId) - addAcquisitionAttempt( - acquisitionAttempts, - acquisitionId, - event.attemptId, - ) + currentAcquisitions.get(demandKey) ?? + reusableAcquisitions.get(demandKey)?.acquisitionId ?? + attemptKey + currentAcquisitions.set(demandKey, acquisitionId) + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) } break case `applyAuthoritativeRows`: { - const acquisitionId = attemptAcquisitions.get(event.attemptId) + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) if ( acquisitionId !== undefined && - currentAcquisitions.get(event.demandId) === acquisitionId + currentAcquisitions.get(demandKey) === acquisitionId ) { - reusableAcquisitions.set(event.demandId, acquisitionId) - currentAcquisitions.delete(event.demandId) + reusableAcquisitions.set(demandKey, { + acquisitionId, + demand: { sourceId: event.sourceId, demandId: event.demandId }, + }) + currentAcquisitions.delete(demandKey) } break } case `truncateSource`: - currentAcquisitions.clear() - reusableAcquisitions.clear() + for (const demandKey of currentAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + currentAcquisitions.delete(demandKey) + } + } + for (const demandKey of reusableAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + reusableAcquisitions.delete(demandKey) + } + } break case `releaseDemand`: { - releaseActiveDemandAttempt( - activeAttempts, - event.demandId, + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, event.attemptId, ) - const acquisitionId = attemptAcquisitions.get(event.attemptId) + releaseActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + const acquisitionId = attemptAcquisitions.get(attemptKey) if ( acquisitionId !== undefined && releaseAcquisitionAttempt( acquisitionAttempts, acquisitionId, - event.attemptId, + attemptKey, ) ) { - if (currentAcquisitions.get(event.demandId) === acquisitionId) { - currentAcquisitions.delete(event.demandId) + if (currentAcquisitions.get(demandKey) === acquisitionId) { + currentAcquisitions.delete(demandKey) } - if (reusableAcquisitions.get(event.demandId) === acquisitionId) { - reusableAcquisitions.delete(event.demandId) + if ( + reusableAcquisitions.get(demandKey)?.acquisitionId === acquisitionId + ) { + reusableAcquisitions.delete(demandKey) } } break } case `applyUnprovenRows`: case `rejectDemand`: + case `settleDemandWithoutEvidence`: case `restartSession`: case `cleanupSession`: case `advanceWindowRevision`: case `scheduleContinuation`: case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: case `stagePublicationRows`: case `commitPublication`: case `beginReplacement`: @@ -828,7 +962,20 @@ export function projectReusableDemands( } } - return [...reusableAcquisitions.keys()].sort() + return [...reusableAcquisitions.values()] + .map(({ demand }) => demand) + .sort((left, right) => + left.sourceId === right.sourceId + ? left.demandId.localeCompare(right.demandId) + : left.sourceId.localeCompare(right.sourceId), + ) +} + +/** Single-source convenience projection retained for existing controls. */ +export function projectReusableDemands( + history: ReadonlyArray, +): Array { + return projectReusableSourceDemands(history).map(({ demandId }) => demandId) } /** @@ -1131,150 +1278,203 @@ export function projectAtomicOrderedPublicationState( } } -/** Derives visible row identity without consulting Collection implementation. */ -export function projectRetainedRowKeys( +/** Derives source-qualified row identity without consulting Collection state. */ +export function projectRetainedSourceRows( history: ReadonlyArray, -): Array { +): Array { assertWellFormedDemandAttempts(history) const activeAttempts: ActiveDemandAttempts = new Map() - const activeAttemptIds = new Set() - const currentAcquisitions = new Map() + const activeAttemptIds = new Set() + const currentAcquisitions = new Map() const reusableRows = new Map< - FullFlowDemandId, - { acquisitionId: FullFlowAttemptId; rows: Set } + ScopedIdentity, + { acquisitionId: ScopedIdentity; rows: Set } >() - const attemptAcquisitions = new Map() + const attemptAcquisitions = new Map() const acquisitionAttempts: AcquisitionAttempts = new Map() - const rowClaims = new Map>() - const attemptRows = new Map>() + const rowClaims = new Map< + ScopedIdentity, + { row: ExpectedPublicRow; attempts: Set } + >() + const attemptRows = new Map>() const claimRows = ( - attemptId: FullFlowAttemptId, + attemptKey: ScopedIdentity, + sourceId: FullFlowSourceId, rowKeys: Iterable, ) => { - let claimed = attemptRows.get(attemptId) + let claimed = attemptRows.get(attemptKey) if (!claimed) { claimed = new Set() - attemptRows.set(attemptId, claimed) + attemptRows.set(attemptKey, claimed) } for (const rowKey of rowKeys) { - claimed.add(rowKey) - let claims = rowClaims.get(rowKey) - if (!claims) { - claims = new Set() - rowClaims.set(rowKey, claims) + const rowIdentity = sourceRowIdentity(sourceId, rowKey) + claimed.add(rowIdentity) + let claim = rowClaims.get(rowIdentity) + if (!claim) { + claim = { row: { sourceId, rowKey }, attempts: new Set() } + rowClaims.set(rowIdentity, claim) } - claims.add(attemptId) + claim.attempts.add(attemptKey) } } - const releaseRows = (attemptId: FullFlowAttemptId) => { - for (const rowKey of attemptRows.get(attemptId) ?? []) { - const claims = rowClaims.get(rowKey) - claims?.delete(attemptId) - if (claims?.size === 0) rowClaims.delete(rowKey) + const releaseRows = (attemptKey: ScopedIdentity) => { + for (const rowIdentity of attemptRows.get(attemptKey) ?? []) { + const claim = rowClaims.get(rowIdentity) + claim?.attempts.delete(attemptKey) + if (claim?.attempts.size === 0) rowClaims.delete(rowIdentity) } - attemptRows.delete(attemptId) + attemptRows.delete(attemptKey) } for (const event of history) { - if (event.type === `requestDemand` && !event.alreadyAborted) { - addActiveDemandAttempt(activeAttempts, event.demandId, event.attemptId) - activeAttemptIds.add(event.attemptId) - const retained = reusableRows.get(event.demandId) - const acquisitionId = - currentAcquisitions.get(event.demandId) ?? - retained?.acquisitionId ?? - event.attemptId - currentAcquisitions.set(event.demandId, acquisitionId) - attemptAcquisitions.set(event.attemptId, acquisitionId) - addAcquisitionAttempt(acquisitionAttempts, acquisitionId, event.attemptId) - if (retained) claimRows(event.attemptId, retained.rows) - } - if (event.type === `applyAuthoritativeRows`) { - const acquisitionId = attemptAcquisitions.get(event.attemptId) - const participants = - acquisitionId === undefined - ? [] - : (acquisitionAttempts.get(acquisitionId) ?? []) - if ( - acquisitionId !== undefined && - currentAcquisitions.get(event.demandId) === acquisitionId - ) { - const rows = new Set(event.rowKeys) - reusableRows.set(event.demandId, { acquisitionId, rows }) - currentAcquisitions.delete(event.demandId) + switch (event.type) { + case `requestDemand`: { + if (event.alreadyAborted) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + addActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + activeAttemptIds.add(attemptKey) + const retained = reusableRows.get(demandKey) + const acquisitionId = + currentAcquisitions.get(demandKey) ?? + retained?.acquisitionId ?? + attemptKey + currentAcquisitions.set(demandKey, acquisitionId) + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) + if (retained) claimRows(attemptKey, event.sourceId, retained.rows) + break } - for (const attemptId of participants) { - if (activeAttemptIds.has(attemptId)) { - claimRows(attemptId, event.rowKeys) + case `applyAuthoritativeRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + const rows = new Set(event.rowKeys) + reusableRows.set(demandKey, { acquisitionId, rows }) + currentAcquisitions.delete(demandKey) } - } - } - if (event.type === `applyUnprovenRows`) { - const acquisitionId = attemptAcquisitions.get(event.attemptId) - if ( - acquisitionId !== undefined && - currentAcquisitions.get(event.demandId) === acquisitionId - ) { - currentAcquisitions.delete(event.demandId) - } - const participants = - acquisitionId === undefined - ? [] - : (acquisitionAttempts.get(acquisitionId) ?? []) - for (const attemptId of participants) { - if (activeAttemptIds.has(attemptId)) { - claimRows(attemptId, event.rowKeys) + for (const participant of participants) { + if (activeAttemptIds.has(participant)) { + claimRows(participant, event.sourceId, event.rowKeys) + } } + break } - } - if ( - event.type === `rejectDemand` || - event.type === `settleDemandWithoutEvidence` - ) { - const acquisitionId = attemptAcquisitions.get(event.attemptId) - if ( - acquisitionId !== undefined && - currentAcquisitions.get(event.demandId) === acquisitionId - ) { - currentAcquisitions.delete(event.demandId) + case `applyUnprovenRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + currentAcquisitions.delete(demandKey) + } + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + for (const participant of participants) { + if (activeAttemptIds.has(participant)) { + claimRows(participant, event.sourceId, event.rowKeys) + } + } + break } - } - if (event.type === `truncateSource`) { - currentAcquisitions.clear() - reusableRows.clear() - rowClaims.clear() - attemptRows.clear() - } - if (event.type === `releaseDemand`) { - activeAttemptIds.delete(event.attemptId) - const acquisitionId = attemptAcquisitions.get(event.attemptId) - releaseActiveDemandAttempt( - activeAttempts, - event.demandId, - event.attemptId, - ) - releaseRows(event.attemptId) - if ( - acquisitionId !== undefined && - releaseAcquisitionAttempt( - acquisitionAttempts, - acquisitionId, + case `rejectDemand`: + case `settleDemandWithoutEvidence`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, event.attemptId, ) - ) { - if (currentAcquisitions.get(event.demandId) === acquisitionId) { - currentAcquisitions.delete(event.demandId) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + currentAcquisitions.delete(demandKey) + } + break + } + case `truncateSource`: + for (const scope of currentAcquisitions.keys()) { + if (belongsToSource(scope, event.sourceId)) { + currentAcquisitions.delete(scope) + } } - if (reusableRows.get(event.demandId)?.acquisitionId === acquisitionId) { - reusableRows.delete(event.demandId) + for (const scope of reusableRows.keys()) { + if (belongsToSource(scope, event.sourceId)) { + reusableRows.delete(scope) + } + } + for (const rowIdentity of rowClaims.keys()) { + if (belongsToSource(rowIdentity, event.sourceId)) { + rowClaims.delete(rowIdentity) + for (const rows of attemptRows.values()) rows.delete(rowIdentity) + } + } + break + case `releaseDemand`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + activeAttemptIds.delete(attemptKey) + const acquisitionId = attemptAcquisitions.get(attemptKey) + releaseActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + releaseRows(attemptKey) + if ( + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + attemptKey, + ) + ) { + if (currentAcquisitions.get(demandKey) === acquisitionId) { + currentAcquisitions.delete(demandKey) + } + if (reusableRows.get(demandKey)?.acquisitionId === acquisitionId) { + reusableRows.delete(demandKey) + } } + break } + default: + break } } - return [...rowClaims.keys()].sort() + return sortPublicRows([...rowClaims.values()].map(({ row }) => row)) +} + +/** Single-source convenience projection retained for existing controls. */ +export function projectRetainedRowKeys( + history: ReadonlyArray, +): Array { + return projectRetainedSourceRows(history).map(({ rowKey }) => rowKey) } export type ExpectedSyncReceiptState = `pending` | `resolved` | `rejected` @@ -1413,6 +1613,7 @@ export function projectSyncTransactions( case `settleReplay`: case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: case `startAcquisition`: case `attachAcquisitionOwner`: case `settleAcquisition`: @@ -1619,6 +1820,7 @@ export function projectReplayPublication( case `settleSyncReceipt`: case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: case `startAcquisition`: case `attachAcquisitionOwner`: case `settleAcquisition`: @@ -1648,6 +1850,8 @@ export function projectSourceReadiness( string, { sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId state: `pending` | `resolved` | `rejected` } >() @@ -1660,18 +1864,43 @@ export function projectSourceReadiness( currentSession ??= event.sessionId if (event.sessionId !== currentSession) break cleanedUp = false - demands.set(`${event.sourceId}\u0000${event.demandId}`, { - sourceId: event.sourceId, - state: `pending`, - }) + demands.set( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + { + sourceId: event.sourceId, + demandId: event.demandId, + attemptId: event.attemptId, + state: `pending`, + }, + ) break case `settleSourceDemand`: { if (event.sessionId !== currentSession) break - const demand = demands.get(`${event.sourceId}\u0000${event.demandId}`) + const demand = demands.get( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + ) if (demand) demand.state = event.outcome === `resolve` ? `resolved` : `rejected` break } + case `retireSourceDemand`: + if (event.sessionId !== currentSession) break + demands.delete( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + ) + break case `cleanupSession`: if (event.sessionId === currentSession) { cleanedUp = true diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 2faa04245..10c0d2736 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -74,7 +74,7 @@ const publicationProperties = [ ) const refinementProperties = Array.from( - { length: 9 }, + { length: 11 }, (_, index) => `load-subset-refinement.${1_779_001 + index}`, ) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 70921983b..7eba9f9d2 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -24,7 +24,9 @@ import { projectOrderedPublicationBoundary, projectOrderedSourceProgress, projectRetainedRowKeys, + projectRetainedSourceRows, projectReusableDemands, + projectReusableSourceDemands, projectTransportLoads, } from '../load-subset-full-flow-model.js' import { @@ -2991,6 +2993,7 @@ async function runTruncateCoverageScenario( const request = (ownerId: string, options: LoadSubsetOptions) => { histories.push({ type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session`, demandId: `prefix-${options.limit}`, @@ -3020,6 +3023,7 @@ async function runTruncateCoverageScenario( histories.push({ type: hasMore === undefined ? `applyUnprovenRows` : `applyAuthoritativeRows`, + sourceId: `source`, ownerId, demandId: `prefix-${options.limit}`, attemptId: `${ownerId}-attempt`, @@ -3031,6 +3035,7 @@ async function runTruncateCoverageScenario( pending.get(options)!.reject(new Error(`fresh replay failed`)) histories.push({ type: `rejectDemand`, + sourceId: `source`, ownerId, demandId: `prefix-${options.limit}`, attemptId: `${ownerId}-attempt`, @@ -3065,7 +3070,11 @@ async function runTruncateCoverageScenario( truncate() const truncated = commit() if (truncated !== true) await truncated - histories.push({ type: `truncateSource`, sessionId: `session` }) + histories.push({ + type: `truncateSource`, + sessionId: `session`, + sourceId: `source`, + }) expectModel() const freshLoad = request(`fresh`, freshOptions) @@ -3103,6 +3112,7 @@ async function runTruncateCoverageScenario( source._sync.unloadSubset(options) histories.push({ type: `releaseDemand`, + sourceId: `source`, ownerId: options === initialOptions ? `initial` @@ -3269,6 +3279,7 @@ it(`keeps adapter release obligations distinct across attempts by one owner`, () const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner`, sessionId: `session`, demandId: `demand`, @@ -3277,6 +3288,7 @@ it(`keeps adapter release obligations distinct across attempts by one owner`, () }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner`, sessionId: `session`, demandId: `demand`, @@ -3285,12 +3297,14 @@ it(`keeps adapter release obligations distinct across attempts by one owner`, () }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner`, demandId: `demand`, attemptId: `attempt-1`, }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner`, demandId: `demand`, attemptId: `attempt-2`, @@ -3298,17 +3312,142 @@ it(`keeps adapter release obligations distinct across attempts by one owner`, () ] expect(projectAdapterLifecycle(history)).toEqual([ - { type: `invoke`, ownerId: `owner`, attemptId: `attempt-1` }, - { type: `invoke`, ownerId: `owner`, attemptId: `attempt-2` }, - { type: `release`, ownerId: `owner`, attemptId: `attempt-1` }, - { type: `release`, ownerId: `owner`, attemptId: `attempt-2` }, + { + type: `invoke`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-1`, + }, + { + type: `invoke`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-2`, + }, + { + type: `release`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-1`, + }, + { + type: `release`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-2`, + }, ]) }) +let sourceIdentityHarnessId = 0 + +it(`keeps identical demand and row identities local to each source`, async () => { + type Row = { id: string } + type Result = { hasMore: false; appliedRowKeys: ReadonlyArray } + const createSource = (sourceId: string) => { + const result = createDeferred() + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const collection = createCollection({ + id: `source-identity-${sourceIdentityHarnessId++}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { loadSubset: () => result.promise } + }, + }, + }) + const options = { limit: 1 } + const load = collection._sync.loadSubset(options) + if (load === true) throw new Error(`Expected a controlled async load`) + return { + sourceId, + collection, + options, + load, + settle: async () => { + begin() + write({ type: `insert`, value: { id: `shared-row` } }) + const applied = commit() + if (applied !== true) await applied + result.resolve({ hasMore: false, appliedRowKeys: [`shared-row`] }) + await load + }, + } + } + const sourceA = createSource(`source-a`) + const sourceB = createSource(`source-b`) + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + rowKeys: [`shared-row`], + }) + const history: Array = [ + request(sourceA.sourceId), + request(sourceB.sourceId), + ] + const actualRows = () => + [sourceA, sourceB].flatMap(({ sourceId, collection }) => + Array.from(collection.keys(), (rowKey) => ({ sourceId, rowKey })), + ) + + try { + await sourceA.settle() + history.push(settle(sourceA.sourceId)) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectReusableSourceDemands(history)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + ]) + + await sourceB.settle() + history.push(settle(sourceB.sourceId)) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectTransportLoads(history)).toBe(2) + + sourceA.collection._sync.unloadSubset(sourceA.options) + history.push({ + type: `releaseDemand`, + sourceId: sourceA.sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectReusableSourceDemands(history)).toEqual([ + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) + } finally { + await Promise.all([ + sourceA.collection.cleanup(), + sourceB.collection.cleanup(), + ]) + } +}) + it(`derives shared row and evidence lifetime from active attempts`, () => { const sharedHistory: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-a`, sessionId: `session`, demandId: `shared`, @@ -3317,6 +3456,7 @@ it(`derives shared row and evidence lifetime from active attempts`, () => { }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-b`, sessionId: `session`, demandId: `shared`, @@ -3325,6 +3465,7 @@ it(`derives shared row and evidence lifetime from active attempts`, () => { }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-a`, demandId: `shared`, attemptId: `attempt-a`, @@ -3332,6 +3473,7 @@ it(`derives shared row and evidence lifetime from active attempts`, () => { }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `shared`, attemptId: `attempt-a`, @@ -3344,6 +3486,7 @@ it(`derives shared row and evidence lifetime from active attempts`, () => { ...sharedHistory, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-c`, sessionId: `session`, demandId: `shared`, @@ -3358,6 +3501,7 @@ it(`derives shared row and evidence lifetime from active attempts`, () => { ...sharedHistory, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-b`, demandId: `shared`, attemptId: `attempt-b`, @@ -3370,6 +3514,7 @@ it(`keeps an additional demand active until its final attempt releases`, () => { const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-a`, sessionId: `session`, demandId: `other`, @@ -3378,6 +3523,7 @@ it(`keeps an additional demand active until its final attempt releases`, () => { }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-b`, sessionId: `session`, demandId: `other`, @@ -3398,6 +3544,7 @@ it(`keeps an additional demand active until its final attempt releases`, () => { }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `other`, attemptId: `attempt-a`, @@ -3418,6 +3565,7 @@ it(`does not release physical work when an already-aborted demand skips adapter const ownerId = `aborted-owner` const requestEvent: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session-1`, demandId: `all-rows`, @@ -3428,6 +3576,7 @@ it(`does not release physical work when an already-aborted demand skips adapter requestEvent, { type: `releaseDemand`, + sourceId: `source`, ownerId, demandId: `all-rows`, attemptId: `aborted-attempt`, @@ -3959,6 +4108,7 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-1`, sessionId: `session-1`, demandId: `all-rows`, @@ -3967,6 +4117,7 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-1`, demandId: `all-rows`, attemptId: `attempt-1`, @@ -3974,6 +4125,7 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-1`, demandId: `all-rows`, attemptId: `attempt-1`, @@ -3985,6 +4137,7 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-2`, sessionId: `session-2`, demandId: `all-rows`, @@ -3993,6 +4146,7 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-2`, demandId: `all-rows`, attemptId: `attempt-2`, @@ -4070,6 +4224,7 @@ it(`does not let an ordered continuation from a cleaned session start new work a const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-1`, sessionId: `session-1`, demandId: `top-1`, @@ -4090,6 +4245,7 @@ it(`does not let an ordered continuation from a cleaned session start new work a }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-2`, sessionId: `session-2`, demandId: `top-1`, @@ -5699,7 +5855,7 @@ async function runOrderedBoundaryProvenanceScenario( rows: [{ key: addedRow.id, orderValue: addedRow.rank }], }, { type: `commitPublication`, publicationId: `additional-publication` }, - { type: `truncateSource`, sessionId: `session` }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, { type: `stagePublicationRows`, publicationId: `failed-replacement`, @@ -5715,6 +5871,7 @@ async function runOrderedBoundaryProvenanceScenario( }, { type: `rejectDemand`, + sourceId: `source`, ownerId: `ordered-owner`, demandId: `ordered-window`, attemptId: `ordered-attempt`, @@ -6318,6 +6475,7 @@ async function runAtomicOrderedReplayScenario( expect(released?.options.signal?.aborted).toBe(true) history.push({ type: `releaseDemand`, + sourceId: `source`, ownerId: `other-owner`, demandId: `other`, attemptId: `other-attempt`, @@ -6339,6 +6497,7 @@ async function runAtomicOrderedReplayScenario( if (scenario.otherDemand !== `none`) { history.push({ type: `requestDemand`, + sourceId: `source`, ownerId: `other-owner`, sessionId: `atomic-session`, demandId: `other`, @@ -6404,6 +6563,7 @@ async function runAtomicOrderedReplayScenario( subscription.releaseSnapshot(otherWhere) history.push({ type: `releaseDemand`, + sourceId: `source`, ownerId: `other-owner`, demandId: `other`, attemptId: `other-attempt`, diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index d91f30e59..f1fbd7a01 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -12,7 +12,9 @@ import { projectOrderedPublicationBoundary, projectReplayPublication, projectRetainedRowKeys, + projectRetainedSourceRows, projectReusableDemands, + projectReusableSourceDemands, projectSourceReadiness, projectSyncTransactions, projectTransportLoads, @@ -90,6 +92,7 @@ type DemandLifecycleCase = { expected: Array<{ type: `invoke` | `release` ownerId: string + sourceId: string attemptId: string }> } @@ -112,6 +115,7 @@ function enumerateDemandLifecycles(): Array { ...history, { type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session`, demandId: `demand`, @@ -126,6 +130,7 @@ function enumerateDemandLifecycles(): Array { { type: `invoke`, ownerId, + sourceId: `source`, attemptId: `${ownerId}-attempt`, }, ], @@ -140,6 +145,7 @@ function enumerateDemandLifecycles(): Array { ...history, { type: `releaseDemand`, + sourceId: `source`, ownerId, demandId: `demand`, attemptId: `${ownerId}-attempt`, @@ -150,6 +156,7 @@ function enumerateDemandLifecycles(): Array { { type: `release`, ownerId, + sourceId: `source`, attemptId: `${ownerId}-attempt`, }, ], @@ -192,6 +199,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, attemptId = `${ownerId}-attempt`, ): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session`, demandId: `exact-demand`, @@ -209,6 +217,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ...concurrent, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `exact-demand`, attemptId: `owner-a-attempt`, @@ -221,12 +230,14 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ...concurrent, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `exact-demand`, attemptId: `owner-a-attempt`, }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-b`, demandId: `exact-demand`, attemptId: `owner-b-attempt`, @@ -239,6 +250,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ...concurrent, { type: `settleDemandWithoutEvidence`, + sourceId: `source`, demandId: `exact-demand`, attemptId: `owner-a-attempt`, }, @@ -250,6 +262,7 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ...concurrent, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-a`, demandId: `exact-demand`, attemptId: `owner-a-attempt`, @@ -260,12 +273,272 @@ it(`shares concurrent exact demand and retries after evidence-free settlement`, ).toBe(1) }) +it(`scopes identical demand attempts, rows, and evidence to their source`, () => { + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + rowKeys: [`shared-row`], + }) + const sourceASettled = [ + request(`source-a`), + request(`source-b`), + settle(`source-a`), + ] + + expect(projectTransportLoads(sourceASettled)).toBe(2) + expect(projectRetainedSourceRows(sourceASettled)).toEqual([ + { sourceId: `source-a`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(sourceASettled)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + ]) + + const bothSettled = [...sourceASettled, settle(`source-b`)] + expect(projectRetainedSourceRows(bothSettled)).toEqual([ + { sourceId: `source-a`, rowKey: `shared-row` }, + { sourceId: `source-b`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(bothSettled)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) + + const sourceATruncated = [ + ...bothSettled, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: `source-a`, + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(sourceATruncated)).toEqual([ + { sourceId: `source-b`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(sourceATruncated)).toEqual([ + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) +}) + +it(`fences stale same-source settlement from a fresh demand generation`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + ...oldRequest, + attemptId: `fresh-attempt`, + } + const beforeFreshSettlement: ReadonlyArray = [ + oldRequest, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, + freshRequest, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + ] + + expect(projectTransportLoads(beforeFreshSettlement)).toBe(2) + expect(projectRetainedSourceRows(beforeFreshSettlement)).toEqual([ + { sourceId: `source`, rowKey: `stale-row` }, + ]) + expect(projectReusableSourceDemands(beforeFreshSettlement)).toEqual([]) + + const oldReleased = [ + ...beforeFreshSettlement, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `old-attempt`, + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(oldReleased)).toEqual([]) + expect(projectReusableSourceDemands(oldReleased)).toEqual([]) + + const freshSettled = [ + ...oldReleased, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `fresh-attempt`, + rowKeys: [`fresh-row`], + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(freshSettled)).toEqual([ + { sourceId: `source`, rowKey: `fresh-row` }, + ]) + expect(projectReusableSourceDemands(freshSettled)).toEqual([ + { sourceId: `source`, demandId: `demand` }, + ]) +}) + +for (const campaign of refinementCampaigns(1_779_010)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ maxLength: 6 }), { + minLength: 2, + maxLength: 2, + }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + ], + campaign.options, + )( + `source identity scopes equal demand histories (${campaign.label})`, + (sourceIds, demandId, attemptId, rowKey) => { + const [sourceA, sourceB] = sourceIds as [string, string] + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId, + rowKeys: [rowKey], + }) + const settled = [ + request(sourceA), + request(sourceB), + settle(sourceA), + settle(sourceB), + ] + const surviving = [ + ...settled, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: sourceA, + } satisfies LoadSubsetFullFlowEvent, + ] + + expect(projectTransportLoads(settled)).toBe(2) + expect(projectRetainedSourceRows(settled)).toEqual( + [sourceA, sourceB] + .sort((left, right) => left.localeCompare(right)) + .map((sourceId) => ({ sourceId, rowKey })), + ) + expect(projectRetainedSourceRows(surviving)).toEqual([ + { sourceId: sourceB, rowKey }, + ]) + expect(projectReusableSourceDemands(surviving)).toEqual([ + { sourceId: sourceB, demandId }, + ]) + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_011)) { + fcTest.prop( + [ + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + fc.uniqueArray(fc.string({ maxLength: 6 }), { + minLength: 2, + maxLength: 2, + }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + ], + campaign.options, + )( + `truncate fences stale settlement from the next demand generation (${campaign.label})`, + (sourceId, demandId, attemptIds, staleRowKey, freshRowKey) => { + const [oldAttemptId, freshAttemptId] = attemptIds as [string, string] + const request = (attemptId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const oldSettlesThenReleases: ReadonlyArray = [ + request(oldAttemptId), + { type: `truncateSource`, sessionId: `session`, sourceId }, + request(freshAttemptId), + { + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: oldAttemptId, + rowKeys: [staleRowKey], + }, + { + type: `releaseDemand`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: oldAttemptId, + }, + ] + const freshSettles = [ + ...oldSettlesThenReleases, + { + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: freshAttemptId, + rowKeys: [freshRowKey], + } satisfies LoadSubsetFullFlowEvent, + ] + + expect(projectTransportLoads(oldSettlesThenReleases)).toBe(2) + expect(projectRetainedSourceRows(oldSettlesThenReleases)).toEqual([]) + expect(projectReusableSourceDemands(oldSettlesThenReleases)).toEqual([]) + expect(projectRetainedSourceRows(freshSettles)).toEqual([ + { sourceId, rowKey: freshRowKey }, + ]) + expect(projectReusableSourceDemands(freshSettles)).toEqual([ + { sourceId, demandId }, + ]) + }, + ) +} + it(`retains a row until its last independent demand claim releases`, () => { const request = ( demandId: string, attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, + sourceId: `source`, ownerId: attemptId, sessionId: `session`, demandId, @@ -277,6 +550,7 @@ it(`retains a row until its last independent demand claim releases`, () => { attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `applyUnprovenRows`, + sourceId: `source`, ownerId: attemptId, demandId, attemptId, @@ -287,6 +561,7 @@ it(`retains a row until its last independent demand claim releases`, () => { attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `releaseDemand`, + sourceId: `source`, ownerId: attemptId, demandId, attemptId, @@ -316,6 +591,7 @@ it(`attaches late rows only to attempts that shared the settling acquisition`, ( attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session`, demandId: `shared`, @@ -327,12 +603,14 @@ it(`attaches late rows only to attempts that shared the settling acquisition`, ( attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `releaseDemand`, + sourceId: `source`, ownerId, demandId: `shared`, attemptId, }) const lateSettlement: LoadSubsetFullFlowEvent = { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `old-owner`, demandId: `shared`, attemptId: `old-attempt`, @@ -364,6 +642,7 @@ it(`retires an ownerless acquisition without disturbing another cohort for the s const demandId = `shared` const request = (attemptId: string): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, + sourceId: `source`, ownerId: attemptId, sessionId: `session`, demandId, @@ -372,10 +651,11 @@ it(`retires an ownerless acquisition without disturbing another cohort for the s }) const history: ReadonlyArray = [ request(`attempt-a`), - { type: `truncateSource`, sessionId: `session` }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, request(`attempt-b`), { type: `releaseDemand`, + sourceId: `source`, ownerId: `attempt-b`, demandId, attemptId: `attempt-b`, @@ -383,6 +663,7 @@ it(`retires an ownerless acquisition without disturbing another cohort for the s request(`attempt-c`), { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `attempt-b`, demandId, attemptId: `attempt-b`, @@ -398,6 +679,7 @@ it(`retires an ownerless acquisition without disturbing another cohort for the s ...history, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `attempt-a`, demandId, attemptId: `attempt-a`, @@ -441,6 +723,7 @@ it.each([ const requests = attempts.map( ({ ownerId, attemptId }) => ({ type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session`, demandId, @@ -450,6 +733,7 @@ it.each([ ) const settlement: LoadSubsetFullFlowEvent = { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: attempts[0]!.ownerId, demandId, attemptId: attempts[0]!.attemptId, @@ -457,6 +741,7 @@ it.each([ } const releases = releaseOrder.map((index) => ({ type: `releaseDemand`, + sourceId: `source`, ownerId: attempts[index]!.ownerId, demandId, attemptId: attempts[index]!.attemptId, @@ -475,6 +760,7 @@ it.each([ expect(projectReusableDemands(history)).toEqual(active ? [demandId] : []) const peerRequest: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId: `peer`, sessionId: `session`, demandId, @@ -526,6 +812,7 @@ it.each([ ...releases, { type: `requestDemand`, + sourceId: `source`, ownerId: `fresh-owner`, sessionId: `session`, demandId, @@ -544,6 +831,7 @@ it.each([ name: `authoritative`, event: { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -554,6 +842,7 @@ it.each([ name: `unproven`, event: { type: `applyUnprovenRows`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -564,6 +853,7 @@ it.each([ name: `rejected`, event: { type: `rejectDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -573,6 +863,7 @@ it.each([ name: `evidence-free`, event: { type: `settleDemandWithoutEvidence`, + sourceId: `source`, demandId: `exact-demand`, attemptId: `old-attempt`, }, @@ -581,6 +872,7 @@ it.each([ name: `released`, event: { type: `releaseDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -596,15 +888,17 @@ it.each([ projectTransportLoads([ { type: `requestDemand`, + sourceId: `source`, ownerId: `old-owner`, sessionId: `session`, demandId: `exact-demand`, attemptId: `old-attempt`, alreadyAborted: false, }, - { type: `truncateSource`, sessionId: `session` }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, { type: `requestDemand`, + sourceId: `source`, ownerId: `fresh-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -614,6 +908,7 @@ it.each([ event, { type: `requestDemand`, + sourceId: `source`, ownerId: `peer-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -628,6 +923,7 @@ it.each([ it(`scopes reusable evidence to the physical attempt when an owner is reused`, () => { const oldRequest: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId: `stable-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -640,6 +936,7 @@ it(`scopes reusable evidence to the physical attempt when an owner is reused`, ( } const oldSettlement: LoadSubsetFullFlowEvent = { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `stable-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -652,13 +949,18 @@ it(`scopes reusable evidence to the physical attempt when an owner is reused`, ( } const staleRelease: LoadSubsetFullFlowEvent = { type: `releaseDemand`, + sourceId: `source`, ownerId: `stable-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, } const beforeFreshSettlement = [ oldRequest, - { type: `truncateSource`, sessionId: `session` } as const, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: `source`, + } as const, freshRequest, oldSettlement, ] @@ -693,6 +995,7 @@ it(`does not rebuild coverage when a released attempt settles after its replacem projectReusableDemands([ { type: `requestDemand`, + sourceId: `source`, ownerId: `old-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -701,12 +1004,14 @@ it(`does not rebuild coverage when a released attempt settles after its replacem }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, }, { type: `requestDemand`, + sourceId: `source`, ownerId: `fresh-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -715,6 +1020,7 @@ it(`does not rebuild coverage when a released attempt settles after its replacem }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -727,6 +1033,7 @@ it(`does not rebuild coverage when a released attempt settles after its replacem it(`keeps fresh same-epoch work shared after an older rejected attempt releases`, () => { const oldRequest: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId: `old-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -735,6 +1042,7 @@ it(`keeps fresh same-epoch work shared after an older rejected attempt releases` } const freshRequest: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId: `fresh-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -747,6 +1055,7 @@ it(`keeps fresh same-epoch work shared after an older rejected attempt releases` oldRequest, { type: `rejectDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -754,6 +1063,7 @@ it(`keeps fresh same-epoch work shared after an older rejected attempt releases` freshRequest, { type: `releaseDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `old-attempt`, @@ -771,6 +1081,7 @@ it(`rejects histories that reuse one demand attempt identity`, () => { const history: Array = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `old-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -779,12 +1090,14 @@ it(`rejects histories that reuse one demand attempt identity`, () => { }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `reused-attempt`, }, { type: `requestDemand`, + sourceId: `source`, ownerId: `fresh-owner`, sessionId: `session`, demandId: `exact-demand`, @@ -793,6 +1106,7 @@ it(`rejects histories that reuse one demand attempt identity`, () => { }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `old-owner`, demandId: `exact-demand`, attemptId: `reused-attempt`, @@ -812,6 +1126,7 @@ it(`rejects histories that settle one demand attempt twice`, () => { const history: Array = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner`, sessionId: `session`, demandId: `demand`, @@ -820,11 +1135,13 @@ it(`rejects histories that settle one demand attempt twice`, () => { }, { type: `settleDemandWithoutEvidence`, + sourceId: `source`, demandId: `demand`, attemptId: `attempt`, }, { type: `rejectDemand`, + sourceId: `source`, ownerId: `owner`, demandId: `demand`, attemptId: `attempt`, @@ -850,6 +1167,7 @@ function renameHistoryIds( ...event, ownerId: `${event.ownerId}-${suffix}`, sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, attemptId: `${event.attemptId}-${suffix}`, } @@ -860,23 +1178,32 @@ function renameHistoryIds( return { ...event, ownerId: `${event.ownerId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, attemptId: `${event.attemptId}-${suffix}`, } case `truncateSource`: - return { ...event, sessionId: `${event.sessionId}-${suffix}` } + return { + ...event, + sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + } case `settleDemandWithoutEvidence`: return { ...event, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, attemptId: `${event.attemptId}-${suffix}`, } case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: return { ...event, sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, } case `cleanupSession`: return { ...event, sessionId: `${event.sessionId}-${suffix}` } @@ -990,6 +1317,118 @@ function removeRenamingSuffix(value: string, suffix: string): string { : value } +function normalizeSourceReadiness( + observation: ReturnType, + suffix: string, +) { + return { + ...observation, + pendingSources: observation.pendingSources.map((sourceId) => + removeRenamingSuffix(sourceId, suffix), + ), + failedSources: observation.failedSources.map((sourceId) => + removeRenamingSuffix(sourceId, suffix), + ), + } +} + +it(`settles source readiness by exact demand attempt`, () => { + const pendingReplacement: ReadonlyArray = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-a`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-b`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-a`, + outcome: `resolve`, + }, + ] + expect(projectSourceReadiness(pendingReplacement)).toEqual({ + status: `loading`, + pendingSources: [`source`], + failedSources: [], + }) + expect( + projectSourceReadiness([ + ...pendingReplacement, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-b`, + outcome: `resolve`, + }, + ]), + ).toEqual({ status: `ready`, pendingSources: [], failedSources: [] }) +}) + +it(`retires source demand attempts without crossing source identity`, () => { + const survivingSource: ReadonlyArray = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + outcome: `reject`, + }, + ] + expect(projectSourceReadiness(survivingSource)).toEqual({ + status: `loading`, + pendingSources: [`source-b`], + failedSources: [], + }) + expect( + projectSourceReadiness([ + ...survivingSource, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + outcome: `resolve`, + }, + ]), + ).toEqual({ status: `ready`, pendingSources: [], failedSources: [] }) +}) + for (const campaign of refinementCampaigns(1_779_002)) { fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( `source demand names are observationally erased (${campaign.label})`, @@ -1000,18 +1439,21 @@ for (const campaign of refinementCampaigns(1_779_002)) { sessionId: `session`, sourceId: `source-a`, demandId: `demand-a`, + attemptId: `attempt-a`, }, { type: `registerSourceDemand`, sessionId: `session`, sourceId: `source-b`, demandId: `demand-b`, + attemptId: `attempt-b`, }, { type: `settleSourceDemand`, sessionId: `session`, sourceId: `source-a`, demandId: `demand-a`, + attemptId: `attempt-a`, outcome: `resolve`, }, ] @@ -1020,6 +1462,7 @@ for (const campaign of refinementCampaigns(1_779_002)) { history, suffix, projectSourceReadiness, + normalizeSourceReadiness, ) }, ) @@ -1032,6 +1475,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { const demandHistory: Array = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner`, sessionId: `session`, demandId: `demand`, @@ -1040,6 +1484,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner`, demandId: `demand`, attemptId: `attempt`, @@ -1047,6 +1492,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner`, demandId: `demand`, attemptId: `attempt`, @@ -1055,6 +1501,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { const continuationHistory: Array = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner`, sessionId: `session`, demandId: `demand`, @@ -1073,6 +1520,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { const evidenceFreeHistory: Array = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `evidence-free-owner`, sessionId: `session`, demandId: `evidence-free-demand`, @@ -1081,6 +1529,7 @@ for (const campaign of refinementCampaigns(1_779_003)) { }, { type: `settleDemandWithoutEvidence`, + sourceId: `source`, demandId: `evidence-free-demand`, attemptId: `evidence-free-attempt`, }, @@ -1435,47 +1884,51 @@ function sourceErasureHistories(): Array> { sessionId: string, sourceId: string, demandId: string, + attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `registerSourceDemand`, sessionId, sourceId, demandId, + attemptId, }) const settle = ( sessionId: string, sourceId: string, demandId: string, + attemptId: string, outcome: `resolve` | `reject`, ): LoadSubsetFullFlowEvent => ({ type: `settleSourceDemand`, sessionId, sourceId, demandId, + attemptId, outcome, }) return [ [ - register(`session-a`, `source-a`, `demand-a`), - register(`session-a`, `source-b`, `demand-b`), - settle(`session-a`, `source-a`, `demand-a`, `resolve`), - settle(`session-a`, `source-b`, `demand-b`, `reject`), + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), + register(`session-a`, `source-b`, `demand-b`, `attempt-b`), + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `resolve`), + settle(`session-a`, `source-b`, `demand-b`, `attempt-b`, `reject`), ], [ - register(`session-a`, `source-a`, `demand-a`), + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), { type: `cleanupSession`, sessionId: `session-a` }, - settle(`session-a`, `source-a`, `demand-a`, `resolve`), + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `resolve`), ], [ - register(`session-a`, `source-a`, `demand-a`), + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), { type: `restartSession`, previousSessionId: `session-a`, nextSessionId: `session-b`, }, - settle(`session-a`, `source-a`, `demand-a`, `reject`), - register(`session-b`, `source-b`, `demand-b`), - settle(`session-b`, `source-b`, `demand-b`, `resolve`), + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `reject`), + register(`session-b`, `source-b`, `demand-b`, `attempt-b`), + settle(`session-b`, `source-b`, `demand-b`, `attempt-b`, `resolve`), ], ] } @@ -1487,6 +1940,7 @@ function demandErasureHistories(): Array> { alreadyAborted = false, ): LoadSubsetFullFlowEvent => ({ type: `requestDemand`, + sourceId: `source`, ownerId, sessionId: `session-a`, demandId: `demand-a`, @@ -1498,6 +1952,7 @@ function demandErasureHistories(): Array> { attemptId: string, ): LoadSubsetFullFlowEvent => ({ type: `releaseDemand`, + sourceId: `source`, ownerId, demandId: `demand-a`, attemptId, @@ -1508,6 +1963,7 @@ function demandErasureHistories(): Array> { request(`owner-a`, `attempt-a`), { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, @@ -1519,6 +1975,7 @@ function demandErasureHistories(): Array> { request(`owner-a`, `attempt-a`), { type: `applyUnprovenRows`, + sourceId: `source`, ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, @@ -1530,6 +1987,7 @@ function demandErasureHistories(): Array> { request(`owner-a`, `attempt-a`), { type: `rejectDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, @@ -1540,6 +1998,7 @@ function demandErasureHistories(): Array> { request(`owner-a`, `attempt-a`), { type: `settleDemandWithoutEvidence`, + sourceId: `source`, demandId: `demand-a`, attemptId: `attempt-a`, }, @@ -1550,6 +2009,7 @@ function demandErasureHistories(): Array> { request(`owner-a`, `attempt-a`), { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, @@ -1557,10 +2017,15 @@ function demandErasureHistories(): Array> { ], [ request(`owner-a`, `attempt-a`), - { type: `truncateSource`, sessionId: `session-a` }, + { + type: `truncateSource`, + sessionId: `session-a`, + sourceId: `source`, + }, request(`owner-b`, `attempt-b`), { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-a`, demandId: `demand-a`, attemptId: `attempt-a`, @@ -1568,6 +2033,7 @@ function demandErasureHistories(): Array> { }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-b`, demandId: `demand-a`, attemptId: `attempt-b`, @@ -1596,6 +2062,7 @@ function demandErasureHistories(): Array> { }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-b`, sessionId: `session-b`, demandId: `demand-b`, @@ -1610,6 +2077,47 @@ function demandErasureHistories(): Array> { }, { type: `runContinuation`, taskId: `task-b` }, ], + [ + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner`, + sessionId: `session-a`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner`, + sessionId: `session-a`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + rowKeys: [`row`], + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-b`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + rowKeys: [`row`], + }, + { + type: `truncateSource`, + sessionId: `session-a`, + sourceId: `source-a`, + }, + ], ] } @@ -1723,6 +2231,7 @@ function erasedIdentityReferences( case `requestDemand`: add(eventIndex, `ownerId`, event.ownerId) add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) add(eventIndex, `attemptId`, event.attemptId) break @@ -1731,14 +2240,19 @@ function erasedIdentityReferences( case `rejectDemand`: case `releaseDemand`: add(eventIndex, `ownerId`, event.ownerId) + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) add(eventIndex, `attemptId`, event.attemptId) break case `settleDemandWithoutEvidence`: + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) add(eventIndex, `attemptId`, event.attemptId) break case `truncateSource`: + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `sessionId`, event.sessionId) + break case `cleanupSession`: case `advanceWindowRevision`: add(eventIndex, `sessionId`, event.sessionId) @@ -1769,8 +2283,11 @@ function erasedIdentityReferences( break case `registerSourceDemand`: case `settleSourceDemand`: + case `retireSourceDemand`: add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) break case `startAcquisition`: add(eventIndex, `acquisitionId`, event.acquisitionId) @@ -1875,6 +2392,7 @@ function publicationErasureHistories(): Array> { const relatedRows = [{ key: `related`, orderValue: 3 }] const requestRelated: LoadSubsetFullFlowEvent = { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-related`, sessionId: `session`, demandId: `related`, @@ -1945,6 +2463,7 @@ function publicationErasureHistories(): Array> { }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-related`, demandId: `related`, attemptId: `attempt-related`, @@ -2031,6 +2550,7 @@ for (const campaign of refinementCampaigns(1_779_009)) { history, suffix, projectSourceReadiness, + normalizeSourceReadiness, ) } @@ -2045,6 +2565,16 @@ for (const campaign of refinementCampaigns(1_779_009)) { suffix, projectRetainedRowKeys, ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedSourceRows, + (rows, renamingSuffix) => + rows.map(({ sourceId, rowKey }) => ({ + sourceId: removeRenamingSuffix(sourceId, renamingSuffix), + rowKey, + })), + ) expectObservationPreservedAfterEveryPrefix( history, suffix, @@ -2054,6 +2584,16 @@ for (const campaign of refinementCampaigns(1_779_009)) { removeRenamingSuffix(demandId, renamingSuffix), ), ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableSourceDemands, + (demands, renamingSuffix) => + demands.map(({ sourceId, demandId }) => ({ + sourceId: removeRenamingSuffix(sourceId, renamingSuffix), + demandId: removeRenamingSuffix(demandId, renamingSuffix), + })), + ) expectObservationPreservedAfterEveryPrefix( history, suffix, diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index 375f0ae24..26f59bc91 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -23,12 +23,14 @@ it.each([`resolve`, `reject`, `cleanup`] as const)( sessionId, sourceId: leftId, demandId: `all`, + attemptId: `left-attempt`, }, { type: `registerSourceDemand`, sessionId, sourceId: rightId, demandId: `all`, + attemptId: `right-attempt`, }, ] const createSource = ( @@ -96,6 +98,7 @@ it.each([`resolve`, `reject`, `cleanup`] as const)( sessionId, sourceId: leftId, demandId: `all`, + attemptId: `left-attempt`, outcome: `resolve`, }) await flushPromises() @@ -114,6 +117,7 @@ it.each([`resolve`, `reject`, `cleanup`] as const)( sessionId, sourceId: rightId, demandId: `all`, + attemptId: `right-attempt`, outcome: `resolve`, }) await flushPromises() @@ -131,6 +135,7 @@ it.each([`resolve`, `reject`, `cleanup`] as const)( sessionId, sourceId: rightId, demandId: `all`, + attemptId: `right-attempt`, outcome: secondOutcome, }) await flushPromises() From 8985a40682ac7bc2d12156b5ee313f623a935793 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:01:56 -0600 Subject: [PATCH 145/327] test(db): qualify publication demands by source --- ...ubscription-replay-oracle.property.test.ts | 19 +- .../db/tests/load-subset-full-flow-model.ts | 67 ++++-- ...d-subset-full-flow-oracle.property.test.ts | 28 ++- ...d-subset-refinement-model.property.test.ts | 222 +++++++++++++++++- 4 files changed, 305 insertions(+), 31 deletions(-) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index cb28169a3..03b2e6bb8 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -8945,6 +8945,7 @@ describe(`CollectionSubscription replay oracle`, () => { { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `ordered`, rows: [], }, @@ -8961,6 +8962,7 @@ describe(`CollectionSubscription replay oracle`, () => { { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `other`, rows: [], }, @@ -8968,6 +8970,7 @@ describe(`CollectionSubscription replay oracle`, () => { ] const expectedBoundary = () => projectAtomicOrderedPublicationState(history, { + sourceId: `source`, demandId: `ordered`, direction: `asc`, initialWindowSize: 1, @@ -9043,7 +9046,10 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `beginReplacement`, publicationId: `replacement`, - demandIds: [`ordered`, `other`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `other` }, + ], }) const orderedReplay = replayLoads.find(({ options }) => options.orderBy) @@ -9061,6 +9067,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `stagePublicationRows`, publicationId: `replacement`, + sourceId: `source`, demandId: `ordered`, rows: [{ key: `new-ordered`, orderValue: 1 }], }) @@ -9071,6 +9078,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `ordered`, outcome: `success`, extent: `exhausted`, @@ -9088,6 +9096,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `other`, outcome: `success`, extent: `exhausted`, @@ -9097,6 +9106,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `other`, outcome: `failure`, }) @@ -9121,7 +9131,10 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `beginReplacement`, publicationId: `failed-replacement`, - demandIds: [`ordered`, `other`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `other` }, + ], }) const nextOrderedReplay = nextReplayLoads.find( ({ options }) => options.orderBy, @@ -9136,6 +9149,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `ordered`, outcome: `failure`, }) @@ -9146,6 +9160,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `other`, outcome: `success`, extent: `exhausted`, diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 1cfbf3031..3d305f060 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -26,6 +26,11 @@ export type FullFlowPublishedOrderRow = { orderValue: number } +export type FullFlowSourceDemand = { + sourceId: FullFlowSourceId + demandId: FullFlowDemandId +} + export type OrderedContinuationEvidencePage = { requestedPrefix: number appliedKeys: ReadonlyArray @@ -347,6 +352,7 @@ export type LoadSubsetFullFlowEvent = | { type: `stagePublicationRows` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId rows: ReadonlyArray } @@ -357,17 +363,19 @@ export type LoadSubsetFullFlowEvent = | { type: `beginReplacement` publicationId: FullFlowPublicationId - demandIds: ReadonlyArray + demands: ReadonlyArray } | { type: `settleReplacement` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId outcome: `failure` | `abort` } | { type: `settleReplacement` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId outcome: `success` extent: `exhausted` | `continues` @@ -986,6 +994,7 @@ export function projectReusableDemands( export function projectOrderedPublicationBoundary( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` prefixSize: number @@ -993,8 +1002,9 @@ export function projectOrderedPublicationBoundary( ): FullFlowPublishedOrderRow | undefined { const staged = new Map< FullFlowPublicationId, - Map> + Map> >() + const targetDemand = sourceDemandIdentity(options.sourceId, options.demandId) let committedRows: ReadonlyArray = [] for (const event of history) { @@ -1004,13 +1014,16 @@ export function projectOrderedPublicationBoundary( publication = new Map() staged.set(event.publicationId, publication) } - publication.set(event.demandId, event.rows) + publication.set( + sourceDemandIdentity(event.sourceId, event.demandId), + event.rows, + ) continue } if (event.type === `commitPublication`) { const publication = staged.get(event.publicationId) - if (publication?.has(options.demandId)) { - committedRows = publication.get(options.demandId) ?? [] + if (publication?.has(targetDemand)) { + committedRows = publication.get(targetDemand) ?? [] } } } @@ -1042,6 +1055,7 @@ export function projectOrderedPublicationBoundary( export function projectAtomicOrderedPublications( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` initialWindowSize: number @@ -1071,6 +1085,7 @@ export type AtomicOrderedPublicationProjection = { export function projectAtomicOrderedPublicationState( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` initialWindowSize: number @@ -1079,12 +1094,12 @@ export function projectAtomicOrderedPublicationState( assertWellFormedDemandAttempts(history) const staged = new Map< FullFlowPublicationId, - Map> + Map> >() const attempts = new Map< FullFlowPublicationId, Map< - FullFlowDemandId, + ScopedIdentity, | { outcome: `success`; publishable: boolean } | { outcome: `failure` | `abort`; publishable: false } | undefined @@ -1097,6 +1112,7 @@ export function projectAtomicOrderedPublicationState( let currentReplacement: FullFlowPublicationId | undefined let retainedSize = options.initialWindowSize let closed = false + const targetDemand = sourceDemandIdentity(options.sourceId, options.demandId) const sortRows = (rows: ReadonlyArray) => [...rows].sort((left, right) => { @@ -1113,7 +1129,7 @@ export function projectAtomicOrderedPublicationState( publicationId: FullFlowPublicationId, ): AtomicOrderedPublicationState | undefined => { const publication = staged.get(publicationId) - const orderedRows = publication?.get(options.demandId) + const orderedRows = publication?.get(targetDemand) if (!publication || !orderedRows) return undefined const orderedPrefix = sortRows(orderedRows).slice(0, retainedSize) @@ -1164,7 +1180,7 @@ export function projectAtomicOrderedPublicationState( } const current = attempts.get(currentReplacement) - const ordered = current?.get(options.demandId) + const ordered = current?.get(targetDemand) const activeDemandFailed = [...activeAdditionalDemands.keys()].some( (demandId) => current?.get(demandId)?.outcome !== `success`, ) @@ -1191,7 +1207,10 @@ export function projectAtomicOrderedPublicationState( publication = new Map() staged.set(event.publicationId, publication) } - publication.set(event.demandId, event.rows) + publication.set( + sourceDemandIdentity(event.sourceId, event.demandId), + event.rows, + ) break } case `commitPublication`: { @@ -1203,7 +1222,12 @@ export function projectAtomicOrderedPublicationState( case `beginReplacement`: attempts.set( event.publicationId, - new Map(event.demandIds.map((demandId) => [demandId, undefined])), + new Map( + event.demands.map(({ sourceId, demandId }) => [ + sourceDemandIdentity(sourceId, demandId), + undefined, + ]), + ), ) currentReplacement = event.publicationId retainsPreviousPublication = true @@ -1213,9 +1237,10 @@ export function projectAtomicOrderedPublicationState( break case `settleReplacement`: { const attempt = attempts.get(event.publicationId) - if (!attempt?.has(event.demandId)) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + if (!attempt?.has(demandKey)) break attempt.set( - event.demandId, + demandKey, event.outcome === `success` ? { outcome: `success`, @@ -1228,7 +1253,7 @@ export function projectAtomicOrderedPublicationState( } case `establishReplacementCoverage`: { if (event.publicationId !== currentReplacement) break - const ordered = attempts.get(event.publicationId)?.get(options.demandId) + const ordered = attempts.get(event.publicationId)?.get(targetDemand) if (ordered?.outcome === `success`) { ordered.publishable = true finishCurrentReplacement() @@ -1236,11 +1261,15 @@ export function projectAtomicOrderedPublicationState( break } case `requestDemand`: - if (!event.alreadyAborted && event.demandId !== options.demandId) { + if ( + !event.alreadyAborted && + (event.sourceId !== options.sourceId || + event.demandId !== options.demandId) + ) { addActiveDemandAttempt( activeAdditionalDemands, - event.demandId, - event.attemptId, + sourceDemandIdentity(event.sourceId, event.demandId), + sourceAttemptIdentity(event.sourceId, event.attemptId), ) } break @@ -1251,8 +1280,8 @@ export function projectAtomicOrderedPublicationState( case `releaseDemand`: releaseActiveDemandAttempt( activeAdditionalDemands, - event.demandId, - event.attemptId, + sourceDemandIdentity(event.sourceId, event.demandId), + sourceAttemptIdentity(event.sourceId, event.attemptId), ) break case `truncateSource`: diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 7eba9f9d2..5a6fdfb05 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -3533,12 +3533,14 @@ it(`keeps an additional demand active until its final attempt releases`, () => { { type: `stagePublicationRows`, publicationId: `next`, + sourceId: `source`, demandId: `ordered`, rows: [{ key: `o`, orderValue: 0 }], }, { type: `stagePublicationRows`, publicationId: `next`, + sourceId: `source`, demandId: `other`, rows: [{ key: `x`, orderValue: 1 }], }, @@ -3554,6 +3556,7 @@ it(`keeps an additional demand active until its final attempt releases`, () => { expect( projectAtomicOrderedPublicationState(history, { + sourceId: `source`, demandId: `ordered`, direction: `asc`, initialWindowSize: 1, @@ -5825,6 +5828,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `initial-publication`, + sourceId: `source`, demandId: `ordered-window`, rows: orderedForDirection.slice(0, prefixSize).map((row) => ({ key: row.id, @@ -5840,6 +5844,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `additional-publication`, + sourceId: `source`, demandId: `ordered-window`, rows: expectedOrderedPrefix.map((row) => ({ key: row.id, @@ -5851,6 +5856,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `additional-publication`, + sourceId: `source`, demandId: `unordered-retention`, rows: [{ key: addedRow.id, orderValue: addedRow.rank }], }, @@ -5859,6 +5865,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `ordered-window`, rows: [ { @@ -5878,6 +5885,7 @@ async function runOrderedBoundaryProvenanceScenario( }, ] const expectedBoundary = projectOrderedPublicationBoundary(history, { + sourceId: `source`, demandId: `ordered-window`, direction: scenario.direction, prefixSize, @@ -6247,6 +6255,7 @@ async function runAtomicOrderedReplayScenario( { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `ordered`, rows: toModelRows(initialRows), }, @@ -6345,12 +6354,14 @@ async function runAtomicOrderedReplayScenario( const expectedPublicationProjection = () => projectAtomicOrderedPublicationState(history, { + sourceId: `source`, demandId: `ordered`, direction: scenario.direction, initialWindowSize, }) const expectedPublications = () => projectAtomicOrderedPublications(history, { + sourceId: `source`, demandId: `ordered`, direction: scenario.direction, initialWindowSize, @@ -6387,9 +6398,10 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `beginReplacement`, publicationId, - demandIds: acquisitions.map((acquisition) => - acquisition === ordered ? `ordered` : `other`, - ), + demands: acquisitions.map((acquisition) => ({ + sourceId: `source`, + demandId: acquisition === ordered ? `ordered` : `other`, + })), }) expectPublicationHistory() return { publicationId, acquisitions, ordered } satisfies PendingAttempt @@ -6414,6 +6426,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: replay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows(rows), }) @@ -6453,6 +6466,7 @@ async function runAtomicOrderedReplayScenario( ? { type: `settleReplacement`, publicationId: replay.publicationId, + sourceId: `source`, demandId, outcome: settledOutcome, extent: isOrdered ? extent : `exhausted`, @@ -6460,6 +6474,7 @@ async function runAtomicOrderedReplayScenario( : { type: `settleReplacement`, publicationId: replay.publicationId, + sourceId: `source`, demandId, outcome: settledOutcome, }, @@ -6510,6 +6525,7 @@ async function runAtomicOrderedReplayScenario( { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `other`, rows: toModelRows(initialOtherRows), }, @@ -6524,6 +6540,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: firstReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([obsoleteRow]), }) @@ -6555,6 +6572,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `other`, rows: toModelRows([replacementOtherRow]), }) @@ -6577,6 +6595,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([sourceDelta]), }) @@ -6588,6 +6607,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([partialRow]), }) @@ -6605,6 +6625,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([partialRow, continuationRow]), }) @@ -6670,6 +6691,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([...finalRows, continuationRow]), }) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index f1fbd7a01..b8bd34fed 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -780,18 +780,25 @@ it.each([ { type: `stagePublicationRows`, publicationId: `publication`, + sourceId: `source`, demandId: `ordered`, rows: [{ key: `o`, orderValue: 0 }], }, { type: `stagePublicationRows`, publicationId: `publication`, + sourceId: `source`, demandId, rows: [{ key: `x`, orderValue: 1 }], }, { type: `commitPublication`, publicationId: `publication` }, ], - { demandId: `ordered`, direction: `asc`, initialWindowSize: 1 }, + { + sourceId: `source`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }, ) expect(publication.currentPublication?.rows.map(({ key }) => key)).toEqual( active ? [`o`, `x`] : [`o`], @@ -826,6 +833,169 @@ it.each([ expect(projectTransportLoads(fullyReleasedBeforeSettlement)).toBe(2) }) +it(`keeps a same-name publication demand active on its surviving source`, () => { + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared`, + attemptId: `same-attempt`, + alreadyAborted: false, + }) + const projection = projectAtomicOrderedPublicationState( + [ + request(`source-a`), + request(`source-b`), + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `other-ordered-source`, + demandId: `ordered`, + rows: [{ key: `wrong-ordered-row`, orderValue: -1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `source-b-row`, orderValue: 1 }], + }, + { + type: `releaseDemand`, + sourceId: `source-a`, + ownerId: `owner`, + demandId: `shared`, + attemptId: `same-attempt`, + }, + { type: `commitPublication`, publicationId: `publication` }, + ], + { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }, + ) + + expect(projection.currentPublication?.rows.map(({ key }) => key)).toEqual([ + `ordered-row`, + `source-b-row`, + ]) +}) + +it(`settles same-name replacement demands independently by source`, () => { + const history: Array = [ + { + type: `stagePublicationRows`, + publicationId: `initial`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `old-ordered-row`, orderValue: 0 }], + }, + { type: `commitPublication`, publicationId: `initial` }, + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `new-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `source-b-row`, orderValue: 2 }], + }, + { + type: `beginReplacement`, + publicationId: `replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + { sourceId: `source-b`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] + const options = { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc` as const, + initialWindowSize: 1, + } + + expect( + projectAtomicOrderedPublicationState( + history, + options, + ).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`old-ordered-row`]) + + history.push({ + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `source-b`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }) + expect( + projectAtomicOrderedPublicationState( + history, + options, + ).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`new-ordered-row`, `source-a-row`, `source-b-row`]) +}) + it.each([ { name: `authoritative`, @@ -1265,6 +1435,7 @@ function renameHistoryIds( return { ...event, publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, } case `commitPublication`: @@ -1277,12 +1448,16 @@ function renameHistoryIds( return { ...event, publicationId: `${event.publicationId}-${suffix}`, - demandIds: event.demandIds.map((demandId) => `${demandId}-${suffix}`), + demands: event.demands.map(({ sourceId, demandId }) => ({ + sourceId: `${sourceId}-${suffix}`, + demandId: `${demandId}-${suffix}`, + })), } case `settleReplacement`: return { ...event, publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, } default: @@ -2302,6 +2477,7 @@ function erasedIdentityReferences( break case `stagePublicationRows`: add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) break case `commitPublication`: @@ -2310,12 +2486,24 @@ function erasedIdentityReferences( break case `beginReplacement`: add(eventIndex, `publicationId`, event.publicationId) - event.demandIds.forEach((demandId, demandIndex) => - add(eventIndex, `demandId`, demandId, `demandIds.${demandIndex}`), - ) + event.demands.forEach(({ sourceId, demandId }, demandIndex) => { + add( + eventIndex, + `sourceId`, + sourceId, + `demands.${demandIndex}.sourceId`, + ) + add( + eventIndex, + `demandId`, + demandId, + `demands.${demandIndex}.demandId`, + ) + }) break case `settleReplacement`: add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) add(eventIndex, `demandId`, event.demandId) break case `establishPublication`: @@ -2405,6 +2593,7 @@ function publicationErasureHistories(): Array> { { type: `stagePublicationRows`, publicationId: `publication-a`, + sourceId: `source`, demandId: `ordered`, rows: orderedRows, }, @@ -2418,6 +2607,7 @@ function publicationErasureHistories(): Array> { { type: `stagePublicationRows`, publicationId: `publication-a`, + sourceId: `source`, demandId: `ordered`, rows: orderedRows, }, @@ -2429,23 +2619,29 @@ function publicationErasureHistories(): Array> { { type: `stagePublicationRows`, publicationId: `publication-b`, + sourceId: `source`, demandId: `ordered`, rows: orderedRows.slice(1), }, { type: `stagePublicationRows`, publicationId: `publication-b`, + sourceId: `source`, demandId: `related`, rows: relatedRows, }, { type: `beginReplacement`, publicationId: `publication-b`, - demandIds: [`ordered`, `related`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `related` }, + ], }, { type: `settleReplacement`, publicationId: `publication-b`, + sourceId: `source`, demandId: `related`, outcome: `success`, extent: `exhausted`, @@ -2453,6 +2649,7 @@ function publicationErasureHistories(): Array> { { type: `settleReplacement`, publicationId: `publication-b`, + sourceId: `source`, demandId: `ordered`, outcome: `success`, extent: `continues`, @@ -2473,6 +2670,7 @@ function publicationErasureHistories(): Array> { { type: `stagePublicationRows`, publicationId: `publication-a`, + sourceId: `source`, demandId: `ordered`, rows: orderedRows, }, @@ -2484,17 +2682,22 @@ function publicationErasureHistories(): Array> { { type: `beginReplacement`, publicationId: `publication-b`, - demandIds: [`ordered`, `related`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `related` }, + ], }, { type: `settleReplacement`, publicationId: `publication-b`, + sourceId: `source`, demandId: `related`, outcome: `abort`, }, { type: `settleReplacement`, publicationId: `publication-b`, + sourceId: `source`, demandId: `ordered`, outcome: `failure`, }, @@ -2502,6 +2705,7 @@ function publicationErasureHistories(): Array> { { type: `settleReplacement`, publicationId: `publication-b`, + sourceId: `source`, demandId: `ordered`, outcome: `success`, extent: `exhausted`, @@ -2644,6 +2848,8 @@ for (const campaign of refinementCampaigns(1_779_009)) { renamingSuffix: string, ) => projectAtomicOrderedPublicationState(prefix, { + sourceId: + renamingSuffix === `` ? `source` : `source-${renamingSuffix}`, demandId: renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, direction: `asc`, @@ -2659,6 +2865,8 @@ for (const campaign of refinementCampaigns(1_779_009)) { suffix, (prefix, renamingSuffix) => projectOrderedPublicationBoundary(prefix, { + sourceId: + renamingSuffix === `` ? `source` : `source-${renamingSuffix}`, demandId: renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, direction: `asc`, From 58f2f86930307413f269bff727a649f045372468 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:12:09 -0600 Subject: [PATCH 146/327] test(db): scope publication target events --- .../db/tests/load-subset-full-flow-model.ts | 18 ++- ...d-subset-full-flow-oracle.property.test.ts | 9 +- ...d-subset-refinement-model.property.test.ts | 126 +++++++++++++++++- 3 files changed, 150 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 3d305f060..408c0c977 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -383,9 +383,13 @@ export type LoadSubsetFullFlowEvent = | { type: `establishReplacementCoverage` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId } | { type: `resizeOrderedWindow` + sourceId: FullFlowSourceId + demandId: FullFlowDemandId size: number } @@ -1233,6 +1237,12 @@ export function projectAtomicOrderedPublicationState( retainsPreviousPublication = true break case `resizeOrderedWindow`: + if ( + event.sourceId !== options.sourceId || + event.demandId !== options.demandId + ) { + break + } retainedSize = Math.max(retainedSize, event.size) break case `settleReplacement`: { @@ -1252,7 +1262,13 @@ export function projectAtomicOrderedPublicationState( break } case `establishReplacementCoverage`: { - if (event.publicationId !== currentReplacement) break + if ( + event.publicationId !== currentReplacement || + event.sourceId !== options.sourceId || + event.demandId !== options.demandId + ) { + break + } const ordered = attempts.get(event.publicationId)?.get(targetDemand) if (ordered?.outcome === `success`) { ordered.publishable = true diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 5a6fdfb05..b1c8d2fb7 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -6562,7 +6562,12 @@ async function runAtomicOrderedReplayScenario( ? ([2, 0] as const) : ([0, 2] as const) for (const size of resizeSizes) { - history.push({ type: `resizeOrderedWindow`, size }) + history.push({ + type: `resizeOrderedWindow`, + sourceId: `source`, + demandId: `ordered`, + size, + }) subscription.ensureOrderedWindowSize(size) expectPublicationHistory() } @@ -6758,6 +6763,8 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `establishReplacementCoverage`, publicationId: currentReplay.publicationId, + sourceId: `source`, + demandId: `ordered`, }) await flushPromises() expectPublicationHistory() diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index b8bd34fed..3b3b99e9e 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -996,6 +996,104 @@ it(`settles same-name replacement demands independently by source`, () => { ).toEqual([`new-ordered-row`, `source-a-row`, `source-b-row`]) }) +it.each([`a-first`, `b-first`] as const)( + `keeps ordered boundaries source-qualified when staged %s`, + (stageOrder) => { + const stages: Array = [ + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `ordered`, + rows: [{ key: `row-a`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `ordered`, + rows: [{ key: `row-b`, orderValue: 2 }], + }, + ] + if (stageOrder === `b-first`) stages.reverse() + const history = [ + ...stages, + { type: `commitPublication`, publicationId: `publication` } as const, + ] + const boundary = (sourceId: string) => + projectOrderedPublicationBoundary(history, { + sourceId, + demandId: `ordered`, + direction: `asc`, + prefixSize: 1, + })?.key + + expect(boundary(`source-a`)).toBe(`row-a`) + expect(boundary(`source-b`)).toBe(`row-b`) + }, +) + +it(`applies target events only to their named source and demand`, () => { + const target = { sourceId: `source-a`, demandId: `ordered` } as const + const base: Array = [ + { + type: `stagePublicationRows`, + publicationId: `initial`, + ...target, + rows: [{ key: `old-row`, orderValue: 0 }], + }, + { type: `commitPublication`, publicationId: `initial` }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + ...target, + rows: [ + { key: `new-row-a`, orderValue: 1 }, + { key: `new-row-b`, orderValue: 2 }, + ], + }, + { + type: `beginReplacement`, + publicationId: `replacement`, + demands: [target], + }, + ] + const settle: LoadSubsetFullFlowEvent = { + type: `settleReplacement`, + publicationId: `replacement`, + ...target, + outcome: `success`, + extent: `continues`, + } + const establish = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `establishReplacementCoverage`, + publicationId: `replacement`, + sourceId, + demandId: `ordered`, + }) + const resize = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `resizeOrderedWindow`, + sourceId, + demandId: `ordered`, + size: 2, + }) + const rows = (history: ReadonlyArray) => + projectAtomicOrderedPublicationState(history, { + ...target, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key) + + expect(rows([...base, settle, establish(`source-b`)])).toEqual([`old-row`]) + expect(rows([...base, settle, establish(`source-a`)])).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-b`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-a`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`, `new-row-b`]) +}) + it.each([ { name: `authoritative`, @@ -1439,10 +1537,16 @@ function renameHistoryIds( demandId: `${event.demandId}-${suffix}`, } case `commitPublication`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + } case `establishReplacementCoverage`: return { ...event, publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, } case `beginReplacement`: return { @@ -1460,6 +1564,12 @@ function renameHistoryIds( sourceId: `${event.sourceId}-${suffix}`, demandId: `${event.demandId}-${suffix}`, } + case `resizeOrderedWindow`: + return { + ...event, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } default: return event } @@ -2481,8 +2591,12 @@ function erasedIdentityReferences( add(eventIndex, `demandId`, event.demandId) break case `commitPublication`: + add(eventIndex, `publicationId`, event.publicationId) + break case `establishReplacementCoverage`: add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) break case `beginReplacement`: add(eventIndex, `publicationId`, event.publicationId) @@ -2507,7 +2621,10 @@ function erasedIdentityReferences( add(eventIndex, `demandId`, event.demandId) break case `establishPublication`: + break case `resizeOrderedWindow`: + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) break } } @@ -2601,7 +2718,12 @@ function publicationErasureHistories(): Array> { type: `commitPublication`, publicationId: `publication-a`, }, - { type: `resizeOrderedWindow`, size: 2 }, + { + type: `resizeOrderedWindow`, + sourceId: `source`, + demandId: `ordered`, + size: 2, + }, ], [ { @@ -2657,6 +2779,8 @@ function publicationErasureHistories(): Array> { { type: `establishReplacementCoverage`, publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, }, { type: `releaseDemand`, From 0171b9cff2dc9772de79498f8d21df95c3662ee2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:19:43 -0600 Subject: [PATCH 147/327] test(db): distinguish publication target demands --- ...d-subset-refinement-model.property.test.ts | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 3b3b99e9e..fb7953342 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -1065,16 +1065,22 @@ it(`applies target events only to their named source and demand`, () => { outcome: `success`, extent: `continues`, } - const establish = (sourceId: string): LoadSubsetFullFlowEvent => ({ + const establish = ( + sourceId: string, + demandId = `ordered`, + ): LoadSubsetFullFlowEvent => ({ type: `establishReplacementCoverage`, publicationId: `replacement`, sourceId, - demandId: `ordered`, + demandId, }) - const resize = (sourceId: string): LoadSubsetFullFlowEvent => ({ + const resize = ( + sourceId: string, + demandId = `ordered`, + ): LoadSubsetFullFlowEvent => ({ type: `resizeOrderedWindow`, sourceId, - demandId: `ordered`, + demandId, size: 2, }) const rows = (history: ReadonlyArray) => @@ -1085,10 +1091,16 @@ it(`applies target events only to their named source and demand`, () => { }).currentPublication?.rows.map(({ key }) => key) expect(rows([...base, settle, establish(`source-b`)])).toEqual([`old-row`]) + expect(rows([...base, settle, establish(`source-a`, `other`)])).toEqual([ + `old-row`, + ]) expect(rows([...base, settle, establish(`source-a`)])).toEqual([`new-row-a`]) expect( rows([...base, resize(`source-b`), settle, establish(`source-a`)]), ).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-a`, `other`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`]) expect( rows([...base, resize(`source-a`), settle, establish(`source-a`)]), ).toEqual([`new-row-a`, `new-row-b`]) From d1dc5944bf47e7d57722698f0c75e11720d5498d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:31:09 -0600 Subject: [PATCH 148/327] test(db): close publication identity gaps --- ...d-subset-refinement-model.property.test.ts | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index fb7953342..1adc3d2ba 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -861,12 +861,19 @@ it(`keeps a same-name publication demand active on its surviving source`, () => demandId: `ordered`, rows: [{ key: `wrong-ordered-row`, orderValue: -1 }], }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, { type: `stagePublicationRows`, publicationId: `publication`, sourceId: `source-b`, demandId: `shared`, - rows: [{ key: `source-b-row`, orderValue: 1 }], + rows: [{ key: `source-b-row`, orderValue: 2 }], }, { type: `releaseDemand`, @@ -891,6 +898,44 @@ it(`keeps a same-name publication demand active on its surviving source`, () => ]) }) +it(`treats a same-name demand from another source as additional`, () => { + const history: Array = [ + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `ordered`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `ordered`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `ordered`, + rows: [{ key: `source-b-row`, orderValue: 2 }], + }, + { type: `commitPublication`, publicationId: `publication` }, + ] + + expect( + projectAtomicOrderedPublicationState(history, { + sourceId: `source-a`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`source-a-row`, `source-b-row`]) +}) + it(`settles same-name replacement demands independently by source`, () => { const history: Array = [ { @@ -1068,9 +1113,10 @@ it(`applies target events only to their named source and demand`, () => { const establish = ( sourceId: string, demandId = `ordered`, + publicationId = `replacement`, ): LoadSubsetFullFlowEvent => ({ type: `establishReplacementCoverage`, - publicationId: `replacement`, + publicationId, sourceId, demandId, }) @@ -1094,6 +1140,9 @@ it(`applies target events only to their named source and demand`, () => { expect(rows([...base, settle, establish(`source-a`, `other`)])).toEqual([ `old-row`, ]) + expect( + rows([...base, settle, establish(`source-a`, `ordered`, `obsolete`)]), + ).toEqual([`old-row`]) expect(rows([...base, settle, establish(`source-a`)])).toEqual([`new-row-a`]) expect( rows([...base, resize(`source-b`), settle, establish(`source-a`)]), From af8f0c098ddb0f0e38d9c7f7e877220ca512e659 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:40:37 -0600 Subject: [PATCH 149/327] test(db): fence retired readiness attempts --- ...source-readiness-refinement-oracle.test.ts | 201 +++++++++++++++++- 1 file changed, 200 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index 26f59bc91..c11805e43 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -2,13 +2,212 @@ import { expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BTreeIndex } from '../../src/index.js' -import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { + createLiveQueryCollection, + eq, + toArray, +} from '../../src/query/index.js' import { projectSourceReadiness } from '../load-subset-full-flow-model.js' import { flushPromises } from '../utils.js' import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' type Row = { id: string; group: string } +it.each([`resolve`, `reject`] as const)( + `fences a retired source-demand attempt when it settles late: %s`, + async (oldOutcome) => { + type Parent = { id: string; group: string } + type Child = { id: string; group: string } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const sessionId = `session` + const parentId = `readiness-generation-parent-${oldOutcome}` + const childId = `readiness-generation-child-${oldOutcome}` + const oldAttemptId = `old-attempt` + const freshAttemptId = `fresh-attempt` + let parentBegin!: () => void + let parentWrite!: (message: { + type: `update` + value: Parent + previousValue: Parent + }) => void + let parentCommit!: () => true | Promise + const oldParent: Parent = { id: `parent`, group: `old` } + const freshParent: Parent = { ...oldParent, group: `fresh` } + const parent = createCollection({ + id: parentId, + getKey: (row) => row.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + parentBegin = begin + parentWrite = write + parentCommit = commit + begin() + write({ type: `insert`, value: oldParent }) + commit() + markReady() + }, + }, + }) + let childBegin!: () => void + let childWrite!: (message: { type: `insert`; value: Child }) => void + let childCommit!: () => true | Promise + const pending: Array>> = [] + const child = createCollection({ + id: childId, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + childBegin = begin + childWrite = write + childCommit = commit + markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `readiness-generation-live-${oldOutcome}`, + query: (q) => + q.from({ parent }).select(({ parent: parentRow }) => ({ + id: parentRow.id, + children: toArray( + q + .from({ child }) + .where(({ child: childRow }) => + eq(childRow.group, parentRow.group), + ), + ), + })), + startSync: true, + }) + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + }, + ] + let preloadState: `pending` | `resolved` | `rejected` = `pending` + const preload = live.preload() + void preload.then( + () => { + preloadState = `resolved` + }, + () => { + preloadState = `rejected` + }, + ) + + try { + await flushPromises() + expect(pending).toHaveLength(1) + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`pending`) + + parentBegin() + parentWrite({ + type: `update`, + value: freshParent, + previousValue: oldParent, + }) + const parentApplied = parentCommit() + if (parentApplied !== true) await parentApplied + history.push( + { + type: `retireSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + }, + { + type: `registerSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: freshAttemptId, + }, + ) + await flushPromises() + + expect(pending).toHaveLength(2) + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`pending`) + + if (oldOutcome === `resolve`) { + pending[0]!.resolve({ hasMore: false, appliedRowKeys: [] }) + } else { + pending[0]!.reject(new Error(`retired source demand failed`)) + } + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + outcome: oldOutcome, + }) + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`pending`) + expect(live.utils.lastSubsetError).toBeUndefined() + + const freshChild: Child = { id: `fresh-child`, group: `fresh` } + childBegin() + childWrite({ type: `insert`, value: freshChild }) + const childApplied = childCommit() + if (childApplied !== true) await childApplied + pending[1]!.resolve({ + hasMore: false, + appliedRowKeys: [freshChild.id], + }) + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: freshAttemptId, + outcome: `resolve`, + }) + await preload + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`resolved`) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray).toEqual([ + expect.objectContaining({ + id: `parent`, + children: [expect.objectContaining({ id: `fresh-child` })], + }), + ]) + } finally { + for (const request of pending) { + request.resolve({ hasMore: false, appliedRowKeys: [] }) + } + await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await Promise.all([parent.cleanup(), child.cleanup()]) + } + }, +) + it.each([`resolve`, `reject`, `cleanup`] as const)( `matches cross-source initial readiness through %s`, async (secondOutcome) => { From acab6df4b48179ec410ca8ef9666328e01b8ac02 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:51:33 -0600 Subject: [PATCH 150/327] test(db): bind readiness to exact acquisitions --- ...source-readiness-refinement-oracle.test.ts | 98 ++++++++++++++----- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index c11805e43..91d37130d 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -2,6 +2,7 @@ import { expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BTreeIndex } from '../../src/index.js' +import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' import { createLiveQueryCollection, eq, @@ -10,6 +11,7 @@ import { import { projectSourceReadiness } from '../load-subset-full-flow-model.js' import { flushPromises } from '../utils.js' import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { LoadSubsetOptions } from '../../src/types.js' type Row = { id: string; group: string } @@ -18,9 +20,9 @@ it.each([`resolve`, `reject`] as const)( async (oldOutcome) => { type Parent = { id: string; group: string } type Child = { id: string; group: string } - type Result = { - hasMore: boolean - appliedRowKeys: ReadonlyArray + type PendingRequest = { + options: LoadSubsetOptions + rows: ReturnType>> } const sessionId = `session` const parentId = `readiness-generation-parent-${oldOutcome}` @@ -54,7 +56,8 @@ it.each([`resolve`, `reject`] as const)( let childBegin!: () => void let childWrite!: (message: { type: `insert`; value: Child }) => void let childCommit!: () => true | Promise - const pending: Array>> = [] + const pending: Array = [] + const unloads: Array = [] const child = createCollection({ id: childId, getKey: (row) => row.id, @@ -69,12 +72,27 @@ it.each([`resolve`, `reject`] as const)( childCommit = commit markReady() return { - loadSubset: () => { - const request = createDeferred() - pending.push(request) - return request.promise + loadSubset: (options) => { + const rows = createDeferred>() + pending.push({ options, rows }) + return rows.promise.then(async (acquiredRows) => { + if (acquiredRows.length > 0) { + childBegin() + for (const row of acquiredRows) { + childWrite({ type: `insert`, value: row }) + } + const applied = childCommit() + if (applied !== true) await applied + } + return { + hasMore: false, + appliedRowKeys: acquiredRows.map((row) => row.id), + } + }) + }, + unloadSubset: (options) => { + unloads.push(options) }, - unloadSubset: () => {}, } }, }, @@ -113,10 +131,25 @@ it.each([`resolve`, `reject`] as const)( preloadState = `rejected` }, ) + const requestedGroups = (options: LoadSubsetOptions): Array => + extractSimpleComparisons(options.where).flatMap((comparison) => { + if (comparison.field.join(`.`) !== `group`) return [] + if (comparison.operator === `eq`) { + return typeof comparison.value === `string` ? [comparison.value] : [] + } + if (comparison.operator !== `in` || !Array.isArray(comparison.value)) { + return [] + } + return comparison.value.filter( + (value): value is string => typeof value === `string`, + ) + }) + let liveCleaned = false try { await flushPromises() expect(pending).toHaveLength(1) + expect(requestedGroups(pending[0]!.options)).toEqual([`old`]) expect(live.status).toBe(projectSourceReadiness(history).status) expect(preloadState).toBe(`pending`) @@ -128,6 +161,17 @@ it.each([`resolve`, `reject`] as const)( }) const parentApplied = parentCommit() if (parentApplied !== true) await parentApplied + await flushPromises() + + expect(pending).toHaveLength(2) + expect(requestedGroups(pending[0]!.options)).toEqual([`old`]) + expect(requestedGroups(pending[1]!.options)).toEqual([`fresh`]) + expect(pending[0]!.options.signal?.aborted).toBe(true) + expect(pending[1]!.options.signal?.aborted).toBe(false) + expect( + unloads.filter((options) => options === pending[0]!.options), + ).toHaveLength(1) + expect(unloads).not.toContain(pending[1]!.options) history.push( { type: `retireSourceDemand`, @@ -144,16 +188,13 @@ it.each([`resolve`, `reject`] as const)( attemptId: freshAttemptId, }, ) - await flushPromises() - - expect(pending).toHaveLength(2) expect(live.status).toBe(projectSourceReadiness(history).status) expect(preloadState).toBe(`pending`) if (oldOutcome === `resolve`) { - pending[0]!.resolve({ hasMore: false, appliedRowKeys: [] }) + pending[0]!.rows.resolve([]) } else { - pending[0]!.reject(new Error(`retired source demand failed`)) + pending[0]!.rows.reject(new Error(`retired source demand failed`)) } history.push({ type: `settleSourceDemand`, @@ -170,14 +211,8 @@ it.each([`resolve`, `reject`] as const)( expect(live.utils.lastSubsetError).toBeUndefined() const freshChild: Child = { id: `fresh-child`, group: `fresh` } - childBegin() - childWrite({ type: `insert`, value: freshChild }) - const childApplied = childCommit() - if (childApplied !== true) await childApplied - pending[1]!.resolve({ - hasMore: false, - appliedRowKeys: [freshChild.id], - }) + expect(child.get(freshChild.id)).toBeUndefined() + pending[1]!.rows.resolve([freshChild]) history.push({ type: `settleSourceDemand`, sessionId, @@ -192,17 +227,32 @@ it.each([`resolve`, `reject`] as const)( expect(live.status).toBe(projectSourceReadiness(history).status) expect(preloadState).toBe(`resolved`) expect(live.utils.lastSubsetError).toBeUndefined() + expect(child.get(freshChild.id)).toEqual( + expect.objectContaining(freshChild), + ) expect(live.toArray).toEqual([ expect.objectContaining({ id: `parent`, children: [expect.objectContaining({ id: `fresh-child` })], }), ]) + expect(pending[1]!.options.signal?.aborted).toBe(false) + expect(unloads).not.toContain(pending[1]!.options) + + await live.cleanup() + liveCleaned = true + expect(pending[1]!.options.signal?.aborted).toBe(true) + expect( + unloads.filter((options) => options === pending[1]!.options), + ).toHaveLength(1) } finally { for (const request of pending) { - request.resolve({ hasMore: false, appliedRowKeys: [] }) + request.rows.resolve([]) } - await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await Promise.all([ + preload.catch(() => undefined), + liveCleaned ? Promise.resolve() : live.cleanup(), + ]) await Promise.all([parent.cleanup(), child.cleanup()]) } }, From 6a425782c3f9fe5572c741aaf161c3f9170a2188 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:54:28 -0600 Subject: [PATCH 151/327] test(db): assert exact readiness unloads --- ...-subset-source-readiness-refinement-oracle.test.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index 91d37130d..839162e32 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -168,10 +168,7 @@ it.each([`resolve`, `reject`] as const)( expect(requestedGroups(pending[1]!.options)).toEqual([`fresh`]) expect(pending[0]!.options.signal?.aborted).toBe(true) expect(pending[1]!.options.signal?.aborted).toBe(false) - expect( - unloads.filter((options) => options === pending[0]!.options), - ).toHaveLength(1) - expect(unloads).not.toContain(pending[1]!.options) + expect(unloads).toEqual([pending[0]!.options]) history.push( { type: `retireSourceDemand`, @@ -237,14 +234,12 @@ it.each([`resolve`, `reject`] as const)( }), ]) expect(pending[1]!.options.signal?.aborted).toBe(false) - expect(unloads).not.toContain(pending[1]!.options) + expect(unloads).toEqual([pending[0]!.options]) await live.cleanup() liveCleaned = true expect(pending[1]!.options.signal?.aborted).toBe(true) - expect( - unloads.filter((options) => options === pending[1]!.options), - ).toHaveLength(1) + expect(unloads).toEqual([pending[0]!.options, pending[1]!.options]) } finally { for (const request of pending) { request.rows.resolve([]) From 5c60c3cb8fe71944639167587919593c082ec9c5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 07:58:59 -0600 Subject: [PATCH 152/327] test(db): observe readiness abort order --- ...source-readiness-refinement-oracle.test.ts | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index 839162e32..e166ae4fe 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -57,7 +57,10 @@ it.each([`resolve`, `reject`] as const)( let childWrite!: (message: { type: `insert`; value: Child }) => void let childCommit!: () => true | Promise const pending: Array = [] - const unloads: Array = [] + const unloads: Array<{ + options: LoadSubsetOptions + abortedAtUnload: boolean | undefined + }> = [] const child = createCollection({ id: childId, getKey: (row) => row.id, @@ -91,7 +94,10 @@ it.each([`resolve`, `reject`] as const)( }) }, unloadSubset: (options) => { - unloads.push(options) + unloads.push({ + options, + abortedAtUnload: options.signal?.aborted, + }) }, } }, @@ -144,6 +150,15 @@ it.each([`resolve`, `reject`] as const)( (value): value is string => typeof value === `string`, ) }) + const expectUnloads = ( + ...expectedOptions: ReadonlyArray + ): void => { + expect(unloads).toHaveLength(expectedOptions.length) + for (const [index, options] of expectedOptions.entries()) { + expect(unloads[index]!.options).toBe(options) + expect(unloads[index]!.abortedAtUnload).toBe(true) + } + } let liveCleaned = false try { @@ -168,7 +183,7 @@ it.each([`resolve`, `reject`] as const)( expect(requestedGroups(pending[1]!.options)).toEqual([`fresh`]) expect(pending[0]!.options.signal?.aborted).toBe(true) expect(pending[1]!.options.signal?.aborted).toBe(false) - expect(unloads).toEqual([pending[0]!.options]) + expectUnloads(pending[0]!.options) history.push( { type: `retireSourceDemand`, @@ -234,12 +249,12 @@ it.each([`resolve`, `reject`] as const)( }), ]) expect(pending[1]!.options.signal?.aborted).toBe(false) - expect(unloads).toEqual([pending[0]!.options]) + expectUnloads(pending[0]!.options) await live.cleanup() liveCleaned = true expect(pending[1]!.options.signal?.aborted).toBe(true) - expect(unloads).toEqual([pending[0]!.options, pending[1]!.options]) + expectUnloads(pending[0]!.options, pending[1]!.options) } finally { for (const request of pending) { request.rows.resolve([]) From ff32c7e4d40f7ed24b3ecafdff073acc658c9954 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 08:22:32 -0600 Subject: [PATCH 153/327] fix(db): reconcile demand release during replay --- packages/db/src/collection/subscription.ts | 49 ++- .../db/tests/load-subset-full-flow-model.ts | 37 +- ...d-subset-refinement-model.property.test.ts | 400 ++++++++++++++++++ ...source-readiness-refinement-oracle.test.ts | 89 ++-- 4 files changed, 529 insertions(+), 46 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index c4b0f4ec8..9029f6a53 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1354,6 +1354,45 @@ export class CollectionSubscription return changes } + /** Apply logical demand release to the public baseline of a private replay. */ + private reconcileBufferedOrderedPublicationOnRelease(): Array< + ChangeMessage + > { + const publication = this.truncateReplaySession?.publicationState + const ordered = publication?.ordered + const window = this.orderedWindow + if (!publication || !ordered || !window) return [] + + const orderedRows = [...ordered.candidateRows] + .sort((left, right) => window.totalOrder.compareEntries(left, right)) + .slice(0, ordered.prefixSize) + const desired = new Map(orderedRows) + const additionalFilters = this.activeAdditionalFilters() + for (const [key, row] of publication.publishedRows) { + if (additionalFilters.some((filter) => filter(row))) { + desired.set(key, row) + } + } + + const lastOrderedRow = orderedRows.at(-1) + const nextOrdered: OrderedPublicationState = { + prefixSize: orderedRows.length, + boundary: + lastOrderedRow === undefined + ? undefined + : window.totalOrder.boundary(lastOrderedRow[1], lastOrderedRow[0]), + candidateRows: ordered.candidateRows, + } + publication.publishedRows = new Map(desired) + publication.sentKeys = new Set(desired.keys()) + publication.ordered = nextOrdered + this.orderedPublication = { + ...nextOrdered, + candidateRows: new Map(nextOrdered.candidateRows), + } + return this.diffPublishedRows(desired) + } + /** * Evolve a failed replay's last good ordered publication without admitting * rows installed by the rejected replacement. Later source deltas form a @@ -2348,10 +2387,12 @@ export class CollectionSubscription releaseFailure = { error } } finally { this.collectReleasedDemand(demand) - if (this.orderedWindow && !this.isBufferingForTruncate) { - const changes = this.stalePublication?.ordered - ? this.reconcileStaleOrderedPublication([]) - : this.reconcileOrderedWindow() + if (this.orderedWindow) { + const changes = this.isBufferingForTruncate + ? this.reconcileBufferedOrderedPublicationOnRelease() + : this.stalePublication?.ordered + ? this.reconcileStaleOrderedPublication([]) + : this.reconcileOrderedWindow() if (changes.length > 0) this.callback(changes) } } diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 408c0c977..699bba65e 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -1114,6 +1114,7 @@ export function projectAtomicOrderedPublicationState( let currentPublication: AtomicOrderedPublicationState | undefined let retainsPreviousPublication = false let currentReplacement: FullFlowPublicationId | undefined + let currentPublicationId: FullFlowPublicationId | undefined let retainedSize = options.initialWindowSize let closed = false const targetDemand = sourceDemandIdentity(options.sourceId, options.demandId) @@ -1131,12 +1132,13 @@ export function projectAtomicOrderedPublicationState( const publicationState = ( publicationId: FullFlowPublicationId, + orderedPrefixSize = retainedSize, ): AtomicOrderedPublicationState | undefined => { const publication = staged.get(publicationId) const orderedRows = publication?.get(targetDemand) if (!publication || !orderedRows) return undefined - const orderedPrefix = sortRows(orderedRows).slice(0, retainedSize) + const orderedPrefix = sortRows(orderedRows).slice(0, orderedPrefixSize) const desired = new Map(orderedPrefix.map((row) => [row.key, row] as const)) for (const demandId of activeAdditionalDemands.keys()) { for (const row of publication.get(demandId) ?? []) { @@ -1150,12 +1152,16 @@ export function projectAtomicOrderedPublicationState( } } - const publish = (publicationId: FullFlowPublicationId) => { - const next = publicationState(publicationId) + const publish = ( + publicationId: FullFlowPublicationId, + orderedPrefixSize?: number, + ) => { + const next = publicationState(publicationId, orderedPrefixSize) if (!next) return const previous = publications.at(-1) if (previous === undefined && next.rows.length === 0) { currentPublication = next + currentPublicationId = publicationId return } if ( @@ -1167,10 +1173,12 @@ export function projectAtomicOrderedPublicationState( ) ) { currentPublication = next + currentPublicationId = publicationId return } publications.push(next.rows) currentPublication = next + currentPublicationId = publicationId } const finishCurrentReplacement = () => { @@ -1294,11 +1302,24 @@ export function projectAtomicOrderedPublicationState( case `rejectDemand`: break case `releaseDemand`: - releaseActiveDemandAttempt( - activeAdditionalDemands, - sourceDemandIdentity(event.sourceId, event.demandId), - sourceAttemptIdentity(event.sourceId, event.attemptId), - ) + if ( + releaseActiveDemandAttempt( + activeAdditionalDemands, + sourceDemandIdentity(event.sourceId, event.demandId), + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) && + currentPublicationId !== undefined + ) { + // A private replacement may have grown the target window. Releasing + // another demand filters the last complete public prefix; it cannot + // expose rows known only to the private replacement. + publish( + currentPublicationId, + currentReplacement === undefined + ? retainedSize + : currentPublication?.orderedPrefixSize, + ) + } break case `truncateSource`: case `restartSession`: diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index 1adc3d2ba..b0da8dab0 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -532,6 +532,406 @@ for (const campaign of refinementCampaigns(1_779_011)) { ) } +type LegalOrderAction = + | `release-old` + | `release-peer` + | `settle-old` + | `settle-fresh` + +function interleaveLegalOrderChains( + left: ReadonlyArray, + right: ReadonlyArray, +): Array> { + if (left.length === 0) return [[...right]] + if (right.length === 0) return [[...left]] + + return [ + ...interleaveLegalOrderChains(left.slice(1), right).map((suffix) => [ + left[0]!, + ...suffix, + ]), + ...interleaveLegalOrderChains(left, right.slice(1)).map((suffix) => [ + right[0]!, + ...suffix, + ]), + ] +} + +function legalOrderBaseHistory(): Array { + return [ + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-old`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-peer`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-peer`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-b`, + ownerId: `owner-peer`, + demandId: `shared`, + attemptId: `attempt-peer`, + rowKeys: [`peer-row`], + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + outcome: `resolve`, + }, + { + type: `stagePublicationRows`, + publicationId: `initial-publication`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `old-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `initial-publication`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `peer-row`, orderValue: 2 }], + }, + { type: `commitPublication`, publicationId: `initial-publication` }, + { + type: `stagePublicationRows`, + publicationId: `old-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `obsolete-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `stale-row`, orderValue: 1 }], + }, + { + type: `beginReplacement`, + publicationId: `old-replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source-a` }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-fresh`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-fresh`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-fresh`, + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `fresh-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `fresh-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `peer-row`, orderValue: 2 }], + }, + { + type: `beginReplacement`, + publicationId: `fresh-replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + { sourceId: `source-b`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `source-b`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] +} + +function legalOrderEvents( + action: LegalOrderAction, + oldOutcome: `resolve` | `reject`, +): Array { + switch (action) { + case `release-old`: + return [ + { + type: `releaseDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + ] + case `release-peer`: + return [ + { + type: `releaseDemand`, + sourceId: `source-b`, + ownerId: `owner-peer`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + ] + case `settle-old`: + return [ + oldOutcome === `resolve` + ? { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + rowKeys: [`stale-row`], + } + : { + type: `rejectDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + outcome: oldOutcome, + }, + oldOutcome === `resolve` + ? { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + } + : { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `failure`, + }, + ] + case `settle-fresh`: + return [ + { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner-fresh`, + demandId: `shared`, + attemptId: `attempt-fresh`, + rowKeys: [`fresh-row`], + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-fresh`, + outcome: `resolve`, + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] + } +} + +it(`enumerates legal release and settlement orders across every refinement projection`, () => { + const publicationOptions = { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc` as const, + initialWindowSize: 1, + } + + for (const releaseOrder of [ + [`release-old`, `release-peer`], + [`release-peer`, `release-old`], + ] as const) { + for (const settlementOrder of [ + [`settle-old`, `settle-fresh`], + [`settle-fresh`, `settle-old`], + ] as const) { + for (const actions of interleaveLegalOrderChains( + releaseOrder, + settlementOrder, + )) { + for (const oldOutcome of [`resolve`, `reject`] as const) { + for ( + let prefixLength = 0; + prefixLength <= actions.length; + prefixLength++ + ) { + const prefix = actions.slice(0, prefixLength) + const history = [ + ...legalOrderBaseHistory(), + ...prefix.flatMap((action) => + legalOrderEvents(action, oldOutcome), + ), + ] + const diagnostic = JSON.stringify({ + releaseOrder, + settlementOrder, + actions, + oldOutcome, + prefixLength, + }) + const oldReleased = prefix.includes(`release-old`) + const peerReleased = prefix.includes(`release-peer`) + const oldSettled = prefix.includes(`settle-old`) + const freshSettled = prefix.includes(`settle-fresh`) + const replacementComplete = oldSettled && freshSettled + const expectedRows = [ + ...(freshSettled + ? [{ sourceId: `source-a`, rowKey: `fresh-row` }] + : []), + ...(oldSettled && oldOutcome === `resolve` && !oldReleased + ? [{ sourceId: `source-a`, rowKey: `stale-row` }] + : []), + ...(!peerReleased + ? [{ sourceId: `source-b`, rowKey: `peer-row` }] + : []), + ] + const expectedEvidence = [ + ...(freshSettled + ? [{ sourceId: `source-a`, demandId: `shared` }] + : []), + ...(!peerReleased + ? [{ sourceId: `source-b`, demandId: `shared` }] + : []), + ] + const expectedPublicationRows = replacementComplete + ? [ + `fresh-ordered-row`, + `fresh-row`, + ...(!peerReleased ? [`peer-row`] : []), + ] + : [`old-ordered-row`, ...(!peerReleased ? [`peer-row`] : [])] + + expect(projectTransportLoads(history), diagnostic).toBe(3) + expect(projectRetainedSourceRows(history), diagnostic).toEqual( + expectedRows, + ) + expect(projectReusableSourceDemands(history), diagnostic).toEqual( + expectedEvidence, + ) + expect(projectSourceReadiness(history), diagnostic).toEqual({ + status: freshSettled ? `ready` : `loading`, + pendingSources: freshSettled ? [] : [`source-a`], + failedSources: [], + }) + const publication = projectAtomicOrderedPublicationState( + history, + publicationOptions, + ) + expect( + publication.currentPublication?.rows.map(({ key }) => key), + diagnostic, + ).toEqual(expectedPublicationRows) + expect(publication.retainsPreviousPublication, diagnostic).toBe( + !replacementComplete, + ) + } + } + } + } + } +}) + it(`retains a row until its last independent demand claim releases`, () => { const request = ( demandId: string, diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts index e166ae4fe..7dde2e4e7 100644 --- a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -15,9 +15,14 @@ import type { LoadSubsetOptions } from '../../src/types.js' type Row = { id: string; group: string } -it.each([`resolve`, `reject`] as const)( - `fences a retired source-demand attempt when it settles late: %s`, - async (oldOutcome) => { +it.each([ + { oldOutcome: `resolve`, settlementOrder: `old-first` }, + { oldOutcome: `reject`, settlementOrder: `old-first` }, + { oldOutcome: `resolve`, settlementOrder: `fresh-first` }, + { oldOutcome: `reject`, settlementOrder: `fresh-first` }, +] as const)( + `fences a retired source-demand attempt across $settlementOrder $oldOutcome settlement`, + async ({ oldOutcome, settlementOrder }) => { type Parent = { id: string; group: string } type Child = { id: string; group: string } type PendingRequest = { @@ -25,8 +30,9 @@ it.each([`resolve`, `reject`] as const)( rows: ReturnType>> } const sessionId = `session` - const parentId = `readiness-generation-parent-${oldOutcome}` - const childId = `readiness-generation-child-${oldOutcome}` + const caseId = `${oldOutcome}-${settlementOrder}` + const parentId = `readiness-generation-parent-${caseId}` + const childId = `readiness-generation-child-${caseId}` const oldAttemptId = `old-attempt` const freshAttemptId = `fresh-attempt` let parentBegin!: () => void @@ -104,7 +110,7 @@ it.each([`resolve`, `reject`] as const)( }, }) const live = createLiveQueryCollection({ - id: `readiness-generation-live-${oldOutcome}`, + id: `readiness-generation-live-${caseId}`, query: (q) => q.from({ parent }).select(({ parent: parentRow }) => ({ id: parentRow.id, @@ -203,36 +209,51 @@ it.each([`resolve`, `reject`] as const)( expect(live.status).toBe(projectSourceReadiness(history).status) expect(preloadState).toBe(`pending`) - if (oldOutcome === `resolve`) { - pending[0]!.rows.resolve([]) - } else { - pending[0]!.rows.reject(new Error(`retired source demand failed`)) + const freshChild: Child = { id: `fresh-child`, group: `fresh` } + const settleOld = async () => { + if (oldOutcome === `resolve`) { + pending[0]!.rows.resolve([]) + } else { + pending[0]!.rows.reject(new Error(`retired source demand failed`)) + } + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + outcome: oldOutcome, + }) + await flushPromises() + } + const settleFresh = async () => { + expect(child.get(freshChild.id)).toBeUndefined() + pending[1]!.rows.resolve([freshChild]) + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: freshAttemptId, + outcome: `resolve`, + }) + await flushPromises() + } + const settlements = + settlementOrder === `old-first` + ? [settleOld, settleFresh] + : [settleFresh, settleOld] + for (const settle of settlements) { + await settle() + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe( + projectSourceReadiness(history).status === `ready` + ? `resolved` + : `pending`, + ) + expect(live.utils.lastSubsetError).toBeUndefined() } - history.push({ - type: `settleSourceDemand`, - sessionId, - sourceId: childId, - demandId: `children`, - attemptId: oldAttemptId, - outcome: oldOutcome, - }) - await flushPromises() - - expect(live.status).toBe(projectSourceReadiness(history).status) - expect(preloadState).toBe(`pending`) - expect(live.utils.lastSubsetError).toBeUndefined() - const freshChild: Child = { id: `fresh-child`, group: `fresh` } - expect(child.get(freshChild.id)).toBeUndefined() - pending[1]!.rows.resolve([freshChild]) - history.push({ - type: `settleSourceDemand`, - sessionId, - sourceId: childId, - demandId: `children`, - attemptId: freshAttemptId, - outcome: `resolve`, - }) await preload await flushPromises() From 3260b29d2153c502cf844f1bd361a19d3e38d5da Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 08:46:17 -0600 Subject: [PATCH 154/327] test(db): close legal order audit gaps --- ...d-subset-full-flow-oracle.property.test.ts | 34 ++-- ...d-subset-refinement-model.property.test.ts | 159 ++++++++++++++---- 2 files changed, 146 insertions(+), 47 deletions(-) diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index b1c8d2fb7..0efc04aa2 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -6881,21 +6881,23 @@ const mixedDemandSettlementScenarios: ReadonlyArray }, ], ), - { - direction, - resizeOrder: `grow-shrink` as const, - overlap: false, - currentOutcome: `resolve` as const, - currentExtent: `exhausted` as const, - settleCurrentFirst: false, - sourceDelta: false, - otherDemand: `active` as const, - otherOutcome: `reject` as const, - demandSettlementOrder: `ordered-first` as const, - releaseAfterOrdered: true, - }, ]) +const releaseDuringPrivateReplayScenarios: ReadonlyArray = + ([`asc`, `desc`] as const).map((direction) => ({ + direction, + resizeOrder: `grow-shrink`, + overlap: false, + currentOutcome: `resolve`, + currentExtent: `exhausted`, + settleCurrentFirst: false, + sourceDelta: false, + otherDemand: `active`, + otherOutcome: `reject`, + demandSettlementOrder: `ordered-first`, + releaseAfterOrdered: true, + })) + it(`does not reuse caller or public continuation state when an active replacement has no progress`, async () => { for (const direction of [`asc`, `desc`] as const) { for (const callerContinuation of [ @@ -6972,6 +6974,12 @@ it(`keeps mixed demand settlements inside one replacement epoch`, async () => { } }) +it(`removes a released peer from the public baseline while replay remains private`, async () => { + for (const scenario of releaseDuringPrivateReplayScenarios) { + await runAtomicOrderedReplayScenario(scenario) + } +}) + it(`discards pending replacement epochs on teardown`, async () => { for (const direction of [`asc`, `desc`] as const) { for (const overlap of [false, true]) { diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts index b0da8dab0..9282c1e75 100644 --- a/packages/db/tests/query/load-subset-refinement-model.property.test.ts +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -851,56 +851,145 @@ it(`enumerates legal release and settlement orders across every refinement proje settlementOrder, )) { for (const oldOutcome of [`resolve`, `reject`] as const) { + const concreteSteps = actions.flatMap((action) => + legalOrderEvents(action, oldOutcome).map((event) => ({ + action, + event, + })), + ) for ( let prefixLength = 0; - prefixLength <= actions.length; + prefixLength <= concreteSteps.length; prefixLength++ ) { - const prefix = actions.slice(0, prefixLength) - const history = [ - ...legalOrderBaseHistory(), - ...prefix.flatMap((action) => - legalOrderEvents(action, oldOutcome), - ), - ] + const prefix = concreteSteps.slice(0, prefixLength) + const prefixEvents = prefix.map(({ event }) => event) + const history = [...legalOrderBaseHistory(), ...prefixEvents] const diagnostic = JSON.stringify({ releaseOrder, settlementOrder, actions, oldOutcome, prefixLength, + prefix: prefix.map(({ action, event }) => ({ + action, + event: event.type, + })), }) - const oldReleased = prefix.includes(`release-old`) - const peerReleased = prefix.includes(`release-peer`) - const oldSettled = prefix.includes(`settle-old`) - const freshSettled = prefix.includes(`settle-fresh`) - const replacementComplete = oldSettled && freshSettled + const eventIndex = ( + predicate: (event: LoadSubsetFullFlowEvent) => boolean, + ) => prefixEvents.findIndex(predicate) + const oldReleased = eventIndex( + (event) => + event.type === `releaseDemand` && + event.attemptId === `attempt-old`, + ) + const peerReleased = eventIndex( + (event) => + event.type === `releaseDemand` && + event.attemptId === `attempt-peer`, + ) + const oldRowsApplied = eventIndex( + (event) => + event.type === `applyAuthoritativeRows` && + event.attemptId === `attempt-old`, + ) + const freshRowsApplied = eventIndex( + (event) => + event.type === `applyAuthoritativeRows` && + event.attemptId === `attempt-fresh`, + ) + const freshSourceSettled = eventIndex( + (event) => + event.type === `settleSourceDemand` && + event.attemptId === `attempt-fresh`, + ) + const oldReplacementSettled = eventIndex( + (event) => + event.type === `settleReplacement` && + event.publicationId === `old-replacement` && + event.sourceId === `source-a`, + ) + const freshReplacementSettled = eventIndex( + (event) => + event.type === `settleReplacement` && + event.publicationId === `fresh-replacement` && + event.sourceId === `source-a`, + ) + const replacementComplete = + oldReplacementSettled >= 0 && freshReplacementSettled >= 0 + const replacementCompletionIndex = Math.max( + oldReplacementSettled, + freshReplacementSettled, + ) const expectedRows = [ - ...(freshSettled + ...(freshRowsApplied >= 0 ? [{ sourceId: `source-a`, rowKey: `fresh-row` }] : []), - ...(oldSettled && oldOutcome === `resolve` && !oldReleased + ...(oldRowsApplied >= 0 && oldReleased < 0 ? [{ sourceId: `source-a`, rowKey: `stale-row` }] : []), - ...(!peerReleased + ...(peerReleased < 0 ? [{ sourceId: `source-b`, rowKey: `peer-row` }] : []), ] const expectedEvidence = [ - ...(freshSettled + ...(freshRowsApplied >= 0 ? [{ sourceId: `source-a`, demandId: `shared` }] : []), - ...(!peerReleased + ...(peerReleased < 0 ? [{ sourceId: `source-b`, demandId: `shared` }] : []), ] - const expectedPublicationRows = replacementComplete - ? [ - `fresh-ordered-row`, - `fresh-row`, - ...(!peerReleased ? [`peer-row`] : []), - ] - : [`old-ordered-row`, ...(!peerReleased ? [`peer-row`] : [])] + const oldOrderedRow = { + key: `old-ordered-row`, + orderValue: 0, + } + const freshOrderedRow = { + key: `fresh-ordered-row`, + orderValue: 0, + } + const freshRow = { key: `fresh-row`, orderValue: 1 } + const peerRow = { key: `peer-row`, orderValue: 2 } + const initialPublication = [oldOrderedRow, peerRow] + const publicationTransitions: Array<{ + index: number + rows: Array<{ key: string; orderValue: number }> + }> = [] + if (replacementComplete) { + publicationTransitions.push({ + index: replacementCompletionIndex, + rows: [ + freshOrderedRow, + freshRow, + ...(peerReleased < 0 || + peerReleased > replacementCompletionIndex + ? [peerRow] + : []), + ], + }) + } + if (peerReleased >= 0) { + publicationTransitions.push({ + index: peerReleased, + rows: + replacementComplete && + replacementCompletionIndex < peerReleased + ? [freshOrderedRow, freshRow] + : [oldOrderedRow], + }) + } + publicationTransitions.sort( + (left, right) => left.index - right.index, + ) + const expectedPublications = [ + initialPublication, + ...publicationTransitions.map(({ rows }) => rows), + ] + const expectedCurrentRows = expectedPublications.at(-1)! + const expectedOrderedBoundary = replacementComplete + ? freshOrderedRow + : oldOrderedRow expect(projectTransportLoads(history), diagnostic).toBe(3) expect(projectRetainedSourceRows(history), diagnostic).toEqual( @@ -910,21 +999,23 @@ it(`enumerates legal release and settlement orders across every refinement proje expectedEvidence, ) expect(projectSourceReadiness(history), diagnostic).toEqual({ - status: freshSettled ? `ready` : `loading`, - pendingSources: freshSettled ? [] : [`source-a`], + status: freshSourceSettled >= 0 ? `ready` : `loading`, + pendingSources: freshSourceSettled >= 0 ? [] : [`source-a`], failedSources: [], }) const publication = projectAtomicOrderedPublicationState( history, publicationOptions, ) - expect( - publication.currentPublication?.rows.map(({ key }) => key), - diagnostic, - ).toEqual(expectedPublicationRows) - expect(publication.retainsPreviousPublication, diagnostic).toBe( - !replacementComplete, - ) + expect(publication, diagnostic).toEqual({ + publications: expectedPublications, + currentPublication: { + rows: expectedCurrentRows, + orderedPrefixSize: 1, + orderedBoundary: expectedOrderedBoundary, + }, + retainsPreviousPublication: !replacementComplete, + }) } } } From 31c62cac1556bb823afedbec1d688a78f091e3a5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 09:22:37 -0600 Subject: [PATCH 155/327] test(db): pin exact write provenance --- ...ubscription-replay-oracle.property.test.ts | 139 ++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index 03b2e6bb8..e59a00399 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -2757,6 +2757,145 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`keeps an ordinary same-key write authoritative while an unordered request is pending`, async () => { + type Row = { id: `a` | `x`; rank: number } + type Outcome = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + type Phase = `initial` | `replay` | `additional` | `probe` + + const replayFailure = new Error(`sibling replay failed`) + const additionalLoad = createDeferred() + const loads: Array<{ phase: Phase; options: LoadSubsetOptions }> = [] + let phase: Phase = `initial` + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + + const collection = createCollection({ + id: `ordinary-write-during-unordered-request`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + + const apply = ( + row: Row, + signal: AbortSignal | undefined, + ): Outcome => { + begin() + write({ type: `insert`, value: row }) + commit(signal) + return { hasMore: false, appliedRowKeys: [row.id] } + } + + return { + loadSubset: (options) => { + loads.push({ phase, options }) + if (phase === `initial`) { + return options.orderBy + ? Promise.resolve(apply({ id: `a`, rank: 1 }, options.signal)) + : Promise.resolve({ + hasMore: false, + appliedRowKeys: [] as const, + }) + } + if (phase === `replay`) { + return options.orderBy + ? Promise.resolve(apply({ id: `x`, rank: 0 }, options.signal)) + : Promise.reject(replayFailure) + } + if (phase === `additional`) return additionalLoad.promise + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [] as const, + }) + }, + unloadSubset: () => {}, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderBy: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ] + const seedWhere = new Func(`eq`, [new PropRef([`id`]), new Value(`a`)]) + const additionalWhere = new Func(`eq`, [ + new PropRef([`id`]), + new Value(`x`), + ]) + const visible = new Set() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.add(key) + } + }) + subscription.setOrderByIndex(index) + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + subscription.requestSnapshot({ where: seedWhere }) + await flushPromises() + expect([...visible]).toEqual([`a`]) + + phase = `replay` + begin() + truncate() + commit() + await flushPromises() + await flushPromises() + expect(subscription.lastError).toBe(replayFailure) + expect(subscription.orderedBoundaryKey).toBe(`a`) + expect([...visible]).toEqual([`a`]) + + phase = `additional` + subscription.requestSnapshot({ where: additionalWhere }) + await flushPromises() + expect(loads.at(-1)).toMatchObject({ phase: `additional` }) + expect(loads.at(-1)?.options.orderBy).toBeUndefined() + + begin() + write({ type: `update`, value: { id: `x`, rank: -1 } }) + commit() + expect(subscription.orderedBoundaryKey).toBe(`x`) + expect([...visible].sort()).toEqual([`a`, `x`]) + + additionalLoad.reject(new Error(`sibling acquisition failed`)) + await flushPromises() + subscription.releaseSnapshot(additionalWhere) + expect(subscription.orderedBoundaryKey).toBe(`x`) + expect([...visible].sort()).toEqual([`a`, `x`]) + + phase = `probe` + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + await flushPromises() + expect(loads.at(-1)).toMatchObject({ + phase: `probe`, + options: { cursor: { lastKey: `x` } }, + }) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it.each([ `sync`, `async`, From d6c6a9a3d86583bc7300f7f84ce605192ca6976a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 09:49:40 -0600 Subject: [PATCH 156/327] fix(db): bound ordered source work --- packages/db/package.json | 2 +- packages/db/src/collection/change-events.ts | 40 +- packages/db/src/indexes/basic-index.ts | 27 +- packages/db/src/indexes/btree-index.ts | 36 +- packages/db/src/indexes/reverse-index.ts | 27 ++ packages/db/src/query/live/ARCHITECTURE.md | 15 + packages/db/src/query/live/window-state.ts | 43 +- packages/db/tests/oracle-config.ts | 2 + .../ordered-work-oracle.property.test.ts | 431 ++++++++++++++++++ 9 files changed, 574 insertions(+), 49 deletions(-) create mode 100644 packages/db/tests/query/ordered-work-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 5db484027..ef18359ed 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index d78f2f45a..d1db269e6 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -1,3 +1,4 @@ +import { compareKeys } from '@tanstack/db-ivm' import { createSingleRowRefProxy, toExpression, @@ -363,26 +364,25 @@ function getOrderedKeys( return index.takeFromStart(limit ?? index.keyCount, filterFn) } - // Reversing a value index also reverses keys inside an equal-value - // bucket, but query TotalOrder keeps its public-key tie-break ascending. - // Refine all matching indexed rows locally so a limit cannot cut the - // wrong side of a tied boundary. - const totalOrder = new TotalOrder(orderBy, collection) - const indexedEntries = index - .takeFromStart(index.keyCount, filterFn) - .flatMap((key) => { - const value = collection.get(key) - return value === undefined ? [] : [{ key, value }] - }) - indexedEntries.sort((left, right) => - totalOrder.compareEntries( - [left.key, left.value], - [right.key, right.value], - ), - ) - return indexedEntries - .slice(0, limit ?? indexedEntries.length) - .map(({ key }) => key) + // Reversing a value index must not reverse the public-key suffix of the + // query's total order. Walk value buckets in reverse value order, sort + // keys inside each bucket ascending, and stop after the first bucket + // that proves the requested prefix. The complete boundary bucket must + // be inspected because filtering can otherwise select the wrong key. + if (limit === 0) return [] + const keys: Array = [] + for (const [, bucket] of index.orderedBuckets()) { + const matchingKeys = [...bucket].sort(compareKeys).filter(filterFn) + const remaining = + limit === undefined ? undefined : limit - keys.length + keys.push( + ...(remaining === undefined + ? matchingKeys + : matchingKeys.slice(0, remaining)), + ) + if (limit !== undefined && keys.length === limit) break + } + return keys } } } diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 8eac6f926..17a4dbb44 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -521,19 +521,32 @@ export class BasicIndex< } get orderedEntriesArray(): Array<[any, Set]> { - return this.sortedValues.map((value) => [ + return Array.from(this.orderedBuckets(), ([value, keys]) => [ value, - this.valueMap.get(value) ?? new Set(), + keys as Set, ]) } get orderedEntriesArrayReversed(): Array<[any, Set]> { - const result: Array<[any, Set]> = [] - for (let i = this.sortedValues.length - 1; i >= 0; i--) { - const value = this.sortedValues[i] - result.push([value, this.valueMap.get(value) ?? new Set()]) + return Array.from(this.orderedBucketsReversed(), ([value, keys]) => [ + value, + keys as Set, + ]) + } + + *orderedBuckets(): IterableIterator]> { + for (const value of this.sortedValues) { + yield [value, this.valueMap.get(value) ?? new Set()] + } + } + + *orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + for (let index = this.sortedValues.length - 1; index >= 0; index--) { + const value = this.sortedValues[index] + yield [value, this.valueMap.get(value) ?? new Set()] } - return result } get valueMapData(): Map> { diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 6379b91b5..883730bb4 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -441,21 +441,39 @@ export class BTreeIndex< } get orderedEntriesArray(): Array<[any, Set]> { - return this.orderedEntries - .keysArray() - .map((key) => [ - denormalizeUndefined(key), - this.valueMap.get(key) ?? new Set(), - ]) + return Array.from(this.orderedBuckets(), ([value, keys]) => [ + value, + keys as Set, + ]) } get orderedEntriesArrayReversed(): Array<[any, Set]> { - return this.takeReversedFromEnd(this.orderedEntries.size).map((key) => [ - denormalizeUndefined(key), - this.valueMap.get(key) ?? new Set(), + return Array.from(this.orderedBucketsReversed(), ([value, keys]) => [ + value, + keys as Set, ]) } + *orderedBuckets(): IterableIterator]> { + let pair = this.orderedEntries.nextHigherPair(undefined) + while (pair !== undefined) { + const value = pair[0] + yield [denormalizeUndefined(value), this.valueMap.get(value) ?? new Set()] + pair = this.orderedEntries.nextHigherPair(value) + } + } + + *orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + let pair = this.orderedEntries.nextLowerPair(undefined) + while (pair !== undefined) { + const value = pair[0] + yield [denormalizeUndefined(value), this.valueMap.get(value) ?? new Set()] + pair = this.orderedEntries.nextLowerPair(value) + } + } + get valueMapData(): Map> { // Return a new Map with denormalized keys const result = new Map>() diff --git a/packages/db/src/indexes/reverse-index.ts b/packages/db/src/indexes/reverse-index.ts index 6ca61636e..9a6bb1aa9 100644 --- a/packages/db/src/indexes/reverse-index.ts +++ b/packages/db/src/indexes/reverse-index.ts @@ -3,6 +3,13 @@ import type { OrderByDirection } from '../query/ir' import type { IndexInterface, IndexOperation, IndexStats } from './base-index' import type { RangeQueryOptions } from './btree-index' +interface OrderedBucketIndex { + orderedBuckets: () => IterableIterator]> + orderedBucketsReversed: () => IterableIterator< + readonly [unknown, ReadonlySet] + > +} + export class ReverseIndex< TKey extends string | number, > implements IndexInterface { @@ -67,6 +74,26 @@ export class ReverseIndex< return this.originalIndex.orderedEntriesArray } + orderedBuckets(): IterableIterator]> { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + orderedIndex.orderedBucketsReversed?.() ?? + this.originalIndex.orderedEntriesArrayReversed[Symbol.iterator]() + ) + } + + orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + orderedIndex.orderedBuckets?.() ?? + this.originalIndex.orderedEntriesArray[Symbol.iterator]() + ) + } + // All operations below delegate to the original index supports(operation: IndexOperation): boolean { diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index b1f082bd7..4a39a9b9c 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1266,6 +1266,21 @@ count source reads or snapshots, sorts or total-order refinements, and predicate compilations independently. A stable result and request trace can still hide repeated local work. +`WindowState` takes at most one ordered source snapshot per collection state +revision. Boundary, coverage, publication, and reconciliation reads share that +snapshot; the next committed source batch invalidates it. The query predicate +is compiled once with the window and is evaluated over the shared ordered +snapshot, so another view of the same revision does not rescan, resort, or +recompile it. + +A descending single-column index walks indexed-value buckets in query order. +It evaluates complete buckets until the requested filtered prefix is known, +orders public keys ascending within each bucket, and stops after the sufficient +boundary bucket. Rows in worse buckets cannot add source reads or total-order +refinement work. An all-tied source is the deliberate worst case: the one +boundary bucket is the whole source and must be inspected before the public-key +suffix can choose top-K. + Runtime reference identity has a different lifetime again. Objects use weak identity, but JavaScript symbols cannot be weak keys. Stable equality for the same live symbol therefore retains one strong entry per distinct symbol for the diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index 6df3fe1a5..14b73fb7b 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -29,11 +29,17 @@ export class WindowState< private readonly candidateKeys = new Set() private readonly provenanceKeys = new Set() private readonly admittedKeys = new Set() + private sourceSnapshot: + | { + revision: number + rows: Array> + } + | undefined constructor( private readonly collection: CollectionImpl, orderBy: OrderBy, - private readonly where: BasicExpression | undefined, + where: BasicExpression | undefined, targetSize: number, private readonly expandSourceOrderTies = false, ) { @@ -390,14 +396,13 @@ export class WindowState< allowedKeys: ReadonlySet | undefined, limit?: number, ): Array> { - const rows = this.collection.currentStateAsChanges({ - ...(this.where && { where: this.where }), - orderBy: this.totalOrder.orderBy, - }) as Array> | undefined + const rows = this.readSourceSnapshot().filter(({ value }) => + this.matchesWhere(value), + ) const allowed = allowedKeys === undefined - ? (rows ?? []) - : (rows ?? []).filter((change) => allowedKeys.has(change.key)) + ? rows + : rows.filter((change) => allowedKeys.has(change.key)) if (limit === undefined) return allowed return this.expandSourceOrderTies ? this.prefixThroughTieClass(allowed, limit) @@ -430,13 +435,27 @@ export class WindowState< allowedKeys: ReadonlySet | undefined, limit?: number, ): Array> { - const rows = this.collection.currentStateAsChanges({ - orderBy: this.totalOrder.orderBy, - }) as Array> | undefined + const rows = this.readSourceSnapshot() const allowed = allowedKeys === undefined - ? (rows ?? []) - : (rows ?? []).filter((change) => allowedKeys.has(change.key)) + ? rows + : rows.filter((change) => allowedKeys.has(change.key)) return limit === undefined ? allowed : allowed.slice(0, limit) } + + private readSourceSnapshot(): Array> { + const revision = this.collection._stateRevision + if ( + this.sourceSnapshot !== undefined && + this.sourceSnapshot.revision === revision + ) { + return this.sourceSnapshot.rows + } + + const rows = this.collection.currentStateAsChanges({ + orderBy: this.totalOrder.orderBy, + }) as Array> + this.sourceSnapshot = { revision, rows } + return rows + } } diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 10c0d2736..be5864732 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -45,6 +45,8 @@ const staticOracleProperties = [ `load-subset.distinct-window-predicate`, `load-subset.ordered-window`, `load-subset.rejected-waiter`, + `ordered-work.reverse-prefix`, + `ordered-work.snapshot-reuse`, `pagination.async-cursor`, `pagination.multi-order`, `pagination.nullable-cursor`, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts new file mode 100644 index 000000000..02a4823f5 --- /dev/null +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -0,0 +1,431 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it, vi } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { localOnlyCollectionOptions } from '../../src/local-only.js' +import { eq } from '../../src/query/builder/functions.js' +import { PropRef } from '../../src/query/ir.js' +import { TotalOrder } from '../../src/query/total-order.js' +import { WindowState } from '../../src/query/live/window-state.js' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' +import type { CollectionImpl } from '../../src/collection/index.js' +import type { + ChangeMessage, + CurrentStateAsChangesOptions, +} from '../../src/types.js' +import type { OrderBy, OrderByDirection } from '../../src/query/ir.js' + +type RankedRow = { + id: string + rank: number + included: boolean +} + +type OrderedWork = { + keys: Array + sourceReads: Array + totalOrderComparisons: number +} + +function orderedWorkCampaigns(property: string, fixedSeed: number) { + return [ + { + label: `fixed seed ${fixedSeed}`, + options: { numRuns: oracleRuns(40), seed: fixedSeed }, + }, + { + label: `random or replayed seed`, + options: oraclePropertyOptions(40, property), + }, + ] as const +} + +function orderBy(direction: OrderByDirection): OrderBy { + return [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction, nulls: `first` }, + }, + ] +} + +async function observeDescendingPrefix( + rows: ReadonlyArray, + limit: number, +): Promise { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-${Math.random()}`, + getKey: (row) => row.id, + initialData: [...rows], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(String(key)) + return originalGet(key) + } + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + + try { + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: orderBy(`desc`), + limit, + })! + + return { + keys: changes.map(({ key }) => String(key)), + sourceReads, + totalOrderComparisons: compareEntries.mock.calls.length, + } + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } +} + +function createReversePrefixRows(options: { + leadingRejects: number + limit: number + extraBoundaryMatches: number + boundaryRejects: number + trailingRows: number +}): { + rows: Array + expectedKeys: Array + expectedSourceReads: number +} { + const leading = Array.from( + { length: options.leadingRejects }, + (_, index): RankedRow => ({ + id: `leading-${index.toString().padStart(2, `0`)}`, + rank: 100 + index, + included: false, + }), + ) + const matchingBoundary = Array.from( + { length: options.limit + options.extraBoundaryMatches }, + (_, index): RankedRow => ({ + id: `boundary-match-${index.toString().padStart(2, `0`)}`, + rank: 50, + included: true, + }), + ).reverse() + const rejectedBoundary = Array.from( + { length: options.boundaryRejects }, + (_, index): RankedRow => ({ + id: `boundary-reject-${index.toString().padStart(2, `0`)}`, + rank: 50, + included: false, + }), + ) + const trailing = Array.from( + { length: options.trailingRows }, + (_, index): RankedRow => ({ + id: `trailing-${index.toString().padStart(3, `0`)}`, + rank: 10 - index, + included: true, + }), + ) + const expectedKeys = matchingBoundary + .map(({ id }) => id) + .sort() + .slice(0, options.limit) + + return { + rows: [...trailing, ...rejectedBoundary, ...matchingBoundary, ...leading], + expectedKeys, + // Every row through the boundary bucket is tested once. The selected rows + // are then read once more to materialize their change messages. + expectedSourceReads: + leading.length + + matchingBoundary.length + + rejectedBoundary.length + + options.limit, + } +} + +describe(`ordered source work oracle`, () => { + it(`does not read worse reverse-index buckets after filling top-K`, async () => { + const scenario = createReversePrefixRows({ + leadingRejects: 2, + limit: 2, + extraBoundaryMatches: 1, + boundaryRejects: 2, + trailingRows: 40, + }) + + const observed = await observeDescendingPrefix(scenario.rows, 2) + + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) + expect(observed.sourceReads).not.toContain(`trailing-000`) + expect(observed.totalOrderComparisons).toBe(0) + }) + + for (const campaign of orderedWorkCampaigns( + `ordered-work.reverse-prefix`, + 1_780_101, + )) { + fcTest.prop( + [ + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 1, max: 5 }), + fc.integer({ min: 0, max: 5 }), + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 0, max: 60 }), + ], + campaign.options, + )( + `bounds reverse-index reads at the sufficient bucket (${campaign.label})`, + async ( + leadingRejects, + limit, + extraBoundaryMatches, + boundaryRejects, + trailingRows, + ) => { + const scenario = createReversePrefixRows({ + leadingRejects, + limit, + extraBoundaryMatches, + boundaryRejects, + trailingRows, + }) + const observed = await observeDescendingPrefix(scenario.rows, limit) + + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + } + + it(`reads the complete tied boundary when every candidate is tied`, async () => { + const rows = Array.from( + { length: 25 }, + (_, index): RankedRow => ({ + id: `tied-${index.toString().padStart(2, `0`)}`, + rank: 1, + included: index % 2 === 0, + }), + ).reverse() + + const observed = await observeDescendingPrefix(rows, 3) + + expect(observed.keys).toEqual([`tied-00`, `tied-02`, `tied-04`]) + expect(observed.sourceReads).toHaveLength(rows.length + 3) + expect(observed.totalOrderComparisons).toBe(0) + }) +}) + +type SnapshotFixture = { + collection: CollectionImpl + snapshotRevisions: Array + replace: (row: RankedRow) => void +} + +function createSnapshotFixture( + initialRows: ReadonlyArray, +): SnapshotFixture { + let rows = new Map(initialRows.map((row) => [row.id, row])) + let revision = 0 + const snapshotRevisions: Array = [] + const collection = { + compareOptions: { stringSort: `lexical` }, + get _stateRevision() { + return revision + }, + currentStateAsChanges: (options: CurrentStateAsChangesOptions) => { + snapshotRevisions.push(revision) + return [...rows] + .filter(([, value]) => options.where === undefined || value.included) + .sort((left, right) => + left[1].rank === right[1].rank + ? left[0].localeCompare(right[0]) + : left[1].rank - right[1].rank, + ) + .map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + + return { + collection, + snapshotRevisions, + replace: (row) => { + rows = new Map(rows).set(row.id, row) + revision++ + }, + } +} + +function observeWindow( + window: WindowState, +) { + return { + localPrefixSize: window.localPrefixSize, + rowsNeeded: window.rowsNeeded(), + publication: window.publicationEntries().map(([key]) => key), + boundary: window.boundary(), + requestBoundary: window.requestBoundary(), + progressBoundary: window.progressBoundary(), + changes: window.reconcile(new Map()).map(({ key }) => key), + } +} + +function createCoveredWindow(fixture: SnapshotFixture, size: number) { + const window = new WindowState( + fixture.collection, + orderBy(`asc`), + eq(new PropRef([`included`]), true), + size, + ) + window.recordInitialCoverage(undefined, true) + return window +} + +it(`reuses one ordered source snapshot until the collection revision changes`, () => { + const fixture = createSnapshotFixture([ + { id: `a`, rank: 1, included: true }, + { id: `b`, rank: 2, included: true }, + { id: `hidden`, rank: 0, included: false }, + ]) + const window = createCoveredWindow(fixture, 2) + + expect(observeWindow(window)).toMatchObject({ + localPrefixSize: 2, + rowsNeeded: 0, + publication: [`a`, `b`], + }) + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(fixture.snapshotRevisions).toEqual([0]) + + fixture.replace({ id: `b`, rank: -1, included: true }) + + expect(observeWindow(window)).toMatchObject({ publication: [`b`, `a`] }) + expect(observeWindow(window)).toMatchObject({ publication: [`b`, `a`] }) + expect(fixture.snapshotRevisions).toEqual([0, 1]) +}) + +it(`invalidates the ordered snapshot after a committed collection write`, async () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-revision-write`, + getKey: (row) => row.id, + initialData: [ + { id: `a`, rank: 1, included: true }, + { id: `b`, rank: 2, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + const snapshotRevisions: Array = [] + const originalSnapshot = collection.currentStateAsChanges.bind(collection) + collection.currentStateAsChanges = (options) => { + snapshotRevisions.push(collection._stateRevision) + return originalSnapshot(options) + } + const window = new WindowState( + collection, + orderBy(`asc`), + eq(new PropRef([`included`]), true), + 2, + ) + window.recordInitialCoverage(undefined, true) + + expect(observeWindow(window).publication).toEqual([`a`, `b`]) + expect(observeWindow(window).publication).toEqual([`a`, `b`]) + const initialRevision = collection._stateRevision + expect(snapshotRevisions).toEqual([initialRevision]) + + collection.update(`b`, (draft) => { + draft.rank = -1 + }) + + expect(observeWindow(window).publication).toEqual([`b`, `a`]) + expect(observeWindow(window).publication).toEqual([`b`, `a`]) + expect(collection._stateRevision).toBeGreaterThan(initialRevision) + expect(snapshotRevisions).toEqual([ + initialRevision, + collection._stateRevision, + ]) + } finally { + await collection.cleanup() + } +}) + +for (const campaign of orderedWorkCampaigns( + `ordered-work.snapshot-reuse`, + 1_780_102, +)) { + fcTest.prop( + [ + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 1, + maxLength: 12, + }), + fc.integer({ min: 1, max: 8 }), + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 0, + maxLength: 8, + }), + ], + campaign.options, + )( + `takes at most one ordered snapshot per source revision (${campaign.label})`, + (initialRanks, observationCount, replacementRanks) => { + const fixture = createSnapshotFixture( + initialRanks.map((rank, index) => ({ + id: `row-${index}`, + rank, + included: index % 3 !== 0, + })), + ) + const window = createCoveredWindow( + fixture, + Math.min(3, initialRanks.length), + ) + + for (let index = 0; index < observationCount; index++) { + observeWindow(window) + } + for (let index = 0; index < replacementRanks.length; index++) { + fixture.replace({ + id: `row-${index % initialRanks.length}`, + rank: replacementRanks[index]!, + included: index % 2 === 0, + }) + for (let repeat = 0; repeat < observationCount; repeat++) { + observeWindow(window) + } + } + + expect(fixture.snapshotRevisions).toEqual( + Array.from( + { length: replacementRanks.length + 1 }, + (_, index) => index, + ), + ) + }, + ) +} From a4455d87a55856f5840c31f5c40258129d001dc0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 10:15:19 -0600 Subject: [PATCH 157/327] fix(db): preserve ordered tie classes --- packages/db/src/collection/change-events.ts | 22 ++ packages/db/src/indexes/auto-index.ts | 5 +- packages/db/src/indexes/base-index.ts | 10 +- packages/db/src/indexes/basic-index.ts | 80 ++++-- packages/db/src/indexes/btree-index.ts | 73 +++--- packages/db/src/indexes/reverse-index.ts | 9 + packages/db/src/query/live/ARCHITECTURE.md | 9 + .../ordered-work-oracle.property.test.ts | 246 +++++++++++++++++- 8 files changed, 391 insertions(+), 63 deletions(-) diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index d1db269e6..ff5a051f9 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -364,6 +364,28 @@ function getOrderedKeys( return index.takeFromStart(limit ?? index.keyCount, filterFn) } + // Public custom indexes predate lazy bucket iteration. Preserve their + // semantics with the full TotalOrder refinement instead of assuming + // their materialized entries expose complete comparator tie classes. + if (!index.supportsOrderedBucketIteration) { + const totalOrder = new TotalOrder(orderBy, collection) + const indexedEntries = index + .takeFromStart(index.keyCount, filterFn) + .flatMap((key) => { + const value = collection.get(key) + return value === undefined ? [] : [{ key, value }] + }) + indexedEntries.sort((left, right) => + totalOrder.compareEntries( + [left.key, left.value], + [right.key, right.value], + ), + ) + return indexedEntries + .slice(0, limit ?? indexedEntries.length) + .map(({ key }) => key) + } + // Reversing a value index must not reverse the public-key suffix of the // query's total order. Walk value buckets in reverse value order, sort // keys inside each bucket ascending, and stop after the first bucket diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index 350b469a4..3c9f5c9f0 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -71,7 +71,10 @@ export function ensureIndexForField< }, { name: `auto:${fieldPath.join(`.`)}`, - options: compareFn ? { compareFn, compareOptions: compareOpts } : {}, + options: { + compareOptions: compareOpts, + ...(compareFn && { compareFn }), + }, }, ) } catch (error) { diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 26cb09887..7cc7a0090 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -174,12 +174,20 @@ export abstract class BaseIndex< /** * Checks if the compare options match the index's compare options. - * The direction is ignored because the index can be reversed if the direction is different. + * Reversing an index also reverses null placement, so opposite directions + * are compatible only when their requested null placement is opposite too. */ matchesCompareOptions(compareOptions: CompareOptions): boolean { + const reversesDirection = + this.compareOptions.direction !== compareOptions.direction const thisCompareOptionsWithoutDirection = { ...this.compareOptions, direction: undefined, + nulls: reversesDirection + ? this.compareOptions.nulls === `first` + ? `last` + : `first` + : this.compareOptions.nulls, } const compareOptionsWithoutDirection = { ...compareOptions, diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 17a4dbb44..645249137 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -1,12 +1,10 @@ import { areSameValueZeroEqual, defaultComparator, + makeComparator, normalizeValue, } from '../utils/comparison.js' -import { - deleteInSortedArray, - findInsertPositionInArray, -} from '../utils/array-utils.js' +import { findInsertPositionInArray } from '../utils/array-utils.js' import { BaseIndex } from './base-index.js' import type { CompareOptions } from '../query/builder/types.js' import type { BasicExpression } from '../query/ir.js' @@ -68,11 +66,11 @@ export class BasicIndex< options?: any, ) { super(id, expression, name, options) - this.compareFn = options?.compareFn ?? defaultComparator - this.hasCustomComparator = options?.compareFn != null if (options?.compareOptions) { this.compareOptions = options!.compareOptions } + this.compareFn = options?.compareFn ?? makeComparator(this.compareOptions) + this.hasCustomComparator = options?.compareFn != null } protected initialize(_options?: BasicIndexOptions): void {} @@ -151,7 +149,23 @@ export class BasicIndex< if (keySet.size === 0) { // No more keys for this value, remove from map and sorted array this.valueMap.delete(normalizedValue) - deleteInSortedArray(this.sortedValues, normalizedValue, this.compareFn) + const firstEqual = findInsertPositionInArray( + this.sortedValues, + normalizedValue, + this.compareFn, + ) + for ( + let index = firstEqual; + index < this.sortedValues.length; + index++ + ) { + const candidate = this.sortedValues[index] + if (this.compareFn(candidate, normalizedValue) !== 0) break + if (areSameValueZeroEqual(candidate, normalizedValue)) { + this.sortedValues.splice(index, 1) + break + } + } } } } @@ -521,31 +535,59 @@ export class BasicIndex< } get orderedEntriesArray(): Array<[any, Set]> { - return Array.from(this.orderedBuckets(), ([value, keys]) => [ + return this.sortedValues.map((value) => [ value, - keys as Set, + this.valueMap.get(value) ?? new Set(), ]) } get orderedEntriesArrayReversed(): Array<[any, Set]> { - return Array.from(this.orderedBucketsReversed(), ([value, keys]) => [ - value, - keys as Set, - ]) + const result: Array<[any, Set]> = [] + for (let index = this.sortedValues.length - 1; index >= 0; index--) { + const value = this.sortedValues[index] + result.push([value, this.valueMap.get(value) ?? new Set()]) + } + return result } *orderedBuckets(): IterableIterator]> { - for (const value of this.sortedValues) { - yield [value, this.valueMap.get(value) ?? new Set()] - } + yield* this.groupOrderedBuckets(this.sortedValues) } *orderedBucketsReversed(): IterableIterator< readonly [unknown, ReadonlySet] > { - for (let index = this.sortedValues.length - 1; index >= 0; index--) { - const value = this.sortedValues[index] - yield [value, this.valueMap.get(value) ?? new Set()] + const reversedValues = function* (values: ReadonlyArray) { + for (let index = values.length - 1; index >= 0; index--) { + yield values[index] + } + } + yield* this.groupOrderedBuckets(reversedValues(this.sortedValues)) + } + + private *groupOrderedBuckets( + values: Iterable, + ): IterableIterator]> { + let hasGroup = false + let groupValue: unknown + let groupKeys = new Set() + + for (const value of values) { + if (!hasGroup) { + groupValue = value + hasGroup = true + } else if (this.compareFn(groupValue, value) !== 0) { + yield [groupValue, groupKeys] + groupKeys = new Set() + groupValue = value + } + for (const key of this.valueMap.get(value) ?? []) { + groupKeys.add(key) + } + } + + if (hasGroup) { + yield [groupValue, groupKeys] } } diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 883730bb4..6d87ed698 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -2,8 +2,8 @@ import { compareKeys } from '@tanstack/db-ivm' import { BTree } from '../utils/btree.js' import { areSameValueZeroEqual, - defaultComparator, denormalizeUndefined, + makeComparator, normalizeForBTree, } from '../utils/comparison.js' import { BaseIndex } from './base-index.js' @@ -46,12 +46,12 @@ export class BTreeIndex< ]) // Internal data structures - private to hide implementation details - // The `orderedEntries` B+ tree is used for efficient range queries - // The `valueMap` is used for O(1) lookups of PKs by indexed value - private orderedEntries: BTree // we don't associate values with the keys of the B+ tree (the keys are indexed values) - private valueMap = new Map>() // instead we store a mapping of indexed values to a set of PKs + // The `orderedEntries` B+ tree groups every key whose indexed values compare + // equal. `valueMap` keeps exact values separate for equality lookups. + private orderedEntries: BTree> + private valueMap = new Map>() private indexedKeys = new Set() - private compareFn: (a: any, b: any) => number = defaultComparator + private compareFn!: (a: any, b: any) => number constructor( id: number, @@ -61,8 +61,12 @@ export class BTreeIndex< ) { super(id, expression, name, options) - // Get the base compare function - const baseCompareFn = options?.compareFn ?? defaultComparator + if (options?.compareOptions) { + this.compareOptions = options!.compareOptions + } + + const baseCompareFn = + options?.compareFn ?? makeComparator(this.compareOptions) this.hasCustomComparator = options?.compareFn != null // Wrap it to denormalize sentinels before comparison @@ -71,9 +75,6 @@ export class BTreeIndex< this.compareFn = (a: any, b: any) => baseCompareFn(denormalizeUndefined(a), denormalizeUndefined(b)) - if (options?.compareOptions) { - this.compareOptions = options!.compareOptions - } this.orderedEntries = new BTree(this.compareFn) } @@ -104,13 +105,16 @@ export class BTreeIndex< private addToBucket(key: TKey, normalizedValue: unknown): void { const keySet = this.valueMap.get(normalizedValue) if (keySet) { - // Add to existing set keySet.add(key) } else { - // Create new set for this value - const newKeySet = new Set([key]) - this.valueMap.set(normalizedValue, newKeySet) - this.orderedEntries.set(normalizedValue, undefined) + this.valueMap.set(normalizedValue, new Set([key])) + } + + const orderedKeySet = this.orderedEntries.get(normalizedValue) + if (orderedKeySet) { + orderedKeySet.add(key) + } else { + this.orderedEntries.set(normalizedValue, new Set([key])) } } @@ -140,16 +144,16 @@ export class BTreeIndex< private removeFromBucket(key: TKey, normalizedValue: unknown): void { const keySet = this.valueMap.get(normalizedValue) - if (keySet) { - keySet.delete(key) + if (!keySet?.delete(key)) return - // If set is now empty, remove the entry entirely - if (keySet.size === 0) { - this.valueMap.delete(normalizedValue) + if (keySet.size === 0) { + this.valueMap.delete(normalizedValue) + } - // Remove from ordered entries - this.orderedEntries.delete(normalizedValue) - } + const orderedKeySet = this.orderedEntries.get(normalizedValue) + orderedKeySet?.delete(key) + if (orderedKeySet?.size === 0) { + this.orderedEntries.delete(normalizedValue) } } @@ -276,7 +280,7 @@ export class BTreeIndex< fromKey, toKey, toInclusive, - (indexedValue, _) => { + (indexedValue, keys) => { // Only exclude the boundary when an exclusive lower bound was // actually provided. Without a `from` bound, `fromKey` defaults to // the minimum key and must not be dropped. Compare against the @@ -292,10 +296,7 @@ export class BTreeIndex< return } - const keys = this.valueMap.get(indexedValue) - if (keys) { - keys.forEach((key) => result.add(key)) - } + keys.forEach((key) => result.add(key)) }, ) @@ -329,22 +330,20 @@ export class BTreeIndex< */ private takeInternal( n: number, - nextPair: (k?: any) => [any, any] | undefined, + nextPair: (k?: any) => [any, Set] | undefined, from: any, filterFn?: (key: TKey) => boolean, reversed: boolean = false, ): Array { const keysInResult: Set = new Set() const result: Array = [] - let pair: [any, any] | undefined + let pair: [any, Set] | undefined let key = from // Use as-is - it's already normalized by the caller while ((pair = nextPair(key)) !== undefined && result.length < n) { key = pair[0] - const keys = this.valueMap.get(key) as - | Set> - | undefined - if (keys && keys.size > 0) { + const keys = pair[1] + if (keys.size > 0) { // Sort keys for deterministic order, reverse if needed const sorted = Array.from(keys).sort(compareKeys) if (reversed) sorted.reverse() @@ -458,7 +457,7 @@ export class BTreeIndex< let pair = this.orderedEntries.nextHigherPair(undefined) while (pair !== undefined) { const value = pair[0] - yield [denormalizeUndefined(value), this.valueMap.get(value) ?? new Set()] + yield [denormalizeUndefined(value), pair[1]] pair = this.orderedEntries.nextHigherPair(value) } } @@ -469,7 +468,7 @@ export class BTreeIndex< let pair = this.orderedEntries.nextLowerPair(undefined) while (pair !== undefined) { const value = pair[0] - yield [denormalizeUndefined(value), this.valueMap.get(value) ?? new Set()] + yield [denormalizeUndefined(value), pair[1]] pair = this.orderedEntries.nextLowerPair(value) } } diff --git a/packages/db/src/indexes/reverse-index.ts b/packages/db/src/indexes/reverse-index.ts index 9a6bb1aa9..e5dc0c9b0 100644 --- a/packages/db/src/indexes/reverse-index.ts +++ b/packages/db/src/indexes/reverse-index.ts @@ -74,6 +74,15 @@ export class ReverseIndex< return this.originalIndex.orderedEntriesArray } + get supportsOrderedBucketIteration(): boolean { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + typeof orderedIndex.orderedBuckets === `function` && + typeof orderedIndex.orderedBucketsReversed === `function` + ) + } + orderedBuckets(): IterableIterator]> { const orderedIndex = this.originalIndex as IndexInterface & Partial> diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 4a39a9b9c..dbe79074d 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1281,6 +1281,15 @@ refinement work. An all-tied source is the deliberate worst case: the one boundary bucket is the whole source and must be inspected before the public-key suffix can choose top-K. +An ordered bucket is a comparator-equivalence class, not an exact Map-key +bucket. Distinct values such as `null` and `undefined`, or values equated by a +custom comparator, contribute all of their public keys to the same tie class. +Reversing an index also reverses its null placement. The optimizer may reuse a +reverse index only when the requested direction and null placement describe +that reversed order; otherwise it creates a matching index or falls back to a +full `TotalOrder` refinement. Public custom indexes without lazy bucket +iteration keep that full-refinement fallback. + Runtime reference identity has a different lifetime again. Objects use weak identity, but JavaScript symbols cannot be weak keys. Stable equality for the same live symbol therefore retains one strong entry per distinct symbol for the diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 02a4823f5..23479d075 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1,13 +1,16 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { ReverseIndex } from '../../src/indexes/reverse-index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' import { eq } from '../../src/query/builder/functions.js' import { PropRef } from '../../src/query/ir.js' import { TotalOrder } from '../../src/query/total-order.js' import { WindowState } from '../../src/query/live/window-state.js' import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' +import type * as DbIvm from '@tanstack/db-ivm' import type { CollectionImpl } from '../../src/collection/index.js' import type { ChangeMessage, @@ -15,6 +18,19 @@ import type { } from '../../src/types.js' import type { OrderBy, OrderByDirection } from '../../src/query/ir.js' +const keyComparisonCounter = vi.hoisted(() => ({ count: 0 })) + +vi.mock(`@tanstack/db-ivm`, async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + compareKeys: (left: string | number, right: string | number) => { + keyComparisonCounter.count++ + return actual.compareKeys(left, right) + }, + } +}) + type RankedRow = { id: string rank: number @@ -24,6 +40,8 @@ type RankedRow = { type OrderedWork = { keys: Array sourceReads: Array + expectedKeyComparisons: number + keyComparisons: number totalOrderComparisons: number } @@ -40,11 +58,14 @@ function orderedWorkCampaigns(property: string, fixedSeed: number) { ] as const } -function orderBy(direction: OrderByDirection): OrderBy { +function orderBy( + direction: OrderByDirection, + nulls: `first` | `last` = `first`, +): OrderBy { return [ { expression: new PropRef([`rank`]), - compareOptions: { direction, nulls: `first` }, + compareOptions: { direction, nulls }, }, ] } @@ -63,7 +84,30 @@ async function observeDescendingPrefix( try { await collection.preload() - collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + }, + }, + }) as BTreeIndex + + let expectedKeyComparisons = 0 + let expectedMatches = 0 + for (const [, bucket] of index.orderedBucketsReversed()) { + const orderedKeys = [...bucket] + orderedKeys.sort((left, right) => { + expectedKeyComparisons++ + return left < right ? -1 : left > right ? 1 : 0 + }) + expectedMatches += orderedKeys.filter( + (key) => rows.find((row) => row.id === key)?.included === true, + ).length + if (expectedMatches >= limit) break + } const sourceReads: Array = [] const originalGet = collection.get.bind(collection) @@ -74,6 +118,7 @@ async function observeDescendingPrefix( const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { + keyComparisonCounter.count = 0 const changes = collection.currentStateAsChanges({ where: eq(new PropRef([`included`]), true), orderBy: orderBy(`desc`), @@ -83,6 +128,8 @@ async function observeDescendingPrefix( return { keys: changes.map(({ key }) => String(key)), sourceReads, + expectedKeyComparisons, + keyComparisons: keyComparisonCounter.count, totalOrderComparisons: compareEntries.mock.calls.length, } } finally { @@ -140,7 +187,6 @@ function createReversePrefixRows(options: { .map(({ id }) => id) .sort() .slice(0, options.limit) - return { rows: [...trailing, ...rejectedBoundary, ...matchingBoundary, ...leading], expectedKeys, @@ -169,6 +215,7 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) expect(observed.sourceReads).not.toContain(`trailing-000`) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }) @@ -205,6 +252,7 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, ) @@ -221,11 +269,174 @@ describe(`ordered source work oracle`, () => { ).reverse() const observed = await observeDescendingPrefix(rows, 3) - expect(observed.keys).toEqual([`tied-00`, `tied-02`, `tied-04`]) expect(observed.sourceReads).toHaveLength(rows.length + 3) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }) + + it(`keeps comparator-equivalent BTree values in one ordered tie class`, async () => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const rows: Array = [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-nullish-tie`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `last`), + limit: rows.length, + })! + + expect(changes.map(({ key }) => key)).toEqual([ + `one`, + `null`, + `undefined`, + ]) + } finally { + await collection.cleanup() + } + }) + + it(`does not reverse an index with incompatible null placement`, async () => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-null-placement`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `first`), + })! + + expect(changes.map(({ key }) => key)).toEqual([ + `null`, + `undefined`, + `one`, + ]) + } finally { + await collection.cleanup() + } + }) + + it(`keeps custom indexes on the materialized reverse-order fallback`, async () => { + const rows: Array = [ + { id: `later`, rank: 1, included: true }, + { id: `tie-b`, rank: 2, included: true }, + { id: `tie-a`, rank: 2, included: true }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-index-fallback`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + }, + }, + }) as BTreeIndex + const customIndex = new Proxy(index, { + get(target, property) { + if ( + property === `orderedBuckets` || + property === `orderedBucketsReversed` + ) { + return undefined + } + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) + collection.indexes.set(index.id, customIndex) + expect(new ReverseIndex(customIndex).supportsOrderedBucketIteration).toBe( + false, + ) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `first`), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual([`tie-a`, `tie-b`]) + } finally { + await collection.cleanup() + } + }) + + it(`groups comparator-equivalent values in every built-in index direction`, () => { + type TextRow = { id: string; value: string } + const rows: Array = [ + { id: `upper`, value: `A` }, + { id: `lower`, value: `a` }, + { id: `later`, value: `b` }, + ] + + for (const IndexType of [BasicIndex, BTreeIndex]) { + const index = new IndexType( + 1, + new PropRef([`value`]), + undefined, + { + compareFn: (left: string, right: string) => + left.toLowerCase().localeCompare(right.toLowerCase()), + }, + ) + index.build(rows.map((row) => [row.id, row])) + + expect( + [...index.orderedBuckets()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`lower`, `upper`], [`later`]]) + expect( + [...index.orderedBucketsReversed()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`later`], [`lower`, `upper`]]) + expect( + [...new ReverseIndex(index).orderedBuckets()].map(([, keys]) => + [...keys].sort(), + ), + ).toEqual([[`later`], [`lower`, `upper`]]) + + index.remove(`upper`, rows[0]) + expect( + [...index.orderedBuckets()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`lower`], [`later`]]) + } + }) }) type SnapshotFixture = { @@ -324,6 +535,31 @@ it(`reuses one ordered source snapshot until the collection revision changes`, ( expect(fixture.snapshotRevisions).toEqual([0, 1]) }) +it(`compiles the ordered predicate once for the lifetime of a window`, () => { + const fixture = createSnapshotFixture([ + { id: `a`, rank: 1, included: true }, + { id: `hidden`, rank: 0, included: false }, + ]) + let compilationReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) compilationReads++ + return Reflect.get(target, property, receiver) + }, + }) + const window = new WindowState(fixture.collection, orderBy(`asc`), where, 1) + const readsAfterConstruction = compilationReads + expect(readsAfterConstruction).toBeGreaterThan(0) + window.recordInitialCoverage(undefined, true) + + observeWindow(window) + observeWindow(window) + fixture.replace({ id: `a`, rank: 2, included: true }) + observeWindow(window) + + expect(compilationReads).toBe(readsAfterConstruction) +}) + it(`invalidates the ordered snapshot after a committed collection write`, async () => { const collection = createCollection( localOnlyCollectionOptions({ From 354646c48a1370c962432c38f1941abc20345fa2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 10:35:50 -0600 Subject: [PATCH 158/327] fix(db): order direct index tie keys --- packages/db/src/collection/change-events.ts | 36 ++-- packages/db/src/query/live/ARCHITECTURE.md | 10 +- packages/db/tests/oracle-config.ts | 1 + .../ordered-work-oracle.property.test.ts | 203 +++++++++++++++++- 4 files changed, 222 insertions(+), 28 deletions(-) diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index ff5a051f9..7f0a54332 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -22,10 +22,26 @@ import type { SubscribeChangesOptions, } from '../types' import type { CollectionImpl } from './index.js' +import type { IndexInterface } from '../indexes/base-index.js' import type { SingleRowRefProxy } from '../query/builder/ref-proxy' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { WithVirtualProps } from '../virtual-props.js' +type OrderedBucketIndex = { + orderedBuckets: () => IterableIterator]> +} + +function getOrderedBuckets( + index: IndexInterface, +): IterableIterator]> | undefined { + if (index instanceof ReverseIndex && !index.supportsOrderedBucketIteration) { + return + } + return ( + index as IndexInterface & Partial> + ).orderedBuckets?.() +} + /** * Returns the current state of the collection as an array of changes * @param collection - The collection to get changes from @@ -357,17 +373,12 @@ function getOrderedKeys( return whereFilter?.(value) ?? true } - // Take the keys that match the filter and limit - // if no limit is provided `index.keyCount` is used, - // i.e. we will take all keys that match the filter - if (!(index instanceof ReverseIndex)) { - return index.takeFromStart(limit ?? index.keyCount, filterFn) - } + const orderedBuckets = getOrderedBuckets(index) // Public custom indexes predate lazy bucket iteration. Preserve their // semantics with the full TotalOrder refinement instead of assuming // their materialized entries expose complete comparator tie classes. - if (!index.supportsOrderedBucketIteration) { + if (!orderedBuckets) { const totalOrder = new TotalOrder(orderBy, collection) const indexedEntries = index .takeFromStart(index.keyCount, filterFn) @@ -386,14 +397,13 @@ function getOrderedKeys( .map(({ key }) => key) } - // Reversing a value index must not reverse the public-key suffix of the - // query's total order. Walk value buckets in reverse value order, sort - // keys inside each bucket ascending, and stop after the first bucket - // that proves the requested prefix. The complete boundary bucket must - // be inspected because filtering can otherwise select the wrong key. + // Value order comes from the matching index or its reverse view. The + // public-key suffix remains ascending in both directions. Stop after + // the first complete bucket that proves the requested prefix because + // filtering can otherwise select the wrong key from a boundary tie. if (limit === 0) return [] const keys: Array = [] - for (const [, bucket] of index.orderedBuckets()) { + for (const [, bucket] of orderedBuckets) { const matchingKeys = [...bucket].sort(compareKeys).filter(filterFn) const remaining = limit === undefined ? undefined : limit - keys.length diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index dbe79074d..70d6ee715 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1273,10 +1273,12 @@ is compiled once with the window and is evaluated over the shared ordered snapshot, so another view of the same revision does not rescan, resort, or recompile it. -A descending single-column index walks indexed-value buckets in query order. -It evaluates complete buckets until the requested filtered prefix is known, -orders public keys ascending within each bucket, and stops after the sufficient -boundary bucket. Rows in worse buckets cannot add source reads or total-order +A compatible single-column built-in index walks indexed-value buckets in query +order, whether the matching view is direct or reversed. It evaluates complete +buckets until the requested filtered prefix is known, orders public keys +ascending within each bucket, and stops after the sufficient boundary bucket. +The public-key suffix does not depend on index insertion order or query +direction. Rows in worse buckets cannot add source reads or total-order refinement work. An all-tied source is the deliberate worst case: the one boundary bucket is the whole source and must be inspected before the public-key suffix can choose top-K. diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index be5864732..767cb6413 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -45,6 +45,7 @@ const staticOracleProperties = [ `load-subset.distinct-window-predicate`, `load-subset.ordered-window`, `load-subset.rejected-waiter`, + `ordered-work.public-key-suffix`, `ordered-work.reverse-prefix`, `ordered-work.snapshot-reuse`, `pagination.async-cursor`, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 23479d075..e23ddb031 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -73,6 +73,7 @@ function orderBy( async function observeDescendingPrefix( rows: ReadonlyArray, limit: number, + indexKind: `basic` | `btree` = `btree`, ): Promise { const collection = createCollection( localOnlyCollectionOptions({ @@ -85,7 +86,7 @@ async function observeDescendingPrefix( try { await collection.preload() const index = collection.createIndex((row) => row.rank, { - indexType: BTreeIndex, + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, options: { compareOptions: { direction: `asc`, @@ -93,7 +94,7 @@ async function observeDescendingPrefix( stringSort: `locale`, }, }, - }) as BTreeIndex + }) as BasicIndex | BTreeIndex let expectedKeyComparisons = 0 let expectedMatches = 0 @@ -230,6 +231,7 @@ describe(`ordered source work oracle`, () => { fc.integer({ min: 0, max: 5 }), fc.integer({ min: 0, max: 8 }), fc.integer({ min: 0, max: 60 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), ], campaign.options, )( @@ -240,6 +242,7 @@ describe(`ordered source work oracle`, () => { extraBoundaryMatches, boundaryRejects, trailingRows, + indexKind, ) => { const scenario = createReversePrefixRows({ leadingRejects, @@ -248,7 +251,11 @@ describe(`ordered source work oracle`, () => { boundaryRejects, trailingRows, }) - const observed = await observeDescendingPrefix(scenario.rows, limit) + const observed = await observeDescendingPrefix( + scenario.rows, + limit, + indexKind, + ) expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) @@ -379,6 +386,20 @@ describe(`ordered source work oracle`, () => { ) { return undefined } + if (property === `orderedEntriesArray`) { + return [ + [1, new Set([`later`])], + [2, new Set([`tie-a`])], + [2, new Set([`tie-b`])], + ] + } + if (property === `orderedEntriesArrayReversed`) { + return [ + [2, new Set([`tie-b`])], + [2, new Set([`tie-a`])], + [1, new Set([`later`])], + ] + } const value = Reflect.get(target, property, target) as unknown return typeof value === `function` ? value.bind(target) : value }, @@ -388,12 +409,18 @@ describe(`ordered source work oracle`, () => { false, ) - const changes = collection.currentStateAsChanges({ - orderBy: orderBy(`desc`, `first`), - limit: 2, - })! - - expect(changes.map(({ key }) => key)).toEqual([`tie-a`, `tie-b`]) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `first`), + limit: 1, + })! + + expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } } finally { await collection.cleanup() } @@ -431,10 +458,164 @@ describe(`ordered source work oracle`, () => { ), ).toEqual([[`later`], [`lower`, `upper`]]) - index.remove(`upper`, rows[0]) + index.remove(`lower`, rows[1]) + expect([...index.equalityLookup(`A`)]).toEqual([`upper`]) + expect([...index.equalityLookup(`a`)]).toEqual([]) expect( [...index.orderedBuckets()].map(([, keys]) => [...keys].sort()), - ).toEqual([[`lower`], [`later`]]) + ).toEqual([[`upper`], [`later`]]) + } + }) + + it.each([ + { + name: `BasicIndex`, + IndexType: BasicIndex, + direction: `asc`, + expectedKeys: [`a`, `z`], + }, + { + name: `BasicIndex`, + IndexType: BasicIndex, + direction: `desc`, + expectedKeys: [`m`, `a`], + }, + { + name: `BTreeIndex`, + IndexType: BTreeIndex, + direction: `asc`, + expectedKeys: [`a`, `z`], + }, + { + name: `BTreeIndex`, + IndexType: BTreeIndex, + direction: `desc`, + expectedKeys: [`m`, `a`], + }, + ] as const)( + `keeps the public-key suffix ascending for $name in $direction order`, + async ({ name, IndexType, direction, expectedKeys }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-key-suffix-${name}-${direction}`, + getKey: (row) => row.id, + initialData: [{ id: `m`, rank: 2, included: true }], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: IndexType }) + const z = collection.insert({ id: `z`, rank: 1, included: true }) + await z.isPersisted.promise + const a = collection.insert({ id: `a`, rank: 1, included: true }) + await a.isPersisted.promise + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + } finally { + await collection.cleanup() + } + }, + ) + + for (const campaign of orderedWorkCampaigns( + `ordered-work.public-key-suffix`, + 1_780_102, + )) { + fcTest.prop( + [ + fc.uniqueArray(fc.integer({ min: 0, max: 999 }), { + minLength: 2, + maxLength: 8, + }), + fc.integer({ min: 1, max: 8 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + fc.constantFrom(`asc`, `desc`), + ], + campaign.options, + )( + `orders dynamic tie keys for every built-in path (${campaign.label})`, + async (keyNumbers, requestedLimit, indexKind, direction) => { + const keys = keyNumbers.map( + (key) => `key-${key.toString().padStart(3, `0`)}`, + ) + const limit = Math.min(requestedLimit, keys.length) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-key-property-${Math.random()}`, + getKey: (row) => row.id, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + }) + for (const key of keys) { + const transaction = collection.insert({ + id: key, + rank: 1, + included: true, + }) + await transaction.isPersisted.promise + } + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction), + limit, + })! + + expect(changes.map(({ key }) => key)).toEqual( + [...keys].sort().slice(0, limit), + ) + } finally { + await collection.cleanup() + } + }, + ) + } + + it(`retains requested comparison metadata on an automatic index`, async () => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-auto-index-options`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + + try { + await collection.preload() + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `first`), + limit: 2, + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual([`null`, `undefined`]) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() } }) }) From 0281422303bdfe681e6c2b2d250915c385705a3b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 10:55:09 -0600 Subject: [PATCH 159/327] test(db): close ordered work audit gaps --- packages/db/tests/oracle-config.ts | 1 + .../ordered-work-oracle.property.test.ts | 401 ++++++++++++------ 2 files changed, 270 insertions(+), 132 deletions(-) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 767cb6413..7f18b5a41 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -45,6 +45,7 @@ const staticOracleProperties = [ `load-subset.distinct-window-predicate`, `load-subset.ordered-window`, `load-subset.rejected-waiter`, + `ordered-work.forward-prefix`, `ordered-work.public-key-suffix`, `ordered-work.reverse-prefix`, `ordered-work.snapshot-reuse`, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index e23ddb031..dc658de05 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -6,6 +6,7 @@ import { BTreeIndex } from '../../src/indexes/btree-index.js' import { ReverseIndex } from '../../src/indexes/reverse-index.js' import { localOnlyCollectionOptions } from '../../src/local-only.js' import { eq } from '../../src/query/builder/functions.js' +import { compileSingleRowExpression } from '../../src/query/compiler/evaluators.js' import { PropRef } from '../../src/query/ir.js' import { TotalOrder } from '../../src/query/total-order.js' import { WindowState } from '../../src/query/live/window-state.js' @@ -70,10 +71,29 @@ function orderBy( ] } -async function observeDescendingPrefix( +const orderedIndexCompatibilityCases = ([`asc`, `desc`] as const).flatMap( + (indexDirection) => + ([`first`, `last`] as const).flatMap((indexNulls) => + ([`asc`, `desc`] as const).flatMap((queryDirection) => + ([`first`, `last`] as const).map((queryNulls) => ({ + indexDirection, + indexNulls, + queryDirection, + queryNulls, + compatible: + indexDirection === queryDirection + ? indexNulls === queryNulls + : indexNulls !== queryNulls, + })), + ), + ), +) + +async function observeOrderedPrefix( rows: ReadonlyArray, limit: number, indexKind: `basic` | `btree` = `btree`, + direction: OrderByDirection = `desc`, ): Promise { const collection = createCollection( localOnlyCollectionOptions({ @@ -98,7 +118,11 @@ async function observeDescendingPrefix( let expectedKeyComparisons = 0 let expectedMatches = 0 - for (const [, bucket] of index.orderedBucketsReversed()) { + const expectedBuckets = + direction === `asc` + ? index.orderedBuckets() + : index.orderedBucketsReversed() + for (const [, bucket] of expectedBuckets) { const orderedKeys = [...bucket] orderedKeys.sort((left, right) => { expectedKeyComparisons++ @@ -122,7 +146,7 @@ async function observeDescendingPrefix( keyComparisonCounter.count = 0 const changes = collection.currentStateAsChanges({ where: eq(new PropRef([`included`]), true), - orderBy: orderBy(`desc`), + orderBy: orderBy(direction, direction === `asc` ? `last` : `first`), limit, })! @@ -141,22 +165,27 @@ async function observeDescendingPrefix( } } -function createReversePrefixRows(options: { - leadingRejects: number - limit: number - extraBoundaryMatches: number - boundaryRejects: number - trailingRows: number -}): { +function createOrderedPrefixRows( + options: { + leadingRejects: number + limit: number + extraBoundaryMatches: number + boundaryRejects: number + trailingRows: number + }, + direction: OrderByDirection = `desc`, +): { rows: Array expectedKeys: Array expectedSourceReads: number } { + const rank = (descendingRank: number) => + direction === `desc` ? descendingRank : -descendingRank const leading = Array.from( { length: options.leadingRejects }, (_, index): RankedRow => ({ id: `leading-${index.toString().padStart(2, `0`)}`, - rank: 100 + index, + rank: rank(100 + index), included: false, }), ) @@ -164,7 +193,7 @@ function createReversePrefixRows(options: { { length: options.limit + options.extraBoundaryMatches }, (_, index): RankedRow => ({ id: `boundary-match-${index.toString().padStart(2, `0`)}`, - rank: 50, + rank: rank(50), included: true, }), ).reverse() @@ -172,7 +201,7 @@ function createReversePrefixRows(options: { { length: options.boundaryRejects }, (_, index): RankedRow => ({ id: `boundary-reject-${index.toString().padStart(2, `0`)}`, - rank: 50, + rank: rank(50), included: false, }), ) @@ -180,7 +209,7 @@ function createReversePrefixRows(options: { { length: options.trailingRows }, (_, index): RankedRow => ({ id: `trailing-${index.toString().padStart(3, `0`)}`, - rank: 10 - index, + rank: rank(10 - index), included: true, }), ) @@ -202,67 +231,93 @@ function createReversePrefixRows(options: { } describe(`ordered source work oracle`, () => { - it(`does not read worse reverse-index buckets after filling top-K`, async () => { - const scenario = createReversePrefixRows({ - leadingRejects: 2, - limit: 2, - extraBoundaryMatches: 1, - boundaryRejects: 2, - trailingRows: 40, - }) + it.each([ + { indexKind: `basic`, direction: `asc` }, + { indexKind: `basic`, direction: `desc` }, + { indexKind: `btree`, direction: `asc` }, + { indexKind: `btree`, direction: `desc` }, + ] as const)( + `does not read worse $indexKind index buckets in $direction order`, + async ({ indexKind, direction }) => { + const scenario = createOrderedPrefixRows( + { + leadingRejects: 2, + limit: 2, + extraBoundaryMatches: 1, + boundaryRejects: 2, + trailingRows: 40, + }, + direction, + ) - const observed = await observeDescendingPrefix(scenario.rows, 2) + const observed = await observeOrderedPrefix( + scenario.rows, + 2, + indexKind, + direction, + ) - expect(observed.keys).toEqual(scenario.expectedKeys) - expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) - expect(observed.sourceReads).not.toContain(`trailing-000`) - expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) - expect(observed.totalOrderComparisons).toBe(0) - }) + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) + expect(observed.sourceReads).not.toContain(`trailing-000`) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) - for (const campaign of orderedWorkCampaigns( - `ordered-work.reverse-prefix`, - 1_780_101, - )) { - fcTest.prop( - [ - fc.integer({ min: 0, max: 8 }), - fc.integer({ min: 1, max: 5 }), - fc.integer({ min: 0, max: 5 }), - fc.integer({ min: 0, max: 8 }), - fc.integer({ min: 0, max: 60 }), - fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), - ], - campaign.options, - )( - `bounds reverse-index reads at the sufficient bucket (${campaign.label})`, - async ( - leadingRejects, - limit, - extraBoundaryMatches, - boundaryRejects, - trailingRows, - indexKind, - ) => { - const scenario = createReversePrefixRows({ + for (const direction of [`asc`, `desc`] as const) { + const property = + direction === `asc` + ? `ordered-work.forward-prefix` + : `ordered-work.reverse-prefix` + const seed = direction === `asc` ? 1_780_103 : 1_780_101 + for (const campaign of orderedWorkCampaigns(property, seed)) { + fcTest.prop( + [ + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 1, max: 5 }), + fc.integer({ min: 0, max: 5 }), + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 0, max: 60 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + ], + campaign.options, + )( + `bounds ${direction} index reads at the sufficient bucket (${campaign.label})`, + async ( leadingRejects, limit, extraBoundaryMatches, boundaryRejects, trailingRows, - }) - const observed = await observeDescendingPrefix( - scenario.rows, - limit, indexKind, - ) + ) => { + const scenario = createOrderedPrefixRows( + { + leadingRejects, + limit, + extraBoundaryMatches, + boundaryRejects, + trailingRows, + }, + direction, + ) + const observed = await observeOrderedPrefix( + scenario.rows, + limit, + indexKind, + direction, + ) - expect(observed.keys).toEqual(scenario.expectedKeys) - expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) - expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) - expect(observed.totalOrderComparisons).toBe(0) - }, - ) + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toHaveLength( + scenario.expectedSourceReads, + ) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + } } it(`reads the complete tied boundary when every candidate is tied`, async () => { @@ -275,7 +330,7 @@ describe(`ordered source work oracle`, () => { }), ).reverse() - const observed = await observeDescendingPrefix(rows, 3) + const observed = await observeOrderedPrefix(rows, 3) expect(observed.keys).toEqual([`tied-00`, `tied-02`, `tied-04`]) expect(observed.sourceReads).toHaveLength(rows.length + 3) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) @@ -352,79 +407,151 @@ describe(`ordered source work oracle`, () => { } }) - it(`keeps custom indexes on the materialized reverse-order fallback`, async () => { - const rows: Array = [ - { id: `later`, rank: 1, included: true }, - { id: `tie-b`, rank: 2, included: true }, - { id: `tie-a`, rank: 2, included: true }, - ] - const collection = createCollection( - localOnlyCollectionOptions({ - id: `ordered-work-custom-index-fallback`, - getKey: (row) => row.id, - initialData: rows, - }), - ) + it.each(orderedIndexCompatibilityCases)( + `matches index $indexDirection/nulls-$indexNulls to query $queryDirection/nulls-$queryNulls: $compatible`, + async ({ + indexDirection, + indexNulls, + queryDirection, + queryNulls, + compatible, + }) => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-index-compatibility-${indexDirection}-${indexNulls}-${queryDirection}-${queryNulls}`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ], + }), + ) - try { - await collection.preload() - const index = collection.createIndex((row) => row.rank, { - indexType: BTreeIndex, - options: { - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { + compareOptions: { + direction: indexDirection, + nulls: indexNulls, + stringSort: `locale`, + }, }, - }, - }) as BTreeIndex - const customIndex = new Proxy(index, { - get(target, property) { - if ( - property === `orderedBuckets` || - property === `orderedBucketsReversed` - ) { - return undefined - } - if (property === `orderedEntriesArray`) { - return [ - [1, new Set([`later`])], - [2, new Set([`tie-a`])], - [2, new Set([`tie-b`])], - ] - } - if (property === `orderedEntriesArrayReversed`) { - return [ - [2, new Set([`tie-b`])], - [2, new Set([`tie-a`])], - [1, new Set([`later`])], - ] - } - const value = Reflect.get(target, property, target) as unknown - return typeof value === `function` ? value.bind(target) : value - }, - }) - collection.indexes.set(index.id, customIndex) - expect(new ReverseIndex(customIndex).supportsOrderedBucketIteration).toBe( - false, + }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(queryDirection, queryNulls), + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual( + compatible + ? queryNulls === `first` + ? [`null`, `undefined`, `one`] + : [`one`, `null`, `undefined`] + : undefined, + ) + } finally { + await collection.cleanup() + } + }, + ) + + it.each([`asc`, `desc`] as const)( + `keeps custom indexes on the materialized %s-order fallback`, + async (direction) => { + const tieRank = direction === `asc` ? 1 : 2 + const laterRank = direction === `asc` ? 2 : 1 + const rows: Array = [ + { id: `later`, rank: laterRank, included: true }, + { id: `tie-b`, rank: tieRank, included: true }, + { id: `tie-a`, rank: tieRank, included: true }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-index-fallback-${direction}`, + getKey: (row) => row.id, + initialData: rows, + }), ) - const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { - const changes = collection.currentStateAsChanges({ - orderBy: orderBy(`desc`, `first`), - limit: 1, - })! + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + }, + }, + }) as BTreeIndex + const customIndex = new Proxy(index, { + get(target, property) { + if ( + property === `orderedBuckets` || + property === `orderedBucketsReversed` + ) { + return undefined + } + if (property === `orderedEntriesArray`) { + return [ + ...(direction === `desc` + ? [[laterRank, new Set([`later`])]] + : []), + [tieRank, new Set([`tie-b`])], + [tieRank, new Set([`tie-a`])], + ...(direction === `asc` + ? [[laterRank, new Set([`later`])]] + : []), + ] + } + if (property === `orderedEntriesArrayReversed`) { + return [ + ...(direction === `asc` + ? [[laterRank, new Set([`later`])]] + : []), + [tieRank, new Set([`tie-b`])], + [tieRank, new Set([`tie-a`])], + ...(direction === `desc` + ? [[laterRank, new Set([`later`])]] + : []), + ] + } + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) + collection.indexes.set(index.id, customIndex) + if (direction === `desc`) { + expect( + new ReverseIndex(customIndex).supportsOrderedBucketIteration, + ).toBe(false) + } - expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) - expect(compareEntries).toHaveBeenCalled() + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction, direction === `asc` ? `last` : `first`), + limit: 1, + })! + + expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } } finally { - compareEntries.mockRestore() + await collection.cleanup() } - } finally { - await collection.cleanup() - } - }) + }, + ) it(`groups comparator-equivalent values in every built-in index direction`, () => { type TextRow = { id: string; value: string } @@ -721,6 +848,16 @@ it(`compiles the ordered predicate once for the lifetime of a window`, () => { { id: `a`, rank: 1, included: true }, { id: `hidden`, rank: 0, included: false }, ]) + let referenceCompilationReads = 0 + const referenceWhere = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) + }, + }) + compileSingleRowExpression(referenceWhere) + expect(referenceCompilationReads).toBeGreaterThan(0) + let compilationReads = 0 const where = new Proxy(eq(new PropRef([`included`]), true), { get(target, property, receiver) { @@ -730,7 +867,7 @@ it(`compiles the ordered predicate once for the lifetime of a window`, () => { }) const window = new WindowState(fixture.collection, orderBy(`asc`), where, 1) const readsAfterConstruction = compilationReads - expect(readsAfterConstruction).toBeGreaterThan(0) + expect(readsAfterConstruction).toBe(referenceCompilationReads) window.recordInitialCoverage(undefined, true) observeWindow(window) From ccd5e92e82fd9ea5b0e6772df1d444b2d936dcdf Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 11:25:04 -0600 Subject: [PATCH 160/327] test(db): preserve ordered oracle domains --- .../ordered-work-oracle.property.test.ts | 263 ++++++++++++++++-- 1 file changed, 239 insertions(+), 24 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index dc658de05..850425c1f 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -13,6 +13,7 @@ import { WindowState } from '../../src/query/live/window-state.js' import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' import type * as DbIvm from '@tanstack/db-ivm' import type { CollectionImpl } from '../../src/collection/index.js' +import type { CompareOptions } from '../../src/query/builder/types.js' import type { ChangeMessage, CurrentStateAsChangesOptions, @@ -38,6 +39,10 @@ type RankedRow = { included: boolean } +type PublicKeyRankedRow = Omit & { + id: string | number +} + type OrderedWork = { keys: Array sourceReads: Array @@ -71,6 +76,28 @@ function orderBy( ] } +function orderByWithOptions(compareOptions: CompareOptions): OrderBy { + return [{ expression: new PropRef([`rank`]), compareOptions }] +} + +function comparePublicKeys( + left: string | number, + right: string | number, +): number { + if (typeof left !== typeof right) { + return typeof left === `string` ? -1 : 1 + } + if (typeof left === `number` && typeof right === `number`) { + const leftIsNaN = Number.isNaN(left) + const rightIsNaN = Number.isNaN(right) + if (leftIsNaN || rightIsNaN) { + if (leftIsNaN && rightIsNaN) return 0 + return leftIsNaN ? 1 : -1 + } + } + return left < right ? -1 : left > right ? 1 : 0 +} + const orderedIndexCompatibilityCases = ([`asc`, `desc`] as const).flatMap( (indexDirection) => ([`first`, `last`] as const).flatMap((indexNulls) => @@ -177,7 +204,7 @@ function createOrderedPrefixRows( ): { rows: Array expectedKeys: Array - expectedSourceReads: number + expectedSourceReads: Array } { const rank = (descendingRank: number) => direction === `desc` ? descendingRank : -descendingRank @@ -217,16 +244,25 @@ function createOrderedPrefixRows( .map(({ id }) => id) .sort() .slice(0, options.limit) + const expectedCandidateReads = [ + ...leading, + ...matchingBoundary, + ...rejectedBoundary, + ] + .sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) return { rows: [...trailing, ...rejectedBoundary, ...matchingBoundary, ...leading], expectedKeys, // Every row through the boundary bucket is tested once. The selected rows // are then read once more to materialize their change messages. - expectedSourceReads: - leading.length + - matchingBoundary.length + - rejectedBoundary.length + - options.limit, + expectedSourceReads: [...expectedCandidateReads, ...expectedKeys], } } @@ -258,8 +294,7 @@ describe(`ordered source work oracle`, () => { ) expect(observed.keys).toEqual(scenario.expectedKeys) - expect(observed.sourceReads).toHaveLength(scenario.expectedSourceReads) - expect(observed.sourceReads).not.toContain(`trailing-000`) + expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -310,9 +345,7 @@ describe(`ordered source work oracle`, () => { ) expect(observed.keys).toEqual(scenario.expectedKeys) - expect(observed.sourceReads).toHaveLength( - scenario.expectedSourceReads, - ) + expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -332,7 +365,12 @@ describe(`ordered source work oracle`, () => { const observed = await observeOrderedPrefix(rows, 3) expect(observed.keys).toEqual([`tied-00`, `tied-02`, `tied-04`]) - expect(observed.sourceReads).toHaveLength(rows.length + 3) + expect(observed.sourceReads).toEqual([ + ...rows.map(({ id }) => id).sort(comparePublicKeys), + `tied-00`, + `tied-02`, + `tied-04`, + ]) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }) @@ -650,6 +688,61 @@ describe(`ordered source work oracle`, () => { }, ) + it.each( + ( + [ + { name: `BasicIndex`, IndexType: BasicIndex }, + { name: `BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ name, IndexType }) => + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { domain: `number`, keys: [10, 2] }, + { domain: `mixed`, keys: [10, `2`, 2, `10`] }, + ].map(({ domain, keys }) => ({ + name, + IndexType, + direction, + domain, + keys, + })), + ), + ), + )( + `keeps $domain public keys in compareKeys order for $name in $direction order`, + async ({ name, IndexType, direction, keys }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-${name}-${direction}-${keys.join(`-`)}`, + getKey: (row) => row.id, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: IndexType }) + for (const key of keys) { + const transaction = collection.insert({ + id: key, + rank: 1, + included: true, + }) + await transaction.isPersisted.promise + } + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction), + })! + + expect(changes.map(({ key }) => key)).toEqual( + [...keys].sort(comparePublicKeys), + ) + } finally { + await collection.cleanup() + } + }, + ) + for (const campaign of orderedWorkCampaigns( `ordered-work.public-key-suffix`, 1_780_102, @@ -660,20 +753,28 @@ describe(`ordered source work oracle`, () => { minLength: 2, maxLength: 8, }), - fc.integer({ min: 1, max: 8 }), + fc.integer({ min: 1, max: 16 }), fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), fc.constantFrom(`asc`, `desc`), + fc.constantFrom<`string` | `number` | `mixed`>( + `string`, + `number`, + `mixed`, + ), ], campaign.options, )( `orders dynamic tie keys for every built-in path (${campaign.label})`, - async (keyNumbers, requestedLimit, indexKind, direction) => { - const keys = keyNumbers.map( - (key) => `key-${key.toString().padStart(3, `0`)}`, - ) + async (keyNumbers, requestedLimit, indexKind, direction, keyDomain) => { + const keys: Array = + keyDomain === `string` + ? keyNumbers.map((key) => `key-${key}`) + : keyDomain === `number` + ? keyNumbers + : keyNumbers.flatMap((key) => [String(key), key]) const limit = Math.min(requestedLimit, keys.length) const collection = createCollection( - localOnlyCollectionOptions({ + localOnlyCollectionOptions({ id: `ordered-work-key-property-${Math.random()}`, getKey: (row) => row.id, }), @@ -699,7 +800,7 @@ describe(`ordered source work oracle`, () => { })! expect(changes.map(({ key }) => key)).toEqual( - [...keys].sort().slice(0, limit), + [...keys].sort(comparePublicKeys).slice(0, limit), ) } finally { await collection.cleanup() @@ -710,8 +811,15 @@ describe(`ordered source work oracle`, () => { it(`retains requested comparison metadata on an automatic index`, async () => { type NullableRankedRow = Omit & { - rank: number | null | undefined + rank: string | null | undefined } + const compareOptions = { + direction: `desc`, + nulls: `first`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + } satisfies CompareOptions const collection = createCollection( localOnlyCollectionOptions({ id: `ordered-work-auto-index-options`, @@ -719,7 +827,8 @@ describe(`ordered source work oracle`, () => { initialData: [ { id: `undefined`, rank: undefined, included: true }, { id: `null`, rank: null, included: true }, - { id: `one`, rank: 1, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + { id: `item-10`, rank: `item-10`, included: true }, ], autoIndex: `eager`, defaultIndexType: BTreeIndex, @@ -731,12 +840,17 @@ describe(`ordered source work oracle`, () => { const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { const changes = collection.currentStateAsChanges({ - orderBy: orderBy(`desc`, `first`), - limit: 2, + orderBy: orderByWithOptions(compareOptions), + limit: 4, optimizedOnly: true, }) - expect(changes?.map(({ key }) => key)).toEqual([`null`, `undefined`]) + expect(changes?.map(({ key }) => key)).toEqual([ + `null`, + `undefined`, + `item-10`, + `item-2`, + ]) expect(compareEntries).not.toHaveBeenCalled() } finally { compareEntries.mockRestore() @@ -745,6 +859,107 @@ describe(`ordered source work oracle`, () => { await collection.cleanup() } }) + + it.each([ + { + name: `the same locale options`, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: true, + }, + { + name: `lexical string order`, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + compatible: false, + }, + { + name: `another locale`, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `de`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another numeric option`, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: false, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another sensitivity option`, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `accent` }, + }, + compatible: false, + }, + ] satisfies Array<{ + name: string + compareOptions: CompareOptions + compatible: boolean + }>)( + `matches an index against $name: $compatible`, + async ({ compareOptions, compatible }) => { + type TextRankedRow = Omit & { rank: string } + const indexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + } satisfies CompareOptions + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-string-options-${Math.random()}`, + getKey: (row) => row.id, + initialData: [ + { id: `item-10`, rank: `item-10`, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: indexCompareOptions }, + }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderByWithOptions(compareOptions), + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual( + compatible ? [`item-2`, `item-10`] : undefined, + ) + } finally { + await collection.cleanup() + } + }, + ) }) type SnapshotFixture = { From 7d14514e663cd06fe940345ef165dbedc0453664 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 11:46:15 -0600 Subject: [PATCH 161/327] test(db): cover reversed ordered domains --- .../ordered-work-oracle.property.test.ts | 229 +++++++++++------- 1 file changed, 144 insertions(+), 85 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 850425c1f..8fb5ceffc 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -17,6 +17,7 @@ import type { CompareOptions } from '../../src/query/builder/types.js' import type { ChangeMessage, CurrentStateAsChangesOptions, + StringCollationConfig, } from '../../src/types.js' import type { OrderBy, OrderByDirection } from '../../src/query/ir.js' @@ -80,6 +81,20 @@ function orderByWithOptions(compareOptions: CompareOptions): OrderBy { return [{ expression: new PropRef([`rank`]), compareOptions }] } +const publicKeyIndexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, +} satisfies CompareOptions + +function publicKeyOrderBy(direction: OrderByDirection): OrderBy { + return orderByWithOptions({ + ...publicKeyIndexCompareOptions, + direction, + nulls: direction === `asc` ? `last` : `first`, + }) +} + function comparePublicKeys( left: string | number, right: string | number, @@ -116,6 +131,69 @@ const orderedIndexCompatibilityCases = ([`asc`, `desc`] as const).flatMap( ), ) +const stringComparisonVariants = [ + { + name: `the same locale options`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: true, + }, + { + name: `lexical string order`, + collation: { stringSort: `lexical` }, + compatible: false, + }, + { + name: `another locale`, + collation: { + stringSort: `locale`, + locale: `de`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another numeric option`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: false, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another sensitivity option`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `accent` }, + }, + compatible: false, + }, +] satisfies Array<{ + name: string + collation: StringCollationConfig + compatible: boolean +}> + +const orderedStringCompatibilityCases = ([`asc`, `desc`] as const).flatMap( + (queryDirection) => + stringComparisonVariants.map(({ name, collation, compatible }) => ({ + name, + queryDirection, + compareOptions: { + ...collation, + direction: queryDirection, + nulls: + queryDirection === `asc` ? (`last` as const) : (`first` as const), + } satisfies CompareOptions, + compatible, + })), +) + async function observeOrderedPrefix( rows: ReadonlyArray, limit: number, @@ -670,18 +748,28 @@ describe(`ordered source work oracle`, () => { try { await collection.preload() - collection.createIndex((row) => row.rank, { indexType: IndexType }) + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) const z = collection.insert({ id: `z`, rank: 1, included: true }) await z.isPersisted.promise const a = collection.insert({ id: `a`, rank: 1, included: true }) await a.isPersisted.promise - const changes = collection.currentStateAsChanges({ - orderBy: orderBy(direction), - limit: 2, - })! + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: 2, + optimizedOnly: true, + })! - expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } } finally { await collection.cleanup() } @@ -697,7 +785,8 @@ describe(`ordered source work oracle`, () => { ).flatMap(({ name, IndexType }) => ([`asc`, `desc`] as const).flatMap((direction) => [ - { domain: `number`, keys: [10, 2] }, + { domain: `signed number`, keys: [1, -2] }, + { domain: `case-sensitive string`, keys: [`a`, `A`] }, { domain: `mixed`, keys: [10, `2`, 2, `10`] }, ].map(({ domain, keys }) => ({ name, @@ -720,7 +809,10 @@ describe(`ordered source work oracle`, () => { try { await collection.preload() - collection.createIndex((row) => row.rank, { indexType: IndexType }) + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) for (const key of keys) { const transaction = collection.insert({ id: key, @@ -730,13 +822,21 @@ describe(`ordered source work oracle`, () => { await transaction.isPersisted.promise } - const changes = collection.currentStateAsChanges({ - orderBy: orderBy(direction), - })! + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: keys.length, + optimizedOnly: true, + })! - expect(changes.map(({ key }) => key)).toEqual( - [...keys].sort(comparePublicKeys), - ) + expect(changes.map(({ key }) => key)).toEqual( + [...keys].sort(comparePublicKeys), + ) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } } finally { await collection.cleanup() } @@ -749,7 +849,7 @@ describe(`ordered source work oracle`, () => { )) { fcTest.prop( [ - fc.uniqueArray(fc.integer({ min: 0, max: 999 }), { + fc.uniqueArray(fc.integer({ min: -999, max: 999 }), { minLength: 2, maxLength: 8, }), @@ -768,7 +868,9 @@ describe(`ordered source work oracle`, () => { async (keyNumbers, requestedLimit, indexKind, direction, keyDomain) => { const keys: Array = keyDomain === `string` - ? keyNumbers.map((key) => `key-${key}`) + ? keyNumbers.map((key, index) => + index % 2 === 0 ? `key-${key}` : `Key-${key}`, + ) : keyDomain === `number` ? keyNumbers : keyNumbers.flatMap((key) => [String(key), key]) @@ -784,6 +886,7 @@ describe(`ordered source work oracle`, () => { await collection.preload() collection.createIndex((row) => row.rank, { indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, }) for (const key of keys) { const transaction = collection.insert({ @@ -794,14 +897,24 @@ describe(`ordered source work oracle`, () => { await transaction.isPersisted.promise } - const changes = collection.currentStateAsChanges({ - orderBy: orderBy(direction), - limit, - })! - - expect(changes.map(({ key }) => key)).toEqual( - [...keys].sort(comparePublicKeys).slice(0, limit), + const compareEntries = vi.spyOn( + TotalOrder.prototype, + `compareEntries`, ) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit, + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual( + [...keys].sort(comparePublicKeys).slice(0, limit), + ) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } } finally { await collection.cleanup() } @@ -860,67 +973,9 @@ describe(`ordered source work oracle`, () => { } }) - it.each([ - { - name: `the same locale options`, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, - locale: `en`, - localeOptions: { numeric: true, sensitivity: `base` }, - }, - compatible: true, - }, - { - name: `lexical string order`, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `lexical`, - }, - compatible: false, - }, - { - name: `another locale`, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, - locale: `de`, - localeOptions: { numeric: true, sensitivity: `base` }, - }, - compatible: false, - }, - { - name: `another numeric option`, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, - locale: `en`, - localeOptions: { numeric: false, sensitivity: `base` }, - }, - compatible: false, - }, - { - name: `another sensitivity option`, - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, - locale: `en`, - localeOptions: { numeric: true, sensitivity: `accent` }, - }, - compatible: false, - }, - ] satisfies Array<{ - name: string - compareOptions: CompareOptions - compatible: boolean - }>)( - `matches an index against $name: $compatible`, - async ({ compareOptions, compatible }) => { + it.each(orderedStringCompatibilityCases)( + `matches an index against $name in $queryDirection order: $compatible`, + async ({ queryDirection, compareOptions, compatible }) => { type TextRankedRow = Omit & { rank: string } const indexCompareOptions = { direction: `asc`, @@ -953,7 +1008,11 @@ describe(`ordered source work oracle`, () => { }) expect(changes?.map(({ key }) => key)).toEqual( - compatible ? [`item-2`, `item-10`] : undefined, + compatible + ? queryDirection === `asc` + ? [`item-2`, `item-10`] + : [`item-10`, `item-2`] + : undefined, ) } finally { await collection.cleanup() From 05d2f30f116f58069893a76f26968f79428959c4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 12:09:09 -0600 Subject: [PATCH 162/327] fix(db): distrust opaque index ordering --- packages/db/src/collection/change-events.ts | 2 +- packages/db/src/indexes/base-index.ts | 10 +- packages/db/src/indexes/basic-index.ts | 9 +- packages/db/src/query/live/ARCHITECTURE.md | 6 +- packages/db/tests/oracle-config.ts | 1 + .../ordered-work-oracle.property.test.ts | 220 +++++++++++++++--- 6 files changed, 202 insertions(+), 46 deletions(-) diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index 7f0a54332..3a9b971cf 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -363,7 +363,7 @@ function getOrderedKeys( // Find the index const index = findIndexForField(collection, fieldPath, compareOpts) - if (index && index.supports(`gt`)) { + if (index && index.supports(`gt`) && index.supportsRangeOptimization) { // Use index optimization const filterFn = (key: TKey): boolean => { const value = collection.get(key) diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 7cc7a0090..a5126a032 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -70,11 +70,11 @@ export interface IndexInterface< supports: (operation: IndexOperation) => boolean /** - * Whether range lookups (gt/gte/lt/lte) on this index can be trusted to - * return every matching key. Range traversal relies on the index ordering, so - * it is unsafe when the index uses a custom comparator, whose order may not - * match the WHERE evaluator's relational operators. Callers must fall back to - * a full scan when this is `false`. + * Whether range lookups (gt/gte/lt/lte) and ordered traversal on this index + * can be trusted to match query comparison semantics. Both rely on the index + * ordering, so they are unsafe when the index uses a custom comparator whose + * order may not match the WHERE or ORDER BY evaluator. Callers must fall back + * to a full scan when this is `false`. */ get supportsRangeOptimization(): boolean diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 645249137..66a8a3a00 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -230,8 +230,13 @@ export class BasicIndex< } } - // Build sorted array from unique values - this.sortedValues = Array.from(this.valueMap.keys()).sort(this.compareFn) + // Array.sort always moves bare undefined elements to the end without + // consulting the comparator. Wrap values while sorting so null placement + // and comparator-equivalent null/undefined tie classes stay authoritative. + this.sortedValues = Array.from(this.valueMap.keys()) + .map((value) => ({ value })) + .sort((left, right) => this.compareFn(left.value, right.value)) + .map(({ value }) => value) this.updateTimestamp() } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 70d6ee715..e8bec9886 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1289,8 +1289,10 @@ custom comparator, contribute all of their public keys to the same tie class. Reversing an index also reverses its null placement. The optimizer may reuse a reverse index only when the requested direction and null placement describe that reversed order; otherwise it creates a matching index or falls back to a -full `TotalOrder` refinement. Public custom indexes without lazy bucket -iteration keep that full-refinement fallback. +full `TotalOrder` refinement. A built-in index configured with a custom +comparator also keeps the full-refinement fallback: comparison metadata cannot +prove that an opaque comparator has the query's order. Public custom indexes +without lazy bucket iteration keep the same fallback. Runtime reference identity has a different lifetime again. Objects use weak identity, but JavaScript symbols cannot be weak keys. Stable equality for the diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 7f18b5a41..91ac1644f 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -46,6 +46,7 @@ const staticOracleProperties = [ `load-subset.ordered-window`, `load-subset.rejected-waiter`, `ordered-work.forward-prefix`, + `ordered-work.custom-comparator-fallback`, `ordered-work.public-key-suffix`, `ordered-work.reverse-prefix`, `ordered-work.snapshot-reuse`, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 8fb5ceffc..de5340a02 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -113,20 +113,23 @@ function comparePublicKeys( return left < right ? -1 : left > right ? 1 : 0 } -const orderedIndexCompatibilityCases = ([`asc`, `desc`] as const).flatMap( - (indexDirection) => - ([`first`, `last`] as const).flatMap((indexNulls) => - ([`asc`, `desc`] as const).flatMap((queryDirection) => - ([`first`, `last`] as const).map((queryNulls) => ({ - indexDirection, - indexNulls, - queryDirection, - queryNulls, - compatible: - indexDirection === queryDirection - ? indexNulls === queryNulls - : indexNulls !== queryNulls, - })), +const orderedIndexCompatibilityCases = ([`basic`, `btree`] as const).flatMap( + (indexKind) => + ([`asc`, `desc`] as const).flatMap((indexDirection) => + ([`first`, `last`] as const).flatMap((indexNulls) => + ([`asc`, `desc`] as const).flatMap((queryDirection) => + ([`first`, `last`] as const).map((queryNulls) => ({ + indexKind, + indexDirection, + indexNulls, + queryDirection, + queryNulls, + compatible: + indexDirection === queryDirection + ? indexNulls === queryNulls + : indexNulls !== queryNulls, + })), + ), ), ), ) @@ -179,19 +182,22 @@ const stringComparisonVariants = [ compatible: boolean }> -const orderedStringCompatibilityCases = ([`asc`, `desc`] as const).flatMap( - (queryDirection) => - stringComparisonVariants.map(({ name, collation, compatible }) => ({ - name, - queryDirection, - compareOptions: { - ...collation, - direction: queryDirection, - nulls: - queryDirection === `asc` ? (`last` as const) : (`first` as const), - } satisfies CompareOptions, - compatible, - })), +const orderedStringCompatibilityCases = ([`basic`, `btree`] as const).flatMap( + (indexKind) => + ([`asc`, `desc`] as const).flatMap((queryDirection) => + stringComparisonVariants.map(({ name, collation, compatible }) => ({ + name, + indexKind, + queryDirection, + compareOptions: { + ...collation, + direction: queryDirection, + nulls: + queryDirection === `asc` ? (`last` as const) : (`first` as const), + } satisfies CompareOptions, + compatible, + })), + ), ) async function observeOrderedPrefix( @@ -524,8 +530,9 @@ describe(`ordered source work oracle`, () => { }) it.each(orderedIndexCompatibilityCases)( - `matches index $indexDirection/nulls-$indexNulls to query $queryDirection/nulls-$queryNulls: $compatible`, + `matches $indexKind index $indexDirection/nulls-$indexNulls to query $queryDirection/nulls-$queryNulls: $compatible`, async ({ + indexKind, indexDirection, indexNulls, queryDirection, @@ -550,7 +557,7 @@ describe(`ordered source work oracle`, () => { try { await collection.preload() collection.createIndex((row) => row.rank, { - indexType: BTreeIndex, + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, options: { compareOptions: { direction: indexDirection, @@ -669,6 +676,140 @@ describe(`ordered source work oracle`, () => { }, ) + it.each( + ( + [ + { name: `BasicIndex`, IndexType: BasicIndex }, + { name: `BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ name, IndexType }) => + ([`asc`, `desc`] as const).map((direction) => ({ + name, + IndexType, + direction, + })), + ), + )( + `fully refines a $name custom comparator in $direction order`, + async ({ name, IndexType, direction }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-comparator-${name}-${direction}`, + getKey: (row) => row.id, + initialData: [ + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: true }, + { id: `three`, rank: 3, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual( + direction === `asc` ? [`one`, `two`] : [`three`, `two`], + ) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + for (const campaign of orderedWorkCampaigns( + `ordered-work.custom-comparator-fallback`, + 1_780_104, + )) { + fcTest.prop( + [ + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 2, + maxLength: 8, + }), + fc.integer({ min: 1, max: 8 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + fc.constantFrom(`asc`, `desc`), + ], + campaign.options, + )( + `fully refines generated custom comparator indexes (${campaign.label})`, + async (ranks, requestedLimit, indexKind, direction) => { + const rows = ranks.map( + (rank, index): RankedRow => ({ + id: `row-${index.toString().padStart(2, `0`)}`, + rank, + included: true, + }), + ) + const limit = Math.min(requestedLimit, rows.length) + const expectedKeys = [...rows] + .sort((left, right) => { + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, limit) + .map(({ id }) => id) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-comparator-property-${Math.random()}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + + const compareEntries = vi.spyOn( + TotalOrder.prototype, + `compareEntries`, + ) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + } + it(`groups comparator-equivalent values in every built-in index direction`, () => { type TextRow = { id: string; value: string } const rows: Array = [ @@ -786,6 +927,7 @@ describe(`ordered source work oracle`, () => { ([`asc`, `desc`] as const).flatMap((direction) => [ { domain: `signed number`, keys: [1, -2] }, + { domain: `NaN number`, keys: [Number.NaN, 2, -1] }, { domain: `case-sensitive string`, keys: [`a`, `A`] }, { domain: `mixed`, keys: [10, `2`, 2, `10`] }, ].map(({ domain, keys }) => ({ @@ -849,10 +991,16 @@ describe(`ordered source work oracle`, () => { )) { fcTest.prop( [ - fc.uniqueArray(fc.integer({ min: -999, max: 999 }), { - minLength: 2, - maxLength: 8, - }), + fc.uniqueArray( + fc.oneof( + fc.integer({ min: -999, max: 999 }), + fc.constant(Number.NaN), + ), + { + minLength: 2, + maxLength: 8, + }, + ), fc.integer({ min: 1, max: 16 }), fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), fc.constantFrom(`asc`, `desc`), @@ -974,8 +1122,8 @@ describe(`ordered source work oracle`, () => { }) it.each(orderedStringCompatibilityCases)( - `matches an index against $name in $queryDirection order: $compatible`, - async ({ queryDirection, compareOptions, compatible }) => { + `matches a $indexKind index against $name in $queryDirection order: $compatible`, + async ({ indexKind, queryDirection, compareOptions, compatible }) => { type TextRankedRow = Omit & { rank: string } const indexCompareOptions = { direction: `asc`, @@ -998,7 +1146,7 @@ describe(`ordered source work oracle`, () => { try { await collection.preload() collection.createIndex((row) => row.rank, { - indexType: BTreeIndex, + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, options: { compareOptions: indexCompareOptions }, }) From 082803935d5e4e20d6f8fd008d5a47f69b44618d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 12:28:37 -0600 Subject: [PATCH 163/327] test(db): count ordered bucket work --- .../ordered-work-oracle.property.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index de5340a02..a11f2ddc0 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -47,6 +47,8 @@ type PublicKeyRankedRow = Omit & { type OrderedWork = { keys: Array sourceReads: Array + expectedBucketYields: number + bucketYields: number expectedKeyComparisons: number keyComparisons: number totalOrderComparisons: number @@ -229,11 +231,13 @@ async function observeOrderedPrefix( let expectedKeyComparisons = 0 let expectedMatches = 0 + let expectedBucketYields = 0 const expectedBuckets = direction === `asc` ? index.orderedBuckets() : index.orderedBucketsReversed() for (const [, bucket] of expectedBuckets) { + expectedBucketYields++ const orderedKeys = [...bucket] orderedKeys.sort((left, right) => { expectedKeyComparisons++ @@ -245,6 +249,23 @@ async function observeOrderedPrefix( if (expectedMatches >= limit) break } + let bucketYields = 0 + const originalOrderedBuckets = index.orderedBuckets.bind(index) + const originalOrderedBucketsReversed = + index.orderedBucketsReversed.bind(index) + index.orderedBuckets = function* () { + for (const bucket of originalOrderedBuckets()) { + bucketYields++ + yield bucket + } + } + index.orderedBucketsReversed = function* () { + for (const bucket of originalOrderedBucketsReversed()) { + bucketYields++ + yield bucket + } + } + const sourceReads: Array = [] const originalGet = collection.get.bind(collection) collection.get = (key) => { @@ -264,6 +285,8 @@ async function observeOrderedPrefix( return { keys: changes.map(({ key }) => String(key)), sourceReads, + expectedBucketYields, + bucketYields, expectedKeyComparisons, keyComparisons: keyComparisonCounter.count, totalOrderComparisons: compareEntries.mock.calls.length, @@ -379,6 +402,7 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -430,6 +454,7 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -455,6 +480,7 @@ describe(`ordered source work oracle`, () => { `tied-02`, `tied-04`, ]) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }) From 69ba72f297daec3e7fa860137e01e12431064068 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 12:49:58 -0600 Subject: [PATCH 164/327] test(db): observe ordered index work --- .../ordered-work-oracle.property.test.ts | 258 ++++++++++++++++-- 1 file changed, 241 insertions(+), 17 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index a11f2ddc0..5cbaca225 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -47,13 +47,127 @@ type PublicKeyRankedRow = Omit & { type OrderedWork = { keys: Array sourceReads: Array + expectedValueReads: number + valueReads: number + expectedBucketReads: number + bucketReads: number + expectedCursorCalls: number + cursorCalls: number expectedBucketYields: number bucketYields: number + unexpectedTraversalCalls: number expectedKeyComparisons: number keyComparisons: number totalOrderComparisons: number } +type OrderedReadProbe = { + getValueReads: () => number + getBucketReads: () => number + getCursorCalls: () => number + getUnexpectedTraversalCalls: () => number + restore: () => void +} + +function isArrayIndex(property: PropertyKey): boolean { + if (typeof property !== `string` || property.length === 0) return false + const index = Number(property) + return Number.isSafeInteger(index) && index >= 0 && String(index) === property +} + +function observeOrderedIndexReads( + index: BasicIndex | BTreeIndex, + indexKind: `basic` | `btree`, + direction: OrderByDirection, +): OrderedReadProbe { + let valueReads = 0 + let bucketReads = 0 + let cursorCalls = 0 + let unexpectedTraversalCalls = 0 + + if (indexKind === `basic`) { + const internals = index as unknown as { + sortedValues: Array + valueMap: Map> + } + const sortedValues = internals.sortedValues + const valueMap = internals.valueMap + internals.sortedValues = new Proxy(sortedValues, { + get(target, property, receiver) { + if (isArrayIndex(property)) valueReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + internals.valueMap = new Proxy(valueMap, { + get(target, property) { + const member = Reflect.get(target, property, target) as unknown + if (property === `get`) { + return (value: unknown) => { + bucketReads++ + return target.get(value) + } + } + if (typeof member === `function`) { + return (...args: Array) => { + unexpectedTraversalCalls++ + return member.apply(target, args) + } + } + return member + }, + }) + return { + getValueReads: () => valueReads, + getBucketReads: () => bucketReads, + getCursorCalls: () => cursorCalls, + getUnexpectedTraversalCalls: () => unexpectedTraversalCalls, + restore: () => { + internals.sortedValues = sortedValues + internals.valueMap = valueMap + }, + } + } + + const internals = index as unknown as { + orderedEntries: { + nextHigherPair: (key?: unknown) => readonly [unknown, unknown] | undefined + nextLowerPair: (key?: unknown) => readonly [unknown, unknown] | undefined + } + } + const orderedEntries = internals.orderedEntries + const expectedMethod = + direction === `asc` ? `nextHigherPair` : `nextLowerPair` + internals.orderedEntries = new Proxy(orderedEntries, { + get(target, property) { + if (property !== expectedMethod) unexpectedTraversalCalls++ + const member = Reflect.get(target, property, target) as unknown + if (typeof member !== `function`) return member + return (...args: Array) => { + if (property === expectedMethod) cursorCalls++ + const result = member.apply(target, args) as + | readonly [unknown, unknown] + | undefined + if (property === `nextHigherPair` || property === `nextLowerPair`) { + if (result !== undefined) { + valueReads++ + bucketReads++ + } + } + return result + } + }, + }) + return { + getValueReads: () => valueReads, + getBucketReads: () => bucketReads, + getCursorCalls: () => cursorCalls, + getUnexpectedTraversalCalls: () => unexpectedTraversalCalls, + restore: () => { + internals.orderedEntries = orderedEntries + }, + } +} + function orderedWorkCampaigns(property: string, fixedSeed: number) { return [ { @@ -232,23 +346,39 @@ async function observeOrderedPrefix( let expectedKeyComparisons = 0 let expectedMatches = 0 let expectedBucketYields = 0 - const expectedBuckets = - direction === `asc` - ? index.orderedBuckets() - : index.orderedBucketsReversed() - for (const [, bucket] of expectedBuckets) { - expectedBucketYields++ - const orderedKeys = [...bucket] - orderedKeys.sort((left, right) => { - expectedKeyComparisons++ - return left < right ? -1 : left > right ? 1 : 0 - }) - expectedMatches += orderedKeys.filter( - (key) => rows.find((row) => row.id === key)?.included === true, - ).length - if (expectedMatches >= limit) break + if (limit > 0) { + const expectedBuckets = + direction === `asc` + ? index.orderedBuckets() + : index.orderedBucketsReversed() + for (const [, bucket] of expectedBuckets) { + expectedBucketYields++ + const orderedKeys = [...bucket] + orderedKeys.sort((left, right) => { + expectedKeyComparisons++ + return left < right ? -1 : left > right ? 1 : 0 + }) + expectedMatches += orderedKeys.filter( + (key) => rows.find((row) => row.id === key)?.included === true, + ).length + if (expectedMatches >= limit) break + } } + // Observe private value traversal and bucket construction independently + // from public generator yields. A generator can materialize all private + // values or groups before yielding only the requested prefix. + const readProbe = observeOrderedIndexReads(index, indexKind, direction) + const distinctValueCount = new Set(rows.map(({ rank }) => rank)).size + const expectedValueReads = + indexKind === `btree` || limit === 0 + ? expectedBucketYields + : Math.min( + distinctValueCount, + expectedBucketYields + + (expectedBucketYields < distinctValueCount ? 1 : 0), + ) + let bucketYields = 0 const originalOrderedBuckets = index.orderedBuckets.bind(index) const originalOrderedBucketsReversed = @@ -285,14 +415,22 @@ async function observeOrderedPrefix( return { keys: changes.map(({ key }) => String(key)), sourceReads, + expectedValueReads, + valueReads: readProbe.getValueReads(), + expectedBucketReads: expectedBucketYields, + bucketReads: readProbe.getBucketReads(), + expectedCursorCalls: indexKind === `btree` ? expectedBucketYields : 0, + cursorCalls: readProbe.getCursorCalls(), expectedBucketYields, bucketYields, + unexpectedTraversalCalls: readProbe.getUnexpectedTraversalCalls(), expectedKeyComparisons, keyComparisons: keyComparisonCounter.count, totalOrderComparisons: compareEntries.mock.calls.length, } } finally { compareEntries.mockRestore() + readProbe.restore() } } finally { await collection.cleanup() @@ -369,7 +507,8 @@ function createOrderedPrefixRows( expectedKeys, // Every row through the boundary bucket is tested once. The selected rows // are then read once more to materialize their change messages. - expectedSourceReads: [...expectedCandidateReads, ...expectedKeys], + expectedSourceReads: + options.limit === 0 ? [] : [...expectedCandidateReads, ...expectedKeys], } } @@ -402,7 +541,11 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -418,7 +561,7 @@ describe(`ordered source work oracle`, () => { fcTest.prop( [ fc.integer({ min: 0, max: 8 }), - fc.integer({ min: 1, max: 5 }), + fc.integer({ min: 0, max: 5 }), fc.integer({ min: 0, max: 5 }), fc.integer({ min: 0, max: 8 }), fc.integer({ min: 0, max: 60 }), @@ -454,7 +597,11 @@ describe(`ordered source work oracle`, () => { expect(observed.keys).toEqual(scenario.expectedKeys) expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }, @@ -480,11 +627,73 @@ describe(`ordered source work oracle`, () => { `tied-02`, `tied-04`, ]) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) expect(observed.totalOrderComparisons).toBe(0) }) + it.each([ + { direction: `asc`, indexNulls: `first` }, + { direction: `desc`, indexNulls: `last` }, + ] as const)( + `stops Basic $direction traversal after a multi-value nullish tie`, + async ({ direction, indexNulls }) => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-basic-nullish-${direction}`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: true }, + ], + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BasicIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: indexNulls, + stringSort: `locale`, + }, + }, + }) as BasicIndex + const readProbe = observeOrderedIndexReads(index, `basic`, direction) + + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction, `first`), + limit: 1, + })! + + expect(changes.map(({ key }) => key)).toEqual([`null`]) + // The two exact nullish values form one comparator bucket. Basic + // reads one worse value to close that group, but it must not scan the + // second worse value or construct either worse bucket. + expect(readProbe.getValueReads()).toBe(3) + expect(readProbe.getBucketReads()).toBe(2) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + } finally { + readProbe.restore() + } + } finally { + await collection.cleanup() + } + }, + ) + it(`keeps comparator-equivalent BTree values in one ordered tie class`, async () => { type NullableRankedRow = Omit & { rank: number | null | undefined @@ -618,6 +827,7 @@ describe(`ordered source work oracle`, () => { const laterRank = direction === `asc` ? 2 : 1 const rows: Array = [ { id: `later`, rank: laterRank, included: true }, + { id: `tie-c`, rank: tieRank, included: true }, { id: `tie-b`, rank: tieRank, included: true }, { id: `tie-a`, rank: tieRank, included: true }, ] @@ -641,6 +851,7 @@ describe(`ordered source work oracle`, () => { }, }, }) as BTreeIndex + const requestedCounts: Array = [] const customIndex = new Proxy(index, { get(target, property) { if ( @@ -654,6 +865,7 @@ describe(`ordered source work oracle`, () => { ...(direction === `desc` ? [[laterRank, new Set([`later`])]] : []), + [tieRank, new Set([`tie-c`])], [tieRank, new Set([`tie-b`])], [tieRank, new Set([`tie-a`])], ...(direction === `asc` @@ -666,6 +878,7 @@ describe(`ordered source work oracle`, () => { ...(direction === `asc` ? [[laterRank, new Set([`later`])]] : []), + [tieRank, new Set([`tie-c`])], [tieRank, new Set([`tie-b`])], [tieRank, new Set([`tie-a`])], ...(direction === `desc` @@ -674,6 +887,16 @@ describe(`ordered source work oracle`, () => { ] } const value = Reflect.get(target, property, target) as unknown + if ( + typeof value === `function` && + (property === `takeFromStart` || + property === `takeReversedFromEnd`) + ) { + return (count: number, ...args: Array) => { + requestedCounts.push(count) + return value.apply(target, [count, ...args]) + } + } return typeof value === `function` ? value.bind(target) : value }, }) @@ -692,6 +915,7 @@ describe(`ordered source work oracle`, () => { })! expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) + expect(requestedCounts).toEqual([index.keyCount]) expect(compareEntries).toHaveBeenCalled() } finally { compareEntries.mockRestore() From 47fac4ac18187d42ae2ad2a67fbfd9fe22d46138 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 13:03:03 -0600 Subject: [PATCH 165/327] fix(db): skip empty ordered source setup --- packages/db/src/collection/change-events.ts | 7 +- packages/db/src/query/live/ARCHITECTURE.md | 24 ++-- packages/db/tests/oracle-config.ts | 2 + .../ordered-work-oracle.property.test.ts | 111 +++++++++++++++++- 4 files changed, 131 insertions(+), 13 deletions(-) diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index 3a9b971cf..9d23ebaf6 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -103,6 +103,12 @@ export function currentStateAsChanges< throw new Error(`limit cannot be used without orderBy`) } + // An empty ordered window has no source work. Return before compiling its + // predicate or finding, creating, and traversing an order index. + if (options.limit === 0) { + return [] + } + // First check if orderBy is present (optionally with limit) if (options.orderBy) { // Create where filter function if present @@ -401,7 +407,6 @@ function getOrderedKeys( // public-key suffix remains ascending in both directions. Stop after // the first complete bucket that proves the requested prefix because // filtering can otherwise select the wrong key from a boundary tie. - if (limit === 0) return [] const keys: Array = [] for (const [, bucket] of orderedBuckets) { const matchingKeys = [...bucket].sort(compareKeys).filter(filterFn) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index e8bec9886..07c65e4f2 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -697,17 +697,19 @@ in-flight Promise guard owns the request until settlement. An ordered window with an active limit of zero creates no ordered transport demand. Its coordinator remains alive so a later window change can load from the same order, but neither the initial offset nor a result deficit may turn -the empty window into a positive request. The dedupe helper applies the same -law when adapters call it directly: a zero-width request establishes no -coverage and owns no physical acquisition. This also holds when no usable -order index exists: core defers the full-snapshot fallback until the window -first becomes positive. One successful or pending fallback covers that -subscription session. A synchronous throw or rejected fallback clears only -that subscription's guard, so the same live query can retry. Cleanup creates a -new subscription and a late settlement from the old one cannot clear the new -guard. Truncate replay belongs to the subscription's retained demand; the live -coordinator must not add a second fallback while that replay is in flight. A -replay that starts after an earlier rejection reclaims the same subscription +the empty window into a positive request. Its local source snapshot also +returns before predicate compilation, source enumeration, sorting, or index +creation. The dedupe helper applies the same law when adapters call it +directly: a zero-width request establishes no coverage and owns no physical +acquisition. This also holds when no usable order index exists: core defers the +full-snapshot fallback until the window first becomes positive. One successful +or pending fallback covers that subscription session. A synchronous throw or +rejected fallback clears only that subscription's guard, so the same live query +can retry. Cleanup creates a new subscription and a late settlement from the +old one cannot clear the new guard. Truncate replay belongs to the +subscription's retained demand; the live coordinator must not add a second +fallback while that replay is in flight. A replay that starts after an earlier +rejection reclaims the same subscription guard. Success keeps it claimed, so replacement publication cannot schedule a duplicate full-source fallback; rejection releases it for a later retry. Cleanup also aborts and settles the subscription-visible acquisition before a diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 91ac1644f..0402c2480 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -45,9 +45,11 @@ const staticOracleProperties = [ `load-subset.distinct-window-predicate`, `load-subset.ordered-window`, `load-subset.rejected-waiter`, + `ordered-work.forward-exhaustion`, `ordered-work.forward-prefix`, `ordered-work.custom-comparator-fallback`, `ordered-work.public-key-suffix`, + `ordered-work.reverse-exhaustion`, `ordered-work.reverse-prefix`, `ordered-work.snapshot-reuse`, `pagination.async-cursor`, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 5cbaca225..c4f607e3b 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -419,7 +419,10 @@ async function observeOrderedPrefix( valueReads: readProbe.getValueReads(), expectedBucketReads: expectedBucketYields, bucketReads: readProbe.getBucketReads(), - expectedCursorCalls: indexKind === `btree` ? expectedBucketYields : 0, + expectedCursorCalls: + indexKind === `btree` + ? expectedBucketYields + Number(expectedMatches < limit) + : 0, cursorCalls: readProbe.getCursorCalls(), expectedBucketYields, bucketYields, @@ -513,6 +516,57 @@ function createOrderedPrefixRows( } describe(`ordered source work oracle`, () => { + it.each([`off`, `eager`] as const)( + `does no setup work for an empty ordered window with auto-indexing %s`, + async (autoIndex) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-empty-${autoIndex}`, + getKey: (row) => row.id, + initialData: [ + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: false }, + { id: `three`, rank: 3, included: true }, + ], + autoIndex, + ...(autoIndex === `eager` && { defaultIndexType: BTreeIndex }), + }), + ) + + try { + await collection.preload() + let whereExpressionReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + whereExpressionReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const entries = vi.spyOn(collection, `entries`) + const get = vi.spyOn(collection, `get`) + const createIndex = vi.spyOn(collection, `createIndex`) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + const indexesBefore = collection.indexes.size + + const changes = collection.currentStateAsChanges({ + where, + orderBy: orderBy(`asc`, `last`), + limit: 0, + }) + + expect(changes).toEqual([]) + expect(whereExpressionReads).toBe(0) + expect(entries).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() + expect(createIndex).not.toHaveBeenCalled() + expect(collection.indexes.size).toBe(indexesBefore) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + await collection.cleanup() + } + }, + ) + it.each([ { indexKind: `basic`, direction: `asc` }, { indexKind: `basic`, direction: `desc` }, @@ -609,6 +663,61 @@ describe(`ordered source work oracle`, () => { } } + for (const direction of [`asc`, `desc`] as const) { + const property = + direction === `asc` + ? `ordered-work.forward-exhaustion` + : `ordered-work.reverse-exhaustion` + const seed = direction === `asc` ? 1_780_105 : 1_780_106 + for (const campaign of orderedWorkCampaigns(property, seed)) { + fcTest.prop( + [ + fc.integer({ min: 0, max: 60 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + ], + campaign.options, + )( + `reads each ${direction} bucket once before proving exhaustion (${campaign.label})`, + async (rowCount, indexKind) => { + const rows = Array.from( + { length: rowCount }, + (_, index): RankedRow => ({ + id: `rejected-${index.toString().padStart(2, `0`)}`, + rank: Math.floor(index / 2), + included: false, + }), + ).reverse() + const expectedSourceReads = [...rows] + .sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) + + const observed = await observeOrderedPrefix( + rows, + 1, + indexKind, + direction, + ) + + expect(observed.keys).toEqual([]) + expect(observed.sourceReads).toEqual(expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + } + } + it(`reads the complete tied boundary when every candidate is tied`, async () => { const rows = Array.from( { length: 25 }, From 37e111c67accf4fc3fd2955296e83fc9f8f1bc89 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 13:16:52 -0600 Subject: [PATCH 166/327] test(db): count custom index source reads --- .../tests/query/ordered-work-oracle.property.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index c4f607e3b..d0c107fdd 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1016,15 +1016,26 @@ describe(`ordered source work oracle`, () => { ).toBe(false) } + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(String(key)) + return originalGet(key) + } const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { const changes = collection.currentStateAsChanges({ orderBy: orderBy(direction, direction === `asc` ? `last` : `first`), limit: 1, })! + const indexReads = + direction === `asc` + ? [`tie-a`, `tie-b`, `tie-c`, `later`] + : [`tie-c`, `tie-b`, `tie-a`, `later`] expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) expect(requestedCounts).toEqual([index.keyCount]) + expect(sourceReads).toEqual([...indexReads, ...indexReads, `tie-a`]) expect(compareEntries).toHaveBeenCalled() } finally { compareEntries.mockRestore() From 1a2ea4b5a2f21dadf1a04a4cc19a40696c02dc0b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 13:36:08 -0600 Subject: [PATCH 167/327] test(db): model custom index fallback work --- .../ordered-work-oracle.property.test.ts | 136 ++++++++++-------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index d0c107fdd..3a1f5228a 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -929,20 +929,51 @@ describe(`ordered source work oracle`, () => { }, ) - it.each([`asc`, `desc`] as const)( - `keeps custom indexes on the materialized %s-order fallback`, - async (direction) => { - const tieRank = direction === `asc` ? 1 : 2 - const laterRank = direction === `asc` ? 2 : 1 - const rows: Array = [ - { id: `later`, rank: laterRank, included: true }, - { id: `tie-c`, rank: tieRank, included: true }, - { id: `tie-b`, rank: tieRank, included: true }, - { id: `tie-a`, rank: tieRank, included: true }, + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + domain: `signed number`, + tieKeys: [1, -2], + rejectedKey: -999, + laterKey: 999, + }, + { + domain: `NaN number`, + tieKeys: [Number.NaN, 2, -1], + rejectedKey: -999, + laterKey: 999, + }, + { + domain: `case-sensitive string`, + tieKeys: [`a`, `A`], + rejectedKey: `rejected`, + laterKey: `later`, + }, + { + domain: `mixed`, + tieKeys: [10, `2`, 2, `10`], + rejectedKey: `rejected`, + laterKey: `later`, + }, + ].map((keyCase) => ({ direction, ...keyCase })), + ), + )( + `fully refines a filtered $domain custom-index fallback in $direction order`, + async ({ direction, domain, tieKeys, rejectedKey, laterKey }) => { + const tieRank = 1 + const rejectedRank = direction === `asc` ? 0 : 2 + const laterRank = direction === `asc` ? 2 : 0 + const rows: Array = [ + { id: laterKey, rank: laterRank, included: true }, + ...tieKeys + .map((id) => ({ id, rank: tieRank, included: true })) + .reverse(), + { id: rejectedKey, rank: rejectedRank, included: false }, ] const collection = createCollection( - localOnlyCollectionOptions({ - id: `ordered-work-custom-index-fallback-${direction}`, + localOnlyCollectionOptions({ + id: `ordered-work-custom-index-fallback-${direction}-${domain}`, getKey: (row) => row.id, initialData: rows, }), @@ -952,14 +983,8 @@ describe(`ordered source work oracle`, () => { await collection.preload() const index = collection.createIndex((row) => row.rank, { indexType: BTreeIndex, - options: { - compareOptions: { - direction: `asc`, - nulls: `last`, - stringSort: `locale`, - }, - }, - }) as BTreeIndex + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex const requestedCounts: Array = [] const customIndex = new Proxy(index, { get(target, property) { @@ -969,32 +994,6 @@ describe(`ordered source work oracle`, () => { ) { return undefined } - if (property === `orderedEntriesArray`) { - return [ - ...(direction === `desc` - ? [[laterRank, new Set([`later`])]] - : []), - [tieRank, new Set([`tie-c`])], - [tieRank, new Set([`tie-b`])], - [tieRank, new Set([`tie-a`])], - ...(direction === `asc` - ? [[laterRank, new Set([`later`])]] - : []), - ] - } - if (property === `orderedEntriesArrayReversed`) { - return [ - ...(direction === `asc` - ? [[laterRank, new Set([`later`])]] - : []), - [tieRank, new Set([`tie-c`])], - [tieRank, new Set([`tie-b`])], - [tieRank, new Set([`tie-a`])], - ...(direction === `desc` - ? [[laterRank, new Set([`later`])]] - : []), - ] - } const value = Reflect.get(target, property, target) as unknown if ( typeof value === `function` && @@ -1016,27 +1015,50 @@ describe(`ordered source work oracle`, () => { ).toBe(false) } - const sourceReads: Array = [] + const orderedTieKeys = [...tieKeys].sort(comparePublicKeys) + const indexTieKeys = + direction === `asc` ? orderedTieKeys : [...orderedTieKeys].reverse() + const indexScanKeys = [rejectedKey, ...indexTieKeys, laterKey] + const matchingIndexKeys = [...indexTieKeys, laterKey] + const rowsByKey = new Map(rows.map((row) => [row.id, row])) + let expectedTotalOrderComparisons = 0 + const expectedKeys = [...matchingIndexKeys] + .sort((left, right) => { + expectedTotalOrderComparisons++ + const leftRow = rowsByKey.get(left)! + const rightRow = rowsByKey.get(right)! + const rankOrder = leftRow.rank - rightRow.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left, right) + }) + .slice(0, 2) + + const sourceReads: Array = [] const originalGet = collection.get.bind(collection) collection.get = (key) => { - sourceReads.push(String(key)) + sourceReads.push(key) return originalGet(key) } const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { const changes = collection.currentStateAsChanges({ - orderBy: orderBy(direction, direction === `asc` ? `last` : `first`), - limit: 1, + where: eq(new PropRef([`included`]), true), + orderBy: publicKeyOrderBy(direction), + limit: 2, })! - const indexReads = - direction === `asc` - ? [`tie-a`, `tie-b`, `tie-c`, `later`] - : [`tie-c`, `tie-b`, `tie-a`, `later`] - expect(changes.map(({ key }) => key)).toEqual([`tie-a`]) + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) expect(requestedCounts).toEqual([index.keyCount]) - expect(sourceReads).toEqual([...indexReads, ...indexReads, `tie-a`]) - expect(compareEntries).toHaveBeenCalled() + expect(sourceReads).toEqual([ + ...indexScanKeys, + ...matchingIndexKeys, + ...expectedKeys, + ]) + expect(compareEntries).toHaveBeenCalledTimes( + expectedTotalOrderComparisons, + ) } finally { compareEntries.mockRestore() } From 403dba5e9f45419220c49d002df48abee244de52 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 14:05:02 -0600 Subject: [PATCH 168/327] test(db): close ordered work oracle gaps --- .../ordered-work-oracle.property.test.ts | 323 ++++++++++++++++++ 1 file changed, 323 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 3a1f5228a..098477459 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -950,6 +950,12 @@ describe(`ordered source work oracle`, () => { rejectedKey: `rejected`, laterKey: `later`, }, + { + domain: `non-ASCII string`, + tieKeys: [`é`, `e`, `Ω`, `ß`], + rejectedKey: `rejected`, + laterKey: `later`, + }, { domain: `mixed`, tieKeys: [10, `2`, 2, `10`], @@ -1068,6 +1074,241 @@ describe(`ordered source work oracle`, () => { }, ) + it.each( + ( + [ + { source: `no index`, IndexType: undefined }, + { source: `opaque BasicIndex`, IndexType: BasicIndex }, + { source: `opaque BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ source, IndexType }) => + ([`asc`, `desc`] as const).map((direction) => ({ + source, + IndexType, + direction, + })), + ), + )( + `does exact one-pass work for the $source fallback in $direction order`, + async ({ source, IndexType, direction }) => { + const rows: Array = [ + { + id: `later`, + rank: direction === `asc` ? 2 : 0, + included: true, + }, + { id: `é`, rank: 1, included: true }, + { id: `e`, rank: 1, included: true }, + { + id: `rejected`, + rank: direction === `asc` ? 0 : 2, + included: false, + }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-full-fallback-${source}-${direction}`, + getKey: (row) => row.id, + initialData: rows, + autoIndex: `off`, + }), + ) + + try { + await collection.preload() + if (IndexType) { + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + } + + let referenceCompilationReads = 0 + const referenceWhere = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + compileSingleRowExpression(referenceWhere) + + let compilationReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) compilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const expectedEntries = [...collection.entries()] + const enumeratedKeys: Array = [] + const originalEntries = collection.entries.bind(collection) + collection.entries = function* () { + for (const entry of originalEntries()) { + enumeratedKeys.push(entry[0]) + yield entry + } + } + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } + + let expectedTotalOrderComparisons = 0 + const expectedKeys = expectedEntries + .map(([, row]) => row) + .filter(({ included }) => included) + .sort((left, right) => { + expectedTotalOrderComparisons++ + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, 2) + .map(({ id }) => id) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where, + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(enumeratedKeys).toEqual(expectedEntries.map(([key]) => key)) + expect(sourceReads).toEqual([ + ...expectedEntries.map(([key]) => key), + ...expectedKeys, + ]) + expect(compilationReads).toBe(referenceCompilationReads) + expect(compareEntries).toHaveBeenCalledTimes( + expectedTotalOrderComparisons, + ) + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + capability: `neither iterator`, + exposeForward: false, + exposeReverse: false, + }, + { + capability: `the forward iterator only`, + exposeForward: true, + exposeReverse: false, + }, + { + capability: `the reverse iterator only`, + exposeForward: false, + exposeReverse: true, + }, + { + capability: `both iterators`, + exposeForward: true, + exposeReverse: true, + }, + ].map((capabilities) => ({ direction, ...capabilities })), + ), + )( + `trusts $capability for a custom index only when it serves $direction order`, + async ({ direction, exposeForward, exposeReverse }) => { + const rows: Array = [ + { + id: `later`, + rank: direction === `asc` ? 2 : 0, + included: true, + }, + { id: `tie-b`, rank: 1, included: true }, + { id: `tie-a`, rank: 1, included: true }, + { + id: `rejected`, + rank: direction === `asc` ? 0 : 2, + included: false, + }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-capabilities-${direction}-${exposeForward}-${exposeReverse}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + const customIndex = new Proxy(index, { + get(target, property) { + if (property === `orderedBuckets` && !exposeForward) { + return undefined + } + if (property === `orderedBucketsReversed` && !exposeReverse) { + return undefined + } + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) + collection.indexes.set(index.id, customIndex) + + const expectedKeys = rows + .filter(({ included }) => included) + .sort((left, right) => { + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, 2) + .map(({ id }) => id) + const usesLazyBuckets = + exposeForward && (direction === `asc` || exposeReverse) + expect( + new ReverseIndex(customIndex).supportsOrderedBucketIteration, + ).toBe(exposeForward && exposeReverse) + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + if (usesLazyBuckets) { + expect(compareEntries).not.toHaveBeenCalled() + } else { + expect(compareEntries).toHaveBeenCalled() + } + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + it.each( ( [ @@ -1321,6 +1562,7 @@ describe(`ordered source work oracle`, () => { { domain: `signed number`, keys: [1, -2] }, { domain: `NaN number`, keys: [Number.NaN, 2, -1] }, { domain: `case-sensitive string`, keys: [`a`, `A`] }, + { domain: `non-ASCII string`, keys: [`é`, `e`, `Ω`, `ß`] }, { domain: `mixed`, keys: [10, `2`, 2, `10`] }, ].map(({ domain, keys }) => ({ name, @@ -1657,6 +1899,87 @@ it(`reuses one ordered source snapshot until the collection revision changes`, ( expect(fixture.snapshotRevisions).toEqual([0, 1]) }) +it(`does no source or ordering work when reusing an unbounded snapshot`, async () => { + const rows: Array = [`é`, `e`, `Ω`, `ß`, `A`].map((id) => ({ + id, + rank: 1, + included: true, + })) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-unbounded-snapshot-reuse`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + const [bucket] = [...index.orderedBuckets()] + const bucketKeys = [...bucket![1]] + let expectedKeyComparisons = 0 + const expectedKeys = bucketKeys.sort((left, right) => { + expectedKeyComparisons++ + return comparePublicKeys(left, right) + }) + + const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + const window = new WindowState( + collection, + publicKeyOrderBy(`asc`), + undefined, + 3, + ) + window.recordInitialCoverage(undefined, true) + + try { + keyComparisonCounter.count = 0 + expect(observeWindow(window)).toMatchObject({ + publication: expectedKeys.slice(0, 3), + }) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(readProbe.getValueReads()).toBe(1) + expect(readProbe.getBucketReads()).toBe(1) + expect(readProbe.getCursorCalls()).toBe(2) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) + expect(compareEntries).not.toHaveBeenCalled() + + const firstReadCount = sourceReads.length + const firstValueReads = readProbe.getValueReads() + const firstBucketReads = readProbe.getBucketReads() + const firstCursorCalls = readProbe.getCursorCalls() + const firstKeyComparisons = keyComparisonCounter.count + + expect(observeWindow(window)).toMatchObject({ + publication: expectedKeys.slice(0, 3), + }) + expect(sourceReads).toHaveLength(firstReadCount) + expect(readProbe.getValueReads()).toBe(firstValueReads) + expect(readProbe.getBucketReads()).toBe(firstBucketReads) + expect(readProbe.getCursorCalls()).toBe(firstCursorCalls) + expect(keyComparisonCounter.count).toBe(firstKeyComparisons) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + readProbe.restore() + } + } finally { + await collection.cleanup() + } +}) + it(`compiles the ordered predicate once for the lifetime of a window`, () => { const fixture = createSnapshotFixture([ { id: `a`, rank: 1, included: true }, From b65fead3dc261da66be474ce47dd59cffaaab957 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 14:34:51 -0600 Subject: [PATCH 169/327] test(db): complete ordered work budgets --- .../ordered-work-oracle.property.test.ts | 272 +++++++++++++++++- 1 file changed, 260 insertions(+), 12 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 098477459..cadff9d0e 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -318,7 +318,7 @@ const orderedStringCompatibilityCases = ([`basic`, `btree`] as const).flatMap( async function observeOrderedPrefix( rows: ReadonlyArray, - limit: number, + limit: number | undefined, indexKind: `basic` | `btree` = `btree`, direction: OrderByDirection = `desc`, ): Promise { @@ -346,7 +346,7 @@ async function observeOrderedPrefix( let expectedKeyComparisons = 0 let expectedMatches = 0 let expectedBucketYields = 0 - if (limit > 0) { + if (limit === undefined || limit > 0) { const expectedBuckets = direction === `asc` ? index.orderedBuckets() @@ -361,7 +361,7 @@ async function observeOrderedPrefix( expectedMatches += orderedKeys.filter( (key) => rows.find((row) => row.id === key)?.included === true, ).length - if (expectedMatches >= limit) break + if (limit !== undefined && expectedMatches >= limit) break } } @@ -373,11 +373,13 @@ async function observeOrderedPrefix( const expectedValueReads = indexKind === `btree` || limit === 0 ? expectedBucketYields - : Math.min( - distinctValueCount, - expectedBucketYields + - (expectedBucketYields < distinctValueCount ? 1 : 0), - ) + : limit === undefined + ? distinctValueCount + : Math.min( + distinctValueCount, + expectedBucketYields + + (expectedBucketYields < distinctValueCount ? 1 : 0), + ) let bucketYields = 0 const originalOrderedBuckets = index.orderedBuckets.bind(index) @@ -421,7 +423,8 @@ async function observeOrderedPrefix( bucketReads: readProbe.getBucketReads(), expectedCursorCalls: indexKind === `btree` - ? expectedBucketYields + Number(expectedMatches < limit) + ? expectedBucketYields + + Number(limit === undefined || expectedMatches < limit) : 0, cursorCalls: readProbe.getCursorCalls(), expectedBucketYields, @@ -745,6 +748,68 @@ describe(`ordered source work oracle`, () => { expect(observed.totalOrderComparisons).toBe(0) }) + it.each( + ([`basic`, `btree`] as const).flatMap((indexKind) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`one tie bucket`, `many buckets`] as const).map((bucketShape) => ({ + indexKind, + direction, + bucketShape, + })), + ), + ), + )( + `does exact unbounded work for $indexKind $direction order with $bucketShape`, + async ({ indexKind, direction, bucketShape }) => { + const rows: Array = + bucketShape === `one tie bucket` + ? [ + { id: `d`, rank: 1, included: true }, + { id: `b`, rank: 1, included: false }, + { id: `c`, rank: 1, included: true }, + { id: `a`, rank: 1, included: true }, + ] + : [ + { id: `d`, rank: 3, included: true }, + { id: `b`, rank: 1, included: false }, + { id: `e`, rank: 3, included: false }, + { id: `c`, rank: 2, included: true }, + { id: `a`, rank: 1, included: true }, + ] + const orderedRows = [...rows].sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + const expectedKeys = orderedRows + .filter(({ included }) => included) + .map(({ id }) => id) + const expectedSourceReads = [ + ...orderedRows.map(({ id }) => id), + ...expectedKeys, + ] + + const observed = await observeOrderedPrefix( + rows, + undefined, + indexKind, + direction, + ) + + expect(observed.keys).toEqual(expectedKeys) + expect(observed.sourceReads).toEqual(expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + it.each([ { direction: `asc`, indexNulls: `first` }, { direction: `desc`, indexNulls: `last` }, @@ -1199,6 +1264,114 @@ describe(`ordered source work oracle`, () => { }, ) + it(`does exact short-circuit work for a multi-term TotalOrder fallback`, async () => { + type MultiTermRow = RankedRow & { secondary: number } + const specs: Array = [ + { id: `d`, rank: 2, secondary: 1, included: true }, + { id: `b`, rank: 1, secondary: 2, included: true }, + { id: `a`, rank: 1, secondary: 2, included: true }, + { id: `c`, rank: 1, secondary: 1, included: true }, + { id: `hidden`, rank: 0, secondary: 0, included: false }, + ] + const reads = { rank: 0, secondary: 0, included: 0 } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-multi-term-fallback`, + getKey: (row) => row.id, + initialData: specs, + autoIndex: `off`, + }), + ) + + try { + await collection.preload() + const originalEntries = collection.entries.bind(collection) + const storedRows = [...originalEntries()].map(([, value]) => value) + collection.entries = function* () { + for (const [key, value] of originalEntries()) { + yield [ + key, + new Proxy(value, { + get(target, property, receiver) { + if (property === `rank`) reads.rank++ + if (property === `secondary`) reads.secondary++ + if (property === `included`) reads.included++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + ] as const + } + } + reads.rank = 0 + reads.secondary = 0 + reads.included = 0 + + let referenceCompilationReads = 0 + compileSingleRowExpression( + new Proxy(new PropRef([`rank`]), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + ) + expect(referenceCompilationReads).toBeGreaterThan(0) + + let termCompilationReads = 0 + const trackedTerm = (propertyName: `rank` | `secondary`) => + new Proxy(new PropRef([propertyName]), { + get(target, property, receiver) { + if (property === `type`) termCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const order: OrderBy = [ + { + expression: trackedTerm(`rank`), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + { + expression: trackedTerm(`secondary`), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ] + + let expectedComparisons = 0 + let expectedRankReads = 0 + let expectedSecondaryReads = 0 + const expectedKeys = storedRows + .filter(({ included }) => included) + .sort((left, right) => { + expectedComparisons++ + expectedRankReads += 2 + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) return rankOrder + expectedSecondaryReads += 2 + const secondaryOrder = left.secondary - right.secondary + return secondaryOrder || comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: order, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(termCompilationReads).toBe(referenceCompilationReads * 2) + expect(reads.included).toBe(specs.length) + expect(reads.rank).toBe(expectedRankReads) + expect(reads.secondary).toBe(expectedSecondaryReads) + expect(compareEntries).toHaveBeenCalledTimes(expectedComparisons) + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }) + it.each( ([`asc`, `desc`] as const).flatMap((direction) => [ @@ -1899,7 +2072,8 @@ it(`reuses one ordered source snapshot until the collection revision changes`, ( expect(fixture.snapshotRevisions).toEqual([0, 1]) }) -it(`does no source or ordering work when reusing an unbounded snapshot`, async () => { +it(`does only exact predicate and boundary work when reusing an unbounded snapshot`, async () => { + const reads = { rank: 0, included: 0 } const rows: Array = [`é`, `e`, `Ω`, `ß`, `A`].map((id) => ({ id, rank: 1, @@ -1930,21 +2104,40 @@ it(`does no source or ordering work when reusing an unbounded snapshot`, async ( const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) const sourceReads: Array = [] const originalGet = collection.get.bind(collection) + const observedValues = new Map< + Parameters[0], + NonNullable> + >() collection.get = (key) => { sourceReads.push(key) - return originalGet(key) + const value = originalGet(key) + if (value === undefined) return + let observed = observedValues.get(key) + if (observed === undefined) { + observed = new Proxy(value, { + get(target, property, receiver) { + if (property === `rank`) reads.rank++ + if (property === `included`) reads.included++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + observedValues.set(key, observed) + } + return observed } const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) const window = new WindowState( collection, publicKeyOrderBy(`asc`), - undefined, + eq(new PropRef([`included`]), true), 3, ) window.recordInitialCoverage(undefined, true) try { keyComparisonCounter.count = 0 + reads.rank = 0 + reads.included = 0 expect(observeWindow(window)).toMatchObject({ publication: expectedKeys.slice(0, 3), }) @@ -1955,12 +2148,16 @@ it(`does no source or ordering work when reusing an unbounded snapshot`, async ( expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) expect(compareEntries).not.toHaveBeenCalled() + expect(reads.included).toBe(rows.length * 5) + expect(reads.rank).toBe(3) const firstReadCount = sourceReads.length const firstValueReads = readProbe.getValueReads() const firstBucketReads = readProbe.getBucketReads() const firstCursorCalls = readProbe.getCursorCalls() const firstKeyComparisons = keyComparisonCounter.count + const firstPredicateReads = reads.included + const firstOrderTermReads = reads.rank expect(observeWindow(window)).toMatchObject({ publication: expectedKeys.slice(0, 3), @@ -1971,6 +2168,8 @@ it(`does no source or ordering work when reusing an unbounded snapshot`, async ( expect(readProbe.getCursorCalls()).toBe(firstCursorCalls) expect(keyComparisonCounter.count).toBe(firstKeyComparisons) expect(compareEntries).not.toHaveBeenCalled() + expect(reads.included - firstPredicateReads).toBe(rows.length * 5) + expect(reads.rank - firstOrderTermReads).toBe(3) } finally { compareEntries.mockRestore() readProbe.restore() @@ -1980,6 +2179,55 @@ it(`does no source or ordering work when reusing an unbounded snapshot`, async ( } }) +it(`does one source-order comparison per row needed to close the boundary tie`, () => { + const reads = { rank: 0 } + const row = (id: string, rank: number): RankedRow => ({ + id, + get rank() { + reads.rank++ + return rank + }, + included: true, + }) + const rows = new Map([ + [`a`, row(`a`, 1)], + [`b`, row(`b`, 2)], + [`c`, row(`c`, 2)], + [`d`, row(`d`, 2)], + [`e`, row(`e`, 3)], + ]) + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, orderBy(`asc`), undefined, 2, true) + window.recordInitialCoverage(undefined, true) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + reads.rank = 0 + expect(window.publicationEntries().map(([key]) => key)).toEqual([ + `a`, + `b`, + `c`, + `d`, + ]) + expect(compareRows).toHaveBeenCalledTimes(3) + expect(reads.rank).toBe(6) + } finally { + compareRows.mockRestore() + } +}) + it(`compiles the ordered predicate once for the lifetime of a window`, () => { const fixture = createSnapshotFixture([ { id: `a`, rank: 1, included: true }, From a53d536973ab051dd2aae6bc1fa403b1e760f9a8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 15:05:25 -0600 Subject: [PATCH 170/327] test(db): close ordered work category gaps --- .../ordered-work-oracle.property.test.ts | 199 +++++++++++++++++- 1 file changed, 196 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index cadff9d0e..e2a46214e 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1325,20 +1325,36 @@ describe(`ordered source work oracle`, () => { return Reflect.get(target, property, receiver) as unknown }, }) + const termComparisons: [number, number] = [0, 0] + const trackedCompareOptions = (term: 0 | 1): CompareOptions => + new Proxy( + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `direction`) termComparisons[term]++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) const order: OrderBy = [ { expression: trackedTerm(`rank`), - compareOptions: { direction: `asc`, nulls: `last` }, + compareOptions: trackedCompareOptions(0), }, { expression: trackedTerm(`secondary`), - compareOptions: { direction: `asc`, nulls: `last` }, + compareOptions: trackedCompareOptions(1), }, ] let expectedComparisons = 0 let expectedRankReads = 0 let expectedSecondaryReads = 0 + let expectedKeyComparisons = 0 const expectedKeys = storedRows .filter(({ included }) => included) .sort((left, right) => { @@ -1348,11 +1364,14 @@ describe(`ordered source work oracle`, () => { if (rankOrder !== 0) return rankOrder expectedSecondaryReads += 2 const secondaryOrder = left.secondary - right.secondary - return secondaryOrder || comparePublicKeys(left.id, right.id) + if (secondaryOrder !== 0) return secondaryOrder + expectedKeyComparisons++ + return comparePublicKeys(left.id, right.id) }) .map(({ id }) => id) const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { + keyComparisonCounter.count = 0 const changes = collection.currentStateAsChanges({ where: eq(new PropRef([`included`]), true), orderBy: order, @@ -1364,6 +1383,11 @@ describe(`ordered source work oracle`, () => { expect(reads.rank).toBe(expectedRankReads) expect(reads.secondary).toBe(expectedSecondaryReads) expect(compareEntries).toHaveBeenCalledTimes(expectedComparisons) + expect(termComparisons).toEqual([ + expectedComparisons, + expectedSecondaryReads / 2, + ]) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) } finally { compareEntries.mockRestore() } @@ -2179,6 +2203,115 @@ it(`does only exact predicate and boundary work when reusing an unbounded snapsh } }) +it(`reuses a multi-term snapshot while extracting each boundary term once`, () => { + type MultiTermWindowRow = RankedRow & { secondary: number } + const reads = { rank: 0, secondary: 0 } + const row = ( + id: string, + rank: number, + secondary: number, + ): MultiTermWindowRow => ({ + id, + get rank() { + reads.rank++ + return rank + }, + get secondary() { + reads.secondary++ + return secondary + }, + included: true, + }) + const rows = new Map([ + [`a`, row(`a`, 1, 2)], + [`b`, row(`b`, 1, 3)], + [`c`, row(`c`, 2, 1)], + ]) + let snapshotCalls = 0 + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => { + snapshotCalls++ + return [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const order: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + { + expression: new PropRef([`secondary`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ] + const window = new WindowState(collection, order, undefined, 2) + window.recordInitialCoverage(undefined, true) + + reads.rank = 0 + reads.secondary = 0 + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(snapshotCalls).toBe(1) + expect(reads).toEqual({ rank: 3, secondary: 3 }) + + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(snapshotCalls).toBe(1) + expect(reads).toEqual({ rank: 6, secondary: 6 }) +}) + +it(`scans each source row once when retaining additional-demand rows`, () => { + const rows = new Map([ + [`a`, { id: `a`, rank: 1, included: true }], + [`b`, { id: `b`, rank: 2, included: true }], + [`c`, { id: `c`, rank: 3, included: true }], + ]) + let snapshotCalls = 0 + let entryReads = 0 + let retentionChecks = 0 + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => { + snapshotCalls++ + return [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: function* () { + for (const entry of rows) { + entryReads++ + yield entry + } + }, + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, orderBy(`asc`), undefined, 1) + window.recordInitialCoverage(undefined, true) + + expect( + window + .reconcile(new Map(), (candidate) => { + retentionChecks++ + return candidate.id === `c` + }) + .map(({ key }) => key), + ).toEqual([`a`, `c`]) + expect(snapshotCalls).toBe(1) + expect(entryReads).toBe(rows.size) + expect(retentionChecks).toBe(rows.size) +}) + it(`does one source-order comparison per row needed to close the boundary tie`, () => { const reads = { rank: 0 } const row = (id: string, rank: number): RankedRow => ({ @@ -2228,6 +2361,66 @@ it(`does one source-order comparison per row needed to close the boundary tie`, } }) +it(`expands a source boundary through a comparator-equivalent string tie`, () => { + type CollatedRow = { id: string; value: string } + const reads = { value: 0 } + const row = (id: string, value: string): CollatedRow => ({ + id, + get value() { + reads.value++ + return value + }, + }) + const rows = new Map([ + [`plain`, row(`plain`, `e`)], + [`accent`, row(`accent`, `é`)], + [`later`, row(`later`, `z`)], + ]) + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const order: OrderBy = [ + { + expression: new PropRef([`value`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ] + const window = new WindowState(collection, order, undefined, 1, true) + window.recordInitialCoverage(undefined, true) + expect( + window.totalOrder.compareRows(rows.get(`plain`)!, rows.get(`accent`)!), + ).toBe(0) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + reads.value = 0 + expect(window.publicationEntries().map(([key]) => key)).toEqual([ + `plain`, + `accent`, + ]) + expect(compareRows).toHaveBeenCalledTimes(2) + expect(reads.value).toBe(4) + } finally { + compareRows.mockRestore() + } +}) + it(`compiles the ordered predicate once for the lifetime of a window`, () => { const fixture = createSnapshotFixture([ { id: `a`, rank: 1, included: true }, From 8215fbd650219205341778d54bc2b35dac650be1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 15:23:51 -0600 Subject: [PATCH 171/327] test(db): decouple ordered work expectations --- .../ordered-work-oracle.property.test.ts | 164 ++++++++++++++++-- 1 file changed, 148 insertions(+), 16 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index e2a46214e..152720d92 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -347,16 +347,24 @@ async function observeOrderedPrefix( let expectedMatches = 0 let expectedBucketYields = 0 if (limit === undefined || limit > 0) { - const expectedBuckets = - direction === `asc` - ? index.orderedBuckets() - : index.orderedBucketsReversed() - for (const [, bucket] of expectedBuckets) { + const keysByRank = new Map>() + const rowsInCollectionOrder = [...rows].sort((left, right) => + comparePublicKeys(left.id, right.id), + ) + for (const { id, rank } of rowsInCollectionOrder) { + const bucket = keysByRank.get(rank) + if (bucket === undefined) keysByRank.set(rank, [id]) + else bucket.push(id) + } + const expectedRanks = [...keysByRank.keys()].sort((left, right) => + direction === `asc` ? left - right : right - left, + ) + for (const rank of expectedRanks) { expectedBucketYields++ - const orderedKeys = [...bucket] + const orderedKeys = [...keysByRank.get(rank)!] orderedKeys.sort((left, right) => { expectedKeyComparisons++ - return left < right ? -1 : left > right ? 1 : 0 + return comparePublicKeys(left, right) }) expectedMatches += orderedKeys.filter( (key) => rows.find((row) => row.id === key)?.included === true, @@ -2299,17 +2307,29 @@ it(`scans each source row once when retaining additional-demand rows`, () => { const window = new WindowState(collection, orderBy(`asc`), undefined, 1) window.recordInitialCoverage(undefined, true) + const reconcile = (publishedRows: ReadonlyMap) => { + entryReads = 0 + retentionChecks = 0 + const changes = window.reconcile(publishedRows, (candidate) => { + retentionChecks++ + return candidate.id === `c` + }) + expect(entryReads).toBe(rows.size) + expect(retentionChecks).toBe(rows.size) + return changes + } + + expect(reconcile(new Map()).map(({ key }) => key)).toEqual([`a`, `c`]) + expect(snapshotCalls).toBe(1) expect( - window - .reconcile(new Map(), (candidate) => { - retentionChecks++ - return candidate.id === `c` - }) - .map(({ key }) => key), - ).toEqual([`a`, `c`]) + reconcile( + new Map([ + [`a`, rows.get(`a`)!], + [`b`, rows.get(`b`)!], + ]), + ).map(({ type, key }) => `${type}:${key}`), + ).toEqual([`delete:b`, `insert:c`]) expect(snapshotCalls).toBe(1) - expect(entryReads).toBe(rows.size) - expect(retentionChecks).toBe(rows.size) }) it(`does one source-order comparison per row needed to close the boundary tie`, () => { @@ -2421,6 +2441,118 @@ it(`expands a source boundary through a comparator-equivalent string tie`, () => } }) +it.each([ + { + name: `one ascending term`, + direction: `asc` as const, + orderArity: 1 as const, + limit: 1, + sourceRows: [ + { id: `tie-a`, primary: `e`, secondary: 0 }, + { id: `tie-b`, primary: `é`, secondary: 0 }, + { id: `later`, primary: `z`, secondary: 0 }, + ], + expectedKeys: [`tie-a`, `tie-b`], + }, + { + name: `one descending term`, + direction: `desc` as const, + orderArity: 1 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `z`, secondary: 0 }, + { id: `tie-a`, primary: `e`, secondary: 0 }, + { id: `tie-b`, primary: `é`, secondary: 0 }, + { id: `later`, primary: `a`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, + { + name: `two ascending terms`, + direction: `asc` as const, + orderArity: 2 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `a`, secondary: 0 }, + { id: `tie-a`, primary: `b`, secondary: `e` }, + { id: `tie-b`, primary: `b`, secondary: `é` }, + { id: `later`, primary: `c`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, + { + name: `two descending terms`, + direction: `desc` as const, + orderArity: 2 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `c`, secondary: 0 }, + { id: `tie-a`, primary: `b`, secondary: `e` }, + { id: `tie-b`, primary: `b`, secondary: `é` }, + { id: `later`, primary: `a`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, +])( + `expands source ties for $name`, + ({ direction, orderArity, limit, sourceRows, expectedKeys }) => { + type SourceTieRow = { + id: string + primary: string + secondary: string | number + } + const rows = new Map( + sourceRows.map((row) => [row.id, row]), + ) + const compareOptions = { + direction, + nulls: direction === `asc` ? (`last` as const) : (`first` as const), + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` as const }, + } satisfies CompareOptions + const order: OrderBy = [ + { + expression: new PropRef([`primary`]), + compareOptions, + }, + ...(orderArity === 2 + ? [ + { + expression: new PropRef([`secondary`]), + compareOptions, + }, + ] + : []), + ] + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, order, undefined, limit, true) + window.recordInitialCoverage(undefined, true) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + expect(window.publicationEntries().map(([key]) => key)).toEqual( + expectedKeys, + ) + expect(compareRows).toHaveBeenCalledTimes(2) + } finally { + compareRows.mockRestore() + } + }, +) + it(`compiles the ordered predicate once for the lifetime of a window`, () => { const fixture = createSnapshotFixture([ { id: `a`, rank: 1, included: true }, From 20d7e9f7dbe10e3d8b6e6e3637d46bc0303330ef Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 15:59:34 -0600 Subject: [PATCH 172/327] fix(db): defer zero window setup --- packages/db/src/collection/subscription.ts | 24 +- packages/db/src/query/live/ARCHITECTURE.md | 9 +- .../src/query/live/collection-subscriber.ts | 4 +- .../ordered-work-oracle.property.test.ts | 316 ++++++++++++++++-- 4 files changed, 311 insertions(+), 42 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 9029f6a53..2ce24ce60 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -2426,6 +2426,18 @@ export class CollectionSubscription orderBy = orderedRequest.orderBy! const where = orderedRequest.where + // Preserve the order for a later positive window without compiling its + // predicate or constructing a coordinator that cannot admit any rows. + if (limit === 0) { + onLoadSubsetResult?.(true, { + where, + orderBy, + limit: 0, + subscription: this, + }) + return + } + this.orderedWindow ??= new WindowState( this.collection, orderBy, @@ -2480,18 +2492,6 @@ export class CollectionSubscription if (changes.length > 0) this.callback(changes) - // A zero window establishes no remote demand, but it must still create the - // ordered coordinator so a later setWindow can load from the same order. - if (limit === 0) { - onLoadSubsetResult?.(true, { - where, - orderBy, - limit: 0, - subscription: this, - }) - return - } - if (!retainedPublication && this.orderedWindow.coversActiveWindow) { // No adapter request was made. Use an impossible zero-window demand so // direct tracking can finish without claiming another demand's outcome. diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 07c65e4f2..01d9dc7df 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -695,10 +695,11 @@ that entry cannot start another ordered request. Once entry returns, the normal in-flight Promise guard owns the request until settlement. An ordered window with an active limit of zero creates no ordered transport -demand. Its coordinator remains alive so a later window change can load from -the same order, but neither the initial offset nor a result deficit may turn -the empty window into a positive request. Its local source snapshot also -returns before predicate compilation, source enumeration, sorting, or index +demand. The subscription freezes the order request so a later window change +can load from the same order, but it does not construct `WindowState` until the +window first becomes positive. Neither the initial offset nor a result deficit +may turn the empty window into a positive request. The zero-width path returns +before predicate or order compilation, source enumeration, sorting, or index creation. The dedupe helper applies the same law when adapters call it directly: a zero-width request establishes no coverage and owns no physical acquisition. This also holds when no usable order index exists: core defers the diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 05921fc69..86c6bc0b8 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -488,8 +488,8 @@ export class CollectionSubscriber< const { dataNeeded, index, offset, limit, refillFromResultDeficit } = orderByInfo - // The ordered subscription keeps its coordinator for later window changes, - // but an empty active window must not start continuation work. + // The ordered subscription keeps its frozen order for later window changes, + // but an empty active window has no coordinator or continuation work. if (limit === 0) return true if (!index) { diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 152720d92..7327e8624 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1,6 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' +import { CollectionSubscription } from '../../src/collection/subscription.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { BTreeIndex } from '../../src/indexes/btree-index.js' import { ReverseIndex } from '../../src/indexes/reverse-index.js' @@ -578,6 +579,71 @@ describe(`ordered source work oracle`, () => { }, ) + it(`defers live ordered setup until a zero window becomes positive`, async () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-live-zero-window`, + getKey: (row) => row.id, + initialData: [{ id: `one`, rank: 1, included: true }], + }), + ) + let subscription: CollectionSubscription | undefined + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) + subscription = new CollectionSubscription(collection, () => {}, {}) + subscription.setOrderByIndex(index) + let orderCompilationReads = 0 + const order: OrderBy = [ + { + expression: new Proxy(new PropRef([`rank`]), { + get(target, property, receiver) { + if (property === `type`) orderCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + compareOptions: publicKeyIndexCompareOptions, + }, + ] + + subscription.requestLimitedSnapshot({ + orderBy: order, + limit: 0, + trackLoadSubsetPromise: false, + }) + expect( + ( + subscription as unknown as { + orderedWindow: WindowState | undefined + } + ).orderedWindow, + ).toBeUndefined() + // Freezing the request reads the expression tag once. It must not also + // construct TotalOrder or compile the frozen expression for no rows. + expect(orderCompilationReads).toBe(1) + + subscription.requestLimitedSnapshot({ + orderBy: order, + limit: 1, + trackLoadSubsetPromise: false, + }) + expect( + ( + subscription as unknown as { + orderedWindow: WindowState | undefined + } + ).orderedWindow, + ).toBeDefined() + } finally { + subscription?.unsubscribe() + await collection.cleanup() + } + }) + it.each([ { indexKind: `basic`, direction: `asc` }, { indexKind: `basic`, direction: `desc` }, @@ -1802,18 +1868,30 @@ describe(`ordered source work oracle`, () => { }) await transaction.isPersisted.promise } + let expectedKeyComparisons = 0 + const expectedKeys = [...keys].sort((left, right) => { + expectedKeyComparisons++ + return comparePublicKeys(left, right) + }) + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) try { + keyComparisonCounter.count = 0 const changes = collection.currentStateAsChanges({ orderBy: publicKeyOrderBy(direction), limit: keys.length, optimizedOnly: true, })! - expect(changes.map(({ key }) => key)).toEqual( - [...keys].sort(comparePublicKeys), - ) + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) expect(compareEntries).not.toHaveBeenCalled() } finally { compareEntries.mockRestore() @@ -1909,6 +1987,116 @@ describe(`ordered source work oracle`, () => { ) } + it.each( + ([`basic`, `btree`] as const).flatMap((indexKind) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`string`, `nullish`] as const).map((orderDomain) => ({ + indexKind, + direction, + orderDomain, + })), + ), + ), + )( + `does exact optimized work for $indexKind $direction $orderDomain order values`, + async ({ indexKind, direction, orderDomain }) => { + type DomainRow = Omit & { + rank: string | number | null | undefined + } + const rows: Array = + orderDomain === `string` + ? [ + { id: `item-10`, rank: `item-10`, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + ] + : [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `two`, rank: 2, included: true }, + { id: `one`, rank: 1, included: true }, + ] + const expectedKeys = + orderDomain === `string` + ? direction === `asc` + ? [`item-2`, `item-10`] + : [`item-10`, `item-2`] + : direction === `asc` + ? [`one`, `two`, `null`, `undefined`] + : [`null`, `undefined`, `two`, `one`] + const indexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` as const }, + } satisfies CompareOptions + const queryCompareOptions = { + ...indexCompareOptions, + direction, + nulls: direction === `asc` ? (`last` as const) : (`first` as const), + } satisfies CompareOptions + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-domain-${indexKind}-${direction}-${orderDomain}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { compareOptions: indexCompareOptions }, + }) as BasicIndex | BTreeIndex + const readProbe = observeOrderedIndexReads(index, indexKind, direction) + const sourceReads: Array = [] + let predicateReads = 0 + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + const value = originalGet(key) + return value === undefined + ? undefined + : new Proxy(value, { + get(target, property, receiver) { + if (property === `included`) predicateReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + } + const bucketCount = + orderDomain === `string` ? 2 : indexKind === `basic` ? 4 : 3 + + try { + keyComparisonCounter.count = 0 + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: orderByWithOptions(queryCompareOptions), + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(predicateReads).toBe(rows.length) + expect(readProbe.getValueReads()).toBe(bucketCount) + expect(readProbe.getBucketReads()).toBe(bucketCount) + expect(readProbe.getCursorCalls()).toBe( + indexKind === `btree` ? bucketCount + 1 : 0, + ) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + expect(keyComparisonCounter.count).toBe( + orderDomain === `nullish` ? 1 : 0, + ) + } finally { + readProbe.restore() + } + } finally { + await collection.cleanup() + } + }, + ) + it(`retains requested comparison metadata on an automatic index`, async () => { type NullableRankedRow = Omit & { rank: string | null | undefined @@ -2125,13 +2313,8 @@ it(`does only exact predicate and boundary work when reusing an unbounded snapsh indexType: BTreeIndex, options: { compareOptions: publicKeyIndexCompareOptions }, }) as BTreeIndex - const [bucket] = [...index.orderedBuckets()] - const bucketKeys = [...bucket![1]] - let expectedKeyComparisons = 0 - const expectedKeys = bucketKeys.sort((left, right) => { - expectedKeyComparisons++ - return comparePublicKeys(left, right) - }) + const expectedKeys = rows.map(({ id }) => id).sort(comparePublicKeys) + const expectedKeyComparisons = Math.max(0, expectedKeys.length - 1) const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) const sourceReads: Array = [] @@ -2276,6 +2459,27 @@ it(`reuses a multi-term snapshot while extracting each boundary term once`, () = }) it(`scans each source row once when retaining additional-demand rows`, () => { + class CountingPublication extends Map { + iterationReads = 0 + membershipReads = 0; + + override *[Symbol.iterator](): Generator< + [string, RankedRow], + undefined, + unknown + > { + for (const entry of super[Symbol.iterator]()) { + this.iterationReads++ + yield entry + } + return undefined + } + + override has(key: string): boolean { + this.membershipReads++ + return super.has(key) + } + } const rows = new Map([ [`a`, { id: `a`, rank: 1, included: true }], [`b`, { id: `b`, rank: 2, included: true }], @@ -2307,7 +2511,7 @@ it(`scans each source row once when retaining additional-demand rows`, () => { const window = new WindowState(collection, orderBy(`asc`), undefined, 1) window.recordInitialCoverage(undefined, true) - const reconcile = (publishedRows: ReadonlyMap) => { + const reconcile = (publishedRows: CountingPublication) => { entryReads = 0 retentionChecks = 0 const changes = window.reconcile(publishedRows, (candidate) => { @@ -2316,14 +2520,19 @@ it(`scans each source row once when retaining additional-demand rows`, () => { }) expect(entryReads).toBe(rows.size) expect(retentionChecks).toBe(rows.size) + expect(publishedRows.iterationReads).toBe(publishedRows.size) + expect(publishedRows.membershipReads).toBe(2) return changes } - expect(reconcile(new Map()).map(({ key }) => key)).toEqual([`a`, `c`]) + expect(reconcile(new CountingPublication()).map(({ key }) => key)).toEqual([ + `a`, + `c`, + ]) expect(snapshotCalls).toBe(1) expect( reconcile( - new Map([ + new CountingPublication([ [`a`, rows.get(`a`)!], [`b`, rows.get(`b`)!], ]), @@ -2467,6 +2676,31 @@ it.each([ ], expectedKeys: [`prefix`, `tie-a`, `tie-b`], }, + { + name: `one ascending numeric term`, + direction: `asc` as const, + orderArity: 1 as const, + limit: 1, + sourceRows: [ + { id: `tie-a`, primary: 1, secondary: 0 }, + { id: `tie-b`, primary: 1, secondary: 0 }, + { id: `later`, primary: 2, secondary: 0 }, + ], + expectedKeys: [`tie-a`, `tie-b`], + }, + { + name: `one descending numeric term`, + direction: `desc` as const, + orderArity: 1 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: 3, secondary: 0 }, + { id: `tie-a`, primary: 1, secondary: 0 }, + { id: `tie-b`, primary: 1, secondary: 0 }, + { id: `later`, primary: 0, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, { name: `two ascending terms`, direction: `asc` as const, @@ -2498,29 +2732,53 @@ it.each([ ({ direction, orderArity, limit, sourceRows, expectedKeys }) => { type SourceTieRow = { id: string - primary: string + primary: string | number secondary: string | number } + const termReads: [number, number] = [0, 0] + const termComparisons: [number, number] = [0, 0] const rows = new Map( - sourceRows.map((row) => [row.id, row]), + sourceRows.map((spec) => [ + spec.id, + { + id: spec.id, + get primary() { + termReads[0]++ + return spec.primary + }, + get secondary() { + termReads[1]++ + return spec.secondary + }, + }, + ]), ) - const compareOptions = { - direction, - nulls: direction === `asc` ? (`last` as const) : (`first` as const), - stringSort: `locale`, - locale: `en`, - localeOptions: { sensitivity: `base` as const }, - } satisfies CompareOptions + const trackedCompareOptions = (term: 0 | 1): CompareOptions => + new Proxy( + { + direction, + nulls: direction === `asc` ? (`last` as const) : (`first` as const), + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` as const }, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `direction`) termComparisons[term]++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) const order: OrderBy = [ { expression: new PropRef([`primary`]), - compareOptions, + compareOptions: trackedCompareOptions(0), }, ...(orderArity === 2 ? [ { expression: new PropRef([`secondary`]), - compareOptions, + compareOptions: trackedCompareOptions(1), }, ] : []), @@ -2543,10 +2801,20 @@ it.each([ const compareRows = vi.spyOn(window.totalOrder, `compareRows`) try { + termReads[0] = 0 + termReads[1] = 0 + termComparisons[0] = 0 + termComparisons[1] = 0 expect(window.publicationEntries().map(([key]) => key)).toEqual( expectedKeys, ) expect(compareRows).toHaveBeenCalledTimes(2) + expect(termReads).toEqual([4, orderArity === 2 ? 2 : 0]) + const optionReadsPerComparison = direction === `asc` ? 1 : 2 + expect(termComparisons).toEqual([ + 2 * optionReadsPerComparison, + orderArity === 2 ? optionReadsPerComparison : 0, + ]) } finally { compareRows.mockRestore() } From 53b819931ed62e62927ff0e4f2ae2beea31a0700 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 16:18:34 -0600 Subject: [PATCH 173/327] test(db): observe publication diff work --- packages/db/src/query/live/window-state.ts | 37 ++++++--- .../ordered-work-oracle.property.test.ts | 82 ++++++++++++------- 2 files changed, 78 insertions(+), 41 deletions(-) diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index 14b73fb7b..d773cce36 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -6,6 +6,29 @@ import type { ChangeMessage } from '../../types.js' import type { BasicExpression, OrderBy } from '../ir.js' import type { TotalOrderBoundary } from '../total-order.js' +/** Compute the exact change set between two materialized publications. */ +export function diffPublications< + TRow extends object, + TKey extends string | number, +>( + publishedRows: ReadonlyMap, + desiredRows: ReadonlyMap, +): Array> { + const changes: Array> = [] + for (const [key, previousValue] of publishedRows) { + const value = desiredRows.get(key) + if (value === undefined) { + changes.push({ type: `delete`, key, value: previousValue }) + } else if (!deepEquals(previousValue, value)) { + changes.push({ type: `update`, key, value, previousValue }) + } + } + for (const [key, value] of desiredRows) { + if (!publishedRows.has(key)) changes.push({ type: `insert`, key, value }) + } + return changes +} + /** * Owns the active ordered demand and its retained local coverage. Rows outside * the retained prefix stay in the source collection until a later window @@ -359,19 +382,7 @@ export class WindowState< } } - const changes: Array> = [] - for (const [key, previousValue] of publishedRows) { - const value = desired.get(key) - if (value === undefined) { - changes.push({ type: `delete`, key, value: previousValue }) - } else if (!deepEquals(previousValue, value)) { - changes.push({ type: `update`, key, value, previousValue }) - } - } - for (const [key, value] of desired) { - if (!publishedRows.has(key)) changes.push({ type: `insert`, key, value }) - } - return changes + return diffPublications(publishedRows, desired) } private readPrefix(): Array> { diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 7327e8624..556b6641c 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -10,7 +10,10 @@ import { eq } from '../../src/query/builder/functions.js' import { compileSingleRowExpression } from '../../src/query/compiler/evaluators.js' import { PropRef } from '../../src/query/ir.js' import { TotalOrder } from '../../src/query/total-order.js' -import { WindowState } from '../../src/query/live/window-state.js' +import { + WindowState, + diffPublications, +} from '../../src/query/live/window-state.js' import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' import type * as DbIvm from '@tanstack/db-ivm' import type { CollectionImpl } from '../../src/collection/index.js' @@ -41,6 +44,30 @@ type RankedRow = { included: boolean } +class CountingMap extends Map { + iterationReads = 0 + membershipReads = 0 + valueReads = 0; + + override *[Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { + for (const entry of super[Symbol.iterator]()) { + this.iterationReads++ + yield entry + } + return undefined + } + + override get(key: TKey): TValue | undefined { + this.valueReads++ + return super.get(key) + } + + override has(key: TKey): boolean { + this.membershipReads++ + return super.has(key) + } +} + type PublicKeyRankedRow = Omit & { id: string | number } @@ -2459,27 +2486,6 @@ it(`reuses a multi-term snapshot while extracting each boundary term once`, () = }) it(`scans each source row once when retaining additional-demand rows`, () => { - class CountingPublication extends Map { - iterationReads = 0 - membershipReads = 0; - - override *[Symbol.iterator](): Generator< - [string, RankedRow], - undefined, - unknown - > { - for (const entry of super[Symbol.iterator]()) { - this.iterationReads++ - yield entry - } - return undefined - } - - override has(key: string): boolean { - this.membershipReads++ - return super.has(key) - } - } const rows = new Map([ [`a`, { id: `a`, rank: 1, included: true }], [`b`, { id: `b`, rank: 2, included: true }], @@ -2511,7 +2517,7 @@ it(`scans each source row once when retaining additional-demand rows`, () => { const window = new WindowState(collection, orderBy(`asc`), undefined, 1) window.recordInitialCoverage(undefined, true) - const reconcile = (publishedRows: CountingPublication) => { + const reconcile = (publishedRows: CountingMap) => { entryReads = 0 retentionChecks = 0 const changes = window.reconcile(publishedRows, (candidate) => { @@ -2525,14 +2531,11 @@ it(`scans each source row once when retaining additional-demand rows`, () => { return changes } - expect(reconcile(new CountingPublication()).map(({ key }) => key)).toEqual([ - `a`, - `c`, - ]) + expect(reconcile(new CountingMap()).map(({ key }) => key)).toEqual([`a`, `c`]) expect(snapshotCalls).toBe(1) expect( reconcile( - new CountingPublication([ + new CountingMap([ [`a`, rows.get(`a`)!], [`b`, rows.get(`b`)!], ]), @@ -2541,6 +2544,29 @@ it(`scans each source row once when retaining additional-demand rows`, () => { expect(snapshotCalls).toBe(1) }) +it(`scans each side of a publication diff exactly once`, () => { + const publishedRows = new CountingMap([ + [`a`, { id: `a`, rank: 2, included: true }], + [`b`, { id: `b`, rank: 2, included: true }], + ]) + const desiredRows = new CountingMap([ + [`a`, { id: `a`, rank: 1, included: true }], + [`c`, { id: `c`, rank: 3, included: true }], + ]) + + expect( + diffPublications(publishedRows, desiredRows).map( + ({ type, key }) => `${type}:${key}`, + ), + ).toEqual([`update:a`, `delete:b`, `insert:c`]) + expect(publishedRows.iterationReads).toBe(publishedRows.size) + expect(publishedRows.membershipReads).toBe(desiredRows.size) + expect(publishedRows.valueReads).toBe(0) + expect(desiredRows.iterationReads).toBe(desiredRows.size) + expect(desiredRows.membershipReads).toBe(0) + expect(desiredRows.valueReads).toBe(publishedRows.size) +}) + it(`does one source-order comparison per row needed to close the boundary tie`, () => { const reads = { rank: 0 } const row = (id: string, rank: number): RankedRow => ({ From 7da4a99fade18c5fe43148b1b264e907eece1a3e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 16:40:28 -0600 Subject: [PATCH 174/327] test(db): observe complete ordered work --- .../ordered-work-oracle.property.test.ts | 96 +++++++++++++++++-- 1 file changed, 90 insertions(+), 6 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 556b6641c..42375b4fc 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -47,16 +47,44 @@ type RankedRow = { class CountingMap extends Map { iterationReads = 0 membershipReads = 0 - valueReads = 0; + valueReads = 0 - override *[Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { - for (const entry of super[Symbol.iterator]()) { + private *countIterator( + iterator: Iterator, + ): Generator { + for (let next = iterator.next(); !next.done; next = iterator.next()) { this.iterationReads++ - yield entry + yield next.value } return undefined } + override [Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(super[Symbol.iterator]()) + } + + override entries(): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(super.entries()) + } + + override keys(): Generator { + return this.countIterator(super.keys()) + } + + override values(): Generator { + return this.countIterator(super.values()) + } + + override forEach( + callback: (value: TValue, key: TKey, map: Map) => void, + thisArg?: unknown, + ): void { + super.forEach((value, key) => { + this.iterationReads++ + callback.call(thisArg, value, key, this) + }) + } + override get(key: TKey): TValue | undefined { this.valueReads++ return super.get(key) @@ -103,6 +131,30 @@ function isArrayIndex(property: PropertyKey): boolean { return Number.isSafeInteger(index) && index >= 0 && String(index) === property } +const traversalMethods = new Set([ + Symbol.iterator, + `entries`, + `keys`, + `values`, + `forEach`, +]) + +function observeUnexpectedTraversals( + target: T, + onTraversal: () => void, +): T { + return new Proxy(target, { + get(inner, property) { + const member = Reflect.get(inner, property, inner) as unknown + if (typeof member !== `function`) return member + return (...args: Array) => { + if (traversalMethods.has(property)) onTraversal() + return Reflect.apply(member, inner, args) as unknown + } + }, + }) +} + function observeOrderedIndexReads( index: BasicIndex | BTreeIndex, indexKind: `basic` | `btree`, @@ -117,9 +169,11 @@ function observeOrderedIndexReads( const internals = index as unknown as { sortedValues: Array valueMap: Map> + indexedKeys: Set } const sortedValues = internals.sortedValues const valueMap = internals.valueMap + const indexedKeys = internals.indexedKeys internals.sortedValues = new Proxy(sortedValues, { get(target, property, receiver) { if (isArrayIndex(property)) valueReads++ @@ -144,6 +198,9 @@ function observeOrderedIndexReads( return member }, }) + internals.indexedKeys = observeUnexpectedTraversals(indexedKeys, () => { + unexpectedTraversalCalls++ + }) return { getValueReads: () => valueReads, getBucketReads: () => bucketReads, @@ -152,6 +209,7 @@ function observeOrderedIndexReads( restore: () => { internals.sortedValues = sortedValues internals.valueMap = valueMap + internals.indexedKeys = indexedKeys }, } } @@ -161,8 +219,12 @@ function observeOrderedIndexReads( nextHigherPair: (key?: unknown) => readonly [unknown, unknown] | undefined nextLowerPair: (key?: unknown) => readonly [unknown, unknown] | undefined } + valueMap: Map> + indexedKeys: Set } const orderedEntries = internals.orderedEntries + const valueMap = internals.valueMap + const indexedKeys = internals.indexedKeys const expectedMethod = direction === `asc` ? `nextHigherPair` : `nextLowerPair` internals.orderedEntries = new Proxy(orderedEntries, { @@ -185,6 +247,12 @@ function observeOrderedIndexReads( } }, }) + internals.valueMap = observeUnexpectedTraversals(valueMap, () => { + unexpectedTraversalCalls++ + }) + internals.indexedKeys = observeUnexpectedTraversals(indexedKeys, () => { + unexpectedTraversalCalls++ + }) return { getValueReads: () => valueReads, getBucketReads: () => bucketReads, @@ -192,6 +260,8 @@ function observeOrderedIndexReads( getUnexpectedTraversalCalls: () => unexpectedTraversalCalls, restore: () => { internals.orderedEntries = orderedEntries + internals.valueMap = valueMap + internals.indexedKeys = indexedKeys }, } } @@ -2545,12 +2615,25 @@ it(`scans each source row once when retaining additional-demand rows`, () => { }) it(`scans each side of a publication diff exactly once`, () => { + const valueReads = { published: 0, desired: 0 } + const countedRow = ( + row: RankedRow, + side: keyof typeof valueReads, + ): RankedRow => + new Proxy(row, { + get(target, property, receiver) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + valueReads[side]++ + } + return Reflect.get(target, property, receiver) as unknown + }, + }) const publishedRows = new CountingMap([ - [`a`, { id: `a`, rank: 2, included: true }], + [`a`, countedRow({ id: `a`, rank: 2, included: true }, `published`)], [`b`, { id: `b`, rank: 2, included: true }], ]) const desiredRows = new CountingMap([ - [`a`, { id: `a`, rank: 1, included: true }], + [`a`, countedRow({ id: `a`, rank: 1, included: true }, `desired`)], [`c`, { id: `c`, rank: 3, included: true }], ]) @@ -2565,6 +2648,7 @@ it(`scans each side of a publication diff exactly once`, () => { expect(desiredRows.iterationReads).toBe(desiredRows.size) expect(desiredRows.membershipReads).toBe(0) expect(desiredRows.valueReads).toBe(publishedRows.size) + expect(valueReads).toEqual({ published: 2, desired: 2 }) }) it(`does one source-order comparison per row needed to close the boundary tie`, () => { From e9146a12400c95d091c4ccd086290b72040c0c36 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 17:04:43 -0600 Subject: [PATCH 175/327] test(db): close ordered work observation gaps --- .../ordered-work-oracle.property.test.ts | 247 ++++++++++++++---- 1 file changed, 203 insertions(+), 44 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 42375b4fc..e855989d4 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -10,6 +10,7 @@ import { eq } from '../../src/query/builder/functions.js' import { compileSingleRowExpression } from '../../src/query/compiler/evaluators.js' import { PropRef } from '../../src/query/ir.js' import { TotalOrder } from '../../src/query/total-order.js' +import { makeComparator } from '../../src/utils/comparison.js' import { WindowState, diffPublications, @@ -44,11 +45,20 @@ type RankedRow = { included: boolean } -class CountingMap extends Map { +class CountingReadonlyMap implements ReadonlyMap { + private readonly valuesByKey: Map iterationReads = 0 membershipReads = 0 valueReads = 0 + constructor(entries: Iterable = []) { + this.valuesByKey = new Map(entries) + } + + get size(): number { + return this.valuesByKey.size + } + private *countIterator( iterator: Iterator, ): Generator { @@ -59,40 +69,44 @@ class CountingMap extends Map { return undefined } - override [Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { - return this.countIterator(super[Symbol.iterator]()) + [Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(this.valuesByKey[Symbol.iterator]()) } - override entries(): Generator<[TKey, TValue], undefined, unknown> { - return this.countIterator(super.entries()) + entries(): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(this.valuesByKey.entries()) } - override keys(): Generator { - return this.countIterator(super.keys()) + keys(): Generator { + return this.countIterator(this.valuesByKey.keys()) } - override values(): Generator { - return this.countIterator(super.values()) + values(): Generator { + return this.countIterator(this.valuesByKey.values()) } - override forEach( - callback: (value: TValue, key: TKey, map: Map) => void, + forEach( + callback: ( + value: TValue, + key: TKey, + map: ReadonlyMap, + ) => void, thisArg?: unknown, ): void { - super.forEach((value, key) => { + this.valuesByKey.forEach((value, key) => { this.iterationReads++ callback.call(thisArg, value, key, this) }) } - override get(key: TKey): TValue | undefined { + get(key: TKey): TValue | undefined { this.valueReads++ - return super.get(key) + return this.valuesByKey.get(key) } - override has(key: TKey): boolean { + has(key: TKey): boolean { this.membershipReads++ - return super.has(key) + return this.valuesByKey.has(key) } } @@ -691,7 +705,7 @@ describe(`ordered source work oracle`, () => { const index = collection.createIndex((row) => row.rank, { indexType: BTreeIndex, options: { compareOptions: publicKeyIndexCompareOptions }, - }) + }) as BTreeIndex subscription = new CollectionSubscription(collection, () => {}, {}) subscription.setOrderByIndex(index) let orderCompilationReads = 0 @@ -707,21 +721,30 @@ describe(`ordered source work oracle`, () => { }, ] - subscription.requestLimitedSnapshot({ - orderBy: order, - limit: 0, - trackLoadSubsetPromise: false, - }) - expect( - ( - subscription as unknown as { - orderedWindow: WindowState | undefined - } - ).orderedWindow, - ).toBeUndefined() - // Freezing the request reads the expression tag once. It must not also - // construct TotalOrder or compile the frozen expression for no rows. - expect(orderCompilationReads).toBe(1) + const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) + try { + subscription.requestLimitedSnapshot({ + orderBy: order, + limit: 0, + trackLoadSubsetPromise: false, + }) + expect( + ( + subscription as unknown as { + orderedWindow: WindowState | undefined + } + ).orderedWindow, + ).toBeUndefined() + // Freezing the request reads the expression tag once. It must not also + // construct TotalOrder or compile the frozen expression for no rows. + expect(orderCompilationReads).toBe(1) + expect(readProbe.getValueReads()).toBe(0) + expect(readProbe.getBucketReads()).toBe(0) + expect(readProbe.getCursorCalls()).toBe(0) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + } finally { + readProbe.restore() + } subscription.requestLimitedSnapshot({ orderBy: order, @@ -741,6 +764,96 @@ describe(`ordered source work oracle`, () => { } }) + it.each([ + { + name: `ascending numbers`, + direction: `asc` as const, + left: 1, + right: 2, + expected: -1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 0, + locale: 0, + localeOptions: 0, + }, + }, + { + name: `ascending strings`, + direction: `asc` as const, + left: `a`, + right: `b`, + expected: -1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 1, + locale: 1, + localeOptions: 1, + }, + }, + { + name: `descending numbers`, + direction: `desc` as const, + left: 1, + right: 2, + expected: 1, + expectedReads: { + direction: 2, + nulls: 2, + stringSort: 1, + locale: 1, + localeOptions: 1, + }, + }, + { + name: `descending strings`, + direction: `desc` as const, + left: `a`, + right: `b`, + expected: 1, + expectedReads: { + direction: 2, + nulls: 2, + stringSort: 1, + locale: 1, + localeOptions: 1, + }, + }, + ])( + `executes the inner comparator once for $name`, + ({ direction, left, right, expected, expectedReads }) => { + const reads = { + direction: 0, + nulls: 0, + stringSort: 0, + locale: 0, + localeOptions: 0, + } + const options = new Proxy( + { + direction, + nulls: `last` as const, + stringSort: `locale` as const, + locale: `en`, + localeOptions: { sensitivity: `base` as const }, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (typeof property === `string` && property in reads) { + reads[property as keyof typeof reads]++ + } + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) + + expect(makeComparator(options)(left, right)).toBe(expected) + expect(reads).toEqual(expectedReads) + }, + ) + it.each([ { indexKind: `basic`, direction: `asc` }, { indexKind: `basic`, direction: `desc` }, @@ -2587,7 +2700,7 @@ it(`scans each source row once when retaining additional-demand rows`, () => { const window = new WindowState(collection, orderBy(`asc`), undefined, 1) window.recordInitialCoverage(undefined, true) - const reconcile = (publishedRows: CountingMap) => { + const reconcile = (publishedRows: CountingReadonlyMap) => { entryReads = 0 retentionChecks = 0 const changes = window.reconcile(publishedRows, (candidate) => { @@ -2601,11 +2714,14 @@ it(`scans each source row once when retaining additional-demand rows`, () => { return changes } - expect(reconcile(new CountingMap()).map(({ key }) => key)).toEqual([`a`, `c`]) + expect(reconcile(new CountingReadonlyMap()).map(({ key }) => key)).toEqual([ + `a`, + `c`, + ]) expect(snapshotCalls).toBe(1) expect( reconcile( - new CountingMap([ + new CountingReadonlyMap([ [`a`, rows.get(`a`)!], [`b`, rows.get(`b`)!], ]), @@ -2615,26 +2731,56 @@ it(`scans each source row once when retaining additional-demand rows`, () => { }) it(`scans each side of a publication diff exactly once`, () => { - const valueReads = { published: 0, desired: 0 } - const countedRow = ( - row: RankedRow, - side: keyof typeof valueReads, - ): RankedRow => + type RowWork = { + valueReads: number + keyReads: number + membershipReads: number + descriptorReads: number + } + const emptyRowWork = (): RowWork => ({ + valueReads: 0, + keyReads: 0, + membershipReads: 0, + descriptorReads: 0, + }) + const rowWork = { + published: emptyRowWork(), + desired: emptyRowWork(), + } + const countedRow = (row: RankedRow, side: keyof typeof rowWork): RankedRow => new Proxy(row, { get(target, property, receiver) { if (typeof property === `string` && Object.hasOwn(target, property)) { - valueReads[side]++ + rowWork[side].valueReads++ } return Reflect.get(target, property, receiver) as unknown }, + ownKeys(target) { + rowWork[side].keyReads++ + return Reflect.ownKeys(target) + }, + has(target, property) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + rowWork[side].membershipReads++ + } + return Reflect.has(target, property) + }, + getOwnPropertyDescriptor(target, property) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + rowWork[side].descriptorReads++ + } + return Reflect.getOwnPropertyDescriptor(target, property) + }, }) - const publishedRows = new CountingMap([ + const publishedRows = new CountingReadonlyMap([ [`a`, countedRow({ id: `a`, rank: 2, included: true }, `published`)], [`b`, { id: `b`, rank: 2, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `published`)], ]) - const desiredRows = new CountingMap([ + const desiredRows = new CountingReadonlyMap([ [`a`, countedRow({ id: `a`, rank: 1, included: true }, `desired`)], [`c`, { id: `c`, rank: 3, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `desired`)], ]) expect( @@ -2648,7 +2794,20 @@ it(`scans each side of a publication diff exactly once`, () => { expect(desiredRows.iterationReads).toBe(desiredRows.size) expect(desiredRows.membershipReads).toBe(0) expect(desiredRows.valueReads).toBe(publishedRows.size) - expect(valueReads).toEqual({ published: 2, desired: 2 }) + expect(rowWork).toEqual({ + published: { + valueReads: 5, + keyReads: 2, + membershipReads: 0, + descriptorReads: 6, + }, + desired: { + valueReads: 5, + keyReads: 2, + membershipReads: 5, + descriptorReads: 6, + }, + }) }) it(`does one source-order comparison per row needed to close the boundary tie`, () => { From e931dcb832fc26b7e4feb3925673820b4064c11a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 17:23:41 -0600 Subject: [PATCH 176/327] test(db): observe ordered work materialization --- .../ordered-work-oracle.property.test.ts | 75 +++++++++++++++++-- 1 file changed, 69 insertions(+), 6 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index e855989d4..eecbd55bb 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -777,6 +777,9 @@ describe(`ordered source work oracle`, () => { stringSort: 0, locale: 0, localeOptions: 0, + ownKeys: 0, + descriptors: 0, + prototype: 0, }, }, { @@ -791,6 +794,9 @@ describe(`ordered source work oracle`, () => { stringSort: 1, locale: 1, localeOptions: 1, + ownKeys: 0, + descriptors: 0, + prototype: 0, }, }, { @@ -805,6 +811,9 @@ describe(`ordered source work oracle`, () => { stringSort: 1, locale: 1, localeOptions: 1, + ownKeys: 1, + descriptors: 5, + prototype: 0, }, }, { @@ -819,6 +828,9 @@ describe(`ordered source work oracle`, () => { stringSort: 1, locale: 1, localeOptions: 1, + ownKeys: 1, + descriptors: 5, + prototype: 0, }, }, ])( @@ -830,6 +842,9 @@ describe(`ordered source work oracle`, () => { stringSort: 0, locale: 0, localeOptions: 0, + ownKeys: 0, + descriptors: 0, + prototype: 0, } const options = new Proxy( { @@ -846,10 +861,37 @@ describe(`ordered source work oracle`, () => { } return Reflect.get(target, property, receiver) as unknown }, + ownKeys(target) { + reads.ownKeys++ + return Reflect.ownKeys(target) + }, + getOwnPropertyDescriptor(target, property) { + reads.descriptors++ + return Reflect.getOwnPropertyDescriptor(target, property) + }, + getPrototypeOf(target) { + reads.prototype++ + return Reflect.getPrototypeOf(target) + }, }, ) - expect(makeComparator(options)(left, right)).toBe(expected) + const descriptorCopies = vi.spyOn(Object, `getOwnPropertyDescriptors`) + const prototypeReads = vi.spyOn(Object, `getPrototypeOf`) + let actual: number + let descriptorCopyCount: number + let prototypeReadCount: number + try { + actual = makeComparator(options)(left, right) + descriptorCopyCount = descriptorCopies.mock.calls.length + prototypeReadCount = prototypeReads.mock.calls.length + } finally { + descriptorCopies.mockRestore() + prototypeReads.mockRestore() + } + expect(descriptorCopyCount).toBe(0) + expect(prototypeReadCount).toBe(0) + expect(actual).toBe(expected) expect(reads).toEqual(expectedReads) }, ) @@ -2783,11 +2825,32 @@ it(`scans each side of a publication diff exactly once`, () => { [`d`, countedRow({ id: `d`, rank: 4, included: true }, `desired`)], ]) - expect( - diffPublications(publishedRows, desiredRows).map( - ({ type, key }) => `${type}:${key}`, - ), - ).toEqual([`update:a`, `delete:b`, `insert:c`]) + const defaultIterator = vi.spyOn(Map.prototype, Symbol.iterator) + const entries = vi.spyOn(Map.prototype, `entries`) + const keys = vi.spyOn(Map.prototype, `keys`) + const values = vi.spyOn(Map.prototype, `values`) + const forEach = vi.spyOn(Map.prototype, `forEach`) + + try { + expect( + diffPublications(publishedRows, desiredRows).map( + ({ type, key }) => `${type}:${key}`, + ), + ).toEqual([`update:a`, `delete:b`, `insert:c`]) + expect( + defaultIterator.mock.calls.length + + entries.mock.calls.length + + keys.mock.calls.length + + values.mock.calls.length + + forEach.mock.calls.length, + ).toBe(2) + } finally { + defaultIterator.mockRestore() + entries.mockRestore() + keys.mockRestore() + values.mockRestore() + forEach.mockRestore() + } expect(publishedRows.iterationReads).toBe(publishedRows.size) expect(publishedRows.membershipReads).toBe(desiredRows.size) expect(publishedRows.valueReads).toBe(0) From 3fa5fd3c8af339293fd47da532b795d2da6f571e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 17:36:48 -0600 Subject: [PATCH 177/327] test(db): prove publication work streams --- .../ordered-work-oracle.property.test.ts | 79 ++++++++++--------- 1 file changed, 42 insertions(+), 37 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index eecbd55bb..4dd9efb93 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -51,7 +51,11 @@ class CountingReadonlyMap implements ReadonlyMap { membershipReads = 0 valueReads = 0 - constructor(entries: Iterable = []) { + constructor( + entries: Iterable = [], + private readonly onIteration?: () => void, + private readonly onMembershipRead?: () => void, + ) { this.valuesByKey = new Map(entries) } @@ -64,6 +68,7 @@ class CountingReadonlyMap implements ReadonlyMap { ): Generator { for (let next = iterator.next(); !next.done; next = iterator.next()) { this.iterationReads++ + this.onIteration?.() yield next.value } return undefined @@ -95,6 +100,7 @@ class CountingReadonlyMap implements ReadonlyMap { ): void { this.valuesByKey.forEach((value, key) => { this.iterationReads++ + this.onIteration?.() callback.call(thisArg, value, key, this) }) } @@ -106,6 +112,7 @@ class CountingReadonlyMap implements ReadonlyMap { has(key: TKey): boolean { this.membershipReads++ + this.onMembershipRead?.() return this.valuesByKey.has(key) } } @@ -2814,43 +2821,41 @@ it(`scans each side of a publication diff exactly once`, () => { return Reflect.getOwnPropertyDescriptor(target, property) }, }) - const publishedRows = new CountingReadonlyMap([ - [`a`, countedRow({ id: `a`, rank: 2, included: true }, `published`)], - [`b`, { id: `b`, rank: 2, included: true }], - [`d`, countedRow({ id: `d`, rank: 4, included: true }, `published`)], - ]) - const desiredRows = new CountingReadonlyMap([ - [`a`, countedRow({ id: `a`, rank: 1, included: true }, `desired`)], - [`c`, { id: `c`, rank: 3, included: true }], - [`d`, countedRow({ id: `d`, rank: 4, included: true }, `desired`)], - ]) - - const defaultIterator = vi.spyOn(Map.prototype, Symbol.iterator) - const entries = vi.spyOn(Map.prototype, `entries`) - const keys = vi.spyOn(Map.prototype, `keys`) - const values = vi.spyOn(Map.prototype, `values`) - const forEach = vi.spyOn(Map.prototype, `forEach`) + let unmatchedDesiredEntries = 0 + let maximumUnmatchedDesiredEntries = 0 + const publishedRows = new CountingReadonlyMap( + [ + [`a`, countedRow({ id: `a`, rank: 2, included: true }, `published`)], + [`b`, { id: `b`, rank: 2, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `published`)], + ], + undefined, + () => { + unmatchedDesiredEntries-- + }, + ) + const desiredRows = new CountingReadonlyMap( + [ + [`a`, countedRow({ id: `a`, rank: 1, included: true }, `desired`)], + [`c`, { id: `c`, rank: 3, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `desired`)], + ], + () => { + unmatchedDesiredEntries++ + maximumUnmatchedDesiredEntries = Math.max( + maximumUnmatchedDesiredEntries, + unmatchedDesiredEntries, + ) + }, + ) - try { - expect( - diffPublications(publishedRows, desiredRows).map( - ({ type, key }) => `${type}:${key}`, - ), - ).toEqual([`update:a`, `delete:b`, `insert:c`]) - expect( - defaultIterator.mock.calls.length + - entries.mock.calls.length + - keys.mock.calls.length + - values.mock.calls.length + - forEach.mock.calls.length, - ).toBe(2) - } finally { - defaultIterator.mockRestore() - entries.mockRestore() - keys.mockRestore() - values.mockRestore() - forEach.mockRestore() - } + expect( + diffPublications(publishedRows, desiredRows).map( + ({ type, key }) => `${type}:${key}`, + ), + ).toEqual([`update:a`, `delete:b`, `insert:c`]) + expect(maximumUnmatchedDesiredEntries).toBe(1) + expect(unmatchedDesiredEntries).toBe(0) expect(publishedRows.iterationReads).toBe(publishedRows.size) expect(publishedRows.membershipReads).toBe(desiredRows.size) expect(publishedRows.valueReads).toBe(0) From e6b6b40bc782c5a59ff57e980262c7ea718fb4b9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 17:51:10 -0600 Subject: [PATCH 178/327] test(db): observe materialized comparator work --- .../ordered-work-oracle.property.test.ts | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 4dd9efb93..cd83212d7 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -903,6 +903,46 @@ describe(`ordered source work oracle`, () => { }, ) + it.each([`asc`, `desc`] as const)( + `executes the materialized inner comparator once for dates in %s order`, + (direction) => { + const getTime = vi.spyOn(Date.prototype, `getTime`) + try { + expect( + makeComparator({ direction, nulls: `last` })( + new Date(0), + new Date(1), + ), + ).toBe(direction === `asc` ? -1 : 1) + // Each valid Date is read once while checking the unorderable case and + // once more for the comparison itself. + expect(getTime).toHaveBeenCalledTimes(4) + } finally { + getTime.mockRestore() + } + }, + ) + + it.each([`asc`, `desc`] as const)( + `executes the materialized inner comparator once for strings in %s order`, + (direction) => { + const localeCompare = vi.spyOn(String.prototype, `localeCompare`) + try { + expect( + makeComparator({ + direction, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + })(`a`, `b`), + ).toBe(direction === `asc` ? -1 : 1) + expect(localeCompare).toHaveBeenCalledTimes(1) + } finally { + localeCompare.mockRestore() + } + }, + ) + it.each([ { indexKind: `basic`, direction: `asc` }, { indexKind: `basic`, direction: `desc` }, From 8cbe7884decc8a9a5cebac4c958f9aeeb66f668d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:01:33 -0600 Subject: [PATCH 179/327] refactor(db): keep comparator work observable --- packages/db/src/utils/comparison.ts | 25 +++++++----- .../ordered-work-oracle.property.test.ts | 40 +++++++++---------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 3c6d22179..59357b5bc 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -77,8 +77,17 @@ export function snapshotUint8ArrayBytes(value: Uint8Array): Uint8Array { * Handles null/undefined, strings, arrays, dates, objects, and primitives * Always sorts null/undefined values first */ -export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { - const { nulls } = opts +const compareAscending = ( + a: any, + b: any, + opts: CompareOptions, + invertNulls: boolean, +): number => { + const nulls = invertNulls + ? opts.nulls === `first` + ? `last` + : `first` + : opts.nulls // Handle null/undefined if (a == null && b == null) return 0 @@ -106,7 +115,7 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { // if a and b are both arrays, compare them element by element if (Array.isArray(a) && Array.isArray(b)) { for (let i = 0; i < Math.min(a.length, b.length); i++) { - const result = ascComparator(a[i], b[i], opts) + const result = compareAscending(a[i], b[i], opts, invertNulls) if (result !== 0) { return result } @@ -148,6 +157,9 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { return 0 } +export const ascComparator = (a: any, b: any, opts: CompareOptions): number => + compareAscending(a, b, opts, false) + /** * Descending comparator function for ordering values * Handles null/undefined as largest values (opposite of ascending) @@ -156,12 +168,7 @@ export const descComparator = ( a: unknown, b: unknown, opts: CompareOptions, -): number => { - return ascComparator(b, a, { - ...opts, - nulls: opts.nulls === `first` ? `last` : `first`, - }) -} +): number => compareAscending(b, a, opts, true) export function makeComparator( opts: CompareOptions, diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index cd83212d7..4188d4d82 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -813,13 +813,13 @@ describe(`ordered source work oracle`, () => { right: 2, expected: 1, expectedReads: { - direction: 2, - nulls: 2, - stringSort: 1, - locale: 1, - localeOptions: 1, - ownKeys: 1, - descriptors: 5, + direction: 1, + nulls: 1, + stringSort: 0, + locale: 0, + localeOptions: 0, + ownKeys: 0, + descriptors: 0, prototype: 0, }, }, @@ -830,13 +830,13 @@ describe(`ordered source work oracle`, () => { right: `b`, expected: 1, expectedReads: { - direction: 2, - nulls: 2, + direction: 1, + nulls: 1, stringSort: 1, locale: 1, localeOptions: 1, - ownKeys: 1, - descriptors: 5, + ownKeys: 0, + descriptors: 0, prototype: 0, }, }, @@ -904,7 +904,7 @@ describe(`ordered source work oracle`, () => { ) it.each([`asc`, `desc`] as const)( - `executes the materialized inner comparator once for dates in %s order`, + `executes the inner comparator's date work once in %s order`, (direction) => { const getTime = vi.spyOn(Date.prototype, `getTime`) try { @@ -924,7 +924,7 @@ describe(`ordered source work oracle`, () => { ) it.each([`asc`, `desc`] as const)( - `executes the materialized inner comparator once for strings in %s order`, + `executes the inner comparator's string work once in %s order`, (direction) => { const localeCompare = vi.spyOn(String.prototype, `localeCompare`) try { @@ -3113,7 +3113,7 @@ it.each([ secondary: string | number } const termReads: [number, number] = [0, 0] - const termComparisons: [number, number] = [0, 0] + const innerComparisons: [number, number] = [0, 0] const rows = new Map( sourceRows.map((spec) => [ spec.id, @@ -3141,7 +3141,7 @@ it.each([ } satisfies CompareOptions, { get(target, property, receiver) { - if (property === `direction`) termComparisons[term]++ + if (property === `nulls`) innerComparisons[term]++ return Reflect.get(target, property, receiver) as unknown }, }, @@ -3180,18 +3180,14 @@ it.each([ try { termReads[0] = 0 termReads[1] = 0 - termComparisons[0] = 0 - termComparisons[1] = 0 + innerComparisons[0] = 0 + innerComparisons[1] = 0 expect(window.publicationEntries().map(([key]) => key)).toEqual( expectedKeys, ) expect(compareRows).toHaveBeenCalledTimes(2) expect(termReads).toEqual([4, orderArity === 2 ? 2 : 0]) - const optionReadsPerComparison = direction === `asc` ? 1 : 2 - expect(termComparisons).toEqual([ - 2 * optionReadsPerComparison, - orderArity === 2 ? optionReadsPerComparison : 0, - ]) + expect(innerComparisons).toEqual([2, orderArity === 2 ? 1 : 0]) } finally { compareRows.mockRestore() } From d92d5d5e6d773e611380c2a83210aed2dea05bc9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:13:52 -0600 Subject: [PATCH 180/327] test(db): observe recursive comparator work --- .../ordered-work-oracle.property.test.ts | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 4188d4d82..db20c771f 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -943,6 +943,65 @@ describe(`ordered source work oracle`, () => { }, ) + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + name: `no common element`, + left: [], + right: [1], + expectedNullReads: 1, + }, + { + name: `first-element difference`, + left: [1, 9], + right: [2, 0], + expectedNullReads: 2, + }, + { + name: `equal prefix then difference`, + left: [1, 2], + right: [1, 3], + expectedNullReads: 3, + }, + { + name: `equal common prefix then length`, + left: [1], + right: [1, 2], + expectedNullReads: 2, + }, + { + name: `nested recursion`, + left: [[1, 2]], + right: [[1, 3]], + expectedNullReads: 4, + }, + ].map((scenario) => ({ direction, ...scenario })), + ), + )( + `visits each array comparison once for $name in $direction order`, + ({ direction, left, right, expectedNullReads }) => { + let nullReads = 0 + const options = new Proxy( + { + direction, + nulls: `last` as const, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `nulls`) nullReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) + + expect(Math.sign(makeComparator(options)(left, right))).toBe( + direction === `asc` ? -1 : 1, + ) + expect(nullReads).toBe(expectedNullReads) + }, + ) + it.each([ { indexKind: `basic`, direction: `asc` }, { indexKind: `basic`, direction: `desc` }, From dde74b4e2ab6da9074b3cd23c85d84f3f902471e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:22:24 -0600 Subject: [PATCH 181/327] perf(db): read comparator arrays once --- packages/db/src/utils/comparison.ts | 7 +++- .../ordered-work-oracle.property.test.ts | 42 +++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 59357b5bc..61d52b8a3 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -114,14 +114,17 @@ const compareAscending = ( // if a and b are both arrays, compare them element by element if (Array.isArray(a) && Array.isArray(b)) { - for (let i = 0; i < Math.min(a.length, b.length); i++) { + const aLength = a.length + const bLength = b.length + const commonLength = Math.min(aLength, bLength) + for (let i = 0; i < commonLength; i++) { const result = compareAscending(a[i], b[i], opts, invertNulls) if (result !== 0) { return result } } // All elements are equal up to the minimum length - return a.length - b.length + return aLength - bLength } // If both are dates, compare them diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index db20c771f..3b297f923 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -951,37 +951,69 @@ describe(`ordered source work oracle`, () => { left: [], right: [1], expectedNullReads: 1, + expectedArrayReads: { lengths: 1, elements: 0 }, }, { name: `first-element difference`, left: [1, 9], right: [2, 0], expectedNullReads: 2, + expectedArrayReads: { lengths: 1, elements: 1 }, }, { name: `equal prefix then difference`, left: [1, 2], right: [1, 3], expectedNullReads: 3, + expectedArrayReads: { lengths: 1, elements: 2 }, }, { name: `equal common prefix then length`, left: [1], right: [1, 2], expectedNullReads: 2, + expectedArrayReads: { lengths: 1, elements: 1 }, }, { name: `nested recursion`, left: [[1, 2]], right: [[1, 3]], expectedNullReads: 4, + expectedArrayReads: { lengths: 2, elements: 3 }, }, ].map((scenario) => ({ direction, ...scenario })), ), )( `visits each array comparison once for $name in $direction order`, - ({ direction, left, right, expectedNullReads }) => { + ({ direction, left, right, expectedNullReads, expectedArrayReads }) => { let nullReads = 0 + const observeArrayReads = ( + value: Array, + reads: { lengths: number; elements: number; other: number }, + ): Array => { + const nested = value.map((element) => + Array.isArray(element) ? observeArrayReads(element, reads) : element, + ) + return new Proxy(nested, { + get(target, property, receiver) { + if (property === `length`) { + reads.lengths++ + } else if ( + typeof property === `string` && + /^(0|[1-9]\d*)$/.test(property) + ) { + reads.elements++ + } else { + reads.other++ + } + return Reflect.get(target, property, receiver) as unknown + }, + }) + } + const leftReads = { lengths: 0, elements: 0, other: 0 } + const rightReads = { lengths: 0, elements: 0, other: 0 } + const observedLeft = observeArrayReads(left, leftReads) + const observedRight = observeArrayReads(right, rightReads) const options = new Proxy( { direction, @@ -995,10 +1027,12 @@ describe(`ordered source work oracle`, () => { }, ) - expect(Math.sign(makeComparator(options)(left, right))).toBe( - direction === `asc` ? -1 : 1, - ) + expect( + Math.sign(makeComparator(options)(observedLeft, observedRight)), + ).toBe(direction === `asc` ? -1 : 1) expect(nullReads).toBe(expectedNullReads) + expect(leftReads).toEqual({ ...expectedArrayReads, other: 0 }) + expect(rightReads).toEqual({ ...expectedArrayReads, other: 0 }) }, ) From 23c84ec949c405bea96aef974a9afd40e43d0654 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:30:50 -0600 Subject: [PATCH 182/327] test(db): observe structural array work --- .../ordered-work-oracle.property.test.ts | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 3b297f923..85b71f087 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -984,12 +984,12 @@ describe(`ordered source work oracle`, () => { ].map((scenario) => ({ direction, ...scenario })), ), )( - `visits each array comparison once for $name in $direction order`, + `does one input-dependent pass for $name in $direction array order`, ({ direction, left, right, expectedNullReads, expectedArrayReads }) => { let nullReads = 0 const observeArrayReads = ( value: Array, - reads: { lengths: number; elements: number; other: number }, + reads: { lengths: number; elements: number; structural: number }, ): Array => { const nested = value.map((element) => Array.isArray(element) ? observeArrayReads(element, reads) : element, @@ -1004,14 +1004,26 @@ describe(`ordered source work oracle`, () => { ) { reads.elements++ } else { - reads.other++ + reads.structural++ } return Reflect.get(target, property, receiver) as unknown }, + ownKeys(target) { + reads.structural++ + return Reflect.ownKeys(target) + }, + getOwnPropertyDescriptor(target, property) { + reads.structural++ + return Reflect.getOwnPropertyDescriptor(target, property) + }, + has(target, property) { + reads.structural++ + return Reflect.has(target, property) + }, }) } - const leftReads = { lengths: 0, elements: 0, other: 0 } - const rightReads = { lengths: 0, elements: 0, other: 0 } + const leftReads = { lengths: 0, elements: 0, structural: 0 } + const rightReads = { lengths: 0, elements: 0, structural: 0 } const observedLeft = observeArrayReads(left, leftReads) const observedRight = observeArrayReads(right, rightReads) const options = new Proxy( @@ -1031,8 +1043,8 @@ describe(`ordered source work oracle`, () => { Math.sign(makeComparator(options)(observedLeft, observedRight)), ).toBe(direction === `asc` ? -1 : 1) expect(nullReads).toBe(expectedNullReads) - expect(leftReads).toEqual({ ...expectedArrayReads, other: 0 }) - expect(rightReads).toEqual({ ...expectedArrayReads, other: 0 }) + expect(leftReads).toEqual({ ...expectedArrayReads, structural: 0 }) + expect(rightReads).toEqual({ ...expectedArrayReads, structural: 0 }) }, ) From 3e6bb2b4d33fea10c17a9ab34d33b8ad6b704e5b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:40:31 -0600 Subject: [PATCH 183/327] test(db): state array work boundary --- packages/db/tests/query/ordered-work-oracle.property.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 85b71f087..91ab41fa6 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -984,7 +984,7 @@ describe(`ordered source work oracle`, () => { ].map((scenario) => ({ direction, ...scenario })), ), )( - `does one input-dependent pass for $name in $direction array order`, + `reads each visited array input once for $name in $direction order`, ({ direction, left, right, expectedNullReads, expectedArrayReads }) => { let nullReads = 0 const observeArrayReads = ( From 9d6a1711fecc5eb9c12769b3634d3ce0f0643312 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 18:55:07 -0600 Subject: [PATCH 184/327] test(db): cover equal comparator arrays --- .../ordered-work-oracle.property.test.ts | 54 ++++++++++++++++++- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 91ab41fa6..8a108fe3a 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -952,6 +952,7 @@ describe(`ordered source work oracle`, () => { right: [1], expectedNullReads: 1, expectedArrayReads: { lengths: 1, elements: 0 }, + ascendingSign: -1, }, { name: `first-element difference`, @@ -959,6 +960,7 @@ describe(`ordered source work oracle`, () => { right: [2, 0], expectedNullReads: 2, expectedArrayReads: { lengths: 1, elements: 1 }, + ascendingSign: -1, }, { name: `equal prefix then difference`, @@ -966,6 +968,7 @@ describe(`ordered source work oracle`, () => { right: [1, 3], expectedNullReads: 3, expectedArrayReads: { lengths: 1, elements: 2 }, + ascendingSign: -1, }, { name: `equal common prefix then length`, @@ -973,6 +976,7 @@ describe(`ordered source work oracle`, () => { right: [1, 2], expectedNullReads: 2, expectedArrayReads: { lengths: 1, elements: 1 }, + ascendingSign: -1, }, { name: `nested recursion`, @@ -980,12 +984,52 @@ describe(`ordered source work oracle`, () => { right: [[1, 3]], expectedNullReads: 4, expectedArrayReads: { lengths: 2, elements: 3 }, + ascendingSign: -1, + }, + { + name: `equal empty arrays`, + left: [], + right: [], + expectedNullReads: 1, + expectedArrayReads: { lengths: 1, elements: 0 }, + ascendingSign: 0, + }, + { + name: `equal primitive arrays`, + left: [1, 2], + right: [1, 2], + expectedNullReads: 3, + expectedArrayReads: { lengths: 1, elements: 2 }, + ascendingSign: 0, + }, + { + name: `equal nested arrays`, + left: [[1, 2]], + right: [[1, 2]], + expectedNullReads: 4, + expectedArrayReads: { lengths: 2, elements: 3 }, + ascendingSign: 0, + }, + { + name: `equal nested prefix then outer difference`, + left: [[1], 2], + right: [[1], 3], + expectedNullReads: 4, + expectedArrayReads: { lengths: 2, elements: 3 }, + ascendingSign: -1, }, ].map((scenario) => ({ direction, ...scenario })), ), )( `reads each visited array input once for $name in $direction order`, - ({ direction, left, right, expectedNullReads, expectedArrayReads }) => { + ({ + direction, + left, + right, + expectedNullReads, + expectedArrayReads, + ascendingSign, + }) => { let nullReads = 0 const observeArrayReads = ( value: Array, @@ -1041,7 +1085,13 @@ describe(`ordered source work oracle`, () => { expect( Math.sign(makeComparator(options)(observedLeft, observedRight)), - ).toBe(direction === `asc` ? -1 : 1) + ).toBe( + ascendingSign === 0 + ? 0 + : direction === `asc` + ? ascendingSign + : -ascendingSign, + ) expect(nullReads).toBe(expectedNullReads) expect(leftReads).toEqual({ ...expectedArrayReads, structural: 0 }) expect(rightReads).toEqual({ ...expectedArrayReads, structural: 0 }) From 89fc7dceb6137ebd6478147c68a06194428df4fb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 19:02:27 -0600 Subject: [PATCH 185/327] test(db): cross nested comparator branches --- .../db/tests/query/ordered-work-oracle.property.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 8a108fe3a..9ba1919f3 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -1018,6 +1018,14 @@ describe(`ordered source work oracle`, () => { expectedArrayReads: { lengths: 2, elements: 3 }, ascendingSign: -1, }, + { + name: `equal nested prefix then outer length`, + left: [[1]], + right: [[1], 2], + expectedNullReads: 3, + expectedArrayReads: { lengths: 2, elements: 2 }, + ascendingSign: -1, + }, ].map((scenario) => ({ direction, ...scenario })), ), )( From 49c2cadc44e8b0911573bfcaa65a88d4af12019a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 19:17:25 -0600 Subject: [PATCH 186/327] test(db): model recursive comparator work --- .../ordered-work-oracle.property.test.ts | 231 ++++++++++-------- 1 file changed, 127 insertions(+), 104 deletions(-) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts index 9ba1919f3..71fa4a0da 100644 --- a/packages/db/tests/query/ordered-work-oracle.property.test.ts +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -943,106 +943,128 @@ describe(`ordered source work oracle`, () => { }, ) + type ComparatorArrayValue = number | Array + type ComparatorArray = Array + type ArrayInputReads = { lengths: number; elements: number } + type ArrayComparisonModel = { + sign: number + nullReads: number + leftReads: ArrayInputReads + rightReads: ArrayInputReads + } + + const modelAscendingArrayComparison = ( + left: ComparatorArrayValue, + right: ComparatorArrayValue, + ): ArrayComparisonModel => { + const leftReads: ArrayInputReads = { lengths: 0, elements: 0 } + const rightReads: ArrayInputReads = { lengths: 0, elements: 0 } + + if (Array.isArray(left) && Array.isArray(right)) { + leftReads.lengths++ + rightReads.lengths++ + const commonLength = Math.min(left.length, right.length) + let nullReads = 1 + + for (let index = 0; index < commonLength; index++) { + leftReads.elements++ + rightReads.elements++ + const child = modelAscendingArrayComparison(left[index]!, right[index]!) + nullReads += child.nullReads + leftReads.lengths += child.leftReads.lengths + leftReads.elements += child.leftReads.elements + rightReads.lengths += child.rightReads.lengths + rightReads.elements += child.rightReads.elements + if (child.sign !== 0) { + return { sign: child.sign, nullReads, leftReads, rightReads } + } + } + + return { + sign: Math.sign(left.length - right.length), + nullReads, + leftReads, + rightReads, + } + } + + const sign = Array.isArray(left) + ? 1 + : Array.isArray(right) + ? -1 + : Math.sign(left - right) + return { sign, nullReads: 1, leftReads, rightReads } + } + + const modelArrayComparison = ( + left: ComparatorArray, + right: ComparatorArray, + direction: `asc` | `desc`, + ): ArrayComparisonModel => { + if (direction === `asc`) { + return modelAscendingArrayComparison(left, right) + } + + const reversed = modelAscendingArrayComparison(right, left) + return { + sign: reversed.sign, + nullReads: reversed.nullReads, + leftReads: reversed.rightReads, + rightReads: reversed.leftReads, + } + } + + const recursiveArrayComparisonScenarios = [ + { name: `empty equality`, left: [], right: [] }, + { name: `no common element`, left: [], right: [1] }, + { name: `primitive equality`, left: [1], right: [1] }, + { name: `primitive difference`, left: [1], right: [2] }, + { name: `equal prefix then difference`, left: [1, 2], right: [1, 3] }, + { name: `equal prefix then length`, left: [1], right: [1, 2] }, + { name: `array then primitive`, left: [[]], right: [1] }, + { name: `primitive then array`, left: [1], right: [[]] }, + { + name: `equal nested prefix then outer difference`, + left: [[1], 2], + right: [[1], 3], + }, + { + name: `equal nested prefix then outer length`, + left: [[1]], + right: [[1], 2], + }, + ].flatMap(({ name, left: initialLeft, right: initialRight }) => { + const scenarios: Array<{ + name: string + left: ComparatorArray + right: ComparatorArray + }> = [] + let left: ComparatorArray = initialLeft + let right: ComparatorArray = initialRight + + for (let depth = 1; depth <= 3; depth++) { + scenarios.push({ name: `${name} at depth ${depth}`, left, right }) + left = [left] + right = [right] + } + return scenarios + }) + it.each( ([`asc`, `desc`] as const).flatMap((direction) => - [ - { - name: `no common element`, - left: [], - right: [1], - expectedNullReads: 1, - expectedArrayReads: { lengths: 1, elements: 0 }, - ascendingSign: -1, - }, - { - name: `first-element difference`, - left: [1, 9], - right: [2, 0], - expectedNullReads: 2, - expectedArrayReads: { lengths: 1, elements: 1 }, - ascendingSign: -1, - }, - { - name: `equal prefix then difference`, - left: [1, 2], - right: [1, 3], - expectedNullReads: 3, - expectedArrayReads: { lengths: 1, elements: 2 }, - ascendingSign: -1, - }, - { - name: `equal common prefix then length`, - left: [1], - right: [1, 2], - expectedNullReads: 2, - expectedArrayReads: { lengths: 1, elements: 1 }, - ascendingSign: -1, - }, - { - name: `nested recursion`, - left: [[1, 2]], - right: [[1, 3]], - expectedNullReads: 4, - expectedArrayReads: { lengths: 2, elements: 3 }, - ascendingSign: -1, - }, - { - name: `equal empty arrays`, - left: [], - right: [], - expectedNullReads: 1, - expectedArrayReads: { lengths: 1, elements: 0 }, - ascendingSign: 0, - }, - { - name: `equal primitive arrays`, - left: [1, 2], - right: [1, 2], - expectedNullReads: 3, - expectedArrayReads: { lengths: 1, elements: 2 }, - ascendingSign: 0, - }, - { - name: `equal nested arrays`, - left: [[1, 2]], - right: [[1, 2]], - expectedNullReads: 4, - expectedArrayReads: { lengths: 2, elements: 3 }, - ascendingSign: 0, - }, - { - name: `equal nested prefix then outer difference`, - left: [[1], 2], - right: [[1], 3], - expectedNullReads: 4, - expectedArrayReads: { lengths: 2, elements: 3 }, - ascendingSign: -1, - }, - { - name: `equal nested prefix then outer length`, - left: [[1]], - right: [[1], 2], - expectedNullReads: 3, - expectedArrayReads: { lengths: 2, elements: 2 }, - ascendingSign: -1, - }, - ].map((scenario) => ({ direction, ...scenario })), + recursiveArrayComparisonScenarios.map((scenario) => ({ + direction, + ...scenario, + })), ), )( `reads each visited array input once for $name in $direction order`, - ({ - direction, - left, - right, - expectedNullReads, - expectedArrayReads, - ascendingSign, - }) => { + ({ direction, left, right }) => { let nullReads = 0 const observeArrayReads = ( - value: Array, + value: ComparatorArray, reads: { lengths: number; elements: number; structural: number }, - ): Array => { + ): ComparatorArray => { const nested = value.map((element) => Array.isArray(element) ? observeArrayReads(element, reads) : element, ) @@ -1055,10 +1077,16 @@ describe(`ordered source work oracle`, () => { /^(0|[1-9]\d*)$/.test(property) ) { reads.elements++ - } else { + } else if (property !== Symbol.toStringTag) { + // Array/scalar comparisons perform constant-time brand checks. + // The work law counts input-dependent traversal, not those checks. reads.structural++ } - return Reflect.get(target, property, receiver) as unknown + return Reflect.get( + target, + property, + receiver, + ) as ComparatorArrayValue }, ownKeys(target) { reads.structural++ @@ -1078,6 +1106,7 @@ describe(`ordered source work oracle`, () => { const rightReads = { lengths: 0, elements: 0, structural: 0 } const observedLeft = observeArrayReads(left, leftReads) const observedRight = observeArrayReads(right, rightReads) + const expected = modelArrayComparison(left, right, direction) const options = new Proxy( { direction, @@ -1093,16 +1122,10 @@ describe(`ordered source work oracle`, () => { expect( Math.sign(makeComparator(options)(observedLeft, observedRight)), - ).toBe( - ascendingSign === 0 - ? 0 - : direction === `asc` - ? ascendingSign - : -ascendingSign, - ) - expect(nullReads).toBe(expectedNullReads) - expect(leftReads).toEqual({ ...expectedArrayReads, structural: 0 }) - expect(rightReads).toEqual({ ...expectedArrayReads, structural: 0 }) + ).toBe(expected.sign) + expect(nullReads).toBe(expected.nullReads) + expect(leftReads).toEqual({ ...expected.leftReads, structural: 0 }) + expect(rightReads).toEqual({ ...expected.rightReads, structural: 0 }) }, ) From 0d4a8e92060169aa2317336600df34284c7de752 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 19:39:35 -0600 Subject: [PATCH 187/327] fix(db): publish source changes through derived mutations --- packages/db/src/query/live/ARCHITECTURE.md | 8 +- .../query/live/collection-config-builder.ts | 5 +- .../tests/live-query-order-only-move.test.ts | 19 +- .../query/includes-publication-oracle.test.ts | 242 +++++++++++++++++- .../query/load-subset-oracle.property.test.ts | 75 ------ 5 files changed, 259 insertions(+), 90 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 01d9dc7df..055ba501d 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -928,8 +928,12 @@ priority merely to make a subset load settle. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are -visible. Rejected, canceled, and obsolete acquisitions establish no coverage. -Sources must honor cancellation before publishing request-scoped rows. +visible. A quiescent live-query graph output uses the same queue bypass so a +direct source change can update a derived Collection while an optimistic +mutation on that Collection persists. The normal optimistic overlay still wins +for conflicting keys, and the graph output remains one coherent publication. +Rejected, canceled, and obsolete acquisitions establish no coverage. Sources +must honor cancellation before publishing request-scoped rows. After those writes are applied, `loadSubset` may resolve with `{ hasMore: boolean | undefined, appliedRowKeys?: readonly Key[] }`. Core diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 813851834..fb0e2215a 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1074,7 +1074,10 @@ export class CollectionConfigBuilder< ) if (hasParentChanges) { - begin() + // The graph has already reached quiescence, so this is one complete + // derived publication. Apply it beneath any pending optimistic + // overlay instead of parking source progress behind that mutation. + begin({ immediate: true }) changesToApply.forEach(this.applyChanges.bind(this, config)) if (hasOrderOnlyMove(changesToApply)) { markLayoutChange(config.collection) diff --git a/packages/db/tests/live-query-order-only-move.test.ts b/packages/db/tests/live-query-order-only-move.test.ts index 5d6bf5181..0fac5f148 100644 --- a/packages/db/tests/live-query-order-only-move.test.ts +++ b/packages/db/tests/live-query-order-only-move.test.ts @@ -43,7 +43,7 @@ async function makeOrderedByAge(source: ReturnType) { const flush = () => new Promise((r) => setTimeout(r, 0)) -describe(`order-only move (RFC #1623 phase 4)`, () => { +describe(`order-only move publication`, () => { it(`republishes the ordered result when a row moves but its value is unchanged`, async () => { const source = makeSource() const lq = await makeOrderedByAge(source) @@ -117,7 +117,7 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { observer.dispose() }) - it(`refreshes a detached observer when an order-only sync is parked`, async () => { + it(`refreshes a detached observer while a separate mutation persists`, async () => { const source = makeSource() const persist = createDeferred() const lq = createLiveQueryCollection({ @@ -158,13 +158,12 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { source.utils.commit() await flush() - const parked = observer.getSnapshot() - expect((parked.data as Array).map((row) => row.id)).toEqual([ - `2`, - `1`, - `3`, - ]) - expect(lq._layoutRevision).toBe(collectionLayoutRevisionBefore) + const whilePersisting = observer.getSnapshot() + expect( + (whilePersisting.data as Array).map((row) => row.id), + ).toEqual([`1`, `3`, `2`]) + expect(lq._layoutRevision).toBeGreaterThan(collectionLayoutRevisionBefore) + const publishedLayoutRevision = lq._layoutRevision persist.resolve() await mutation.isPersisted.promise @@ -176,7 +175,7 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { `3`, `2`, ]) - expect(lq._layoutRevision).toBeGreaterThan(collectionLayoutRevisionBefore) + expect(lq._layoutRevision).toBe(publishedLayoutRevision) observer.dispose() }) diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 2e59db03e..38f4499f2 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -1,6 +1,8 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect } from 'vitest' +import { describe, expect, it } from 'vitest' +import { createDeferred } from '../../src/deferred.js' import { BasicIndex } from '../../src/indexes/basic-index.js' +import { createOptimisticAction } from '../../src/optimistic-action.js' import { createLiveQueryCollection, eq, @@ -37,6 +39,13 @@ type PublishedRow = { type Q2Shape = `passThrough` | `where` | `orderBy` | `select` type Q1Shape = `direct` | `joined` +type PendingPublicationOperation = `insert` | `update` | `delete` +type PendingPublicationDepth = `direct` | `layered` + +type PendingPublicationRow = { + id: number + value: number +} const initialParent: ParentRow = { id: 1, group: 10, value: 0 } const initialChild: ChildRow = { id: 100, parentGroup: 10, value: 1 } @@ -401,6 +410,210 @@ async function expectPublicationMatches( const q2Shapes = [`passThrough`, `where`, `orderBy`, `select`] as const const q1Shapes = [`direct`, `joined`] as const +const pendingPublicationOperations = [`insert`, `update`, `delete`] as const +const pendingPublicationDepths = [`direct`, `layered`] as const + +const optimisticExistingRow: PendingPublicationRow = { id: 1, value: 10 } +const sourceExistingRow: PendingPublicationRow = { id: 2, value: 20 } +const optimisticInsertedRow: PendingPublicationRow = { id: 3, value: 30 } +const sourceInsertedRow: PendingPublicationRow = { id: 4, value: 40 } + +function pendingOperationRow( + operation: PendingPublicationOperation, + owner: `optimistic` | `source`, +): PendingPublicationRow { + if (owner === `optimistic`) { + if (operation === `insert`) return { ...optimisticInsertedRow } + if (operation === `update`) return { ...optimisticExistingRow, value: 11 } + return { ...optimisticExistingRow } + } + + if (operation === `insert`) return { ...sourceInsertedRow } + if (operation === `update`) return { ...sourceExistingRow, value: 21 } + return { ...sourceExistingRow } +} + +function applyPendingOperation( + rows: Map, + operation: PendingPublicationOperation, + row: PendingPublicationRow, +): void { + if (operation === `delete`) rows.delete(row.id) + else rows.set(row.id, { ...row }) +} + +function expectedPendingRows( + rows: ReadonlyMap, +): Array { + return [...rows.values()] + .map((row) => ({ ...row })) + .sort((left, right) => left.id - right.id) +} + +function inversePendingOperation( + operation: PendingPublicationOperation, +): PendingPublicationOperation { + if (operation === `insert`) return `delete` + if (operation === `delete`) return `insert` + return `update` +} + +async function expectSourcePublicationDuringPendingMutation( + optimisticOperation: PendingPublicationOperation, + sourceOperation: PendingPublicationOperation, + depth: PendingPublicationDepth, + sameKey = false, +): Promise { + const initialRows = [optimisticExistingRow, sourceExistingRow] + const source = createControlledCollection( + `pending-publication-source`, + initialRows, + ) + const q1 = createLiveQueryCollection({ + id: `pending-publication-q1-${nextCollectionId++}`, + query: (q) => + q.from({ row: source.collection }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + const q2 = createLiveQueryCollection({ + id: `pending-publication-q2-${nextCollectionId++}`, + query: (q) => + q.from({ row: q1 }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + const target = depth === `direct` ? q1 : q2 + const persistence = createDeferred() + const observedBatches: Array< + Array<{ type: `insert` | `update` | `delete`; key: number }> + > = [] + const callbackSnapshots: Array> = [] + const currentRows = () => + target.toArray + .map((row) => ({ id: row.id, value: row.value })) + .sort((left, right) => left.id - right.id) + + await target.preload() + const subscription = target.subscribeChanges( + (changes) => { + observedBatches.push( + changes.map(({ type, key }) => ({ type, key: Number(key) })), + ) + callbackSnapshots.push(currentRows()) + }, + { includeInitialState: false }, + ) + + const optimisticRow = pendingOperationRow(optimisticOperation, `optimistic`) + const sourceRow = sameKey + ? { ...optimisticRow } + : pendingOperationRow(sourceOperation, `source`) + const insertTarget = target.insert.bind(target) as unknown as ( + row: PendingPublicationRow, + ) => unknown + const mutate = createOptimisticAction({ + onMutate: (operation) => { + if (operation === `insert`) { + insertTarget(optimisticRow) + } else if (operation === `update`) { + target.update(optimisticRow.id, (draft) => { + draft.value = optimisticRow.value + }) + } else { + target.delete(optimisticRow.id) + } + }, + mutationFn: () => persistence.promise, + }) + + const transaction = mutate(optimisticOperation) + const afterOptimistic = new Map( + initialRows.map((row) => [row.id, { ...row }] as const), + ) + applyPendingOperation(afterOptimistic, optimisticOperation, optimisticRow) + + try { + expect(observedBatches).toEqual([ + [{ type: optimisticOperation, key: optimisticRow.id }], + ]) + expect(callbackSnapshots).toEqual([expectedPendingRows(afterOptimistic)]) + expect(currentRows()).toEqual(expectedPendingRows(afterOptimistic)) + + source.write(sourceOperation, sourceRow) + const afterSource = new Map( + initialRows.map((row) => [row.id, { ...row }] as const), + ) + applyPendingOperation(afterSource, sourceOperation, sourceRow) + const whilePending = new Map(afterSource) + applyPendingOperation(whilePending, optimisticOperation, optimisticRow) + + if (sameKey) { + expect(observedBatches).toEqual([ + [{ type: optimisticOperation, key: optimisticRow.id }], + ]) + expect(callbackSnapshots).toEqual([expectedPendingRows(whilePending)]) + } else { + expect(observedBatches).toEqual([ + [{ type: optimisticOperation, key: optimisticRow.id }], + [{ type: sourceOperation, key: sourceRow.id }], + ]) + expect(callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic), + expectedPendingRows(whilePending), + ]) + } + expect(currentRows()).toEqual(expectedPendingRows(whilePending)) + + persistence.resolve() + await transaction.isPersisted.promise + await flushPromises() + + if (sameKey) { + const confirmationBatches = + optimisticOperation === `delete` + ? [] + : [[{ type: `update` as const, key: optimisticRow.id }]] + expect(observedBatches).toEqual([ + [{ type: optimisticOperation, key: optimisticRow.id }], + ...confirmationBatches, + ]) + expect(callbackSnapshots).toEqual([ + expectedPendingRows(afterSource), + ...confirmationBatches.map(() => expectedPendingRows(afterSource)), + ]) + } else { + expect(observedBatches).toEqual([ + [{ type: optimisticOperation, key: optimisticRow.id }], + [{ type: sourceOperation, key: sourceRow.id }], + [ + { + type: inversePendingOperation(optimisticOperation), + key: optimisticRow.id, + }, + ], + ]) + expect(callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic), + expectedPendingRows(whilePending), + expectedPendingRows(afterSource), + ]) + } + expect(currentRows()).toEqual(expectedPendingRows(afterSource)) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await q2.cleanup() + await q1.cleanup() + await source.collection.cleanup() + } +} + describe(`layered-query publication oracle`, () => { const changedValueArbitrary = fc.oneof( fc.integer({ min: -100, max: -1 }), @@ -420,7 +633,7 @@ describe(`layered-query publication oracle`, () => { `includes-publication.parent-scalar.${q1Shape}.${q2Shape}`, ), )( - `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, + `publishes scalar parent updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( { type: `parentScalar`, value }, @@ -529,3 +742,28 @@ describe(`layered-query publication oracle`, () => { }) }) }) + +describe(`source publication across pending derived mutations`, () => { + for (const depth of pendingPublicationDepths) { + for (const optimisticOperation of pendingPublicationOperations) { + for (const sourceOperation of pendingPublicationOperations) { + it(`publishes a disjoint source ${sourceOperation} through a ${depth} query while an optimistic ${optimisticOperation} persists`, async () => { + await expectSourcePublicationDuringPendingMutation( + optimisticOperation, + sourceOperation, + depth, + ) + }) + } + + it(`retains a same-key source ${optimisticOperation} through a ${depth} query while its optimistic confirmation persists`, async () => { + await expectSourcePublicationDuringPendingMutation( + optimisticOperation, + optimisticOperation, + depth, + true, + ) + }) + } + } +}) diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index 5b5dada62..6dd830b0b 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -2,7 +2,6 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' -import { createOptimisticAction } from '../../src/optimistic-action.js' import { createLiveQueryCollection, eq } from '../../src/query/index.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { Func, PropRef, Value } from '../../src/query/ir.js' @@ -73,11 +72,6 @@ type PersistedLoadRow = { projectId: string } -type OptimisticDerivedRow = { - id: string - value: string -} - type CoverageSubject = { loadSubset: LoadSubsetFn reset?: () => void @@ -1877,64 +1871,6 @@ async function expectCleanupRejectsReceiptOnce() { await source.cleanup() } -async function expectDerivedSyncDuringOptimisticMutation(): Promise { - let begin!: () => void - let write!: (message: { type: `insert`; value: OptimisticDerivedRow }) => void - let commit!: () => void - const source = createCollection({ - id: `optimistic-derived-source-${collectionSequence++}`, - getKey: (row) => row.id, - sync: { - sync: (params) => { - begin = params.begin - write = params.write - commit = params.commit - params.markReady() - }, - }, - }) - const derived = createLiveQueryCollection({ - query: (query) => - query - .from({ row: source }) - .select(({ row }) => ({ id: row.id, value: row.value })), - getKey: (row) => row.id, - startSync: true, - }) - const persistence = createDeferred() - // Query collections currently expose read-side virtual properties in their - // insert input type even though the runtime accepts the plain selected row. - const insertDerived = derived.insert.bind(derived) as unknown as ( - row: OptimisticDerivedRow, - ) => ReturnType - const insertOptimistically = createOptimisticAction({ - onMutate: insertDerived, - mutationFn: () => persistence.promise, - }) - - await derived.preload() - const transaction = insertOptimistically({ - id: `optimistic`, - value: `optimistic`, - }) - try { - begin() - write({ type: `insert`, value: { id: `synced`, value: `synced` } }) - commit() - - try { - expect([...derived.keys()].sort()).toEqual([`optimistic`, `synced`]) - } catch (error) { - throw new TraceAssertionError(0, error) - } - } finally { - persistence.resolve() - await transaction.isPersisted.promise - await derived.cleanup() - await source.cleanup() - } -} - async function expectDeduplicatedWaiterHandlesRejection( scenario: RejectedWaiterScenario, ): Promise { @@ -2874,17 +2810,6 @@ describe(`loadSubset coverage oracle`, () => { await expectCleanupRejectsReceiptOnce() }) - it(`publishes synced source rows while a derived mutation persists`, async () => { - await expectAssertionFailure(expectDerivedSyncDuringOptimisticMutation, { - checkpoint: 0, - classify: ({ actual, expected }) => - Array.isArray(actual) && - actual.join(`,`) === `optimistic` && - Array.isArray(expected) && - expected.join(`,`) === `optimistic,synced`, - })() - }) - it( `discovered trace: adjacent ordered windows do not cover their combined window`, expectAssertionFailure( From 43a3a63923723521957f462620934c285fc700fa Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 19:55:11 -0600 Subject: [PATCH 188/327] test(db): require same-stack order publication --- packages/db/tests/live-query-order-only-move.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/live-query-order-only-move.test.ts b/packages/db/tests/live-query-order-only-move.test.ts index 0fac5f148..31116d41f 100644 --- a/packages/db/tests/live-query-order-only-move.test.ts +++ b/packages/db/tests/live-query-order-only-move.test.ts @@ -156,13 +156,14 @@ describe(`order-only move publication`, () => { value: { id: `2`, name: `Bob`, age: 99 }, }) source.utils.commit() - await flush() const whilePersisting = observer.getSnapshot() - expect( - (whilePersisting.data as Array).map((row) => row.id), - ).toEqual([`1`, `3`, `2`]) - expect(lq._layoutRevision).toBeGreaterThan(collectionLayoutRevisionBefore) + expect((whilePersisting.data as Array).map((row) => row.id)).toEqual([ + `1`, + `3`, + `2`, + ]) + expect(lq._layoutRevision).toBe(collectionLayoutRevisionBefore + 1) const publishedLayoutRevision = lq._layoutRevision persist.resolve() From 5a4c758df240053b9c097bd56cc5e0461288ccde Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 20:02:43 -0600 Subject: [PATCH 189/327] test(db): isolate ordered source publication --- .../db/tests/live-query-order-only-move.test.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/live-query-order-only-move.test.ts b/packages/db/tests/live-query-order-only-move.test.ts index 31116d41f..5f43dea6b 100644 --- a/packages/db/tests/live-query-order-only-move.test.ts +++ b/packages/db/tests/live-query-order-only-move.test.ts @@ -134,9 +134,14 @@ describe(`order-only move publication`, () => { { id: string; name: string }, string >(lq as any) + const publications: Array> = [] + const subscription = lq.subscribeChanges( + (changes) => publications.push(changes), + { includeInitialState: false }, + ) const before = observer.getSnapshot() - const collectionLayoutRevisionBefore = lq._layoutRevision + const layoutRevisionBeforeMutation = lq._layoutRevision expect((before.data as Array).map((row) => row.id)).toEqual([ `2`, `1`, @@ -149,6 +154,9 @@ describe(`order-only move publication`, () => { (draft) => void (draft.name = `Pending`), ) expect(mutation.state).toBe(`persisting`) + expect(lq._layoutRevision).toBe(layoutRevisionBeforeMutation) + expect(publications).toEqual([]) + const layoutRevisionBeforeSourceCommit = lq._layoutRevision source.utils.begin() source.utils.write({ @@ -163,7 +171,8 @@ describe(`order-only move publication`, () => { `3`, `2`, ]) - expect(lq._layoutRevision).toBe(collectionLayoutRevisionBefore + 1) + expect(lq._layoutRevision).toBe(layoutRevisionBeforeSourceCommit + 1) + expect(publications).toEqual([[]]) const publishedLayoutRevision = lq._layoutRevision persist.resolve() @@ -177,6 +186,8 @@ describe(`order-only move publication`, () => { `2`, ]) expect(lq._layoutRevision).toBe(publishedLayoutRevision) + expect(publications).toEqual([[]]) + subscription.unsubscribe() observer.dispose() }) From 91896e7216c554c46343e8b6d9cf5b04c51175af Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 20:21:25 -0600 Subject: [PATCH 190/327] fix(db): publish child facades through mutations --- packages/db/src/query/live/ARCHITECTURE.md | 17 +- .../src/query/live/bucket-facade-adapter.ts | 9 +- ...ncludes-collection-oracle.property.test.ts | 399 ++++++++++++++++++ 3 files changed, 415 insertions(+), 10 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 055ba501d..55fe94402 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -928,10 +928,11 @@ priority merely to make a subset load settle. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are -visible. A quiescent live-query graph output uses the same queue bypass so a -direct source change can update a derived Collection while an optimistic -mutation on that Collection persists. The normal optimistic overlay still wins -for conflicting keys, and the graph output remains one coherent publication. +visible. A quiescent live-query graph output uses the same queue bypass for both +the root Collection and Collection-valued child facades. A direct source change +therefore updates the whole derived publication while an optimistic mutation on +either Collection persists. The normal optimistic overlay still wins for +conflicting keys, and the graph output remains one coherent publication. Rejected, canceled, and obsolete acquisitions establish no coverage. Sources must honor cancellation before publishing request-scoped rows. @@ -1077,10 +1078,10 @@ For each scheduled graph turn: 1. enqueue all currently committed input deltas into their D2 inputs; 2. run D2 until it has no pending synchronous work; 3. consolidate the already canonical final-output deltas; -4. install child-facade state through normal Collection transactions while - deferring their subscriber delivery; -5. apply direct root insert, update, and delete writes through one normal - Collection transaction; +4. install child-facade state through queue-bypassing Collection transactions + while deferring their subscriber delivery; +5. apply direct root insert, update, and delete writes through one + queue-bypassing Collection transaction; 6. release the deferred child-facade events after every synchronous read can see the complete root and facade state; 7. allow dependent live-query graphs to run through the existing diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index f350e8885..ca8d90fa4 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -128,7 +128,10 @@ export class BucketFacadeAdapter { this.prepareChange(entry, change) } deferPublication(entry) - sync.begin() + // The graph is already quiescent. Install this complete child + // publication beneath any pending optimistic facade overlay instead + // of parking source progress behind that mutation. + sync.begin({ immediate: true }) for (const change of changes.values()) { this.applyChange(entry, sync, change, compilation.hasOrderBy) } @@ -337,7 +340,9 @@ export class BucketFacadeAdapter { const keys = [...entry.collection.keys()] if (sync && keys.length > 0) { deferPublication(entry) - sync.begin() + // Route retirement is part of the same quiescent graph publication as + // the root change that removed its final consumer. + sync.begin({ immediate: true }) for (const key of keys) sync.write({ type: `delete`, key }) sync.commit() } diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 6cf697bd9..7db76832f 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,5 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' +import { createDeferred } from '../../src/deferred.js' +import { createOptimisticAction } from '../../src/optimistic-action.js' import { add, caseWhen, @@ -34,6 +36,31 @@ type ChildRow = { value: number } +type ProjectedChildChange = { + type: `insert` | `update` | `delete` + key: number + value: ChildRow + previousValue?: ChildRow +} + +function projectChildChange( + change: ChangeMessage, +): ProjectedChildChange { + const projectRow = ({ id, parentGroup, value }: ChildRow): ChildRow => ({ + id, + parentGroup, + value, + }) + return { + type: change.type, + key: Number(change.key), + value: projectRow(change.value), + ...(change.previousValue + ? { previousValue: projectRow(change.previousValue) } + : {}), + } +} + type CollectionAction = | { type: `putParent`; row: ParentRow } | { type: `deleteParent`; id: number } @@ -325,6 +352,80 @@ const exhaustiveActions: ReadonlyArray = [ { type: `deleteChild`, id: 10 }, ] +type PendingFacadeOperation = `insert` | `update` | `delete` +type PendingFacadeOptimisticOperation = Exclude< + PendingFacadeOperation, + `insert` +> + +const pendingFacadeOptimisticOperations = [`update`, `delete`] as const +const pendingFacadeSourceOperations = [`insert`, `update`, `delete`] as const +const pendingFacadeSettlements = [`resolve`, `reject`] as const +const pendingFacadeInitialRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, +] + +function pendingOptimisticFacadeRow( + operation: PendingFacadeOptimisticOperation, +): ChildRow { + if (operation === `update`) { + return { id: 10, parentGroup: 1, value: 11 } + } + return { id: 10, parentGroup: 1, value: 10 } +} + +function pendingSourceFacadeRow(operation: PendingFacadeOperation): ChildRow { + if (operation === `insert`) { + return { id: 40, parentGroup: 1, value: 40 } + } + if (operation === `update`) { + return { id: 20, parentGroup: 1, value: 21 } + } + return { id: 20, parentGroup: 1, value: 20 } +} + +function applyPendingFacadeOperation( + rows: Map, + operation: PendingFacadeOperation, + row: ChildRow, +): void { + if (operation === `delete`) rows.delete(row.id) + else rows.set(row.id, { ...row }) +} + +function expectedPendingFacadeRows( + rows: ReadonlyMap, +): Array { + return [...rows.values()] + .map((row) => ({ ...row })) + .sort((left, right) => left.id - right.id) +} + +function expectedPendingFacadeChange( + before: ReadonlyMap, + after: ReadonlyMap, + key: number, +): ProjectedChildChange { + const previousValue = before.get(key) + const value = after.get(key) + if (!previousValue && value) { + return { type: `insert`, key, value: { ...value } } + } + if (previousValue && !value) { + return { type: `delete`, key, value: { ...previousValue } } + } + if (!previousValue || !value) { + throw new Error(`Expected a visible facade change for key ${key}`) + } + return { + type: `update`, + key, + value: { ...value }, + previousValue: { ...previousValue }, + } +} + describe(`Collection-valued includes oracle`, () => { fcTest.prop( [collectionScenarioArbitrary], @@ -765,6 +866,304 @@ describe(`Collection-valued includes oracle`, () => { }, ) + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + for (const sourceOperation of pendingFacadeSourceOperations) { + fcTest( + `publishes a source ${sourceOperation} while a separate facade ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection( + `pending-facade-parents`, + [{ id: 1, group: 1 }], + ) + const children = createControlledCollection( + `pending-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const rootPublications: Array = [] + const childPublications: Array> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => childPublications.push(batch.map(projectChildChange)), + { includeInitialState: false }, + ) + const optimisticRow = + pendingOptimisticFacadeRow(optimisticOperation) + const sourceRow = pendingSourceFacadeRow(sourceOperation) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + facade.update(optimisticRow.id, (draft) => { + draft.value = optimisticRow.value + }) + } else { + facade.delete(optimisticRow.id) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map( + pendingFacadeInitialRows.map( + (row) => [row.id, { ...row }] as const, + ), + ) + const afterOptimistic = new Map(initialRows) + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + optimisticRow, + ) + const afterSource = new Map(initialRows) + applyPendingFacadeOperation(afterSource, sourceOperation, sourceRow) + const whilePending = new Map(afterSource) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + optimisticRow, + ) + const expectedOptimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + optimisticRow.id, + ) + const expectedSourceChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + sourceRow.id, + ) + const expectedSettlementChange = expectedPendingFacadeChange( + whilePending, + afterSource, + optimisticRow.id, + ) + const childRows = () => + expectedPendingFacadeRows( + new Map( + facade.toArray.map(({ id, parentGroup, value }) => [ + id, + { id, parentGroup, value }, + ]), + ), + ) + + try { + expect(transaction.state).toBe(`persisting`) + expect(childRows()).toEqual( + expectedPendingFacadeRows(afterOptimistic), + ) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([[expectedOptimisticChange]]) + + children.write(sourceOperation, sourceRow) + + expect(live.get(1)!.children).toBe(facade) + expect(childRows()).toEqual( + expectedPendingFacadeRows(whilePending), + ) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedSourceChange], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(childRows()).toEqual( + expectedPendingFacadeRows(afterSource), + ) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedSourceChange], + [expectedSettlementChange], + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `retires unrelated facade rows while a facade update ${settlement}s`, + async () => { + const parents = createControlledCollection(`retiring-facade-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `retiring-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const facadeRows = () => + facade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array< + Array<{ type: `insert` | `update` | `delete`; key: number }> + > = [] + const rootCallbackFacades: Array> = [] + const childPublications: Array> = [] + const childCallbackFacades: Array> = [] + const rootSubscription = live.subscribeChanges( + (batch) => { + rootPublications.push( + batch.map(({ type, key }) => ({ type, key: Number(key) })), + ) + rootCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + childPublications.push(batch.map(projectChildChange)) + childCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(facadeRows()).toEqual([ + { id: 10, parentGroup: 1, value: 11 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + + parents.write(`delete`, { id: 1, group: 1 }) + + expect(live.has(1)).toBe(false) + expect(facadeRows()).toEqual([{ id: 10, parentGroup: 1, value: 11 }]) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(rootCallbackFacades).toEqual([ + [{ id: 10, parentGroup: 1, value: 11 }], + ]) + expect(childPublications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [ + { + type: `delete`, + key: 20, + value: { id: 20, parentGroup: 1, value: 20 }, + }, + ], + ]) + expect(childCallbackFacades).toEqual([ + [ + { id: 10, parentGroup: 1, value: 11 }, + { id: 20, parentGroup: 1, value: 20 }, + ], + [{ id: 10, parentGroup: 1, value: 11 }], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(facadeRows()).toEqual([]) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(childPublications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [ + { + type: `delete`, + key: 20, + value: { id: 20, parentGroup: 1, value: 20 }, + }, + ], + [ + { + type: `delete`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + }, + ], + ]) + expect(childCallbackFacades.at(-1)).toEqual([]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + fcTest( `outer fn.select recomputes nested values after a union branch include changes`, async () => { From b0a38540c2d39f2a68f8ed75d8f84518f5c93fba Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 20:49:34 -0600 Subject: [PATCH 191/327] fix(db): preserve facade source state under mutations --- packages/db/src/query/live/ARCHITECTURE.md | 19 +- .../src/query/live/bucket-facade-adapter.ts | 28 +- ...ncludes-collection-oracle.property.test.ts | 918 +++++++++++++----- 3 files changed, 733 insertions(+), 232 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 55fe94402..cdd5ae44b 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -928,11 +928,14 @@ priority merely to make a subset load settle. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are -visible. A quiescent live-query graph output uses the same queue bypass for both -the root Collection and Collection-valued child facades. A direct source change -therefore updates the whole derived publication while an optimistic mutation on -either Collection persists. The normal optimistic overlay still wins for -conflicting keys, and the graph output remains one coherent publication. +visible. A quiescent live-query graph output uses the same queue bypass for the +root Collection and regular Collection-valued child-facade changes. A facade +retirement is committed earlier in the same FIFO causal prefix; the immediate +root or containing-facade transaction that removed its final route drains that +retirement too. A direct source change therefore updates the whole derived +publication while an optimistic mutation on either Collection persists. The +normal optimistic overlay still wins for conflicting keys, and the graph +output remains one coherent publication. Rejected, canceled, and obsolete acquisitions establish no coverage. Sources must honor cancellation before publishing request-scoped rows. @@ -1078,8 +1081,10 @@ For each scheduled graph turn: 1. enqueue all currently committed input deltas into their D2 inputs; 2. run D2 until it has no pending synchronous work; 3. consolidate the already canonical final-output deltas; -4. install child-facade state through queue-bypassing Collection transactions - while deferring their subscriber delivery; +4. install regular child-facade changes through queue-bypassing Collection + transactions while deferring their subscriber delivery; put route + retirement before the queue-bypassing ancestor that drains its FIFO causal + prefix; 5. apply direct root insert, update, and delete writes through one queue-bypassing Collection transaction; 6. release the deferred child-facade events after every synchronous read can diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index ca8d90fa4..b97ebe015 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -340,9 +340,10 @@ export class BucketFacadeAdapter { const keys = [...entry.collection.keys()] if (sync && keys.length > 0) { deferPublication(entry) - // Route retirement is part of the same quiescent graph publication as - // the root change that removed its final consumer. - sync.begin({ immediate: true }) + // Route retirement precedes the root or containing-facade change that + // removed its final consumer. That later immediate transaction drains + // this earlier transaction as part of the same FIFO causal prefix. + sync.begin() for (const key of keys) sync.write({ type: `delete`, key }) sync.commit() } @@ -419,10 +420,19 @@ export class BucketFacadeAdapter { const key = change.value.publicKey as string | number const previousOrder = entry.currentOrder.get(key) const nextOrder = change.value.order - const orderChanged = sync.collection.has(key) && previousOrder !== nextOrder + // Graph deltas update the synced base. The public Collection view may be + // hiding that row beneath a pending optimistic delete, so it cannot tell + // us whether this delta is an insert, update, or delete of the base row. + const hasSyncedRow = entry.collection._state.syncedData.has(key) + const previousSyncedRow = entry.collection._state.syncedData.get(key) + const orderChanged = hasSyncedRow && previousOrder !== nextOrder + const orderChangeIsVisible = + orderChanged && + !entry.collection._state.optimisticDeletes.has(key) && + !entry.collection._state.optimisticUpserts.has(key) const resolvedRow = this.resolve(change.value.value) const row = - orderChanged && sync.collection.get(key) === resolvedRow + orderChanged && previousSyncedRow === resolvedRow ? { ...resolvedRow } : resolvedRow entry.keys.set(row, key) @@ -432,10 +442,10 @@ export class BucketFacadeAdapter { if (change.inserts > change.deletes) { sync.write({ - type: sync.collection.has(key) ? `update` : `insert`, + type: hasSyncedRow ? `update` : `insert`, value: row, }) - } else if (change.inserts === change.deletes && sync.collection.has(key)) { + } else if (change.inserts === change.deletes && hasSyncedRow) { sync.write({ type: `update`, value: row }) } else if (change.deletes > 0) { sync.write({ type: `delete`, key }) @@ -444,7 +454,9 @@ export class BucketFacadeAdapter { } entry.currentOrder.set(key, nextOrder) - if (hasOrderBy && orderChanged) sync.collection._markLayoutChange() + if (hasOrderBy && orderChangeIsVisible) { + sync.collection._markLayoutChange() + } } /** Resolve and validate every public key before opening a sync transaction. */ diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 7db76832f..8e90ac08a 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -357,10 +357,14 @@ type PendingFacadeOptimisticOperation = Exclude< PendingFacadeOperation, `insert` > +type PendingFacadeKeyRelation = `disjoint-key` | `same-key` +type PendingFacadeShape = `unordered` | `ordered` const pendingFacadeOptimisticOperations = [`update`, `delete`] as const const pendingFacadeSourceOperations = [`insert`, `update`, `delete`] as const const pendingFacadeSettlements = [`resolve`, `reject`] as const +const pendingFacadeKeyRelations = [`disjoint-key`, `same-key`] as const +const pendingFacadeShapes = [`unordered`, `ordered`] as const const pendingFacadeInitialRows: ReadonlyArray = [ { id: 10, parentGroup: 1, value: 10 }, { id: 20, parentGroup: 1, value: 20 }, @@ -375,10 +379,20 @@ function pendingOptimisticFacadeRow( return { id: 10, parentGroup: 1, value: 10 } } -function pendingSourceFacadeRow(operation: PendingFacadeOperation): ChildRow { +function pendingSourceFacadeRow( + operation: PendingFacadeOperation, + keyRelation: PendingFacadeKeyRelation, +): ChildRow { if (operation === `insert`) { return { id: 40, parentGroup: 1, value: 40 } } + if (keyRelation === `same-key`) { + return { + id: 10, + parentGroup: 1, + value: operation === `update` ? 21 : 10, + } + } if (operation === `update`) { return { id: 20, parentGroup: 1, value: 21 } } @@ -396,28 +410,38 @@ function applyPendingFacadeOperation( function expectedPendingFacadeRows( rows: ReadonlyMap, + shape: PendingFacadeShape = `unordered`, ): Array { return [...rows.values()] .map((row) => ({ ...row })) - .sort((left, right) => left.id - right.id) + .sort((left, right) => + shape === `ordered` + ? left.value - right.value || left.id - right.id + : left.id - right.id, + ) } function expectedPendingFacadeChange( before: ReadonlyMap, after: ReadonlyMap, key: number, -): ProjectedChildChange { +): ProjectedChildChange | undefined { const previousValue = before.get(key) const value = after.get(key) + if ( + previousValue?.id === value?.id && + previousValue?.parentGroup === value?.parentGroup && + previousValue?.value === value?.value + ) { + return undefined + } if (!previousValue && value) { return { type: `insert`, key, value: { ...value } } } if (previousValue && !value) { return { type: `delete`, key, value: { ...previousValue } } } - if (!previousValue || !value) { - throw new Error(`Expected a visible facade change for key ${key}`) - } + if (!previousValue || !value) return undefined return { type: `update`, key, @@ -869,301 +893,761 @@ describe(`Collection-valued includes oracle`, () => { for (const settlement of pendingFacadeSettlements) { for (const optimisticOperation of pendingFacadeOptimisticOperations) { for (const sourceOperation of pendingFacadeSourceOperations) { - fcTest( - `publishes a source ${sourceOperation} while a separate facade ${optimisticOperation} ${settlement}s`, - async () => { - const parents = createControlledCollection( - `pending-facade-parents`, - [{ id: 1, group: 1 }], - ) - const children = createControlledCollection( - `pending-facade-children`, - pendingFacadeInitialRows, - ) - const live = createLiveQueryCollection((q) => - q.from({ parent: parents.collection }).select(({ parent }) => ({ - id: parent.id, - children: q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)), - })), - ) - const persistence = createDeferred() - - await live.preload() - const facade = live.get(1)!.children - const rootPublications: Array = [] - const childPublications: Array> = [] - const rootSubscription = live.subscribeChanges( - (batch) => rootPublications.push(batch), - { includeInitialState: false }, - ) - const childSubscription = facade.subscribeChanges( - (batch) => childPublications.push(batch.map(projectChildChange)), - { includeInitialState: false }, - ) - const optimisticRow = - pendingOptimisticFacadeRow(optimisticOperation) - const sourceRow = pendingSourceFacadeRow(sourceOperation) - const mutate = createOptimisticAction({ - onMutate: () => { - if (optimisticOperation === `update`) { - facade.update(optimisticRow.id, (draft) => { - draft.value = optimisticRow.value - }) - } else { - facade.delete(optimisticRow.id) + for (const keyRelation of pendingFacadeKeyRelations) { + if (sourceOperation === `insert` && keyRelation === `same-key`) { + continue + } + for (const shape of pendingFacadeShapes) { + fcTest( + `publishes an ${shape} ${keyRelation} source ${sourceOperation} while a facade ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection( + `pending-facade-parents`, + [{ id: 1, group: 1 }], + ) + const children = createControlledCollection( + `pending-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => + eq(child.parentGroup, parent.group), + ) + return { + id: parent.id, + children: + shape === `ordered` + ? childRows.orderBy(({ child }) => child.value) + : childRows, + } + }), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const childRows = () => + expectedPendingFacadeRows( + new Map( + facade.toArray.map(({ id, parentGroup, value }) => [ + id, + { id, parentGroup, value }, + ]), + ), + shape, + ) + const rootRows = () => + live.toArray.map( + ({ id: parentId, children: rootFacade }) => ({ + id: parentId, + children: expectedPendingFacadeRows( + new Map( + rootFacade.toArray.map( + ({ id: childId, parentGroup, value }) => [ + childId, + { id: childId, parentGroup, value }, + ], + ), + ), + shape, + ), + }), + ) + const rootPublications: Array = [] + const childPublications: Array> = [] + const childCallbackSnapshots: Array<{ + facade: Array + root: ReturnType + }> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + childPublications.push(batch.map(projectChildChange)) + childCallbackSnapshots.push({ + facade: childRows(), + root: rootRows(), + }) + }, + { includeInitialState: false }, + ) + const optimisticRow = + pendingOptimisticFacadeRow(optimisticOperation) + const sourceRow = pendingSourceFacadeRow( + sourceOperation, + keyRelation, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + facade.update(optimisticRow.id, (draft) => { + draft.value = optimisticRow.value + }) + } else { + facade.delete(optimisticRow.id) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map( + pendingFacadeInitialRows.map( + (row) => [row.id, { ...row }] as const, + ), + ) + const afterOptimistic = new Map(initialRows) + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + optimisticRow, + ) + const afterSource = new Map(initialRows) + applyPendingFacadeOperation( + afterSource, + sourceOperation, + sourceRow, + ) + const whilePending = new Map(afterSource) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + optimisticRow, + ) + const expectedOptimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + optimisticRow.id, + ) + const expectedSourceChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + sourceRow.id, + ) + const expectedSettlementChange = expectedPendingFacadeChange( + whilePending, + afterSource, + optimisticRow.id, + ) + const optimisticRows = expectedPendingFacadeRows( + afterOptimistic, + shape, + ) + const pendingRows = expectedPendingFacadeRows( + whilePending, + shape, + ) + const settledRows = expectedPendingFacadeRows( + afterSource, + shape, + ) + const expectedSourcePublications = [ + [expectedOptimisticChange], + ...(expectedSourceChange ? [[expectedSourceChange]] : []), + ] + const expectedSourceSnapshots = [ + { + facade: optimisticRows, + root: [{ id: 1, children: optimisticRows }], + }, + ...(expectedSourceChange + ? [ + { + facade: pendingRows, + root: [{ id: 1, children: pendingRows }], + }, + ] + : []), + ] + const expectedSettledPublications = [ + ...expectedSourcePublications, + ...(expectedSettlementChange + ? [[expectedSettlementChange]] + : []), + ] + const expectedSettledSnapshots = [ + ...expectedSourceSnapshots, + ...(expectedSettlementChange + ? [ + { + facade: settledRows, + root: [{ id: 1, children: settledRows }], + }, + ] + : []), + ] + + try { + expect(transaction.state).toBe(`persisting`) + expect(childRows()).toEqual(optimisticRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + ]) + expect(childCallbackSnapshots).toEqual( + expectedSourceSnapshots.slice(0, 1), + ) + + children.write(sourceOperation, sourceRow) + + expect(live.get(1)!.children).toBe(facade) + expect(childRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([]) + expect(childCallbackSnapshots).toEqual( + expectedSourceSnapshots, + ) + expect(childPublications).toEqual(expectedSourcePublications) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(childRows()).toEqual(settledRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual(expectedSettledPublications) + expect(childCallbackSnapshots).toEqual( + expectedSettledSnapshots, + ) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) } }, - mutationFn: () => persistence.promise, - }) - const transaction = mutate() - const initialRows = new Map( - pendingFacadeInitialRows.map( - (row) => [row.id, { ...row }] as const, - ), - ) - const afterOptimistic = new Map(initialRows) - applyPendingFacadeOperation( - afterOptimistic, - optimisticOperation, - optimisticRow, - ) - const afterSource = new Map(initialRows) - applyPendingFacadeOperation(afterSource, sourceOperation, sourceRow) - const whilePending = new Map(afterSource) - applyPendingFacadeOperation( - whilePending, - optimisticOperation, - optimisticRow, ) - const expectedOptimisticChange = expectedPendingFacadeChange( - initialRows, - afterOptimistic, - optimisticRow.id, - ) - const expectedSourceChange = expectedPendingFacadeChange( - afterOptimistic, - whilePending, - sourceRow.id, - ) - const expectedSettlementChange = expectedPendingFacadeChange( - whilePending, - afterSource, - optimisticRow.id, - ) - const childRows = () => - expectedPendingFacadeRows( - new Map( - facade.toArray.map(({ id, parentGroup, value }) => [ - id, - { id, parentGroup, value }, - ]), - ), - ) - - try { - expect(transaction.state).toBe(`persisting`) - expect(childRows()).toEqual( - expectedPendingFacadeRows(afterOptimistic), - ) - expect(rootPublications).toEqual([]) - expect(childPublications).toEqual([[expectedOptimisticChange]]) + } + } + } + } + } - children.write(sourceOperation, sourceRow) + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + fcTest( + `retires unrelated facade rows while a facade ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection( + `retiring-facade-parents`, + [{ id: 1, group: 1 }], + ) + const children = createControlledCollection( + `retiring-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + const persistence = createDeferred() - expect(live.get(1)!.children).toBe(facade) - expect(childRows()).toEqual( - expectedPendingFacadeRows(whilePending), + await live.preload() + const facade = live.get(1)!.children + const facadeRows = () => + facade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array< + Array<{ type: `insert` | `update` | `delete`; key: number }> + > = [] + const rootCallbackFacades: Array> = [] + const childPublications: Array> = [] + const childCallbackFacades: Array> = [] + const publicationTimeline: Array<`root` | `facade`> = [] + const rootSubscription = live.subscribeChanges( + (batch) => { + publicationTimeline.push(`root`) + rootPublications.push( + batch.map(({ type, key }) => ({ type, key: Number(key) })), ) - expect(rootPublications).toEqual([]) - expect(childPublications).toEqual([ - [expectedOptimisticChange], - [expectedSourceChange], - ]) + rootCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + publicationTimeline.push(`facade`) + childPublications.push(batch.map(projectChildChange)) + childCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + facade.update(10, (draft) => { + draft.value = 11 + }) + } else { + facade.delete(10) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map( + pendingFacadeInitialRows.map( + (row) => [row.id, { ...row }] as const, + ), + ) + const optimisticRow = pendingOptimisticFacadeRow(optimisticOperation) + const afterOptimistic = new Map(initialRows) + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + optimisticRow, + ) + const emptyBase = new Map() + const whilePending = new Map(emptyBase) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + optimisticRow, + ) + const optimisticRows = expectedPendingFacadeRows(afterOptimistic) + const pendingRows = expectedPendingFacadeRows(whilePending) + const expectedOptimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + optimisticRow.id, + )! + const expectedRetirementChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + 20, + )! + const expectedSettlementChange = expectedPendingFacadeChange( + whilePending, + emptyBase, + optimisticRow.id, + ) - const persisted = transaction.isPersisted.promise.catch( - () => undefined, - ) - if (settlement === `resolve`) persistence.resolve() - else persistence.reject(new Error(`facade mutation rejected`)) - await persisted - await flushPromises() + try { + expect(facadeRows()).toEqual(optimisticRows) + expect(childPublications).toEqual([[expectedOptimisticChange]]) + expect(childCallbackFacades).toEqual([optimisticRows]) + expect(publicationTimeline).toEqual([`facade`]) + publicationTimeline.length = 0 + + parents.write(`delete`, { id: 1, group: 1 }) + + expect(live.has(1)).toBe(false) + expect(facadeRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(rootCallbackFacades).toEqual([pendingRows]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedRetirementChange], + ]) + expect(childCallbackFacades).toEqual([optimisticRows, pendingRows]) + expect(publicationTimeline).toEqual([`root`, `facade`]) - expect(childRows()).toEqual( - expectedPendingFacadeRows(afterSource), - ) - expect(rootPublications).toEqual([]) - expect(childPublications).toEqual([ - [expectedOptimisticChange], - [expectedSourceChange], - [expectedSettlementChange], - ]) - } finally { - persistence.resolve() - await transaction.isPersisted.promise.catch(() => undefined) - rootSubscription.unsubscribe() - childSubscription.unsubscribe() - await Promise.all([ - live.cleanup(), - parents.collection.cleanup(), - children.collection.cleanup(), - ]) - } - }, - ) - } + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(facadeRows()).toEqual([]) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedRetirementChange], + ...(expectedSettlementChange ? [[expectedSettlementChange]] : []), + ]) + expect(childCallbackFacades.at(-1)).toEqual([]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) } } for (const settlement of pendingFacadeSettlements) { fcTest( - `retires unrelated facade rows while a facade update ${settlement}s`, + `publishes a nested facade source update while a same-key delete ${settlement}s`, async () => { - const parents = createControlledCollection(`retiring-facade-parents`, [ + const parents = createControlledCollection(`nested-facade-parents`, [ { id: 1, group: 1 }, ]) - const children = createControlledCollection( - `retiring-facade-children`, - pendingFacadeInitialRows, + const children = createControlledCollection(`nested-facade-children`, [ + { id: 100, parentGroup: 1, group: 7 }, + ]) + const grandchildren = createControlledCollection( + `nested-facade-grandchildren`, + [ + { id: 10, parentGroup: 7, value: 10 }, + { id: 20, parentGroup: 7, value: 20 }, + ], ) const live = createLiveQueryCollection((q) => q.from({ parent: parents.collection }).select(({ parent }) => ({ id: parent.id, children: q .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)), + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + id: child.id, + group: child.group, + grandchildren: q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ), + })), })), ) const persistence = createDeferred() await live.preload() - const facade = live.get(1)!.children - const facadeRows = () => - facade.toArray + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(100)!.grandchildren + const grandchildRows = () => + grandchildFacade.toArray .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) .sort((left, right) => left.id - right.id) - const rootPublications: Array< - Array<{ type: `insert` | `update` | `delete`; key: number }> - > = [] - const rootCallbackFacades: Array> = [] - const childPublications: Array> = [] - const childCallbackFacades: Array> = [] + const rootPublications: Array = [] + const childPublications: Array = [] + const grandchildPublications: Array> = [] + const callbackRows: Array> = [] const rootSubscription = live.subscribeChanges( - (batch) => { - rootPublications.push( - batch.map(({ type, key }) => ({ type, key: Number(key) })), - ) - rootCallbackFacades.push(facadeRows()) - }, + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = childFacade.subscribeChanges( + (batch) => childPublications.push(batch), { includeInitialState: false }, ) - const childSubscription = facade.subscribeChanges( + const grandchildSubscription = grandchildFacade.subscribeChanges( (batch) => { - childPublications.push(batch.map(projectChildChange)) - childCallbackFacades.push(facadeRows()) + grandchildPublications.push(batch.map(projectChildChange)) + callbackRows.push(grandchildRows()) }, { includeInitialState: false }, ) const mutate = createOptimisticAction({ - onMutate: () => { - facade.update(10, (draft) => { - draft.value = 11 - }) - }, + onMutate: () => grandchildFacade.delete(10), mutationFn: () => persistence.promise, }) const transaction = mutate() try { - expect(facadeRows()).toEqual([ - { id: 10, parentGroup: 1, value: 11 }, - { id: 20, parentGroup: 1, value: 20 }, - ]) - - parents.write(`delete`, { id: 1, group: 1 }) - - expect(live.has(1)).toBe(false) - expect(facadeRows()).toEqual([{ id: 10, parentGroup: 1, value: 11 }]) - expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) - expect(rootCallbackFacades).toEqual([ - [{ id: 10, parentGroup: 1, value: 11 }], + expect(grandchildRows()).toEqual([ + { id: 20, parentGroup: 7, value: 20 }, ]) - expect(childPublications).toEqual([ - [ - { - type: `update`, - key: 10, - value: { id: 10, parentGroup: 1, value: 11 }, - previousValue: { id: 10, parentGroup: 1, value: 10 }, - }, - ], + expect(grandchildPublications).toEqual([ [ { type: `delete`, - key: 20, - value: { id: 20, parentGroup: 1, value: 20 }, + key: 10, + value: { id: 10, parentGroup: 7, value: 10 }, }, ], ]) - expect(childCallbackFacades).toEqual([ - [ - { id: 10, parentGroup: 1, value: 11 }, - { id: 20, parentGroup: 1, value: 20 }, - ], - [{ id: 10, parentGroup: 1, value: 11 }], + + grandchildren.write(`update`, { + id: 10, + parentGroup: 7, + value: 21, + }) + + expect(grandchildRows()).toEqual([ + { id: 20, parentGroup: 7, value: 20 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(grandchildPublications).toHaveLength(1) + expect(callbackRows).toEqual([ + [{ id: 20, parentGroup: 7, value: 20 }], ]) const persisted = transaction.isPersisted.promise.catch( () => undefined, ) if (settlement === `resolve`) persistence.resolve() - else persistence.reject(new Error(`facade mutation rejected`)) + else persistence.reject(new Error(`nested facade mutation rejected`)) await persisted await flushPromises() - expect(facadeRows()).toEqual([]) - expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) - expect(childPublications).toEqual([ - [ - { - type: `update`, - key: 10, - value: { id: 10, parentGroup: 1, value: 11 }, - previousValue: { id: 10, parentGroup: 1, value: 10 }, - }, - ], + expect(grandchildRows()).toEqual([ + { id: 10, parentGroup: 7, value: 21 }, + { id: 20, parentGroup: 7, value: 20 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(grandchildPublications).toEqual([ [ { type: `delete`, - key: 20, - value: { id: 20, parentGroup: 1, value: 20 }, + key: 10, + value: { id: 10, parentGroup: 7, value: 10 }, }, ], [ { - type: `delete`, + type: `insert`, key: 10, - value: { id: 10, parentGroup: 1, value: 11 }, + value: { id: 10, parentGroup: 7, value: 21 }, }, ], ]) - expect(childCallbackFacades.at(-1)).toEqual([]) + expect(callbackRows.at(-1)).toEqual([ + { id: 10, parentGroup: 7, value: 21 }, + { id: 20, parentGroup: 7, value: 20 }, + ]) } finally { persistence.resolve() await transaction.isPersisted.promise.catch(() => undefined) rootSubscription.unsubscribe() childSubscription.unsubscribe() + grandchildSubscription.unsubscribe() await Promise.all([ live.cleanup(), parents.collection.cleanup(), children.collection.cleanup(), + grandchildren.collection.cleanup(), ]) } }, ) } + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + fcTest( + `retires a nested facade while its ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection(`nested-retire-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `nested-retire-children`, + [{ id: 100, parentGroup: 1, group: 7 }], + ) + const grandchildren = createControlledCollection( + `nested-retire-grandchildren`, + [ + { id: 10, parentGroup: 7, value: 10 }, + { id: 20, parentGroup: 7, value: 20 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + id: child.id, + group: child.group, + grandchildren: q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ), + })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(100)!.grandchildren + const grandchildRows = () => + grandchildFacade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array = [] + const childPublications: Array<{ + type: `insert` | `update` | `delete` + key: number + id: number + group: number + grandchildren: boolean + }> = [] + const childCallbackSnapshots: Array<{ + childIds: Array + grandchildRows: Array + }> = [] + const grandchildPublications: Array> = [] + const grandchildCallbackRows: Array> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = childFacade.subscribeChanges( + (batch) => { + childPublications.push( + ...batch.map(({ type, key, value }) => ({ + type, + key: Number(key), + id: value.id, + group: value.group, + grandchildren: value.grandchildren === grandchildFacade, + })), + ) + childCallbackSnapshots.push({ + childIds: childFacade.toArray.map(({ id }) => id), + grandchildRows: grandchildRows(), + }) + }, + { includeInitialState: false }, + ) + const grandchildSubscription = grandchildFacade.subscribeChanges( + (batch) => { + grandchildPublications.push(batch.map(projectChildChange)) + grandchildCallbackRows.push(grandchildRows()) + }, + { includeInitialState: false }, + ) + const optimisticRow = pendingOptimisticFacadeRow(optimisticOperation) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + grandchildFacade.update(10, (draft) => { + draft.value = optimisticRow.value + }) + } else { + grandchildFacade.delete(10) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map([ + [10, { id: 10, parentGroup: 7, value: 10 }], + [20, { id: 20, parentGroup: 7, value: 20 }], + ]) + const afterOptimistic = new Map(initialRows) + const nestedOptimisticRow = { ...optimisticRow, parentGroup: 7 } + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + nestedOptimisticRow, + ) + const emptyBase = new Map() + const whilePending = new Map(emptyBase) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + nestedOptimisticRow, + ) + const optimisticRows = expectedPendingFacadeRows(afterOptimistic) + const pendingRows = expectedPendingFacadeRows(whilePending) + const optimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + 10, + )! + const retirementChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + 20, + )! + const settlementChange = expectedPendingFacadeChange( + whilePending, + emptyBase, + 10, + ) + + try { + expect(grandchildRows()).toEqual(optimisticRows) + + children.write(`delete`, { + id: 100, + parentGroup: 1, + group: 7, + }) + + expect(live.has(1)).toBe(true) + expect(childFacade.toArray).toEqual([]) + expect(grandchildRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + { + type: `delete`, + key: 100, + id: 100, + group: 7, + grandchildren: true, + }, + ]) + expect(childCallbackSnapshots).toEqual([ + { childIds: [], grandchildRows: pendingRows }, + ]) + expect(grandchildPublications).toEqual([ + [optimisticChange], + [retirementChange], + ]) + expect(grandchildCallbackRows).toEqual([ + optimisticRows, + pendingRows, + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`nested retirement rejected`)) + await persisted + await flushPromises() + + expect(childFacade.toArray).toEqual([]) + expect(grandchildRows()).toEqual([]) + expect(rootPublications).toEqual([]) + expect(grandchildPublications).toEqual([ + [optimisticChange], + [retirementChange], + ...(settlementChange ? [[settlementChange]] : []), + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + grandchildSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + grandchildren.collection.cleanup(), + ]) + } + }, + ) + } + } + fcTest( `outer fn.select recomputes nested values after a union branch include changes`, async () => { From f6c7cae22c16d5319d589afb23ae1a4cc79b0c67 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Sun, 30 Aug 2026 23:38:23 -0600 Subject: [PATCH 192/327] fix(db): preserve facade identity and order metadata --- packages/db/src/collection/mutations.ts | 18 +- packages/db/src/query/live/ARCHITECTURE.md | 15 +- .../src/query/live/bucket-facade-adapter.ts | 37 +- .../tests/query/bucket-facade-adapter.test.ts | 46 ++ ...ncludes-collection-oracle.property.test.ts | 429 +++++++++++++++++- 5 files changed, 508 insertions(+), 37 deletions(-) diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 9c9178978..967898d84 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -37,6 +37,19 @@ import type { TransactionScope } from '../transactions' import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionStateManager } from './state' +function copyNonEnumerableSymbols( + target: T, + source: object, +): T { + for (const symbol of Object.getOwnPropertySymbols(source)) { + const descriptor = Object.getOwnPropertyDescriptor(source, symbol) + if (descriptor && !descriptor.enumerable) { + Object.defineProperty(target, symbol, descriptor) + } + } + return target +} + export class CollectionMutationsManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -369,10 +382,9 @@ export class CollectionMutationsManager< ) // Construct the full modified item by applying the validated update payload to the original item - const modifiedItem = Object.assign( - {}, + const modifiedItem = copyNonEnumerableSymbols( + Object.assign({}, originalItem, validatedUpdatePayload), originalItem, - validatedUpdatePayload, ) // Check if the ID of the item is being changed diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index cdd5ae44b..c8f9d06c9 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -265,6 +265,14 @@ row. A negative aggregate is an invariant violation. This is a specialized use of the existing D2 keyed reduction. It is not a separate contribution-ledger subsystem. +Collection-valued facades keep the canonical public key and order token as +non-enumerable row metadata. Optimistic Collection updates preserve that +metadata when they clone a row. It is adapter state, not part of the selected +query value: projections need not expose a key field, and equality or change +payloads must not acquire one. An order update that reuses the canonical row +object clones that row before replacing its order metadata, so the ordered map +can remove the old position before it installs the new one. + ## Routes and buckets are relations For each materialization edge, the compiler produces these keyed relations: @@ -935,7 +943,12 @@ root or containing-facade transaction that removed its final route drains that retirement too. A direct source change therefore updates the whole derived publication while an optimistic mutation on either Collection persists. The normal optimistic overlay still wins for conflicting keys, and the graph -output remains one coherent publication. +output remains one coherent publication. The overlay replaces the row value, +not the graph-owned key order. A source order move publishes a layout change +when that key remains visible, including beneath an optimistic update. An +optimistic delete hides the key and its order moves. If the synced base is +deleted beneath an optimistic update, the still-visible row moves to the +optimistic-only suffix and that layout change also publishes. Rejected, canceled, and obsolete acquisitions establish no coverage. Sources must honor cancellation before publishing request-scoped rows. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index b97ebe015..fe530a681 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -13,6 +13,9 @@ import type { type FacadeSync = Parameters[`sync`]>[0] +const BUCKET_FACADE_PUBLIC_KEY = Symbol(`bucketFacadePublicKey`) +const BUCKET_FACADE_ORDER = Symbol(`bucketFacadeOrder`) + type PendingRow = { deletes: number inserts: number @@ -372,15 +375,16 @@ export class BucketFacadeAdapter { const collection = createCollection({ id: `__bucket-facade:${this.parentId}:${edgeId}:${bucketKey}`, getKey: (row) => { - const key = keys.get(row) ?? row?.$key + const key = + keys.get(row) ?? row?.[BUCKET_FACADE_PUBLIC_KEY] ?? row?.$key if (typeof key !== `string` && typeof key !== `number`) { throw new Error(`Bucket facade row has no public key`) } return key }, compare: (left, right) => { - const leftOrder = order.get(left) - const rightOrder = order.get(right) + const leftOrder = order.get(left) ?? left?.[BUCKET_FACADE_ORDER] + const rightOrder = order.get(right) ?? right?.[BUCKET_FACADE_ORDER] if (leftOrder === rightOrder) return 0 if (leftOrder === undefined) return 1 if (rightOrder === undefined) return -1 @@ -426,18 +430,26 @@ export class BucketFacadeAdapter { const hasSyncedRow = entry.collection._state.syncedData.has(key) const previousSyncedRow = entry.collection._state.syncedData.get(key) const orderChanged = hasSyncedRow && previousOrder !== nextOrder - const orderChangeIsVisible = - orderChanged && - !entry.collection._state.optimisticDeletes.has(key) && - !entry.collection._state.optimisticUpserts.has(key) const resolvedRow = this.resolve(change.value.value) + // Order metadata lives in a WeakMap keyed by row identity. Never attach a + // new base order to an object that may also back the optimistic overlay. const row = orderChanged && previousSyncedRow === resolvedRow ? { ...resolvedRow } : resolvedRow entry.keys.set(row, key) + // Collection updates clone the public row. Keep its route key on an + // internal symbol so projected facade rows retain their identity. + Object.defineProperty(row, BUCKET_FACADE_PUBLIC_KEY, { + configurable: true, + value: key, + }) if (nextOrder !== undefined) { entry.order.set(row, nextOrder) + Object.defineProperty(row, BUCKET_FACADE_ORDER, { + configurable: true, + value: nextOrder, + }) } if (change.inserts > change.deletes) { @@ -450,11 +462,20 @@ export class BucketFacadeAdapter { } else if (change.deletes > 0) { sync.write({ type: `delete`, key }) entry.currentOrder.delete(key) + // Deleting the synced base moves a still-visible optimistic upsert out + // of the base ordering and into the optimistic-only suffix. + if (hasOrderBy && entry.collection._state.optimisticUpserts.has(key)) { + sync.collection._markLayoutChange() + } return } entry.currentOrder.set(key, nextOrder) - if (hasOrderBy && orderChangeIsVisible) { + if ( + hasOrderBy && + orderChanged && + !entry.collection._state.optimisticDeletes.has(key) + ) { sync.collection._markLayoutChange() } } diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 4b969e211..1122671d1 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -18,6 +18,52 @@ import type { Context } from '../../src/query/builder/types.js' type FacadeSync = Parameters>[`sync`]>[0] describe(`BucketFacadeAdapter`, () => { + it(`moves a row when the graph reuses its object for a new order`, async () => { + const graph = new D2() + const rows = graph.newInput<[string, BucketRow]>() + const activeBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-order-parent`, + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: true }], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const moving = { id: 1, value: `moving` } + const fixed = { id: 2, value: `fixed` } + activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + rows.sendData( + new MultiSet([ + [[bucketKey, { publicKey: moving.id, value: moving, order: `0` }], 1], + [[bucketKey, { publicKey: fixed.id, value: fixed, order: `1` }], 1], + ]), + ) + graph.run() + adapter.flush().publish() + + const facadeRef: BucketFacadeRef = { + [BUCKET_FACADE_REF]: { edgeId: `children`, bucketKey }, + } + const facade = adapter.resolve(facadeRef) as unknown as Collection< + typeof moving, + number + > + expect(facade.toArray.map(({ id }) => id)).toEqual([1, 2]) + + rows.sendData( + new MultiSet([ + [[bucketKey, { publicKey: moving.id, value: moving, order: `0` }], -1], + [[bucketKey, { publicKey: moving.id, value: moving, order: `2` }], 1], + ]), + ) + graph.run() + adapter.flush().publish() + + expect(facade.toArray.map(({ id }) => id)).toEqual([2, 1]) + await adapter.cleanup() + }) + it(`restores facade state when a flush fails after writing`, async () => { const graph = new D2() const rows = graph.newInput<[string, BucketRow]>() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 8e90ac08a..209aca5ee 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -411,14 +411,33 @@ function applyPendingFacadeOperation( function expectedPendingFacadeRows( rows: ReadonlyMap, shape: PendingFacadeShape = `unordered`, + orderRows: ReadonlyMap = rows, ): Array { return [...rows.values()] .map((row) => ({ ...row })) - .sort((left, right) => - shape === `ordered` - ? left.value - right.value || left.id - right.id - : left.id - right.id, - ) + .sort((left, right) => { + if (shape === `unordered`) return left.id - right.id + const leftOrder = orderRows.get(left.id)?.value + const rightOrder = orderRows.get(right.id)?.value + if (leftOrder === rightOrder) return left.id - right.id + if (leftOrder === undefined) return 1 + if (rightOrder === undefined) return -1 + return leftOrder - rightOrder + }) +} + +function projectPendingFacadeRows( + rows: ReadonlyArray, + shape: PendingFacadeShape, +): Array { + const projected = rows.map(({ id, parentGroup, value }) => ({ + id, + parentGroup, + value, + })) + return shape === `ordered` + ? projected + : projected.sort((left, right) => left.id - right.id) } function expectedPendingFacadeChange( @@ -932,28 +951,13 @@ describe(`Collection-valued includes oracle`, () => { await live.preload() const facade = live.get(1)!.children const childRows = () => - expectedPendingFacadeRows( - new Map( - facade.toArray.map(({ id, parentGroup, value }) => [ - id, - { id, parentGroup, value }, - ]), - ), - shape, - ) + projectPendingFacadeRows(facade.toArray, shape) const rootRows = () => live.toArray.map( ({ id: parentId, children: rootFacade }) => ({ id: parentId, - children: expectedPendingFacadeRows( - new Map( - rootFacade.toArray.map( - ({ id: childId, parentGroup, value }) => [ - childId, - { id: childId, parentGroup, value }, - ], - ), - ), + children: projectPendingFacadeRows( + rootFacade.toArray, shape, ), }), @@ -1038,25 +1042,41 @@ describe(`Collection-valued includes oracle`, () => { const optimisticRows = expectedPendingFacadeRows( afterOptimistic, shape, + initialRows, ) const pendingRows = expectedPendingFacadeRows( whilePending, shape, + afterSource, ) const settledRows = expectedPendingFacadeRows( afterSource, shape, + afterSource, ) + const sourceLayoutChanged = + shape === `ordered` && + (optimisticRows.length !== pendingRows.length || + optimisticRows.some( + (row, index) => row.id !== pendingRows[index]?.id, + )) + const expectedSourcePublication = expectedSourceChange + ? [expectedSourceChange] + : sourceLayoutChanged + ? [] + : undefined const expectedSourcePublications = [ [expectedOptimisticChange], - ...(expectedSourceChange ? [[expectedSourceChange]] : []), + ...(expectedSourcePublication + ? [expectedSourcePublication] + : []), ] const expectedSourceSnapshots = [ { facade: optimisticRows, root: [{ id: 1, children: optimisticRows }], }, - ...(expectedSourceChange + ...(expectedSourcePublication ? [ { facade: pendingRows, @@ -1137,6 +1157,365 @@ describe(`Collection-valued includes oracle`, () => { } } + for (const settlement of pendingFacadeSettlements) { + fcTest( + `publishes a non-projected same-key order move while its facade update ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`hidden-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `hidden-order-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const ids = () => facade.toArray.map(({ id }) => id) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackIds: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectChildChange)) + callbackIds.push(ids()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const revisionBeforeSource = facade._layoutRevision + + try { + expect(ids()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + ]) + + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 10, + position: 2, + }) + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`hidden order mutation rejected`)) + await persisted + await flushPromises() + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 10 }, + previousValue: { id: 10, parentGroup: 1, value: 11 }, + }, + ], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + [20, 10], + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `publishes an independent joined order move while a facade update ${settlement}s`, + async () => { + type SortRow = { id: number; childId: number; position: number } + const parents = createControlledCollection(`joined-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`joined-order-children`, [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + const sorts = createControlledCollection( + `joined-order-sorts`, + [ + { id: 100, childId: 10, position: 0 }, + { id: 200, childId: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .join({ sort: sorts.collection }, ({ child, sort }) => + eq(child.id, sort.childId), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ sort }) => sort.position) + .select(({ child }) => child), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const ids = () => facade.toArray.map(({ id }) => id) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackIds: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectChildChange)) + callbackIds.push(ids()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const revisionBeforeSource = facade._layoutRevision + + try { + expect(ids()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + + sorts.write(`update`, { id: 100, childId: 10, position: 2 }) + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`joined order mutation rejected`)) + await persisted + await flushPromises() + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 10 }, + previousValue: { id: 10, parentGroup: 1, value: 11 }, + }, + ], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + [20, 10], + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + sorts.collection.cleanup(), + ]) + } + }, + ) + } + + fcTest( + `does not publish an order token change that preserves facade layout`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`stable-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `stable-order-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 2 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const publications: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => publications.push(batch.map(projectChildChange)), + { includeInitialState: false }, + ) + const revision = facade._layoutRevision + + try { + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 10, + position: 1, + }) + + expect(facade.toArray.map(({ id }) => id)).toEqual([10, 20]) + expect(facade._layoutRevision).toBe(revision) + expect(publications).toEqual([]) + } finally { + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + for (const settlement of pendingFacadeSettlements) { for (const optimisticOperation of pendingFacadeOptimisticOperations) { fcTest( From 89ac7f2a6620682497879a1a22f31fec3b0af73f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 00:20:45 -0600 Subject: [PATCH 193/327] fix(db): validate visible layout changes --- packages/db/src/collection/state.ts | 20 +- packages/db/src/collection/sync.ts | 4 +- packages/db/src/query/live/ARCHITECTURE.md | 16 +- .../src/query/live/bucket-facade-adapter.ts | 44 +- .../tests/live-query-order-only-move.test.ts | 42 ++ ...ncludes-collection-oracle.property.test.ts | 465 ++++++++++++++++++ 6 files changed, 573 insertions(+), 18 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 7f422cfc8..72a4c10ad 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -35,6 +35,8 @@ interface PendingSyncedTransaction< committed: boolean applicationStarted: boolean layoutChanged: boolean + /** Visible key order before a possible layout change. */ + layoutSnapshot?: Array operations: Array> truncate?: boolean deletedKeys: Set @@ -879,11 +881,15 @@ export class CollectionStateManager< hasTruncateSync, hasImmediateSync, layoutChanged, + layoutSnapshots, } = this.pendingSyncedTransactions.reduce( (acc, t) => { if (t.committed) { acc.committedSyncedTransactions.push(t) acc.layoutChanged ||= t.layoutChanged + if (t.layoutSnapshot !== undefined) { + acc.layoutSnapshots.push(t.layoutSnapshot) + } if (t.truncate) { acc.hasTruncateSync = true } @@ -905,6 +911,7 @@ export class CollectionStateManager< hasTruncateSync: false, hasImmediateSync: false, layoutChanged: false, + layoutSnapshots: [] as Array>, }, ) @@ -1458,9 +1465,20 @@ export class CollectionStateManager< } // End batching and emit all events (combines any batched events with sync events) + const visibleKeysAfterCommit = layoutChanged ? [...this.keys()] : [] + const visibleLayoutChanged = + layoutChanged && + (layoutSnapshots.length === 0 || + layoutSnapshots.some( + (before) => + before.length !== visibleKeysAfterCommit.length || + before.some( + (key, index) => key !== visibleKeysAfterCommit[index], + ), + )) let publicationError: { error: unknown } | undefined try { - this.changes.emitEvents(events, true, layoutChanged) + this.changes.emitEvents(events, true, visibleLayoutChanged) } catch (error) { // The state is already committed. Finish this batch and drain any work // queued by earlier listeners before surfacing their publication error. diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index b14766205..70227f355 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -199,7 +199,9 @@ export class CollectionSyncManager< /** Mark the active sync transaction as changing collection layout. */ public markLayoutChange(): void { - this.getActivePendingSyncTransaction().layoutChanged = true + const transaction = this.getActivePendingSyncTransaction() + transaction.layoutChanged = true + transaction.layoutSnapshot ??= [...this.state.keys()] } /** diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index c8f9d06c9..29b6809f6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -945,10 +945,22 @@ publication while an optimistic mutation on either Collection persists. The normal optimistic overlay still wins for conflicting keys, and the graph output remains one coherent publication. The overlay replaces the row value, not the graph-owned key order. A source order move publishes a layout change -when that key remains visible, including beneath an optimistic update. An +only when the complete visible public-key sequence changes after applying the +optimistic overlay. This includes a move beneath an optimistic update. It does +not include a move whose only crossed peers are optimistically deleted. An optimistic delete hides the key and its order moves. If the synced base is deleted beneath an optimistic update, the still-visible row moves to the -optimistic-only suffix and that layout change also publishes. +optimistic-only suffix; that publishes only when the suffix transition changes +the visible sequence. Re-establishing the base applies the inverse rule. A +legal absent-to-present source transition must keep the optimistic value +visible while restoring the graph-owned position in the same publication. +The graph reports a possible layout change before its sync commit. Collection +state captures the visible key sequence at that point and compares it with the +final sequence after the sync writes and active optimistic overlay have both +been applied. The layout revision advances only when those exact sequences +differ. This final check is shared by root Collections and child facades; an +adapter-local order token is evidence to check layout, not proof that public +layout changed. Rejected, canceled, and obsolete acquisitions establish no coverage. Sources must honor cancellation before publishing request-scoped rows. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index fe530a681..c543f6cdd 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -130,13 +130,21 @@ export class BucketFacadeAdapter { for (const change of changes.values()) { this.prepareChange(entry, change) } + const mayChangeVisibleOrder = + compilation.hasOrderBy && + [...changes.values()].some((change) => + this.mayChangeVisibleOrder(entry, change), + ) deferPublication(entry) // The graph is already quiescent. Install this complete child // publication beneath any pending optimistic facade overlay instead // of parking source progress behind that mutation. sync.begin({ immediate: true }) for (const change of changes.values()) { - this.applyChange(entry, sync, change, compilation.hasOrderBy) + this.applyChange(entry, sync, change) + } + if (mayChangeVisibleOrder) { + sync.collection._markLayoutChange() } sync.commit() } @@ -419,7 +427,6 @@ export class BucketFacadeAdapter { entry: FacadeEntry, sync: FacadeSync, change: PendingRow, - hasOrderBy: boolean, ): void { const key = change.value.publicKey as string | number const previousOrder = entry.currentOrder.get(key) @@ -462,22 +469,31 @@ export class BucketFacadeAdapter { } else if (change.deletes > 0) { sync.write({ type: `delete`, key }) entry.currentOrder.delete(key) - // Deleting the synced base moves a still-visible optimistic upsert out - // of the base ordering and into the optimistic-only suffix. - if (hasOrderBy && entry.collection._state.optimisticUpserts.has(key)) { - sync.collection._markLayoutChange() - } return } entry.currentOrder.set(key, nextOrder) - if ( - hasOrderBy && - orderChanged && - !entry.collection._state.optimisticDeletes.has(key) - ) { - sync.collection._markLayoutChange() - } + } + + /** Identify graph changes that can move a visible key. Collection state + * validates the final public sequence before publishing the layout signal. */ + private mayChangeVisibleOrder( + entry: FacadeEntry, + change: PendingRow, + ): boolean { + const key = change.value.publicKey as string | number + const hasSyncedRow = entry.collection._state.syncedData.has(key) + const nextHasSyncedRow = + change.inserts > change.deletes || + (change.inserts === change.deletes && hasSyncedRow) + const orderChanged = + hasSyncedRow && + nextHasSyncedRow && + entry.currentOrder.get(key) !== change.value.order + const movesBetweenBaseAndOptimisticSuffix = + hasSyncedRow !== nextHasSyncedRow && + entry.collection._state.optimisticUpserts.has(key) + return orderChanged || movesBetweenBaseAndOptimisticSuffix } /** Resolve and validate every public key before opening a sync transaction. */ diff --git a/packages/db/tests/live-query-order-only-move.test.ts b/packages/db/tests/live-query-order-only-move.test.ts index 5f43dea6b..ddcf06ae5 100644 --- a/packages/db/tests/live-query-order-only-move.test.ts +++ b/packages/db/tests/live-query-order-only-move.test.ts @@ -222,6 +222,48 @@ describe(`order-only move publication`, () => { observer.dispose() }) + it(`does not publish a move whose only crossed peer is optimistically deleted`, async () => { + const source = makeSource() + const persist = createDeferred() + const lq = createLiveQueryCollection({ + getKey: (row) => row.id, + query: (q) => + q + .from({ p: source }) + .orderBy(({ p }) => p.age, `asc`) + .select(({ p }) => ({ id: p.id, name: p.name })), + onDelete: () => persist.promise, + }) + await lq.preload() + const publications: Array> = [] + const subscription = lq.subscribeChanges( + (changes) => publications.push(changes), + { includeInitialState: false }, + ) + const mutation = lq.delete(`2`) + + expect(mutation.state).toBe(`persisting`) + expect(lq.toArray.map(({ id }) => id)).toEqual([`1`, `3`]) + publications.length = 0 + const revisionBeforeSource = lq._layoutRevision + + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: `1`, name: `Alice`, age: 10 }, + }) + source.utils.commit() + + expect(lq.toArray.map(({ id }) => id)).toEqual([`1`, `3`]) + expect(publications).toEqual([]) + expect(lq._layoutRevision).toBe(revisionBeforeSource) + + persist.resolve() + await mutation.isPersisted.promise + subscription.unsubscribe() + await Promise.all([lq.cleanup(), source.cleanup()]) + }) + it(`does not publish when multiple moves cancel within one transaction`, async () => { const source = makeSource() const lq = await makeOrderedByAge(source) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 209aca5ee..0a8186b5f 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -61,6 +61,26 @@ function projectChildChange( } } +type ProjectedValueChange = { + type: `insert` | `update` | `delete` + key: number + value: number + previousValue?: number +} + +function projectValueChange( + change: ChangeMessage<{ value: number }, string | number>, +): ProjectedValueChange { + return { + type: change.type, + key: Number(change.key), + value: change.value.value, + ...(change.previousValue + ? { previousValue: change.previousValue.value } + : {}), + } +} + type CollectionAction = | { type: `putParent`; row: ParentRow } | { type: `deleteParent`; id: number } @@ -1456,6 +1476,451 @@ describe(`Collection-valued includes oracle`, () => { ) } + for (const settlement of pendingFacadeSettlements) { + fcTest( + `keeps a projected optimistic value visible through a same-key base reinsert that ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`reinsert-order-parents`, [ + { id: 1, group: 1 }, + ]) + const sourceRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ] + const children = createControlledCollection( + `reinsert-order-children`, + sourceRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(transaction.state).toBe(`persisting`) + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + publications.length = 0 + callbackKeys.length = 0 + callbackValues.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`delete`, sourceRows[0]!) + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(publications).toEqual([[]]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(callbackValues).toEqual([[20, 11]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + + children.write(`insert`, sourceRows[0]!) + + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + expect(publications).toEqual([[], []]) + expect(callbackKeys).toEqual([ + [20, 10], + [10, 20], + ]) + expect(callbackValues).toEqual([ + [20, 11], + [11, 20], + ]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 2) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`reinsert mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([10, 20]) + expect(publications).toEqual([ + [], + [], + [ + { + type: `update`, + key: 10, + value: 10, + previousValue: 11, + }, + ], + ]) + expect(callbackKeys).toEqual([ + [20, 10], + [10, 20], + [10, 20], + ]) + expect(callbackValues).toEqual([ + [20, 11], + [11, 20], + [10, 20], + ]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 2) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + for (const targetPosition of [`first`, `last`] as const) { + fcTest( + `publishes a base-to-optimistic-suffix move only when a ${targetPosition} row changes layout and ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`suffix-order-parents`, [ + { id: 1, group: 1 }, + ]) + const target: OrderedSourceChild = { + id: 10, + parentGroup: 1, + value: 10, + position: targetPosition === `first` ? 0 : 1, + } + const peer: OrderedSourceChild = { + id: 20, + parentGroup: 1, + value: 20, + position: targetPosition === `first` ? 1 : 0, + } + const children = createControlledCollection(`suffix-order-children`, [ + target, + peer, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual( + targetPosition === `first` ? [10, 20] : [20, 10], + ) + expect(values()).toEqual( + targetPosition === `first` ? [11, 20] : [20, 11], + ) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`delete`, target) + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(publications).toEqual(targetPosition === `first` ? [[]] : []) + expect(callbackKeys).toEqual( + targetPosition === `first` ? [[20, 10]] : [], + ) + expect(facade._layoutRevision).toBe( + revisionBeforeSource + (targetPosition === `first` ? 1 : 0), + ) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`suffix mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications.at(-1)).toEqual([ + { + type: `delete`, + key: 10, + value: 11, + }, + ]) + expect(callbackKeys.at(-1)).toEqual([20]) + expect(facade._layoutRevision).toBe( + revisionBeforeSource + (targetPosition === `first` ? 1 : 0), + ) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `does not publish a same-source order move across an optimistically deleted peer that ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`hidden-peer-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `hidden-peer-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => facade.delete(10), + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`update`, { + id: 20, + parentGroup: 1, + value: 20, + position: -1, + }) + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications).toEqual([]) + expect(callbackKeys).toEqual([]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`hidden peer mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(publications).toEqual([ + [{ type: `insert`, key: 10, value: 10 }], + ]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `does not publish a joined order move across an optimistically deleted peer that ${settlement}s`, + async () => { + type SortRow = { id: number; childId: number; position: number } + const parents = createControlledCollection(`joined-hidden-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`joined-hidden-children`, [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + const sorts = createControlledCollection( + `joined-hidden-sorts`, + [ + { id: 100, childId: 10, position: 0 }, + { id: 200, childId: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .join({ sort: sorts.collection }, ({ child, sort }) => + eq(child.id, sort.childId), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ sort }) => sort.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => facade.delete(10), + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + sorts.write(`update`, { id: 200, childId: 20, position: -1 }) + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications).toEqual([]) + expect(callbackKeys).toEqual([]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`joined hidden peer rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(publications).toEqual([ + [{ type: `insert`, key: 10, value: 10 }], + ]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + sorts.collection.cleanup(), + ]) + } + }, + ) + } + fcTest( `does not publish an order token change that preserves facade layout`, async () => { From 905606dc262596f19bd3858787bb49bbc8890ae2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 08:04:51 -0600 Subject: [PATCH 194/327] fix(db): compare layout at prefix application --- packages/db/src/collection/state.ts | 21 ++-- packages/db/src/collection/sync.ts | 4 +- packages/db/src/query/live/ARCHITECTURE.md | 14 ++- .../tests/collection-sync-reentrancy.test.ts | 107 ++++++++++++++++++ 4 files changed, 123 insertions(+), 23 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 72a4c10ad..15d55503c 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -35,8 +35,6 @@ interface PendingSyncedTransaction< committed: boolean applicationStarted: boolean layoutChanged: boolean - /** Visible key order before a possible layout change. */ - layoutSnapshot?: Array operations: Array> truncate?: boolean deletedKeys: Set @@ -881,15 +879,11 @@ export class CollectionStateManager< hasTruncateSync, hasImmediateSync, layoutChanged, - layoutSnapshots, } = this.pendingSyncedTransactions.reduce( (acc, t) => { if (t.committed) { acc.committedSyncedTransactions.push(t) acc.layoutChanged ||= t.layoutChanged - if (t.layoutSnapshot !== undefined) { - acc.layoutSnapshots.push(t.layoutSnapshot) - } if (t.truncate) { acc.hasTruncateSync = true } @@ -911,7 +905,6 @@ export class CollectionStateManager< hasTruncateSync: false, hasImmediateSync: false, layoutChanged: false, - layoutSnapshots: [] as Array>, }, ) @@ -930,6 +923,10 @@ export class CollectionStateManager< // non-immediate transactions would be applied later and could overwrite newer state. // Processing all committed transactions together preserves causal ordering. if (!hasPersistingTransaction || hasTruncateSync || hasImmediateSync) { + // Every committed transaction below belongs to one applied causal prefix. + // Compare its final layout with the one public state immediately before + // application, not with snapshots taken when older work first queued. + const visibleKeysBeforeCommit = layoutChanged ? [...this.keys()] : [] // This queue remains authoritative while user callbacks run. Transactions // opened by a callback must not be overwritten by this batch's snapshot. this.pendingSyncedTransactions = uncommittedSyncedTransactions @@ -1468,13 +1465,9 @@ export class CollectionStateManager< const visibleKeysAfterCommit = layoutChanged ? [...this.keys()] : [] const visibleLayoutChanged = layoutChanged && - (layoutSnapshots.length === 0 || - layoutSnapshots.some( - (before) => - before.length !== visibleKeysAfterCommit.length || - before.some( - (key, index) => key !== visibleKeysAfterCommit[index], - ), + (visibleKeysBeforeCommit.length !== visibleKeysAfterCommit.length || + visibleKeysBeforeCommit.some( + (key, index) => key !== visibleKeysAfterCommit[index], )) let publicationError: { error: unknown } | undefined try { diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index 70227f355..b14766205 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -199,9 +199,7 @@ export class CollectionSyncManager< /** Mark the active sync transaction as changing collection layout. */ public markLayoutChange(): void { - const transaction = this.getActivePendingSyncTransaction() - transaction.layoutChanged = true - transaction.layoutSnapshot ??= [...this.state.keys()] + this.getActivePendingSyncTransaction().layoutChanged = true } /** diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 29b6809f6..6a57eda22 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -955,12 +955,14 @@ the visible sequence. Re-establishing the base applies the inverse rule. A legal absent-to-present source transition must keep the optimistic value visible while restoring the graph-owned position in the same publication. The graph reports a possible layout change before its sync commit. Collection -state captures the visible key sequence at that point and compares it with the -final sequence after the sync writes and active optimistic overlay have both -been applied. The layout revision advances only when those exact sequences -differ. This final check is shared by root Collections and child facades; an -adapter-local order token is evidence to check layout, not proof that public -layout changed. +state captures the visible key sequence immediately before the whole committed +causal prefix applies, then compares it with the final sequence after the sync +writes and active optimistic overlay have both been applied. Queued +transaction-local snapshots are not public boundaries: an overlay may change +before a later immediate transaction drains them. The layout revision advances +only when the exact before/after sequences differ. This final check is shared +by root Collections and child facades; an adapter-local order token is evidence +to check layout, not proof that public layout changed. Rejected, canceled, and obsolete acquisitions establish no coverage. Sources must honor cancellation before publishing request-scoped rows. diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 521bba7e8..975c31405 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -194,6 +194,113 @@ const { multiplier, ...replay } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { + it(`compares layout with the public state before an immediate prefix drain`, async () => { + type OrderedRow = Row & { rank: number } + type OrderedSync = Parameters[`sync`]>[0] + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + ops.begin({ immediate: true }) + ops.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 0 }, + }) + ops.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + ops.commit() + ops.markReady() + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + const callbacks: Array<{ + changes: Array + keys: Array + values: Array + }> = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(1, (draft) => { + draft.value = `optimistic-one` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.collection._markLayoutChange() + const secondReceipt = sync.commit() + + expect([...collection.keys()]).toEqual([1, 2, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-one`, + `two`, + `optimistic-three`, + ]) + expect(callbacks).toEqual([]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain) + await Promise.all( + [firstReceipt, secondReceipt] + .filter((receipt) => receipt !== true) + .map((receipt) => receipt), + ) + + updatePersistence.resolve() + insertPersistence.resolve() + await Promise.all([ + update.isPersisted.promise, + insert.isPersisted.promise, + ]) + } finally { + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`preserves sync work opened by a listener until it is committed`, async () => { const harness = createSyncHarness(`listener-opened-sync-work`) const { collection } = harness From 750f82e46cf168efee3a4585a474cc3250f340a5 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 08:47:37 -0600 Subject: [PATCH 195/327] test(db): cover causal prefix layout drains --- .../tests/collection-sync-reentrancy.test.ts | 339 +++++++++++++++++- 1 file changed, 337 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 975c31405..87a5d5386 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -13,6 +13,17 @@ type Row = { type SyncOps = Parameters[`sync`]>[0] +type OrderedRow = Row & { rank: number } +type OrderedSync = Parameters[`sync`]>[0] + +type LayoutCallback = { + changes: Array + keys: Array + values: Array + markedReceiptSettled: boolean + revision: number +} + type ListenerAction = `commit` | `abort` type ListenerScenario = { @@ -92,6 +103,20 @@ function stageInsert( sync.write({ type: `insert`, value: row }) } +function installInitialOrderedRows(sync: OrderedSync): void { + sync.begin({ immediate: true }) + sync.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + sync.commit() + sync.markReady() +} + async function runListenerScenario(scenario: ListenerScenario): Promise { const harness = createSyncHarness( `generated-listener-sync-${generatedHarnessId++}`, @@ -195,8 +220,6 @@ const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { it(`compares layout with the public state before an immediate prefix drain`, async () => { - type OrderedRow = Row & { rank: number } - type OrderedSync = Parameters[`sync`]>[0] const updatePersistence = createDeferred() const insertPersistence = createDeferred() let sync!: OrderedSync @@ -301,6 +324,318 @@ describe(`sync publication reentrancy`, () => { } }) + it(`publishes a parked layout mark when optimistic persistence drains it`, async () => { + const updatePersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-normal-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + }) + let markedReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const receipt = sync.commit() + expect(receipt).not.toBe(true) + if (receipt !== true) { + void receipt.then(() => { + markedReceiptSettled = true + }) + } + + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + await Promise.resolve() + expect(markedReceiptSettled).toBe(false) + expect([...collection.keys()]).toEqual([1, 2]) + + updatePersistence.resolve() + await update.isPersisted.promise + if (receipt !== true) await receipt + + expect([...collection.keys()]).toEqual([2, 1]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `two`, + `one`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [1, 2], + keys: [2, 1], + values: [`two`, `one`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + expect(markedReceiptSettled).toBe(true) + } finally { + updatePersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it.each([`first`, `middle`, `last`] as const)( + `honors a %s-position layout mark in an immediate causal prefix`, + async (markPosition) => { + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-immediate-${markPosition}`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + let firstReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: firstReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: + markPosition === `first` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-a`, rank: 1 }, + }) + if (markPosition === `first`) sync.collection._markLayoutChange() + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + if (firstReceipt !== true) { + void firstReceipt.then(() => { + firstReceiptSettled = true + }) + } + await Promise.resolve() + expect(firstReceiptSettled).toBe(false) + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + + sync.begin() + sync.write({ + type: `update`, + value: + markPosition === `middle` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-b`, rank: 1 }, + }) + if (markPosition === `middle`) sync.collection._markLayoutChange() + const middleReceipt = sync.commit() + expect(middleReceipt).not.toBe(true) + + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: + markPosition === `last` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-c`, rank: 1 }, + }) + if (markPosition === `last`) sync.collection._markLayoutChange() + const lastReceipt = sync.commit() + + expect(lastReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2, 1, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-two`, + `one`, + `optimistic-three`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [1], + keys: [2, 1, 3], + values: [`optimistic-two`, `one`, `optimistic-three`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + + await Promise.all( + [firstReceipt, middleReceipt] + .filter((receipt) => receipt !== true) + .map((receipt) => receipt), + ) + expect(firstReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }, + ) + + it(`honors a parked layout mark when truncate drains its causal prefix`, async () => { + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-truncate-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + let markedReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(1, (draft) => { + draft.value = `optimistic-one` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + if (firstReceipt !== true) { + void firstReceipt.then(() => { + markedReceiptSettled = true + }) + } + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin() + sync.truncate() + sync.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + const truncateReceipt = sync.commit() + + expect(truncateReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2, 1, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `two`, + `optimistic-one`, + `optimistic-three`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [2, 1, 3, 1, 3, 1, 2], + keys: [2, 1, 3], + values: [`two`, `optimistic-one`, `optimistic-three`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + if (firstReceipt !== true) await firstReceipt + expect(markedReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + it(`preserves sync work opened by a listener until it is committed`, async () => { const harness = createSyncHarness(`listener-opened-sync-work`) const { collection } = harness From 84c6e5911c93fe93f9815e987a4fae144cd7cec2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 09:07:19 -0600 Subject: [PATCH 196/327] test(db): preserve multiple prefix layout marks --- .../db/tests/collection-sync-reentrancy.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 87a5d5386..003d459bb 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -407,8 +407,8 @@ describe(`sync publication reentrancy`, () => { } }) - it.each([`first`, `middle`, `last`] as const)( - `honors a %s-position layout mark in an immediate causal prefix`, + it.each([`first`, `middle`, `last`, `first-and-middle`] as const)( + `honors %s layout marks in an immediate causal prefix`, async (markPosition) => { const updatePersistence = createDeferred() const insertPersistence = createDeferred() @@ -451,11 +451,13 @@ describe(`sync publication reentrancy`, () => { sync.write({ type: `update`, value: - markPosition === `first` + markPosition === `first` || markPosition === `first-and-middle` ? { id: 1, value: `one`, rank: 2 } : { id: 2, value: `server-two-a`, rank: 1 }, }) - if (markPosition === `first`) sync.collection._markLayoutChange() + if (markPosition === `first` || markPosition === `first-and-middle`) { + sync.collection._markLayoutChange() + } const firstReceipt = sync.commit() expect(firstReceipt).not.toBe(true) if (firstReceipt !== true) { @@ -480,7 +482,9 @@ describe(`sync publication reentrancy`, () => { ? { id: 1, value: `one`, rank: 2 } : { id: 2, value: `server-two-b`, rank: 1 }, }) - if (markPosition === `middle`) sync.collection._markLayoutChange() + if (markPosition === `middle` || markPosition === `first-and-middle`) { + sync.collection._markLayoutChange() + } const middleReceipt = sync.commit() expect(middleReceipt).not.toBe(true) From 4815abfb27ed75ed2f72a90186d85a0da667f271 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 09:35:52 -0600 Subject: [PATCH 197/327] test(db): cover overlay removal before prefix drains --- .../tests/collection-sync-reentrancy.test.ts | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 003d459bb..c4632f1f6 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -324,6 +324,91 @@ describe(`sync publication reentrancy`, () => { } }) + it(`uses the post-removal public layout before an unmarked prefix drain`, async () => { + const updatePersistence = createDeferred() + const deletePersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-removal-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onDelete: () => deletePersistence.promise, + }) + const callbacks: Array = [] + let parkedReceiptSettled = false + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: parkedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + let deletion: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const parkedReceipt = sync.commit() + expect(parkedReceipt).not.toBe(true) + if (parkedReceipt !== true) { + void parkedReceipt.then(() => { + parkedReceiptSettled = true + }) + } + + deletion = collection.delete(1) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + await Promise.resolve() + expect(parkedReceiptSettled).toBe(false) + expect([...collection.keys()]).toEqual([2]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 2, value: `server-two`, rank: 1 }, + }) + const drainReceipt = sync.commit() + + expect(drainReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-two`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain) + expect(callbacks).toEqual([]) + if (parkedReceipt !== true) await parkedReceipt + expect(parkedReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + deletePersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await deletion?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + it(`publishes a parked layout mark when optimistic persistence drains it`, async () => { const updatePersistence = createDeferred() let sync!: OrderedSync From f0f0eabe2831a3d18c239107e130cd01a51a9cd1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 10:06:03 -0600 Subject: [PATCH 198/327] test(db): isolate reentrant layout prefixes --- .../tests/collection-sync-reentrancy.test.ts | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index c4632f1f6..773baadf8 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -725,6 +725,99 @@ describe(`sync publication reentrancy`, () => { } }) + it(`captures a fresh layout boundary for each reentrant causal prefix`, async () => { + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-reentrant-prefixes`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + }) + let listenerDepth = 0 + let maxListenerDepth = 0 + let queuedRestore = false + let innerReceipt: Promise | undefined + let innerReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + listenerDepth++ + maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: innerReceiptSettled, + revision: collection._layoutRevision, + }) + + if (!queuedRestore) { + queuedRestore = true + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.collection._markLayoutChange() + const receipt = sync.commit() + if (receipt === true) { + throw new Error(`Expected listener-created work to queue`) + } + innerReceipt = receipt + void receipt.then(() => { + innerReceiptSettled = true + }) + } + + listenerDepth-- + }, + { includeInitialState: false }, + ) + + try { + const revisionBeforeDrain = collection._layoutRevision + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + expect(sync.commit()).toBe(true) + + expect([...collection.keys()]).toEqual([1, 2]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 2) + expect(callbacks).toEqual([ + { + changes: [1], + keys: [2, 1], + values: [`two`, `one`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + { + changes: [1], + keys: [1, 2], + values: [`one`, `two`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 2, + }, + ]) + expect(maxListenerDepth).toBe(1) + expect(innerReceipt).toBeDefined() + await innerReceipt + expect(innerReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`preserves sync work opened by a listener until it is committed`, async () => { const harness = createSyncHarness(`listener-opened-sync-work`) const { collection } = harness From 03649f68fff148a503c4033391891272612e4a0a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 10:31:54 -0600 Subject: [PATCH 199/327] fix(db): roll back deferred revision clocks --- packages/db/src/collection/changes.ts | 21 ++++++-- packages/db/src/query/live/ARCHITECTURE.md | 8 ++- ...ncludes-collection-oracle.property.test.ts | 53 ++++++++++++++++--- 3 files changed, 70 insertions(+), 12 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 00523a2f4..6432fda29 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -41,6 +41,8 @@ export class CollectionChangesManager< changes: Array> layoutChanged: boolean }> = [] + private deferredStateRevisionDelta = 0 + private deferredLayoutRevisionDelta = 0 private layoutChangeListeners = new Set<() => void>() /** @@ -107,10 +109,15 @@ export class CollectionChangesManager< forceEmit = false, layoutChanged = false, ): void { - // The visible state was already committed by the caller, so the revision - // advances even when the events below end up batched for later emission. - if (changes.length > 0) this.stateRevision++ - if (layoutChanged) this.layoutRevision++ + // A coherent multi-Collection publication may still roll back. Hold its + // revision clocks with its events so discard leaves no public trace. + if (this.publicationDeferralDepth > 0) { + if (changes.length > 0) this.deferredStateRevisionDelta++ + if (layoutChanged) this.deferredLayoutRevisionDelta++ + } else { + if (changes.length > 0) this.stateRevision++ + if (layoutChanged) this.layoutRevision++ + } // Skip batching for user actions (forceEmit=true) to keep UI responsive if (this.shouldBatchEvents && !forceEmit) { @@ -161,10 +168,16 @@ export class CollectionChangesManager< const publications = this.deferredPublications this.deferredPublications = [] + const stateRevisionDelta = this.deferredStateRevisionDelta + const layoutRevisionDelta = this.deferredLayoutRevisionDelta + this.deferredStateRevisionDelta = 0 + this.deferredLayoutRevisionDelta = 0 if (this.discardDeferredPublications) { this.discardDeferredPublications = false return } + this.stateRevision += stateRevisionDelta + this.layoutRevision += layoutRevisionDelta this.publishEvents( publications.flatMap(({ changes }) => changes), publications.some(({ layoutChanged }) => layoutChanged), diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 6a57eda22..5fab3b369 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1128,8 +1128,12 @@ in-place include repair, and forced secondary events are forbidden. Installed state, synchronous reads, change-event payloads, and downstream queries must all observe the same fully materialized commit. The facade adapter -may defer event delivery across its Collection transactions, but it must not -defer state or index installation. Routing and identity remain inside D2. +may defer public revision clocks and event delivery across its Collection +transactions, but it must not defer state or index installation. A successful +coherent publication advances those clocks before callbacks run. If a later +root or containing-facade application fails, rollback restores the installed +state and discards both the held events and their revision advances. Routing +and identity remain inside D2. ## External boundaries diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 0a8186b5f..810923e93 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,6 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' import { createDeferred } from '../../src/deferred.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { createOptimisticAction } from '../../src/optimistic-action.js' import { add, @@ -791,9 +792,16 @@ describe(`Collection-valued includes oracle`, () => { group: 1, value: 1, } + const initialSibling: NodeRow = { + id: 20, + kind: `child`, + group: 1, + value: 2, + } const nodes = createControlledCollection(`rollback-nodes`, [ initialParent, initialChild, + initialSibling, ]) const live = createLiveQueryCollection((q) => q @@ -805,7 +813,8 @@ describe(`Collection-valued includes oracle`, () => { children: q .from({ child: nodes.collection }) .where(({ child }) => eq(child.kind, `child`)) - .where(({ child }) => eq(child.group, parent.group)), + .where(({ child }) => eq(child.group, parent.group)) + .orderBy(({ child }) => child.value), })), ) @@ -821,6 +830,13 @@ describe(`Collection-valued includes oracle`, () => { (batch) => childPublications.push(...batch), { includeInitialState: false }, ) + const childObserver = createLiveQueryObserver(facade) + let observerNotifications = 0 + childObserver.subscribe(() => observerNotifications++) + observerNotifications = 0 + const observerBeforeFailure = childObserver.getSnapshot() + const childStateRevisionBeforeFailure = facade._stateRevision + const childLayoutRevisionBeforeFailure = facade._layoutRevision const originalGetKey = live.config.getKey live.config.getKey = (row) => { if (row.value === 2) throw new Error(`root key failed`) @@ -836,14 +852,25 @@ describe(`Collection-valued includes oracle`, () => { }, { type: `update`, - value: { ...initialChild, value: 2 }, + value: { ...initialChild, value: 3 }, + }, + { + type: `update`, + value: { ...initialSibling, value: 0 }, }, ]), ).toThrow(`root key failed`) expect(live.get(1)!.value).toBe(1) - expect(facade.get(10)!.value).toBe(1) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) + expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure) + expect(facade._layoutRevision).toBe(childLayoutRevisionBeforeFailure) + expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) + expect(observerNotifications).toBe(0) live.config.getKey = originalGetKey nodes.writeBatch([ @@ -853,15 +880,29 @@ describe(`Collection-valued includes oracle`, () => { }, { type: `update`, - value: { ...initialChild, value: 3 }, + value: { ...initialChild, value: 4 }, + }, + { + type: `update`, + value: { ...initialSibling, value: 3 }, }, ]) expect(live.get(1)!.value).toBe(3) - expect(facade.get(10)!.value).toBe(3) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 20, value: 3 }, + { id: 10, value: 4 }, + ]) expect(rootPublications).toHaveLength(1) - expect(childPublications).toHaveLength(1) + expect(childPublications).toHaveLength(2) + expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure + 1) + expect(facade._layoutRevision).toBe( + childLayoutRevisionBeforeFailure + 1, + ) + expect(childObserver.getSnapshot()).not.toBe(observerBeforeFailure) + expect(observerNotifications).toBe(1) } finally { live.config.getKey = originalGetKey + childObserver.dispose() rootSubscription.unsubscribe() childSubscription.unsubscribe() await Promise.all([live.cleanup(), nodes.collection.cleanup()]) From 0308fc53d907ff9b0745c989d56c9630f8b6a31f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 11:07:07 -0600 Subject: [PATCH 200/327] fix(db): prepare coherent revisions before callbacks --- packages/db/src/collection/changes.ts | 140 +++++++++----- packages/db/src/query/live/ARCHITECTURE.md | 10 +- .../src/query/live/bucket-facade-adapter.ts | 11 +- .../query/live/collection-config-builder.ts | 5 + ...ncludes-collection-oracle.property.test.ts | 171 +++++++++++++++++- 5 files changed, 287 insertions(+), 50 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 6432fda29..015560c6c 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -15,10 +15,35 @@ import type { CollectionStateManager } from './state.js' import type { WithVirtualProps } from '../virtual-props.js' export type PublicationDeferral = { + /** Irrevocably advance held revision clocks without invoking callbacks. */ + prepare: () => void + /** Prepare if needed, then release the held callbacks. */ publish: () => void + /** Discard held clocks and callbacks before preparation. */ discard: () => void } +type PublicationDeferralState< + TOutput extends object, + TKey extends string | number, +> = { + depth: number + discard: boolean + publications: Array<{ + changes: Array> + layoutChanged: boolean + }> + stateRevisionDelta: number + layoutRevisionDelta: number + prepared: + | { + changes: Array> + layoutChanged: boolean + } + | undefined + published: boolean +} + export class CollectionChangesManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -35,14 +60,9 @@ export class CollectionChangesManager< public changeSubscriptions = new Set() public batchedEvents: Array> = [] public shouldBatchEvents = false - private publicationDeferralDepth = 0 - private discardDeferredPublications = false - private deferredPublications: Array<{ - changes: Array> - layoutChanged: boolean - }> = [] - private deferredStateRevisionDelta = 0 - private deferredLayoutRevisionDelta = 0 + private publicationDeferral: + | PublicationDeferralState + | undefined private layoutChangeListeners = new Set<() => void>() /** @@ -111,9 +131,10 @@ export class CollectionChangesManager< ): void { // A coherent multi-Collection publication may still roll back. Hold its // revision clocks with its events so discard leaves no public trace. - if (this.publicationDeferralDepth > 0) { - if (changes.length > 0) this.deferredStateRevisionDelta++ - if (layoutChanged) this.deferredLayoutRevisionDelta++ + const publicationDeferral = this.publicationDeferral + if (publicationDeferral) { + if (changes.length > 0) publicationDeferral.stateRevisionDelta++ + if (layoutChanged) publicationDeferral.layoutRevisionDelta++ } else { if (changes.length > 0) this.stateRevision++ if (layoutChanged) this.layoutRevision++ @@ -140,8 +161,11 @@ export class CollectionChangesManager< this.shouldBatchEvents = false } - if (this.publicationDeferralDepth > 0) { - this.deferredPublications.push({ changes: rawEvents, layoutChanged }) + if (publicationDeferral) { + publicationDeferral.publications.push({ + changes: rawEvents, + layoutChanged, + }) return } @@ -154,39 +178,63 @@ export class CollectionChangesManager< * normal transaction boundaries. */ public deferPublication(): PublicationDeferral { - this.publicationDeferralDepth++ - let closed = false - - const close = (discard: boolean) => { - if (closed) return - closed = true - if (this.publicationDeferralDepth === 0) return - this.discardDeferredPublications ||= discard - - this.publicationDeferralDepth-- - if (this.publicationDeferralDepth > 0) return - - const publications = this.deferredPublications - this.deferredPublications = [] - const stateRevisionDelta = this.deferredStateRevisionDelta - const layoutRevisionDelta = this.deferredLayoutRevisionDelta - this.deferredStateRevisionDelta = 0 - this.deferredLayoutRevisionDelta = 0 - if (this.discardDeferredPublications) { - this.discardDeferredPublications = false - return + const publicationDeferral = this.publicationDeferral ?? { + depth: 0, + discard: false, + publications: [], + stateRevisionDelta: 0, + layoutRevisionDelta: 0, + prepared: undefined, + published: false, + } + this.publicationDeferral = publicationDeferral + publicationDeferral.depth++ + let handleState: `open` | `prepared` | `discarded` = `open` + + const prepare = (discard: boolean) => { + if (handleState !== `open`) return + handleState = discard ? `discarded` : `prepared` + publicationDeferral.discard ||= discard + publicationDeferral.depth-- + if (publicationDeferral.depth > 0) return + + if (this.publicationDeferral === publicationDeferral) { + this.publicationDeferral = undefined + } + if (publicationDeferral.discard) return + + this.stateRevision += publicationDeferral.stateRevisionDelta + this.layoutRevision += publicationDeferral.layoutRevisionDelta + publicationDeferral.prepared = { + changes: publicationDeferral.publications.flatMap( + ({ changes }) => changes, + ), + layoutChanged: publicationDeferral.publications.some( + ({ layoutChanged }) => layoutChanged, + ), } - this.stateRevision += stateRevisionDelta - this.layoutRevision += layoutRevisionDelta - this.publishEvents( - publications.flatMap(({ changes }) => changes), - publications.some(({ layoutChanged }) => layoutChanged), - ) } return { - publish: () => close(false), - discard: () => close(true), + prepare: () => prepare(false), + publish: () => { + if (handleState === `discarded`) return + prepare(false) + if ( + publicationDeferral.depth > 0 || + publicationDeferral.discard || + publicationDeferral.published + ) { + return + } + publicationDeferral.published = true + const publication = publicationDeferral.prepared + publicationDeferral.prepared = undefined + if (publication) { + this.publishEvents(publication.changes, publication.layoutChanged) + } + }, + discard: () => prepare(true), } } @@ -360,7 +408,11 @@ export class CollectionChangesManager< public cleanup(): void { this.batchedEvents = [] this.shouldBatchEvents = false - this.deferredPublications = [] - this.publicationDeferralDepth = 0 + if (this.publicationDeferral) { + this.publicationDeferral.discard = true + this.publicationDeferral.publications = [] + this.publicationDeferral.prepared = undefined + } + this.publicationDeferral = undefined } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 5fab3b369..db8d34b37 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1130,10 +1130,12 @@ Installed state, synchronous reads, change-event payloads, and downstream queries must all observe the same fully materialized commit. The facade adapter may defer public revision clocks and event delivery across its Collection transactions, but it must not defer state or index installation. A successful -coherent publication advances those clocks before callbacks run. If a later -root or containing-facade application fails, rollback restores the installed -state and discards both the held events and their revision advances. Routing -and identity remain inside D2. +coherent publication uses a two-phase release: first advance the clocks of the +root and every changed facade, then deliver any callback. This lets a callback +read another participating Collection without seeing new rows behind an old +revision. If a later root or containing-facade application fails before that +release, rollback restores the installed state and discards both the held +events and their revision advances. Routing and identity remain inside D2. ## External boundaries diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index c543f6cdd..9ad84887a 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -44,6 +44,7 @@ type FacadeSnapshot = { } export type FacadePublication = { + prepare: () => void publish: () => void rollback: () => void } @@ -165,10 +166,18 @@ export class BucketFacadeAdapter { this.pending.clear() this.pendingActivity.clear() + let prepared = false let closed = false + const prepare = () => { + if (prepared || closed) return + prepared = true + for (const publication of publications) publication.prepare() + } return { + prepare, publish: () => { if (closed) return + prepare() closed = true for (const publication of publications) publication.publish() // Drop only the adapter's strong reference. External holders keep an @@ -176,7 +185,7 @@ export class BucketFacadeAdapter { this.retiredEntries.clear() }, rollback: () => { - if (closed) return + if (closed || prepared) return closed = true this.restore(snapshot, deferredEntries) this.retiredEntries.clear() diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index fb0e2215a..068cbe062 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1092,6 +1092,11 @@ export class CollectionConfigBuilder< } pendingChanges = new Map() + // Advance every participating Collection's public clocks before the + // first callback can inspect another Collection from this graph turn. + rootPublication?.prepare() + facadePublication.prepare() + let publicationError: unknown for (const publish of [ rootPublication?.publish, diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 810923e93..37d76cc0d 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -822,8 +822,20 @@ describe(`Collection-valued includes oracle`, () => { const facade = live.get(1)!.children const rootPublications: Array = [] const childPublications: Array = [] + const rootCallbackFacadeSnapshots: Array<{ + rows: Array<{ id: number; value: number }> + stateRevision: number + layoutRevision: number + }> = [] const rootSubscription = live.subscribeChanges( - (batch) => rootPublications.push(...batch), + (batch) => { + rootPublications.push(...batch) + rootCallbackFacadeSnapshots.push({ + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + stateRevision: facade._stateRevision, + layoutRevision: facade._layoutRevision, + }) + }, { includeInitialState: false }, ) const childSubscription = facade.subscribeChanges( @@ -867,6 +879,7 @@ describe(`Collection-valued includes oracle`, () => { ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) + expect(rootCallbackFacadeSnapshots).toEqual([]) expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure) expect(facade._layoutRevision).toBe(childLayoutRevisionBeforeFailure) expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) @@ -894,6 +907,16 @@ describe(`Collection-valued includes oracle`, () => { ]) expect(rootPublications).toHaveLength(1) expect(childPublications).toHaveLength(2) + expect(rootCallbackFacadeSnapshots).toEqual([ + { + rows: [ + { id: 20, value: 3 }, + { id: 10, value: 4 }, + ], + stateRevision: childStateRevisionBeforeFailure + 1, + layoutRevision: childLayoutRevisionBeforeFailure + 1, + }, + ]) expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure + 1) expect(facade._layoutRevision).toBe( childLayoutRevisionBeforeFailure + 1, @@ -970,6 +993,152 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `coherent nested publication advances every revision before callbacks`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` | `grandchild` + parentGroup: number + group: number + value: number + } + const initialRows: Array = [ + { + id: 1, + kind: `parent`, + parentGroup: 0, + group: 1, + value: 1, + }, + { + id: 10, + kind: `child`, + parentGroup: 1, + group: 10, + value: 1, + }, + { + id: 20, + kind: `child`, + parentGroup: 1, + group: 20, + value: 2, + }, + { + id: 100, + kind: `grandchild`, + parentGroup: 10, + group: 100, + value: 1, + }, + { + id: 200, + kind: `grandchild`, + parentGroup: 10, + group: 200, + value: 2, + }, + ] + const nodes = createControlledCollection( + `nested-publication-revisions`, + initialRows, + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.value) + .select(({ child }) => ({ + id: child.id, + value: child.value, + grandchildren: q + .from({ grandchild: nodes.collection }) + .where(({ grandchild }) => + eq(grandchild.kind, `grandchild`), + ) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ) + .orderBy(({ grandchild }) => grandchild.value), + })), + })), + ) + + await live.preload() + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(10)!.grandchildren + const childStateRevision = childFacade._stateRevision + const childLayoutRevision = childFacade._layoutRevision + const grandchildStateRevision = grandchildFacade._stateRevision + const grandchildLayoutRevision = grandchildFacade._layoutRevision + const callbackSnapshots: Array<{ + childRows: Array<{ id: number; value: number }> + childStateRevision: number + childLayoutRevision: number + grandchildRows: Array<{ id: number; value: number }> + grandchildStateRevision: number + grandchildLayoutRevision: number + }> = [] + const subscription = live.subscribeChanges( + () => { + callbackSnapshots.push({ + childRows: childFacade.toArray.map(({ id, value }) => ({ + id, + value, + })), + childStateRevision: childFacade._stateRevision, + childLayoutRevision: childFacade._layoutRevision, + grandchildRows: grandchildFacade.toArray.map(({ id, value }) => ({ + id, + value, + })), + grandchildStateRevision: grandchildFacade._stateRevision, + grandchildLayoutRevision: grandchildFacade._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + + try { + nodes.writeBatch([ + { type: `update`, value: { ...initialRows[0]!, value: 3 } }, + { type: `update`, value: { ...initialRows[1]!, value: 4 } }, + { type: `update`, value: { ...initialRows[2]!, value: 3 } }, + { type: `update`, value: { ...initialRows[3]!, value: 4 } }, + { type: `update`, value: { ...initialRows[4]!, value: 3 } }, + ]) + + expect(callbackSnapshots).toEqual([ + { + childRows: [ + { id: 20, value: 3 }, + { id: 10, value: 4 }, + ], + childStateRevision: childStateRevision + 1, + childLayoutRevision: childLayoutRevision + 1, + grandchildRows: [ + { id: 200, value: 3 }, + { id: 100, value: 4 }, + ], + grandchildStateRevision: grandchildStateRevision + 1, + grandchildLayoutRevision: grandchildLayoutRevision + 1, + }, + ]) + } finally { + subscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + for (const settlement of pendingFacadeSettlements) { for (const optimisticOperation of pendingFacadeOptimisticOperations) { for (const sourceOperation of pendingFacadeSourceOperations) { From d4f110ad103b8b1cd40b9e80352ad74e59fde924 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 11:43:35 -0600 Subject: [PATCH 201/327] fix(db): serialize window callback graph work --- packages/db/src/query/live/ARCHITECTURE.md | 7 + .../query/live/collection-config-builder.ts | 16 +- ...ncludes-collection-oracle.property.test.ts | 205 ++++++++++++++++++ 3 files changed, 223 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index db8d34b37..27b0a0c81 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1137,6 +1137,13 @@ revision. If a later root or containing-facade application fails before that release, rollback restores the installed state and discards both the held events and their revision advances. Routing and identity remain inside D2. +Every graph-turn origin that can publish rows owns a scheduler publication +context through the complete coherent release. This includes direct window +changes as well as source transactions. Work created by a publication callback +joins that context and runs only after the current root and facade callbacks +finish; it cannot start a second graph turn inside the first one or disappear +through the graph's reentrancy guard. + ## External boundaries ### Query-db ownership diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 068cbe062..fc3e292ca 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -7,6 +7,7 @@ import { import { getActivePublicationContext, transactionScopedScheduler, + withPublicationContext, } from '../../scheduler.js' import { getActiveTransaction } from '../../transactions.js' import { deepEquals } from '../../utils.js' @@ -302,7 +303,8 @@ export class CollectionConfigBuilder< } setWindow(options: WindowOptions): true | Promise { - if (!this.windowFn) { + const windowFn = this.windowFn + if (!windowFn) { throw new SetWindowRequiresOrderByError() } @@ -316,8 +318,10 @@ export class CollectionConfigBuilder< const operation: { failed: boolean; error?: unknown } = { failed: false } this.activeWindowOperation = operation try { - this.windowFn(options) - this.maybeRunGraphFn?.() + withPublicationContext(() => { + windowFn(options) + this.maybeRunGraphFn?.() + }) if (operation.failed) throw operation.error this.currentWindow = options } catch (error) { @@ -328,8 +332,10 @@ export class CollectionConfigBuilder< windowOperationGeneration === this.windowOperationGeneration ) { try { - this.windowFn(previousWindow) - this.maybeRunGraphFn?.() + withPublicationContext(() => { + windowFn(previousWindow) + this.maybeRunGraphFn?.() + }) if (windowOperationGeneration === this.windowOperationGeneration) { this.windowOperationGeneration = previousWindowOperationGeneration } diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 37d76cc0d..602e1b70c 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1139,6 +1139,211 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `window callback-created source work follows the current facade publication`, + async () => { + const parents = createControlledCollection(`reentrant-window-parents`, [ + { id: 1, rank: 1, group: 1 }, + { id: 2, rank: 2, group: 2 }, + ]) + const children = createControlledCollection(`reentrant-window-children`, [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const observations: Array<{ + eventValues: Array + visibleValue: number + revision: number + }> = [] + let childSubscription: { unsubscribe: () => void } | undefined + let preparedRevision = -1 + let reentered = false + const rootSubscription = live.subscribeChanges( + () => { + if (reentered) return + reentered = true + const facade = live.get(2)!.children + preparedRevision = facade._stateRevision + childSubscription = facade.subscribeChanges( + (batch) => { + observations.push({ + eventValues: batch.map((change) => change.value.value), + visibleValue: facade.get(20)!.value, + revision: facade._stateRevision, + }) + }, + { includeInitialState: false }, + ) + children.write(`update`, { id: 20, group: 2, value: 3 }) + }, + { includeInitialState: false }, + ) + + try { + const result = live.utils.setWindow({ offset: 0, limit: 2 }) + if (result instanceof Promise) await result + await flushPromises() + + expect(children.collection.get(20)!.value).toBe(3) + expect(live.get(2)!.children.get(20)!.value).toBe(3) + expect(observations).toEqual([ + { + eventValues: [1], + visibleValue: 1, + revision: preparedRevision, + }, + { + eventValues: [3], + visibleValue: 3, + revision: preparedRevision + 1, + }, + ]) + } finally { + rootSubscription.unsubscribe() + childSubscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + for (const turnOrigin of [`source`, `window`] as const) { + for (const callbackAction of [`source-write`, `set-window`] as const) { + fcTest( + `${turnOrigin} graph turns serialize callback ${callbackAction} work`, + async () => { + const parents = createControlledCollection( + `callback-origin-parents`, + [ + { id: 1, rank: 1, group: 1, value: 1 }, + { id: 2, rank: 2, group: 2, value: 1 }, + ], + ) + const children = createControlledCollection( + `callback-origin-children`, + [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const rootLayouts: Array> = [] + const childBatches: Array> = [] + let childSubscription: { unsubscribe: () => void } | undefined + let childRevision = -1 + let actionResult: true | Promise | undefined + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + if (acted) return + acted = true + if (callbackAction === `source-write`) { + const parentId = turnOrigin === `window` ? 2 : 1 + const childId = parentId === 1 ? 10 : 20 + const facade = live.get(parentId)!.children + childRevision = facade._stateRevision + childSubscription = facade.subscribeChanges( + (batch) => { + childBatches.push( + batch.map((change) => change.value.value), + ) + }, + { includeInitialState: false }, + ) + children.write(`update`, { + id: childId, + group: parentId, + value: 3, + }) + } else { + actionResult = live.utils.setWindow( + turnOrigin === `window` + ? { offset: 1, limit: 1 } + : { offset: 0, limit: 2 }, + ) + } + }, + { includeInitialState: false }, + ) + + try { + if (turnOrigin === `source`) { + parents.write(`update`, { + id: 1, + rank: 1, + group: 1, + value: 2, + }) + } else { + const result = live.utils.setWindow({ offset: 0, limit: 2 }) + if (result instanceof Promise) await result + } + if (actionResult instanceof Promise) await actionResult + await flushPromises() + + if (callbackAction === `source-write`) { + const parentId = turnOrigin === `window` ? 2 : 1 + const childId = parentId === 1 ? 10 : 20 + const facade = live.get(parentId)!.children + expect(facade.get(childId)!.value).toBe(3) + expect(childBatches.at(-1)).toEqual([3]) + expect(facade._stateRevision).toBe(childRevision + 1) + } else if (turnOrigin === `source`) { + expect(rootLayouts).toEqual([[1], [1, 2]]) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + } else { + expect(rootLayouts).toEqual([ + [1, 2], + [2], + ]) + expect(live.toArray.map(({ id }) => id)).toEqual([2]) + } + } finally { + rootSubscription.unsubscribe() + childSubscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + for (const settlement of pendingFacadeSettlements) { for (const optimisticOperation of pendingFacadeOptimisticOperations) { for (const sourceOperation of pendingFacadeSourceOperations) { From 47e1462919b76de94098ca12d1841e9332130f50 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 12:07:20 -0600 Subject: [PATCH 202/327] fix(db): preserve reentrant window ownership --- packages/db/src/query/live/ARCHITECTURE.md | 7 ++ .../query/live/collection-config-builder.ts | 4 +- ...ncludes-collection-oracle.property.test.ts | 91 ++++++++++++++++++- 3 files changed, 97 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 27b0a0c81..652a0f468 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1144,6 +1144,13 @@ joins that context and runs only after the current root and facade callbacks finish; it cannot start a second graph turn inside the first one or disappear through the graph's reentrancy guard. +Window metadata follows the same causal order as the published rows. If a +publication callback starts a newer window operation, that newer generation +owns the final public window and the older caller cannot overwrite it when it +resumes. Restoring a rejected window is itself a graph-turn origin: its +callbacks remain inside one publication context, and restoration cannot roll +back a newer nested window generation. + ## External boundaries ### Query-db ownership diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index fc3e292ca..1c89417f2 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -323,7 +323,9 @@ export class CollectionConfigBuilder< this.maybeRunGraphFn?.() }) if (operation.failed) throw operation.error - this.currentWindow = options + if (windowOperationGeneration === this.windowOperationGeneration) { + this.currentWindow = options + } } catch (error) { if ( previousWindow && diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 602e1b70c..171f4c855 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -3,6 +3,7 @@ import { describe, expect } from 'vitest' import { createDeferred } from '../../src/deferred.js' import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { createOptimisticAction } from '../../src/optimistic-action.js' +import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' import { add, caseWhen, @@ -1323,12 +1324,11 @@ describe(`Collection-valued includes oracle`, () => { } else if (turnOrigin === `source`) { expect(rootLayouts).toEqual([[1], [1, 2]]) expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) } else { - expect(rootLayouts).toEqual([ - [1, 2], - [2], - ]) + expect(rootLayouts).toEqual([[1, 2], [2]]) expect(live.toArray.map(({ id }) => id)).toEqual([2]) + expect(live.utils.getWindow()).toEqual({ offset: 1, limit: 1 }) } } finally { rootSubscription.unsubscribe() @@ -1344,6 +1344,89 @@ describe(`Collection-valued includes oracle`, () => { } } + fcTest( + `failed window restoration serializes callback-created source work`, + async () => { + const parents = createControlledCollection(`rollback-window-parents`, [ + { id: 1, rank: 1, group: 1 }, + { id: 2, rank: 2, group: 2 }, + ]) + const children = createControlledCollection(`rollback-window-children`, [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const failure = new Error(`requested window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failRequestedWindow = true + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (failRequestedWindow) { + failRequestedWindow = false + builder.recordSubsetError(failure) + } + }) + + const facade = live.get(1)!.children + const rootLayouts: Array> = [] + const childBatches: Array> = [] + let sawRequestedWindow = false + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + const layout = live.toArray.map(({ id }) => id) + rootLayouts.push(layout) + if (layout.length === 2) sawRequestedWindow = true + if (!sawRequestedWindow || acted || layout.length !== 1) return + acted = true + children.write(`update`, { id: 10, group: 1, value: 3 }) + }, + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + childBatches.push(batch.map((change) => change.value.value)) + }, + { includeInitialState: false }, + ) + + try { + expect(() => live.utils.setWindow({ offset: 0, limit: 2 })).toThrow( + failure, + ) + + expect(rootLayouts).toEqual([[1, 2], [1]]) + expect(live.toArray.map(({ id }) => id)).toEqual([1]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) + expect(live.utils.lastSubsetError).toBe(failure) + expect(facade.get(10)!.value).toBe(3) + expect(childBatches.at(-1)).toEqual([3]) + } finally { + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + for (const settlement of pendingFacadeSettlements) { for (const optimisticOperation of pendingFacadeOptimisticOperations) { for (const sourceOperation of pendingFacadeSourceOperations) { From bbc9edf28ef707c8fb3c451fe2c4724039a0037a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 12:42:33 -0600 Subject: [PATCH 203/327] fix(db): match index collation options semantically (#1788) * fix(db): match index collation options semantically * chore: add changeset for index collation matching * fix(db): narrow locale compare options * fix(db): canonicalize index locale identifiers --- .changeset/fix-index-collation-matching.md | 5 ++ packages/db/src/indexes/base-index.ts | 50 ++++++++++--- .../db/tests/collection-auto-index.test.ts | 6 +- packages/db/tests/collection-indexes.test.ts | 71 +++++++++++++++++++ packages/db/tests/query/join-subquery.test.ts | 63 +++++++++++++++- 5 files changed, 184 insertions(+), 11 deletions(-) create mode 100644 .changeset/fix-index-collation-matching.md diff --git a/.changeset/fix-index-collation-matching.md b/.changeset/fix-index-collation-matching.md new file mode 100644 index 000000000..ffd98c712 --- /dev/null +++ b/.changeset/fix-index-collation-matching.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Match index collation options by their effective values so indexes remain reusable when optional locale fields are omitted, set to `undefined`, or use equivalent locale identifiers. diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 26cb09887..9cb421688 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -6,6 +6,28 @@ import type { RangeQueryOptions } from './btree-index.js' import type { CompareOptions } from '../query/builder/types.js' import type { BasicExpression, OrderByDirection } from '../query/ir.js' +function normalizeLocaleOptions(options: object | undefined): object { + return Object.fromEntries( + Object.entries(options ?? {}).filter(([, value]) => value !== undefined), + ) +} + +function canonicalizeLocale(locale: string | undefined): string | undefined { + return locale === undefined ? undefined : Intl.getCanonicalLocales(locale)[0] +} + +type LocaleCompareOptions = CompareOptions & { + stringSort?: `locale` + locale?: string + localeOptions?: object +} + +function usesLocaleCollation( + options: CompareOptions, +): options is LocaleCompareOptions { + return (options.stringSort ?? DEFAULT_COMPARE_OPTIONS.stringSort) === `locale` +} + /** * Operations that indexes can support, imported from available comparison functions */ @@ -177,18 +199,28 @@ export abstract class BaseIndex< * The direction is ignored because the index can be reversed if the direction is different. */ matchesCompareOptions(compareOptions: CompareOptions): boolean { - const thisCompareOptionsWithoutDirection = { - ...this.compareOptions, - direction: undefined, + const indexCompareOptions = this.compareOptions + const indexUsesLocale = usesLocaleCollation(indexCompareOptions) + const requestedUsesLocale = usesLocaleCollation(compareOptions) + + if ( + indexCompareOptions.nulls !== compareOptions.nulls || + indexUsesLocale !== requestedUsesLocale + ) { + return false } - const compareOptionsWithoutDirection = { - ...compareOptions, - direction: undefined, + + if (!indexUsesLocale || !requestedUsesLocale) { + return true } - return deepEquals( - thisCompareOptionsWithoutDirection, - compareOptionsWithoutDirection, + return ( + canonicalizeLocale(indexCompareOptions.locale) === + canonicalizeLocale(compareOptions.locale) && + deepEquals( + normalizeLocaleOptions(indexCompareOptions.localeOptions), + normalizeLocaleOptions(compareOptions.localeOptions), + ) ) } diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index 4fdaac012..b4e25fdd8 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -252,11 +252,15 @@ describe(`Collection Auto-Indexing`, () => { it(`should create auto-indexes for transformed fields of subqueries when autoIndex is "eager"`, async () => {}) - it(`should not create duplicate auto-indexes for the same field`, async () => { + it(`should not create duplicate auto-indexes when locale options are omitted`, async () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, defaultIndexType: BTreeIndex, + defaultStringCollation: { + stringSort: `locale`, + localeOptions: { sensitivity: undefined }, + }, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index bd5f4868c..a453b8fb7 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -15,6 +15,9 @@ import { } from '../src/query/builder/functions' import { PropRef } from '../src/query/ir' import { BTreeIndex } from '../src/indexes/btree-index.js' +import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' +import { findIndexForField } from '../src/utils/index-optimization.js' +import { makeComparator } from '../src/utils/comparison.js' import { expectIndexUsage, stripVirtualProps, withIndexTracking } from './utils' import type { Collection } from '../src/collection/index.js' import type { MutationFn, PendingMutation } from '../src/types' @@ -161,6 +164,74 @@ describe(`Collection Indexes`, () => { expect(index.indexedKeysSet.size).toBe(5) }) + it(`should match compare options by collation semantics`, () => { + const index = collection.createIndex((row) => row.status) + + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + locale: undefined, + localeOptions: undefined, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + localeOptions: { sensitivity: undefined }, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + direction: `desc`, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + locale: `de-DE`, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + localeOptions: { sensitivity: `base` }, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + nulls: `last`, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + stringSort: `lexical`, + }), + ).toBe(false) + }) + + it(`should reuse an index for equivalent locale identifiers`, () => { + const indexCompareOptions = { + ...DEFAULT_COMPARE_OPTIONS, + locale: `en-us`, + } + const index = collection.createIndex((row) => row.name, { + options: { + compareOptions: indexCompareOptions, + compareFn: makeComparator(indexCompareOptions), + }, + }) + + expect( + findIndexForField(collection, [`name`], { + ...DEFAULT_COMPARE_OPTIONS, + locale: `en-US`, + }), + ).toBe(index) + }) + it(`should create multiple indexes`, () => { const statusIndex = collection.createIndex((row) => row.status) const ageIndex = collection.createIndex((row) => row.age) diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index 3468dedd3..ed3261cc5 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -954,7 +954,68 @@ describe(`Lazy join: subquery whose join key resolves to an indexed collection`, }) }) -describe(`Lazy join without a usable index`, () => { +describe(`Lazy join index availability`, () => { + test(`uses an auto-index with omitted locale options`, async () => { + type Team = { id: string } + type Member = { id: string; teamId: string } + const teams = createCollection( + mockSyncCollectionOptions({ + id: `lazy-default-collation-teams`, + getKey: (team) => team.id, + initialData: [{ id: `t1` }], + }), + ) + const members = createCollection( + mockSyncCollectionOptions({ + id: `lazy-default-collation-members`, + getKey: (member) => member.id, + initialData: [{ id: `m1`, teamId: `t1` }], + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + defaultStringCollation: { + stringSort: `locale`, + localeOptions: { sensitivity: undefined }, + }, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `m1`, teamId: `t1` } }) + commit() + markReady() + return { loadSubset: () => true } + }, + }, + }), + ) + const warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const live = createLiveQueryCollection((q) => + q + .from({ team: teams }) + .leftJoin({ member: members }, ({ team, member }) => + eq(team.id, member.teamId), + ) + .select(({ team, member }) => ({ + id: team.id, + memberId: member.id, + })), + ) + + try { + await live.preload() + expect(live.toArray.map(stripVirtualProps)).toEqual([ + { id: `t1`, memberId: `m1` }, + ]) + expect(members.indexes.size).toBe(1) + expect(warnSpy).not.toHaveBeenCalledWith( + expect.stringContaining(`Join requires an index`), + ) + } finally { + warnSpy.mockRestore() + await Promise.all([live.cleanup(), teams.cleanup(), members.cleanup()]) + } + }) + test(`warns when demand falls back to a full local scan`, async () => { type Team = { id: string } type Member = { id: string; teamId: string } From 3238961cde39248283242852c6445d97c6f68886 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 12:55:34 -0600 Subject: [PATCH 204/327] fix(db): restore nested window generations --- packages/db/src/query/live/ARCHITECTURE.md | 7 +- .../query/live/collection-config-builder.ts | 8 ++ ...ncludes-collection-oracle.property.test.ts | 124 ++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 652a0f468..dd6b44267 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1149,7 +1149,12 @@ publication callback starts a newer window operation, that newer generation owns the final public window and the older caller cannot overwrite it when it resumes. Restoring a rejected window is itself a graph-turn origin: its callbacks remain inside one publication context, and restoration cannot roll -back a newer nested window generation. +back a newer nested window generation. A rejected nested operation restores +its immediate parent's effective window, not an older public snapshot; rows +and window metadata therefore describe the same surviving generation. +If teardown clears the runtime while an accepted window call is unwinding, +that generation remains the desired window for the next sync session. A call +that fails synchronously instead restores its previous effective window. ## External boundaries diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 1c89417f2..79e40ab6d 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -318,12 +318,19 @@ export class CollectionConfigBuilder< const operation: { failed: boolean; error?: unknown } = { failed: false } this.activeWindowOperation = operation try { + // Window metadata is part of the synchronous publication. This also + // gives a nested operation the effective window of its immediate parent + // to restore if the nested operation fails. + this.currentWindow = options withPublicationContext(() => { windowFn(options) this.maybeRunGraphFn?.() }) if (operation.failed) throw operation.error if (windowOperationGeneration === this.windowOperationGeneration) { + // Teardown may clear the runtime while an accepted request is still + // unwinding. Preserve that request as the desired window for the next + // sync session, but never overwrite a newer nested operation. this.currentWindow = options } } catch (error) { @@ -334,6 +341,7 @@ export class CollectionConfigBuilder< windowOperationGeneration === this.windowOperationGeneration ) { try { + this.currentWindow = previousWindow withPublicationContext(() => { windowFn(previousWindow) this.maybeRunGraphFn?.() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 171f4c855..3a3014fc6 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1260,6 +1260,9 @@ describe(`Collection-valued includes oracle`, () => { await live.preload() const rootLayouts: Array> = [] + const rootWindows: Array< + { offset: number; limit: number } | undefined + > = [] const childBatches: Array> = [] let childSubscription: { unsubscribe: () => void } | undefined let childRevision = -1 @@ -1268,6 +1271,7 @@ describe(`Collection-valued includes oracle`, () => { const rootSubscription = live.subscribeChanges( () => { rootLayouts.push(live.toArray.map(({ id }) => id)) + rootWindows.push(live.utils.getWindow()) if (acted) return acted = true if (callbackAction === `source-write`) { @@ -1321,12 +1325,25 @@ describe(`Collection-valued includes oracle`, () => { expect(facade.get(childId)!.value).toBe(3) expect(childBatches.at(-1)).toEqual([3]) expect(facade._stateRevision).toBe(childRevision + 1) + expect(rootWindows).toEqual([ + turnOrigin === `window` + ? { offset: 0, limit: 2 } + : { offset: 0, limit: 1 }, + ]) } else if (turnOrigin === `source`) { expect(rootLayouts).toEqual([[1], [1, 2]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 1 }, + { offset: 0, limit: 2 }, + ]) expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) } else { expect(rootLayouts).toEqual([[1, 2], [2]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 2 }, + { offset: 1, limit: 1 }, + ]) expect(live.toArray.map(({ id }) => id)).toEqual([2]) expect(live.utils.getWindow()).toEqual({ offset: 1, limit: 1 }) } @@ -1344,6 +1361,113 @@ describe(`Collection-valued includes oracle`, () => { } } + fcTest( + `a rejected nested window restores its parent operation's window`, + async () => { + const parents = createControlledCollection(`nested-window-parents`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const nestedFailure = new Error(`nested window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNextGraph = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (failNextGraph) { + failNextGraph = false + builder.recordSubsetError(nestedFailure) + } + }) + + const rootLayouts: Array> = [] + let nestedError: unknown + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + if (acted) return + acted = true + failNextGraph = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + }, + { includeInitialState: false }, + ) + + try { + live.utils.setWindow({ offset: 0, limit: 2 }) + + expect(nestedError).toBe(nestedFailure) + expect(rootLayouts[0]).toEqual([1, 2]) + expect(rootLayouts.at(-1)).toEqual([1, 2]) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) + } finally { + rootSubscription.unsubscribe() + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `an older failed window cannot restore over a newer nested window`, + async () => { + const parents = createControlledCollection(`stale-window-parents`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const outerFailure = new Error(`outer window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const rootLayouts: Array> = [] + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + if (acted) return + acted = true + live.utils.setWindow({ offset: 1, limit: 1 }) + builder.recordSubsetError(outerFailure) + }, + { includeInitialState: false }, + ) + + try { + expect(() => live.utils.setWindow({ offset: 0, limit: 2 })).toThrow( + outerFailure, + ) + + expect(rootLayouts).toEqual([[1, 2], [2]]) + expect(live.toArray.map(({ id }) => id)).toEqual([2]) + expect(live.utils.getWindow()).toEqual({ offset: 1, limit: 1 }) + } finally { + rootSubscription.unsubscribe() + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + fcTest( `failed window restoration serializes callback-created source work`, async () => { From 1de71873a7e253f7ca22554142da6d56e46c8db3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 13:23:33 -0600 Subject: [PATCH 205/327] test(db): preserve nested window ownership --- ...ncludes-collection-oracle.property.test.ts | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 3a3014fc6..bcb20f255 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1422,6 +1422,91 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `a rejected nested window preserves its parent operation outcome`, + async () => { + const parents = createControlledCollection(`parent-window-outcome`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + offset?: number + limit?: number + }) => void + const parentOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit === 2) { + builder.trackSubsetLoadOperationPromise(parentOutcome.promise, `root`) + } + }) + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + }, + { includeInitialState: false }, + ) + + try { + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(parentReady).toBeInstanceOf(Promise) + expect(nestedError).toBe(nestedFailure) + parentOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( + [expect.objectContaining({ demand: { limit: 2 } })], + ) + } finally { + subscription.unsubscribe() + parentOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + fcTest( `an older failed window cannot restore over a newer nested window`, async () => { @@ -1507,6 +1592,8 @@ describe(`Collection-valued includes oracle`, () => { const facade = live.get(1)!.children const rootLayouts: Array> = [] + const rootWindows: Array<{ offset: number; limit: number } | undefined> = + [] const childBatches: Array> = [] let sawRequestedWindow = false let acted = false @@ -1514,6 +1601,7 @@ describe(`Collection-valued includes oracle`, () => { () => { const layout = live.toArray.map(({ id }) => id) rootLayouts.push(layout) + rootWindows.push(live.utils.getWindow()) if (layout.length === 2) sawRequestedWindow = true if (!sawRequestedWindow || acted || layout.length !== 1) return acted = true @@ -1534,6 +1622,10 @@ describe(`Collection-valued includes oracle`, () => { ) expect(rootLayouts).toEqual([[1, 2], [1]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 2 }, + { offset: 0, limit: 1 }, + ]) expect(live.toArray.map(({ id }) => id)).toEqual([1]) expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) expect(live.utils.lastSubsetError).toBe(failure) From b760da0027c60b39a24eb154fc57968668bdd477 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 13:44:24 -0600 Subject: [PATCH 206/327] fix(db): restore nested load operation ownership --- packages/db/src/collection/sync.ts | 6 +- packages/db/src/query/live/ARCHITECTURE.md | 5 +- .../query/live/collection-config-builder.ts | 5 +- ...ncludes-collection-oracle.property.test.ts | 119 ++++++++++++++++++ 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index b14766205..791998715 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -800,6 +800,8 @@ export class CollectionSyncManager< cancel: () => void getOutcomes: () => ReadonlyArray } { + // A failed nested operation restores this owner before rollback work. + const previousOperation = this.activeLoadSubsetOperation const operation: LoadSubsetOperation = { pending: new Set(), outcomes: new Map(), @@ -818,7 +820,9 @@ export class CollectionSyncManager< operation.completed = true this.loadSubsetOperations.delete(operation) if (this.activeLoadSubsetOperation === operation) { - this.activeLoadSubsetOperation = undefined + this.activeLoadSubsetOperation = previousOperation?.completed + ? undefined + : previousOperation } }, getOutcomes: () => diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index dd6b44267..eab96b908 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1151,7 +1151,10 @@ resumes. Restoring a rejected window is itself a graph-turn origin: its callbacks remain inside one publication context, and restoration cannot roll back a newer nested window generation. A rejected nested operation restores its immediate parent's effective window, not an older public snapshot; rows -and window metadata therefore describe the same surviving generation. +and window metadata therefore describe the same surviving generation. It also +restores the parent's imperative load-operation ownership before rollback +publication. Loads started by rollback or by the parent callback after it +catches the nested error must delay and contribute outcomes to the parent. If teardown clears the runtime while an accepted window call is unwinding, that generation remains the desired window for the next sync session. A call that fails synchronously instead restores its previous effective window. diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 79e40ab6d..56d92ee71 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -334,6 +334,10 @@ export class CollectionConfigBuilder< this.currentWindow = options } } catch (error) { + // A rejected nested window returns ownership to its parent before the + // rollback publishes. Work caused by that publication must settle with + // the restored parent operation, not the canceled child. + loadOperation?.cancel() if ( previousWindow && syncSession === this.syncSession && @@ -354,7 +358,6 @@ export class CollectionConfigBuilder< // window rather than replacing it with a rollback failure. } } - loadOperation?.cancel() throw error } finally { this.activeWindowOperation = previousOperation diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index bcb20f255..061e65576 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1507,6 +1507,125 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `a rejected nested window restores its parent operation for follow-up work`, + async () => { + const parents = createControlledCollection(`parent-window-follow-up`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const rollbackOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const afterCatchOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + limit?: number + }) => void + let parentWindowCalls = 0 + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit === 2 && ++parentWindowCalls === 2) { + builder.trackSubsetLoadOperationPromise( + rollbackOutcome.promise, + `rollback`, + ) + } + }) + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + builder.trackSubsetLoadOperationPromise( + afterCatchOutcome.promise, + `after-catch`, + ) + }, + { includeInitialState: false }, + ) + + try { + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(nestedError).toBe(nestedFailure) + expect(parentReady).toBeInstanceOf(Promise) + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + sourceId: `rollback`, + demand: { limit: 2 }, + }), + expect.objectContaining({ + sourceId: `after-catch`, + demand: { limit: 2 }, + }), + ]), + ) + } finally { + subscription.unsubscribe() + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + fcTest( `an older failed window cannot restore over a newer nested window`, async () => { From 613807d753dac30607fd21197d952bb10b6dd597 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:49:01 -0600 Subject: [PATCH 207/327] ci: Version Packages (#1789) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/fix-index-collation-matching.md | 5 -- examples/angular/todos/package.json | 4 +- examples/electron/offline-first/package.json | 10 +-- .../offline-transactions/package.json | 10 +-- .../react-native/shopping-list/package.json | 10 +-- examples/react/next-ssr-e2e/package.json | 4 +- .../react/offline-transactions/package.json | 10 +-- .../react/paced-mutations-demo/package.json | 4 +- examples/react/projects/package.json | 4 +- examples/react/start-ssr-e2e/package.json | 2 +- examples/react/todo/package.json | 8 +- examples/solid/todo/package.json | 8 +- packages/angular-db/CHANGELOG.md | 7 ++ packages/angular-db/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../CHANGELOG.md | 7 ++ .../e2e/app/CHANGELOG.md | 8 ++ .../e2e/app/package.json | 2 +- .../package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../db-sqlite-persistence-core/CHANGELOG.md | 7 ++ .../db-sqlite-persistence-core/package.json | 2 +- packages/db/CHANGELOG.md | 6 ++ packages/db/package.json | 2 +- packages/electric-db-collection/CHANGELOG.md | 7 ++ packages/electric-db-collection/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- .../expo-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../e2e/expo-runtime-app/CHANGELOG.md | 8 ++ .../e2e/expo-runtime-app/package.json | 2 +- .../expo-db-sqlite-persistence/package.json | 2 +- .../node-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../node-db-sqlite-persistence/package.json | 2 +- packages/offline-transactions/CHANGELOG.md | 7 ++ packages/offline-transactions/package.json | 2 +- packages/powersync-db-collection/CHANGELOG.md | 7 ++ packages/powersync-db-collection/package.json | 2 +- packages/query-db-collection/CHANGELOG.md | 7 ++ packages/query-db-collection/package.json | 2 +- packages/react-db/CHANGELOG.md | 7 ++ packages/react-db/package.json | 2 +- .../CHANGELOG.md | 7 ++ .../package.json | 2 +- packages/rxdb-db-collection/CHANGELOG.md | 7 ++ packages/rxdb-db-collection/package.json | 2 +- packages/solid-db/CHANGELOG.md | 7 ++ packages/solid-db/package.json | 2 +- packages/svelte-db/CHANGELOG.md | 7 ++ packages/svelte-db/package.json | 2 +- .../tauri-db-sqlite-persistence/CHANGELOG.md | 7 ++ .../e2e/app/CHANGELOG.md | 8 ++ .../e2e/app/package.json | 2 +- .../tauri-db-sqlite-persistence/package.json | 2 +- packages/trailbase-db-collection/CHANGELOG.md | 7 ++ packages/trailbase-db-collection/package.json | 2 +- packages/vue-db/CHANGELOG.md | 7 ++ packages/vue-db/package.json | 2 +- pnpm-lock.yaml | 74 +++++++++---------- 61 files changed, 268 insertions(+), 103 deletions(-) delete mode 100644 .changeset/fix-index-collation-matching.md diff --git a/.changeset/fix-index-collation-matching.md b/.changeset/fix-index-collation-matching.md deleted file mode 100644 index ffd98c712..000000000 --- a/.changeset/fix-index-collation-matching.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@tanstack/db': patch ---- - -Match index collation options by their effective values so indexes remain reusable when optional locale fields are omitted, set to `undefined`, or use equivalent locale identifiers. diff --git a/examples/angular/todos/package.json b/examples/angular/todos/package.json index aad965305..f875a9436 100644 --- a/examples/angular/todos/package.json +++ b/examples/angular/todos/package.json @@ -28,8 +28,8 @@ "@angular/forms": "^20.3.16", "@angular/platform-browser": "^20.3.16", "@angular/router": "^20.3.16", - "@tanstack/angular-db": "^0.1.87", - "@tanstack/db": "^0.8.6", + "@tanstack/angular-db": "^0.1.88", + "@tanstack/db": "^0.8.7", "rxjs": "^7.8.2", "tslib": "^2.8.1", "zone.js": "~0.15.0" diff --git a/examples/electron/offline-first/package.json b/examples/electron/offline-first/package.json index 04d41fbdb..c40b53e27 100644 --- a/examples/electron/offline-first/package.json +++ b/examples/electron/offline-first/package.json @@ -13,11 +13,11 @@ "postinstall": "prebuild-install --runtime electron --target 40.2.1 --arch arm64 || echo 'prebuild-install failed, try: npx @electron/rebuild'" }, "dependencies": { - "@tanstack/electron-db-sqlite-persistence": "^0.1.31", - "@tanstack/node-db-sqlite-persistence": "^0.2.19", - "@tanstack/offline-transactions": "^1.0.52", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/react-db": "^0.3.6", + "@tanstack/electron-db-sqlite-persistence": "^0.1.32", + "@tanstack/node-db-sqlite-persistence": "^0.2.20", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-query": "^5.90.20", "better-sqlite3": "^12.6.2", "react": "^19.2.4", diff --git a/examples/react-native/offline-transactions/package.json b/examples/react-native/offline-transactions/package.json index 4a8cda4f8..c1d9b21cd 100644 --- a/examples/react-native/offline-transactions/package.json +++ b/examples/react-native/offline-transactions/package.json @@ -15,11 +15,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.6", - "@tanstack/offline-transactions": "^1.0.52", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/react-db": "^0.3.6", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.19", + "@tanstack/db": "^0.8.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.20", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react-native/shopping-list/package.json b/examples/react-native/shopping-list/package.json index 4e6943c15..8a580bbae 100644 --- a/examples/react-native/shopping-list/package.json +++ b/examples/react-native/shopping-list/package.json @@ -18,11 +18,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.6", - "@tanstack/electric-db-collection": "^0.4.6", - "@tanstack/offline-transactions": "^1.0.52", - "@tanstack/react-db": "^0.3.6", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.19", + "@tanstack/db": "^0.8.7", + "@tanstack/electric-db-collection": "^0.4.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/react-db": "^0.3.7", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.20", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react/next-ssr-e2e/package.json b/examples/react/next-ssr-e2e/package.json index b6420d761..680de9dab 100644 --- a/examples/react/next-ssr-e2e/package.json +++ b/examples/react/next-ssr-e2e/package.json @@ -9,8 +9,8 @@ "test:e2e": "pnpm --filter @tanstack/db build && pnpm --filter @tanstack/react-db build && playwright test" }, "dependencies": { - "@tanstack/db": "^0.8.6", - "@tanstack/react-db": "^0.3.6", + "@tanstack/db": "^0.8.7", + "@tanstack/react-db": "^0.3.7", "next": "^16.3.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/offline-transactions/package.json b/examples/react/offline-transactions/package.json index f1663d76c..790e21f41 100644 --- a/examples/react/offline-transactions/package.json +++ b/examples/react/offline-transactions/package.json @@ -8,11 +8,11 @@ "build": "vite build && tsc --noEmit" }, "dependencies": { - "@tanstack/browser-db-sqlite-persistence": "^0.2.19", - "@tanstack/db": "^0.8.6", - "@tanstack/offline-transactions": "^1.0.52", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/react-db": "^0.3.6", + "@tanstack/browser-db-sqlite-persistence": "^0.2.20", + "@tanstack/db": "^0.8.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-query": "^5.90.20", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", diff --git a/examples/react/paced-mutations-demo/package.json b/examples/react/paced-mutations-demo/package.json index 55d4e4cf2..aaed91570 100644 --- a/examples/react/paced-mutations-demo/package.json +++ b/examples/react/paced-mutations-demo/package.json @@ -9,8 +9,8 @@ "preview": "vite preview" }, "dependencies": { - "@tanstack/db": "^0.8.6", - "@tanstack/react-db": "^0.3.6", + "@tanstack/db": "^0.8.7", + "@tanstack/react-db": "^0.3.7", "mitt": "^3.0.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/projects/package.json b/examples/react/projects/package.json index d605ed0db..3251b2464 100644 --- a/examples/react/projects/package.json +++ b/examples/react/projects/package.json @@ -17,8 +17,8 @@ "dependencies": { "@tailwindcss/vite": "^4.1.18", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/react-db": "^0.3.6", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", "@tanstack/react-router-with-query": "^1.130.17", diff --git a/examples/react/start-ssr-e2e/package.json b/examples/react/start-ssr-e2e/package.json index 439a5aba6..b521504a6 100644 --- a/examples/react/start-ssr-e2e/package.json +++ b/examples/react/start-ssr-e2e/package.json @@ -10,7 +10,7 @@ "test:e2e:hosted": "playwright test" }, "dependencies": { - "@tanstack/react-db": "^0.3.6", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-with-db": "^0.1.0", "@tanstack/react-start": "^1.159.5", diff --git a/examples/react/todo/package.json b/examples/react/todo/package.json index 30392b3e3..6264ccbf6 100644 --- a/examples/react/todo/package.json +++ b/examples/react/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.1.27", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.6", + "@tanstack/electric-db-collection": "^0.4.7", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/react-db": "^0.3.6", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.105", + "@tanstack/trailbase-db-collection": "^0.1.106", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/examples/solid/todo/package.json b/examples/solid/todo/package.json index 6b5e333cb..562c72f4e 100644 --- a/examples/solid/todo/package.json +++ b/examples/solid/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.0.36", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.6", + "@tanstack/electric-db-collection": "^0.4.7", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.11", - "@tanstack/solid-db": "^0.2.41", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/solid-db": "^0.2.42", "@tanstack/solid-router": "^1.159.5", "@tanstack/solid-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.105", + "@tanstack/trailbase-db-collection": "^0.1.106", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/packages/angular-db/CHANGELOG.md b/packages/angular-db/CHANGELOG.md index d0d5462db..4573819d5 100644 --- a/packages/angular-db/CHANGELOG.md +++ b/packages/angular-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/angular-db +## 0.1.88 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.1.87 ### Patch Changes diff --git a/packages/angular-db/package.json b/packages/angular-db/package.json index 7dec15154..dbfce25ea 100644 --- a/packages/angular-db/package.json +++ b/packages/angular-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/angular-db", - "version": "0.1.87", + "version": "0.1.88", "description": "Angular integration for @tanstack/db", "author": "Ethan McDaniel", "license": "MIT", diff --git a/packages/browser-db-sqlite-persistence/CHANGELOG.md b/packages/browser-db-sqlite-persistence/CHANGELOG.md index 6a3ec3ead..56534f46e 100644 --- a/packages/browser-db-sqlite-persistence/CHANGELOG.md +++ b/packages/browser-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/browser-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 269ecef94..a5b695fee 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/browser-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Browser wa-sqlite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md index 7ece1d3c7..256149571 100644 --- a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/capacitor-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md index fa88a892a..f063fefbb 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/capacitor-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/capacitor-db-sqlite-persistence@0.2.20 + ## 0.0.31 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json index 30181f666..cab7d1ec6 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.31", + "version": "0.0.32", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/capacitor-db-sqlite-persistence/package.json b/packages/capacitor-db-sqlite-persistence/package.json index 8f5fbe62e..9aecec826 100644 --- a/packages/capacitor-db-sqlite-persistence/package.json +++ b/packages/capacitor-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Capacitor SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md index 886547c6a..595601afd 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/cloudflare-durable-objects-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json index bbbd5a842..70de8645f 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/cloudflare-durable-objects-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Cloudflare Durable Object SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db-sqlite-persistence-core/CHANGELOG.md b/packages/db-sqlite-persistence-core/CHANGELOG.md index 741329479..400abead4 100644 --- a/packages/db-sqlite-persistence-core/CHANGELOG.md +++ b/packages/db-sqlite-persistence-core/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/db-sqlite-persistence-core +## 0.2.20 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.2.19 ### Patch Changes diff --git a/packages/db-sqlite-persistence-core/package.json b/packages/db-sqlite-persistence-core/package.json index 9dd360a64..e4979c894 100644 --- a/packages/db-sqlite-persistence-core/package.json +++ b/packages/db-sqlite-persistence-core/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db-sqlite-persistence-core", - "version": "0.2.19", + "version": "0.2.20", "description": "SQLite persisted collection core for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db/CHANGELOG.md b/packages/db/CHANGELOG.md index a1450fcdf..2b2a07a8e 100644 --- a/packages/db/CHANGELOG.md +++ b/packages/db/CHANGELOG.md @@ -1,5 +1,11 @@ # @tanstack/db +## 0.8.7 + +### Patch Changes + +- Match index collation options by their effective values so indexes remain reusable when optional locale fields are omitted, set to `undefined`, or use equivalent locale identifiers. ([#1788](https://github.com/TanStack/db/pull/1788)) + ## 0.8.6 ### Patch Changes diff --git a/packages/db/package.json b/packages/db/package.json index 4845820a7..6ebfd58d8 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db", - "version": "0.8.6", + "version": "0.8.7", "description": "A reactive client store for building super fast apps on sync", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/electric-db-collection/CHANGELOG.md b/packages/electric-db-collection/CHANGELOG.md index 6d11766f8..db2dfe74b 100644 --- a/packages/electric-db-collection/CHANGELOG.md +++ b/packages/electric-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/electric-db-collection +## 0.4.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.4.6 ### Patch Changes diff --git a/packages/electric-db-collection/package.json b/packages/electric-db-collection/package.json index acc0ceb88..6aa1cba22 100644 --- a/packages/electric-db-collection/package.json +++ b/packages/electric-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electric-db-collection", - "version": "0.4.6", + "version": "0.4.7", "description": "ElectricSQL collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/electron-db-sqlite-persistence/CHANGELOG.md b/packages/electron-db-sqlite-persistence/CHANGELOG.md index 909c5d0e8..0d625c139 100644 --- a/packages/electron-db-sqlite-persistence/CHANGELOG.md +++ b/packages/electron-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/electron-db-sqlite-persistence +## 0.1.32 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.1.31 ### Patch Changes diff --git a/packages/electron-db-sqlite-persistence/package.json b/packages/electron-db-sqlite-persistence/package.json index 7f1e72e87..ef4d4fcb9 100644 --- a/packages/electron-db-sqlite-persistence/package.json +++ b/packages/electron-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electron-db-sqlite-persistence", - "version": "0.1.31", + "version": "0.1.32", "description": "Electron SQLite persisted collection bridge for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/expo-db-sqlite-persistence/CHANGELOG.md b/packages/expo-db-sqlite-persistence/CHANGELOG.md index 63e9334f8..c59705795 100644 --- a/packages/expo-db-sqlite-persistence/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/expo-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md index feed398dc..835b6088d 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/expo-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/expo-db-sqlite-persistence@0.2.20 + ## 0.0.31 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json index f2c0a722f..c874b5be6 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/expo-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.31", + "version": "0.0.32", "main": "index.js", "scripts": { "start": "expo start", diff --git a/packages/expo-db-sqlite-persistence/package.json b/packages/expo-db-sqlite-persistence/package.json index a44121bc2..414efa630 100644 --- a/packages/expo-db-sqlite-persistence/package.json +++ b/packages/expo-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/expo-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/node-db-sqlite-persistence/CHANGELOG.md b/packages/node-db-sqlite-persistence/CHANGELOG.md index 3102ceb00..391f0eb7f 100644 --- a/packages/node-db-sqlite-persistence/CHANGELOG.md +++ b/packages/node-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/node-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/node-db-sqlite-persistence/package.json b/packages/node-db-sqlite-persistence/package.json index 0ccb64255..f1cb10f54 100644 --- a/packages/node-db-sqlite-persistence/package.json +++ b/packages/node-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/node-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Node SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/offline-transactions/CHANGELOG.md b/packages/offline-transactions/CHANGELOG.md index 71f29aa33..f2fc2fa41 100644 --- a/packages/offline-transactions/CHANGELOG.md +++ b/packages/offline-transactions/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/offline-transactions +## 1.0.53 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 1.0.52 ### Patch Changes diff --git a/packages/offline-transactions/package.json b/packages/offline-transactions/package.json index afe03a2ad..cec8ca307 100644 --- a/packages/offline-transactions/package.json +++ b/packages/offline-transactions/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/offline-transactions", - "version": "1.0.52", + "version": "1.0.53", "description": "Offline-first transaction capabilities for TanStack DB", "author": "TanStack", "license": "MIT", diff --git a/packages/powersync-db-collection/CHANGELOG.md b/packages/powersync-db-collection/CHANGELOG.md index c1f400bb4..43d1dbe73 100644 --- a/packages/powersync-db-collection/CHANGELOG.md +++ b/packages/powersync-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/powersync-db-collection +## 0.1.66 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.1.65 ### Patch Changes diff --git a/packages/powersync-db-collection/package.json b/packages/powersync-db-collection/package.json index e4b536b68..cf2ecc9d5 100644 --- a/packages/powersync-db-collection/package.json +++ b/packages/powersync-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/powersync-db-collection", - "version": "0.1.65", + "version": "0.1.66", "description": "PowerSync collection for TanStack DB", "author": "POWERSYNC", "license": "MIT", diff --git a/packages/query-db-collection/CHANGELOG.md b/packages/query-db-collection/CHANGELOG.md index 326ec3134..fcbbaa4ff 100644 --- a/packages/query-db-collection/CHANGELOG.md +++ b/packages/query-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/query-db-collection +## 1.2.12 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 1.2.11 ### Patch Changes diff --git a/packages/query-db-collection/package.json b/packages/query-db-collection/package.json index b5341773f..c067e7fca 100644 --- a/packages/query-db-collection/package.json +++ b/packages/query-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/query-db-collection", - "version": "1.2.11", + "version": "1.2.12", "description": "TanStack Query collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/react-db/CHANGELOG.md b/packages/react-db/CHANGELOG.md index 604690e64..205e5023f 100644 --- a/packages/react-db/CHANGELOG.md +++ b/packages/react-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-db +## 0.3.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.3.6 ### Patch Changes diff --git a/packages/react-db/package.json b/packages/react-db/package.json index e1b8a9c0c..b9612dc39 100644 --- a/packages/react-db/package.json +++ b/packages/react-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-db", - "version": "0.3.6", + "version": "0.3.7", "description": "React integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/react-native-db-sqlite-persistence/CHANGELOG.md b/packages/react-native-db-sqlite-persistence/CHANGELOG.md index 2927be705..1282f97f7 100644 --- a/packages/react-native-db-sqlite-persistence/CHANGELOG.md +++ b/packages/react-native-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-native-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/react-native-db-sqlite-persistence/package.json b/packages/react-native-db-sqlite-persistence/package.json index a47164344..c3549d5f0 100644 --- a/packages/react-native-db-sqlite-persistence/package.json +++ b/packages/react-native-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-native-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "React Native and Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/rxdb-db-collection/CHANGELOG.md b/packages/rxdb-db-collection/CHANGELOG.md index 13cc2d034..1e8d68589 100644 --- a/packages/rxdb-db-collection/CHANGELOG.md +++ b/packages/rxdb-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/rxdb-db-collection +## 0.1.94 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.1.93 ### Patch Changes diff --git a/packages/rxdb-db-collection/package.json b/packages/rxdb-db-collection/package.json index 3db3b69c3..36387acf4 100644 --- a/packages/rxdb-db-collection/package.json +++ b/packages/rxdb-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/rxdb-db-collection", - "version": "0.1.93", + "version": "0.1.94", "description": "Reactive, Offline-First adapter for TanStack DB using RxDB. Sync, Replication and Local-First support.", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/solid-db/CHANGELOG.md b/packages/solid-db/CHANGELOG.md index 3091f962a..c4fa5b37a 100644 --- a/packages/solid-db/CHANGELOG.md +++ b/packages/solid-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/react-db +## 0.2.42 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.2.41 ### Patch Changes diff --git a/packages/solid-db/package.json b/packages/solid-db/package.json index 2d1b4db39..cff5ce25f 100644 --- a/packages/solid-db/package.json +++ b/packages/solid-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/solid-db", - "version": "0.2.41", + "version": "0.2.42", "description": "Solid integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/svelte-db/CHANGELOG.md b/packages/svelte-db/CHANGELOG.md index bf2d8da46..3a205b7f2 100644 --- a/packages/svelte-db/CHANGELOG.md +++ b/packages/svelte-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/svelte-db +## 0.3.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.3.6 ### Patch Changes diff --git a/packages/svelte-db/package.json b/packages/svelte-db/package.json index b1fa049b3..1e5857494 100644 --- a/packages/svelte-db/package.json +++ b/packages/svelte-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/svelte-db", - "version": "0.3.6", + "version": "0.3.7", "description": "Svelte integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/tauri-db-sqlite-persistence/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/CHANGELOG.md index d764cd4a1..4b2026318 100644 --- a/packages/tauri-db-sqlite-persistence/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/tauri-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + ## 0.2.19 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md index 0de77a1af..c1fa5008e 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,13 @@ # @tanstack/tauri-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/tauri-db-sqlite-persistence@0.2.20 + ## 0.0.31 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/package.json b/packages/tauri-db-sqlite-persistence/e2e/app/package.json index 6ed9b5f34..9da824e80 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/package.json +++ b/packages/tauri-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/tauri-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.31", + "version": "0.0.32", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/tauri-db-sqlite-persistence/package.json b/packages/tauri-db-sqlite-persistence/package.json index 70b83ba0a..0d6ffdf8f 100644 --- a/packages/tauri-db-sqlite-persistence/package.json +++ b/packages/tauri-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/tauri-db-sqlite-persistence", - "version": "0.2.19", + "version": "0.2.20", "description": "Tauri SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/trailbase-db-collection/CHANGELOG.md b/packages/trailbase-db-collection/CHANGELOG.md index 50dc48499..ef587279e 100644 --- a/packages/trailbase-db-collection/CHANGELOG.md +++ b/packages/trailbase-db-collection/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/trailbase-db-collection +## 0.1.106 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.1.105 ### Patch Changes diff --git a/packages/trailbase-db-collection/package.json b/packages/trailbase-db-collection/package.json index 1bba98931..3b9bc9f01 100644 --- a/packages/trailbase-db-collection/package.json +++ b/packages/trailbase-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/trailbase-db-collection", - "version": "0.1.105", + "version": "0.1.106", "description": "TrailBase collection for TanStack DB", "author": "Sebastian Jeltsch", "license": "MIT", diff --git a/packages/vue-db/CHANGELOG.md b/packages/vue-db/CHANGELOG.md index 37684c9ed..a152ff8b9 100644 --- a/packages/vue-db/CHANGELOG.md +++ b/packages/vue-db/CHANGELOG.md @@ -1,5 +1,12 @@ # @tanstack/vue-db +## 0.1.9 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + ## 0.1.8 ### Patch Changes diff --git a/packages/vue-db/package.json b/packages/vue-db/package.json index d03e5d57e..e93d02780 100644 --- a/packages/vue-db/package.json +++ b/packages/vue-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/vue-db", - "version": "0.1.8", + "version": "0.1.9", "description": "Vue integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 21c34e415..ce1a271cf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,10 +148,10 @@ importers: specifier: ^20.3.16 version: 20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1)))(rxjs@7.8.2) '@tanstack/angular-db': - specifier: ^0.1.87 + specifier: ^0.1.88 version: link:../../../packages/angular-db '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db rxjs: specifier: ^7.8.2 @@ -209,19 +209,19 @@ importers: examples/electron/offline-first: dependencies: '@tanstack/electron-db-sqlite-persistence': - specifier: ^0.1.31 + specifier: ^0.1.32 version: link:../../../packages/electron-db-sqlite-persistence '@tanstack/node-db-sqlite-persistence': - specifier: ^0.2.19 + specifier: ^0.2.20 version: link:../../../packages/node-db-sqlite-persistence '@tanstack/offline-transactions': - specifier: ^1.0.52 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -300,19 +300,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.52 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.19 + specifier: ^0.2.20 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -397,19 +397,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/electric-db-collection': - specifier: ^0.4.6 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/offline-transactions': - specifier: ^1.0.52 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.19 + specifier: ^0.2.20 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -482,10 +482,10 @@ importers: examples/react/next-ssr-e2e: dependencies: '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db next: specifier: ^16.3.1 @@ -516,19 +516,19 @@ importers: examples/react/offline-transactions: dependencies: '@tanstack/browser-db-sqlite-persistence': - specifier: ^0.2.19 + specifier: ^0.2.20 version: link:../../../packages/browser-db-sqlite-persistence '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.52 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -586,10 +586,10 @@ importers: examples/react/paced-mutations-demo: dependencies: '@tanstack/db': - specifier: ^0.8.6 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db mitt: specifier: ^3.0.1 @@ -626,10 +626,10 @@ importers: specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -759,7 +759,7 @@ importers: examples/react/start-ssr-e2e: dependencies: '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -805,16 +805,16 @@ importers: examples/react/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.6 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.6 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -823,7 +823,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.105 + specifier: ^0.1.106 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6 @@ -926,16 +926,16 @@ importers: examples/solid/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.6 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.11 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/solid-db': - specifier: ^0.2.41 + specifier: ^0.2.42 version: link:../../../packages/solid-db '@tanstack/solid-router': specifier: ^1.159.5 @@ -944,7 +944,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(@tanstack/react-router@1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(solid-js@1.9.11)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.105 + specifier: ^0.1.106 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6 From 49a9de5c43f7a5f7856f2efefa92af53e6a22318 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 14:06:15 -0600 Subject: [PATCH 208/327] test(db): cover waiting nested window ownership --- ...ncludes-collection-oracle.property.test.ts | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 061e65576..6bc876f50 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1626,6 +1626,153 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `a rejected nested window restores a waiting parent operation`, + async () => { + const parents = createControlledCollection(`waiting-window-parent`, [ + { id: 1, rank: 1, value: 1 }, + { id: 2, rank: 2, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id, value: parent.value })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const initialOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const rollbackOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const afterCatchOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + limit?: number + }) => void + let parentWindowCalls = 0 + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit !== 2) return + parentWindowCalls++ + if (parentWindowCalls === 1) { + builder.trackSubsetLoadOperationPromise( + initialOutcome.promise, + `initial`, + ) + } else if (parentWindowCalls === 2) { + builder.trackSubsetLoadOperationPromise( + rollbackOutcome.promise, + `rollback`, + ) + } + }) + + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(parentReady).toBeInstanceOf(Promise) + + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + builder.trackSubsetLoadOperationPromise( + afterCatchOutcome.promise, + `after-catch`, + ) + }, + { includeInitialState: false }, + ) + + try { + parents.write(`update`, { id: 1, rank: 1, value: 3 }) + expect(nestedError).toBe(nestedFailure) + expect(parentWindowCalls).toBe(2) + expect(live.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 1, value: 3 }, + { id: 2, value: 2 }, + ]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) + + let parentSettled = false + void Promise.resolve(parentReady).then(() => { + parentSettled = true + }) + initialOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await flushPromises() + expect(parentSettled).toBe(false) + + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ sourceId: `initial` }), + expect.objectContaining({ sourceId: `rollback` }), + expect.objectContaining({ sourceId: `after-catch` }), + ]), + ) + } finally { + subscription.unsubscribe() + const outcome = { + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted` as const, + } + initialOutcome.resolve(outcome) + rollbackOutcome.resolve(outcome) + afterCatchOutcome.resolve(outcome) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + fcTest( `an older failed window cannot restore over a newer nested window`, async () => { From 684033651de14857018c92e42770648663bbdb5d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 14:21:02 -0600 Subject: [PATCH 209/327] test(db): separate nested window load phases --- ...ncludes-collection-oracle.property.test.ts | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 6bc876f50..fc959f41f 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1643,24 +1643,16 @@ describe(`Collection-valued includes oracle`, () => { await live.preload() const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() - const initialOutcome = createDeferred<{ + type Outcome = { collectionId: string demand: { limit: number } generation: number extent: `exhausted` - }>() - const rollbackOutcome = createDeferred<{ - collectionId: string - demand: { limit: number } - generation: number - extent: `exhausted` - }>() - const afterCatchOutcome = createDeferred<{ - collectionId: string - demand: { limit: number } - generation: number - extent: `exhausted` - }>() + } + const initialOutcome = createDeferred() + const beforeNestedOutcome = createDeferred() + const rollbackOutcome = createDeferred() + const afterCatchOutcome = createDeferred() const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { limit?: number }) => void @@ -1700,6 +1692,10 @@ describe(`Collection-valued includes oracle`, () => { () => { if (acted) return acted = true + builder.trackSubsetLoadOperationPromise( + beforeNestedOutcome.promise, + `before-nested`, + ) failNested = true try { live.utils.setWindow({ offset: 1, limit: 1 }) @@ -1737,12 +1733,24 @@ describe(`Collection-valued includes oracle`, () => { await flushPromises() expect(parentSettled).toBe(false) + beforeNestedOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await flushPromises() + expect(parentSettled).toBe(false) + rollbackOutcome.resolve({ collectionId: `parents`, demand: { limit: 2 }, generation: 1, extent: `exhausted`, }) + await flushPromises() + expect(parentSettled).toBe(false) + afterCatchOutcome.resolve({ collectionId: `parents`, demand: { limit: 2 }, @@ -1750,13 +1758,11 @@ describe(`Collection-valued includes oracle`, () => { extent: `exhausted`, }) await parentReady - expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( - expect.arrayContaining([ - expect.objectContaining({ sourceId: `initial` }), - expect.objectContaining({ sourceId: `rollback` }), - expect.objectContaining({ sourceId: `after-catch` }), - ]), - ) + expect( + live.utils[LIVE_QUERY_INTERNAL] + .getLastWindowOutcomes() + .map(({ sourceId }) => sourceId), + ).toEqual([`initial`, `before-nested`, `rollback`, `after-catch`]) } finally { subscription.unsubscribe() const outcome = { @@ -1766,6 +1772,7 @@ describe(`Collection-valued includes oracle`, () => { extent: `exhausted` as const, } initialOutcome.resolve(outcome) + beforeNestedOutcome.resolve(outcome) rollbackOutcome.resolve(outcome) afterCatchOutcome.resolve(outcome) await Promise.all([live.cleanup(), parents.collection.cleanup()]) From 2fbac765e66260c4a58c97efe2c8712b2f8adb95 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 14:36:13 -0600 Subject: [PATCH 210/327] fix(db): isolate facade publication errors --- packages/db/src/query/live/ARCHITECTURE.md | 3 + .../src/query/live/bucket-facade-adapter.ts | 14 ++- .../query/live/collection-config-builder.ts | 8 +- ...ncludes-collection-oracle.property.test.ts | 99 +++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index eab96b908..93cca2cbc 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1136,6 +1136,9 @@ read another participating Collection without seeing new rows behind an old revision. If a later root or containing-facade application fails before that release, rollback restores the installed state and discards both the held events and their revision advances. Routing and identity remain inside D2. +Once release begins, one subscriber callback failure cannot suppress another +prepared root or facade publication. Release attempts every participant, then +rethrows the first callback failure unchanged, including `null` or `undefined`. Every graph-turn origin that can publish rows owns a scheduler publication context through the complete coherent release. This includes direct window diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 9ad84887a..2d080071d 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -179,10 +179,22 @@ export class BucketFacadeAdapter { if (closed) return prepare() closed = true - for (const publication of publications) publication.publish() + let hasPublicationError = false + let publicationError: unknown + for (const publication of publications) { + try { + publication.publish() + } catch (error) { + if (!hasPublicationError) { + hasPublicationError = true + publicationError = error + } + } + } // Drop only the adapter's strong reference. External holders keep an // empty, ready facade; a later active interval receives a new one. this.retiredEntries.clear() + if (hasPublicationError) throw publicationError }, rollback: () => { if (closed || prepared) return diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 56d92ee71..4e3dba739 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1116,6 +1116,7 @@ export class CollectionConfigBuilder< rootPublication?.prepare() facadePublication.prepare() + let hasPublicationError = false let publicationError: unknown for (const publish of [ rootPublication?.publish, @@ -1125,10 +1126,13 @@ export class CollectionConfigBuilder< try { publish() } catch (error) { - publicationError ??= error + if (!hasPublicationError) { + hasPublicationError = true + publicationError = error + } } } - if (publicationError !== undefined) throw publicationError + if (hasPublicationError) throw publicationError } graph.finalize() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index fc959f41f..966cbad30 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -994,6 +994,105 @@ describe(`Collection-valued includes oracle`, () => { }, ) + for (const { + throwingParentId, + position, + failure, + } of [ + { throwingParentId: 1, position: `first`, failure: `error` }, + { throwingParentId: 2, position: `middle`, failure: `undefined` }, + { throwingParentId: 3, position: `last`, failure: `null` }, + ] as const) { + fcTest( + `a throwing ${position} facade callback does not suppress sibling publications`, + async () => { + const parents = createControlledCollection(`callback-error-parents`, [ + { id: 1, group: 1 }, + { id: 2, group: 2 }, + { id: 3, group: 3 }, + ]) + const children = createControlledCollection(`callback-error-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 2, value: 2 }, + { id: 30, parentGroup: 3, value: 3 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => + eq(child.parentGroup, parent.group), + ), + })), + ) + + await live.preload() + const facades = [1, 2, 3].map((parentId) => ({ + parentId, + collection: live.get(parentId)!.children, + })) + const callbackParentIds: Array = [] + const callbackError = + failure === `error` + ? new Error(`facade ${throwingParentId} callback failed`) + : failure === `undefined` + ? undefined + : null + const subscriptions = facades.map(({ parentId, collection }) => + collection.subscribeChanges( + () => { + callbackParentIds.push(parentId) + if (parentId === throwingParentId) throw callbackError + }, + { includeInitialState: false }, + ), + ) + + try { + let didThrow = false + let publicationError: unknown + try { + children.writeBatch([ + { + type: `update`, + value: { id: 10, parentGroup: 1, value: 11 }, + }, + { + type: `update`, + value: { id: 20, parentGroup: 2, value: 12 }, + }, + { + type: `update`, + value: { id: 30, parentGroup: 3, value: 13 }, + }, + ]) + } catch (error) { + didThrow = true + publicationError = error + } + + expect(didThrow).toBe(true) + expect(publicationError).toBe(callbackError) + expect(callbackParentIds).toEqual([1, 2, 3]) + expect( + facades.map(({ collection }) => collection.toArray[0]!.value), + ).toEqual([11, 12, 13]) + } finally { + for (const subscription of subscriptions) subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + fcTest( `coherent nested publication advances every revision before callbacks`, async () => { From c6ebb6bc496608c3e2c8051aac0accff0a2745e2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 14:47:10 -0600 Subject: [PATCH 211/327] test(db): preserve first facade publication error --- .../db/tests/query/includes-collection-oracle.property.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 966cbad30..d6c7bba82 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1047,6 +1047,9 @@ describe(`Collection-valued includes oracle`, () => { () => { callbackParentIds.push(parentId) if (parentId === throwingParentId) throw callbackError + if (position === `first` && parentId === 3) { + throw new Error(`later facade callback failed`) + } }, { includeInitialState: false }, ), From 70e40abd68f7f86ec704720ebcd722b90ab5b84e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 15:03:18 -0600 Subject: [PATCH 212/327] fix(db): cancel prepared publications on cleanup --- packages/db/src/collection/changes.ts | 11 +++ packages/db/src/query/live/ARCHITECTURE.md | 3 + ...ncludes-collection-oracle.property.test.ts | 91 +++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 015560c6c..cc3217933 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -63,6 +63,9 @@ export class CollectionChangesManager< private publicationDeferral: | PublicationDeferralState | undefined + private preparedPublicationDeferrals = new Set< + PublicationDeferralState + >() private layoutChangeListeners = new Set<() => void>() /** @@ -213,6 +216,7 @@ export class CollectionChangesManager< ({ layoutChanged }) => layoutChanged, ), } + this.preparedPublicationDeferrals.add(publicationDeferral) } return { @@ -228,6 +232,7 @@ export class CollectionChangesManager< return } publicationDeferral.published = true + this.preparedPublicationDeferrals.delete(publicationDeferral) const publication = publicationDeferral.prepared publicationDeferral.prepared = undefined if (publication) { @@ -414,5 +419,11 @@ export class CollectionChangesManager< this.publicationDeferral.prepared = undefined } this.publicationDeferral = undefined + for (const publicationDeferral of this.preparedPublicationDeferrals) { + publicationDeferral.discard = true + publicationDeferral.publications = [] + publicationDeferral.prepared = undefined + } + this.preparedPublicationDeferrals.clear() } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 93cca2cbc..1d3b67977 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1139,6 +1139,9 @@ events and their revision advances. Routing and identity remain inside D2. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. +If a callback cleans up another participant after preparation but before its +release, cleanup cancels that participant's held delivery. No callback may run +later against its cleaned-up state. Every graph-turn origin that can publish rows owns a scheduler publication context through the complete coherent release. This includes direct window diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index d6c7bba82..7c2a94361 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1096,6 +1096,97 @@ describe(`Collection-valued includes oracle`, () => { ) } + fcTest( + `cleanup during root publication suppresses a prepared facade callback`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` + group: number + value: number + } + const nodes = createControlledCollection( + `prepared-facade-cleanup`, + [ + { id: 1, kind: `parent`, group: 1, value: 1 }, + { id: 10, kind: `child`, group: 1, value: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootSnapshots: Array<{ + status: string + rows: Array<{ id: number; value: number }> + }> = [] + const facadeSnapshots: Array<{ + status: string + rows: Array<{ id: number; value: number }> + }> = [] + let cleanupPromise: Promise | undefined + const rootSubscription = live.subscribeChanges( + () => { + rootSnapshots.push({ + status: facade.status, + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + }) + cleanupPromise = facade.cleanup() + }, + { includeInitialState: false }, + ) + const facadeSubscription = facade.subscribeChanges( + () => { + facadeSnapshots.push({ + status: facade.status, + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + }) + }, + { includeInitialState: false }, + ) + + try { + nodes.writeBatch([ + { + type: `update`, + value: { id: 1, kind: `parent`, group: 1, value: 2 }, + }, + { + type: `update`, + value: { id: 10, kind: `child`, group: 1, value: 2 }, + }, + ]) + await cleanupPromise + + expect(rootSnapshots).toEqual([ + { + status: `ready`, + rows: [{ id: 10, value: 2 }], + }, + ]) + expect(facadeSnapshots).toEqual([]) + expect(facade.status).toBe(`cleaned-up`) + expect(facade.toArray).toEqual([]) + } finally { + rootSubscription.unsubscribe() + facadeSubscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + fcTest( `coherent nested publication advances every revision before callbacks`, async () => { From 57c447be7259655969fb9c95db618d641ddca41f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 15:16:09 -0600 Subject: [PATCH 213/327] test(db): cancel every prepared publication --- ...ncludes-collection-oracle.property.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 7c2a94361..f57ae10a4 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1187,6 +1187,44 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest(`cleanup cancels every independently prepared publication`, async () => { + const rows = createControlledCollection(`prepared-publication-cleanup`, [ + { id: 1, value: 1 }, + ]) + await rows.collection.preload() + const callbackValues: Array> = [] + const subscription = rows.collection.subscribeChanges( + (batch) => { + callbackValues.push(batch.map((change) => change.value.value)) + }, + { includeInitialState: false }, + ) + + try { + const firstPublication = rows.collection._deferPublication() + rows.write(`update`, { id: 1, value: 2 }) + firstPublication.prepare() + + const secondPublication = rows.collection._deferPublication() + rows.write(`update`, { id: 1, value: 3 }) + secondPublication.prepare() + + expect(rows.collection.get(1)!.value).toBe(3) + expect(rows.collection.status).toBe(`ready`) + + await rows.collection.cleanup() + firstPublication.publish() + secondPublication.publish() + + expect(callbackValues).toEqual([]) + expect(rows.collection.status).toBe(`cleaned-up`) + expect(rows.collection.toArray).toEqual([]) + } finally { + subscription.unsubscribe() + await rows.collection.cleanup() + } + }) + fcTest( `coherent nested publication advances every revision before callbacks`, async () => { From 6f9c5619c9d55df69d60df60b44ae391856fc42e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 15:32:33 -0600 Subject: [PATCH 214/327] fix(db): reject overlapping prepared publications --- packages/db/src/collection/changes.ts | 27 +++--- packages/db/src/query/live/ARCHITECTURE.md | 5 ++ .../tests/collection-sync-reentrancy.test.ts | 90 +++++++++++++++++++ ...ncludes-collection-oracle.property.test.ts | 5 +- 4 files changed, 114 insertions(+), 13 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index cc3217933..5267a9118 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -63,9 +63,9 @@ export class CollectionChangesManager< private publicationDeferral: | PublicationDeferralState | undefined - private preparedPublicationDeferrals = new Set< - PublicationDeferralState - >() + private preparedPublicationDeferral: + | PublicationDeferralState + | undefined private layoutChangeListeners = new Set<() => void>() /** @@ -181,6 +181,11 @@ export class CollectionChangesManager< * normal transaction boundaries. */ public deferPublication(): PublicationDeferral { + if (this.preparedPublicationDeferral) { + throw new Error( + `Cannot start a publication cycle while another is prepared`, + ) + } const publicationDeferral = this.publicationDeferral ?? { depth: 0, discard: false, @@ -216,7 +221,7 @@ export class CollectionChangesManager< ({ layoutChanged }) => layoutChanged, ), } - this.preparedPublicationDeferrals.add(publicationDeferral) + this.preparedPublicationDeferral = publicationDeferral } return { @@ -232,7 +237,9 @@ export class CollectionChangesManager< return } publicationDeferral.published = true - this.preparedPublicationDeferrals.delete(publicationDeferral) + if (this.preparedPublicationDeferral === publicationDeferral) { + this.preparedPublicationDeferral = undefined + } const publication = publicationDeferral.prepared publicationDeferral.prepared = undefined if (publication) { @@ -419,11 +426,11 @@ export class CollectionChangesManager< this.publicationDeferral.prepared = undefined } this.publicationDeferral = undefined - for (const publicationDeferral of this.preparedPublicationDeferrals) { - publicationDeferral.discard = true - publicationDeferral.publications = [] - publicationDeferral.prepared = undefined + if (this.preparedPublicationDeferral) { + this.preparedPublicationDeferral.discard = true + this.preparedPublicationDeferral.publications = [] + this.preparedPublicationDeferral.prepared = undefined } - this.preparedPublicationDeferrals.clear() + this.preparedPublicationDeferral = undefined } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 1d3b67977..715ae3291 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1142,6 +1142,11 @@ rethrows the first callback failure unchanged, including `null` or `undefined`. If a callback cleans up another participant after preparation but before its release, cleanup cancels that participant's held delivery. No callback may run later against its cleaned-up state. +Nested deferral handles may join one open Collection publication cycle. Once +that cycle is prepared, no independent cycle may begin until it is published, +or canceled by cleanup. An open cycle may instead be discarded. The Collection +rejects prepared-cycle overlap before the newer cycle can install state; +otherwise an older event could be delivered against a newer visible snapshot. Every graph-turn origin that can publish rows owns a scheduler publication context through the complete coherent release. This includes direct window diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 773baadf8..dd3a84785 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -219,6 +219,96 @@ const { multiplier, ...replay } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { + it.each([`open`, `prepared`, `published`] as const)( + `starts a second publication cycle with the first cycle %s`, + async (firstCycleState) => { + const harness = createSyncHarness( + `publication-cycle-${firstCycleState}`, + ) + const { collection } = harness + const callbacks: Array<{ + changes: Array + visibleValue: string + revision: number + }> = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map((change) => change.value.value), + visibleValue: collection.get(1)!.value, + revision: collection._stateRevision, + }) + }, + { includeInitialState: false }, + ) + const initialRevision = collection._stateRevision + const write = (type: `insert` | `update`, value: string) => { + harness.sync.begin({ immediate: true }) + harness.sync.write({ type, value: { id: 1, value } }) + harness.sync.commit() + } + + try { + const firstPublication = collection._deferPublication() + write(`insert`, `first`) + + if (firstCycleState === `open`) { + const secondPublication = collection._deferPublication() + write(`update`, `second`) + firstPublication.prepare() + secondPublication.prepare() + firstPublication.publish() + secondPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`, `second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } else if (firstCycleState === `prepared`) { + firstPublication.prepare() + expect(() => collection._deferPublication()).toThrow( + `Cannot start a publication cycle while another is prepared`, + ) + firstPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + ]) + } else { + firstPublication.prepare() + firstPublication.publish() + const secondPublication = collection._deferPublication() + write(`update`, `second`) + secondPublication.prepare() + secondPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + { + changes: [`second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + it(`compares layout with the public state before an immediate prefix drain`, async () => { const updatePersistence = createDeferred() const insertPersistence = createDeferred() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index f57ae10a4..079f9c3d9 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1187,7 +1187,7 @@ describe(`Collection-valued includes oracle`, () => { }, ) - fcTest(`cleanup cancels every independently prepared publication`, async () => { + fcTest(`cleanup cancels every handle in a prepared publication`, async () => { const rows = createControlledCollection(`prepared-publication-cleanup`, [ { id: 1, value: 1 }, ]) @@ -1203,10 +1203,9 @@ describe(`Collection-valued includes oracle`, () => { try { const firstPublication = rows.collection._deferPublication() rows.write(`update`, { id: 1, value: 2 }) - firstPublication.prepare() - const secondPublication = rows.collection._deferPublication() rows.write(`update`, { id: 1, value: 3 }) + firstPublication.prepare() secondPublication.prepare() expect(rows.collection.get(1)!.value).toBe(3) From b487a91aa71187bdb0e74c7e6f1f7a8b887e64a7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 15:44:59 -0600 Subject: [PATCH 215/327] test(db): allow publication cycles from callbacks --- .../tests/collection-sync-reentrancy.test.ts | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index dd3a84785..e253901fc 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -309,6 +309,62 @@ describe(`sync publication reentrancy`, () => { }, ) + it(`lets a publication callback start the next publication cycle`, async () => { + const harness = createSyncHarness(`publication-cycle-from-callback`) + const { collection } = harness + const callbacks: Array<{ + changes: Array + visibleValue: string + revision: number + }> = [] + const initialRevision = collection._stateRevision + const write = (type: `insert` | `update`, value: string) => { + harness.sync.begin({ immediate: true }) + harness.sync.write({ type, value: { id: 1, value } }) + harness.sync.commit() + } + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map((change) => change.value.value), + visibleValue: collection.get(1)!.value, + revision: collection._stateRevision, + }) + + if (changes[0]?.value.value === `first`) { + const secondPublication = collection._deferPublication() + write(`update`, `second`) + secondPublication.prepare() + secondPublication.publish() + } + }, + { includeInitialState: false }, + ) + + try { + const firstPublication = collection._deferPublication() + write(`insert`, `first`) + firstPublication.prepare() + firstPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + { + changes: [`second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`compares layout with the public state before an immediate prefix drain`, async () => { const updatePersistence = createDeferred() const insertPersistence = createDeferred() From 3748653e932a9875de8617a32e13b85a44b1310f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 16:02:14 -0600 Subject: [PATCH 216/327] test(db): require exact layout sequence comparison --- .../tests/collection-sync-reentrancy.test.ts | 67 +++++++++++++++++++ ...ncludes-collection-oracle.property.test.ts | 67 +++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index e253901fc..e745dd498 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -365,6 +365,73 @@ describe(`sync publication reentrancy`, () => { } }) + it(`publishes an internal layout swap with unchanged endpoints`, async () => { + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-middle-swap`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + ops.begin({ immediate: true }) + for (let id = 1; id <= 4; id++) { + ops.write({ + type: `insert`, + value: { id, value: `value-${id}`, rank: id }, + }) + } + ops.commit() + ops.markReady() + }, + }, + }) + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: false, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + + try { + const revisionBeforeSwap = collection._layoutRevision + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 2, value: `value-2`, rank: 3 }, + }) + sync.write({ + type: `update`, + value: { id: 3, value: `value-3`, rank: 2 }, + }) + sync.collection._markLayoutChange() + expect(sync.commit()).toBe(true) + + expect([...collection.keys()]).toEqual([1, 3, 2, 4]) + expect(collection._layoutRevision).toBe(revisionBeforeSwap + 1) + expect(callbacks).toEqual([ + { + changes: [2, 3], + keys: [1, 3, 2, 4], + values: [`value-1`, `value-3`, `value-2`, `value-4`], + markedReceiptSettled: false, + revision: revisionBeforeSwap + 1, + }, + ]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`compares layout with the public state before an immediate prefix drain`, async () => { const updatePersistence = createDeferred() const insertPersistence = createDeferred() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 079f9c3d9..fb53bb6a7 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -4208,6 +4208,73 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest(`publishes an internal facade swap with unchanged endpoints`, async () => { + type OrderedChild = ChildRow & { position: number } + const parents = createControlledCollection(`middle-swap-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `middle-swap-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 1 }, + { id: 20, parentGroup: 1, value: 20, position: 2 }, + { id: 30, parentGroup: 1, value: 30, position: 3 }, + { id: 40, parentGroup: 1, value: 40, position: 4 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ id: child.id, value: child.value })), + })), + ) + + try { + await live.preload() + const facade = live.get(1)!.children + const revisionBeforeSwap = facade._layoutRevision + const publicationSizes: Array = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (changes) => { + publicationSizes.push(changes.length) + callbackKeys.push(facade.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + + try { + children.writeBatch([ + { + type: `update`, + value: { id: 20, parentGroup: 1, value: 20, position: 3 }, + }, + { + type: `update`, + value: { id: 30, parentGroup: 1, value: 30, position: 2 }, + }, + ]) + + expect(facade.toArray.map(({ id }) => id)).toEqual([10, 30, 20, 40]) + expect(facade._layoutRevision).toBe(revisionBeforeSwap + 1) + expect(publicationSizes).toEqual([0]) + expect(callbackKeys).toEqual([[10, 30, 20, 40]]) + } finally { + subscription.unsubscribe() + } + } finally { + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }) + fcTest( `reconstructs nested conditional includes through guard transitions`, async () => { From 2d55f88ad2bdf8e96c2a95b0a6ea0217273d58dd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 16:19:32 -0600 Subject: [PATCH 217/327] test(db): compare later layout positions --- .../db/tests/collection-sync-reentrancy.test.ts | 14 +++++++------- .../includes-collection-oracle.property.test.ts | 11 +++++++---- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index e745dd498..16ca7afd7 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -376,7 +376,7 @@ describe(`sync publication reentrancy`, () => { sync: (ops) => { sync = ops ops.begin({ immediate: true }) - for (let id = 1; id <= 4; id++) { + for (let id = 1; id <= 5; id++) { ops.write({ type: `insert`, value: { id, value: `value-${id}`, rank: id }, @@ -406,22 +406,22 @@ describe(`sync publication reentrancy`, () => { sync.begin({ immediate: true }) sync.write({ type: `update`, - value: { id: 2, value: `value-2`, rank: 3 }, + value: { id: 3, value: `value-3`, rank: 4 }, }) sync.write({ type: `update`, - value: { id: 3, value: `value-3`, rank: 2 }, + value: { id: 4, value: `value-4`, rank: 3 }, }) sync.collection._markLayoutChange() expect(sync.commit()).toBe(true) - expect([...collection.keys()]).toEqual([1, 3, 2, 4]) + expect([...collection.keys()]).toEqual([1, 2, 4, 3, 5]) expect(collection._layoutRevision).toBe(revisionBeforeSwap + 1) expect(callbacks).toEqual([ { - changes: [2, 3], - keys: [1, 3, 2, 4], - values: [`value-1`, `value-3`, `value-2`, `value-4`], + changes: [3, 4], + keys: [1, 2, 4, 3, 5], + values: [`value-1`, `value-2`, `value-4`, `value-3`, `value-5`], markedReceiptSettled: false, revision: revisionBeforeSwap + 1, }, diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index fb53bb6a7..2d2084d83 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -4220,6 +4220,7 @@ describe(`Collection-valued includes oracle`, () => { { id: 20, parentGroup: 1, value: 20, position: 2 }, { id: 30, parentGroup: 1, value: 30, position: 3 }, { id: 40, parentGroup: 1, value: 40, position: 4 }, + { id: 50, parentGroup: 1, value: 50, position: 5 }, ], ) const live = createLiveQueryCollection((q) => @@ -4251,18 +4252,20 @@ describe(`Collection-valued includes oracle`, () => { children.writeBatch([ { type: `update`, - value: { id: 20, parentGroup: 1, value: 20, position: 3 }, + value: { id: 30, parentGroup: 1, value: 30, position: 4 }, }, { type: `update`, - value: { id: 30, parentGroup: 1, value: 30, position: 2 }, + value: { id: 40, parentGroup: 1, value: 40, position: 3 }, }, ]) - expect(facade.toArray.map(({ id }) => id)).toEqual([10, 30, 20, 40]) + expect(facade.toArray.map(({ id }) => id)).toEqual([ + 10, 20, 40, 30, 50, + ]) expect(facade._layoutRevision).toBe(revisionBeforeSwap + 1) expect(publicationSizes).toEqual([0]) - expect(callbackKeys).toEqual([[10, 30, 20, 40]]) + expect(callbackKeys).toEqual([[10, 20, 40, 30, 50]]) } finally { subscription.unsubscribe() } From ec9375ea41ff4176e052a2cb00fa723ef66dda72 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 16:35:47 -0600 Subject: [PATCH 218/327] test(db): generate layout swap positions --- packages/db/tests/oracle-config.ts | 1 + ...ncludes-collection-oracle.property.test.ts | 207 ++++++++++++------ 2 files changed, 139 insertions(+), 69 deletions(-) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 0402c2480..5505535c6 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -4,6 +4,7 @@ const staticOracleProperties = [ `collection-sync.reentrant-drain`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, + `includes-collection.layout-swap`, `includes-collection.optimistic-child-history`, `includes-collection.public-key-order`, `includes-collection.relationship-history`, diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 2d2084d83..c9f4da490 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -38,6 +38,30 @@ type ChildRow = { value: number } +type LayoutSwapScenario = { + length: number + swapIndex: number +} + +const exhaustiveLayoutSwapScenarios: Array = Array.from( + { length: 9 }, + (_, offset) => offset + 4, +).flatMap((length) => + Array.from({ length: length - 3 }, (_, offset) => ({ + length, + swapIndex: offset + 1, + })), +) + +const layoutSwapScenarioArbitrary: fc.Arbitrary = fc + .integer({ min: 4, max: 12 }) + .chain((length) => + fc.integer({ min: 1, max: length - 3 }).map((swapIndex) => ({ + length, + swapIndex, + })), + ) + type ProjectedChildChange = { type: `insert` | `update` | `delete` key: number @@ -133,6 +157,108 @@ function expectedMaterializations(rows: ReadonlyArray) { } } +async function expectRootAndFacadeLayoutSwap({ + length, + swapIndex, +}: LayoutSwapScenario): Promise { + type OrderedChild = ChildRow & { position: number } + const parents = createControlledCollection(`layout-swap-parents`, [ + { id: 1, group: 1 }, + ]) + const initialRows: Array = Array.from( + { length }, + (_, index) => ({ + id: index + 1, + parentGroup: 1, + value: index + 1, + position: index, + }), + ) + const children = createControlledCollection( + `layout-swap-children`, + initialRows, + ) + const root = createLiveQueryCollection((q) => + q + .from({ child: children.collection }) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ id: child.id, value: child.value })), + ) + const nested = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ id: child.id, value: child.value })), + })), + ) + let rootSubscription: { unsubscribe: () => void } | undefined + let facadeSubscription: { unsubscribe: () => void } | undefined + + try { + await Promise.all([root.preload(), nested.preload()]) + const facade = nested.get(1)!.children + const rootRevision = root._layoutRevision + const facadeRevision = facade._layoutRevision + const rootPublicationSizes: Array = [] + const facadePublicationSizes: Array = [] + const rootCallbackKeys: Array> = [] + const facadeCallbackKeys: Array> = [] + rootSubscription = root.subscribeChanges( + (changes) => { + rootPublicationSizes.push(changes.length) + rootCallbackKeys.push(root.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + facadeSubscription = facade.subscribeChanges( + (changes) => { + facadePublicationSizes.push(changes.length) + facadeCallbackKeys.push(facade.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + const expectedKeys = initialRows.map(({ id }) => id) + ;[expectedKeys[swapIndex], expectedKeys[swapIndex + 1]] = [ + expectedKeys[swapIndex + 1]!, + expectedKeys[swapIndex]!, + ] + const first = initialRows[swapIndex]! + const second = initialRows[swapIndex + 1]! + + children.writeBatch([ + { + type: `update`, + value: { ...first, position: second.position }, + }, + { + type: `update`, + value: { ...second, position: first.position }, + }, + ]) + + expect(root.toArray.map(({ id }) => id)).toEqual(expectedKeys) + expect(facade.toArray.map(({ id }) => id)).toEqual(expectedKeys) + expect(root._layoutRevision).toBe(rootRevision + 1) + expect(facade._layoutRevision).toBe(facadeRevision + 1) + expect(rootPublicationSizes).toEqual([0]) + expect(facadePublicationSizes).toEqual([0]) + expect(rootCallbackKeys).toEqual([expectedKeys]) + expect(facadeCallbackKeys).toEqual([expectedKeys]) + } finally { + rootSubscription?.unsubscribe() + facadeSubscription?.unsubscribe() + await Promise.all([ + root.cleanup(), + nested.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } +} + function createCollectionQuery( parents: Collection, children: Collection, @@ -4208,75 +4334,18 @@ describe(`Collection-valued includes oracle`, () => { }, ) - fcTest(`publishes an internal facade swap with unchanged endpoints`, async () => { - type OrderedChild = ChildRow & { position: number } - const parents = createControlledCollection(`middle-swap-parents`, [ - { id: 1, group: 1 }, - ]) - const children = createControlledCollection( - `middle-swap-children`, - [ - { id: 10, parentGroup: 1, value: 10, position: 1 }, - { id: 20, parentGroup: 1, value: 20, position: 2 }, - { id: 30, parentGroup: 1, value: 30, position: 3 }, - { id: 40, parentGroup: 1, value: 40, position: 4 }, - { id: 50, parentGroup: 1, value: 50, position: 5 }, - ], - ) - const live = createLiveQueryCollection((q) => - q.from({ parent: parents.collection }).select(({ parent }) => ({ - id: parent.id, - children: q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)) - .orderBy(({ child }) => child.position) - .select(({ child }) => ({ id: child.id, value: child.value })), - })), - ) - - try { - await live.preload() - const facade = live.get(1)!.children - const revisionBeforeSwap = facade._layoutRevision - const publicationSizes: Array = [] - const callbackKeys: Array> = [] - const subscription = facade.subscribeChanges( - (changes) => { - publicationSizes.push(changes.length) - callbackKeys.push(facade.toArray.map(({ id }) => id)) - }, - { includeInitialState: false }, - ) - - try { - children.writeBatch([ - { - type: `update`, - value: { id: 30, parentGroup: 1, value: 30, position: 4 }, - }, - { - type: `update`, - value: { id: 40, parentGroup: 1, value: 40, position: 3 }, - }, - ]) - - expect(facade.toArray.map(({ id }) => id)).toEqual([ - 10, 20, 40, 30, 50, - ]) - expect(facade._layoutRevision).toBe(revisionBeforeSwap + 1) - expect(publicationSizes).toEqual([0]) - expect(callbackKeys).toEqual([[10, 20, 40, 30, 50]]) - } finally { - subscription.unsubscribe() - } - } finally { - await Promise.all([ - live.cleanup(), - parents.collection.cleanup(), - children.collection.cleanup(), - ]) - } - }) + fcTest.prop( + [layoutSwapScenarioArbitrary], + { + ...oraclePropertyOptions(20, `includes-collection.layout-swap`), + examples: exhaustiveLayoutSwapScenarios.map( + (scenario) => [scenario] as [LayoutSwapScenario], + ), + }, + )( + `publishes every internal order-only swap through root and facade producers`, + expectRootAndFacadeLayoutSwap, + ) fcTest( `reconstructs nested conditional includes through guard transitions`, From 68366ecaeef6c12a13402b558bd4a68d7519442f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 16:36:00 -0600 Subject: [PATCH 219/327] docs: regenerate API documentation (#1790) Co-authored-by: github-actions[bot] --- docs/reference/classes/BTreeIndex.md | 34 ++++---- docs/reference/classes/BaseIndex.md | 80 +++++++++---------- docs/reference/classes/BasicIndex.md | 34 ++++---- docs/reference/interfaces/IndexInterface.md | 52 ++++++------ docs/reference/interfaces/IndexStats.md | 10 +-- .../type-aliases/IndexConstructor.md | 2 +- .../type-aliases/IndexOperation-1.md | 2 +- docs/reference/type-aliases/IndexOperation.md | 2 +- 8 files changed, 108 insertions(+), 108 deletions(-) diff --git a/docs/reference/classes/BTreeIndex.md b/docs/reference/classes/BTreeIndex.md index e62c8fd56..1f82ddaef 100644 --- a/docs/reference/classes/BTreeIndex.md +++ b/docs/reference/classes/BTreeIndex.md @@ -68,7 +68,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:56](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) #### Inherited from @@ -82,7 +82,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) #### Inherited from @@ -96,7 +96,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -113,7 +113,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) #### Inherited from @@ -127,7 +127,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) #### Inherited from @@ -141,7 +141,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) #### Inherited from @@ -155,7 +155,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) #### Inherited from @@ -183,7 +183,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:39](https://github.com/TanSt protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) #### Inherited from @@ -281,7 +281,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:452](https://github.com/TanS get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -427,7 +427,7 @@ Performs an equality lookup protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -451,7 +451,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -549,7 +549,7 @@ Performs a lookup operation matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -576,7 +576,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -602,7 +602,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -709,7 +709,7 @@ Removes a value from the index supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -891,7 +891,7 @@ The last n items protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -952,7 +952,7 @@ Updates a value in the index protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/classes/BaseIndex.md b/docs/reference/classes/BaseIndex.md index d0ae8deba..5a282c222 100644 --- a/docs/reference/classes/BaseIndex.md +++ b/docs/reference/classes/BaseIndex.md @@ -5,7 +5,7 @@ title: BaseIndex # Abstract Class: BaseIndex\ -Defined in: [packages/db/src/indexes/base-index.ts:91](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L91) +Defined in: [packages/db/src/indexes/base-index.ts:113](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L113) Base abstract class that all index types extend @@ -36,7 +36,7 @@ new BaseIndex( options?): BaseIndex; ``` -Defined in: [packages/db/src/indexes/base-index.ts:110](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L110) +Defined in: [packages/db/src/indexes/base-index.ts:132](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L132) #### Parameters @@ -68,7 +68,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:110](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) *** @@ -78,7 +78,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) *** @@ -88,7 +88,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -101,7 +101,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) *** @@ -111,7 +111,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) *** @@ -121,7 +121,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) *** @@ -131,7 +131,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) *** @@ -141,7 +141,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanSta abstract readonly supportedOperations: Set<"eq" | "gt" | "gte" | "lt" | "lte" | "in" | "like" | "ilike">; ``` -Defined in: [packages/db/src/indexes/base-index.ts:97](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L97) +Defined in: [packages/db/src/indexes/base-index.ts:119](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L119) *** @@ -151,7 +151,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:97](https://github.com/TanSta protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) ## Accessors @@ -163,7 +163,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanSt get abstract indexedKeysSet(): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:155](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L155) +Defined in: [packages/db/src/indexes/base-index.ts:177](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L177) ##### Returns @@ -183,7 +183,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:155](https://github.com/TanSt get abstract keyCount(): number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L148) +Defined in: [packages/db/src/indexes/base-index.ts:170](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L170) ##### Returns @@ -203,7 +203,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanSt get abstract orderedEntriesArray(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:153](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L153) +Defined in: [packages/db/src/indexes/base-index.ts:175](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L175) ##### Returns @@ -223,7 +223,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:153](https://github.com/TanSt get abstract orderedEntriesArrayReversed(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:154](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L154) +Defined in: [packages/db/src/indexes/base-index.ts:176](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L176) ##### Returns @@ -243,7 +243,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:154](https://github.com/TanSt get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -269,7 +269,7 @@ a full scan when this is `false`. get abstract valueMapData(): Map>; ``` -Defined in: [packages/db/src/indexes/base-index.ts:156](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L156) +Defined in: [packages/db/src/indexes/base-index.ts:178](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L178) ##### Returns @@ -287,7 +287,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:156](https://github.com/TanSt abstract add(key, item): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) +Defined in: [packages/db/src/indexes/base-index.ts:146](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L146) #### Parameters @@ -315,7 +315,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanSt abstract build(entries): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:127](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L127) +Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L149) #### Parameters @@ -339,7 +339,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:127](https://github.com/TanSt abstract clear(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:128](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L128) +Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L150) #### Returns @@ -357,7 +357,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:128](https://github.com/TanSt abstract equalityLookup(value): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L149) +Defined in: [packages/db/src/indexes/base-index.ts:171](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L171) #### Parameters @@ -381,7 +381,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanSt protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -401,7 +401,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -419,7 +419,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanSt abstract inArrayLookup(values): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L150) +Defined in: [packages/db/src/indexes/base-index.ts:172](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L172) #### Parameters @@ -443,7 +443,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanSt abstract protected initialize(options?): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:212](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L212) +Defined in: [packages/db/src/indexes/base-index.ts:244](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L244) #### Parameters @@ -463,7 +463,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:212](https://github.com/TanSt abstract lookup(operation, value): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:129](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L129) +Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L151) #### Parameters @@ -491,7 +491,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:129](https://github.com/TanSt matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -518,7 +518,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -544,7 +544,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -568,7 +568,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanSt abstract rangeQuery(options): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L151) +Defined in: [packages/db/src/indexes/base-index.ts:173](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L173) #### Parameters @@ -592,7 +592,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanSt abstract rangeQueryReversed(options): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L152) +Defined in: [packages/db/src/indexes/base-index.ts:174](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L174) #### Parameters @@ -616,7 +616,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanSt abstract remove(key, item): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:125](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L125) +Defined in: [packages/db/src/indexes/base-index.ts:147](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L147) #### Parameters @@ -644,7 +644,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:125](https://github.com/TanSt supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -671,7 +671,7 @@ abstract take( filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) +Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L152) #### Parameters @@ -703,7 +703,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanSt abstract takeFromStart(n, filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:135](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L135) +Defined in: [packages/db/src/indexes/base-index.ts:157](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L157) #### Parameters @@ -734,7 +734,7 @@ abstract takeReversed( filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:139](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L139) +Defined in: [packages/db/src/indexes/base-index.ts:161](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L161) #### Parameters @@ -766,7 +766,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:139](https://github.com/TanSt abstract takeReversedFromEnd(n, filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:144](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L144) +Defined in: [packages/db/src/indexes/base-index.ts:166](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L166) #### Parameters @@ -794,7 +794,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:144](https://github.com/TanSt protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -817,7 +817,7 @@ abstract update( newItem): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:126](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L126) +Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L148) #### Parameters @@ -849,7 +849,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:126](https://github.com/TanSt protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/classes/BasicIndex.md b/docs/reference/classes/BasicIndex.md index 4ca10dfd5..5100dd653 100644 --- a/docs/reference/classes/BasicIndex.md +++ b/docs/reference/classes/BasicIndex.md @@ -74,7 +74,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:64](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) #### Inherited from @@ -88,7 +88,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) #### Inherited from @@ -102,7 +102,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -119,7 +119,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) #### Inherited from @@ -133,7 +133,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) #### Inherited from @@ -147,7 +147,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) #### Inherited from @@ -161,7 +161,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) #### Inherited from @@ -189,7 +189,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:46](https://github.com/TanSt protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) #### Inherited from @@ -287,7 +287,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:530](https://github.com/TanS get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -433,7 +433,7 @@ Performs an equality lookup - O(1) protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -457,7 +457,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -555,7 +555,7 @@ Performs a lookup operation matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -582,7 +582,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -608,7 +608,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -714,7 +714,7 @@ Removes a value from the index supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -872,7 +872,7 @@ Returns the first n items in reverse sorted order (from the end) protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -933,7 +933,7 @@ Updates a value in the index protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/interfaces/IndexInterface.md b/docs/reference/interfaces/IndexInterface.md index 48c8c6bdf..415f73996 100644 --- a/docs/reference/interfaces/IndexInterface.md +++ b/docs/reference/interfaces/IndexInterface.md @@ -5,7 +5,7 @@ title: IndexInterface # Interface: IndexInterface\ -Defined in: [packages/db/src/indexes/base-index.ts:29](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L29) +Defined in: [packages/db/src/indexes/base-index.ts:51](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L51) ## Type Parameters @@ -21,7 +21,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:29](https://github.com/TanSta add: (key, item) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:32](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L32) +Defined in: [packages/db/src/indexes/base-index.ts:54](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L54) #### Parameters @@ -45,7 +45,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:32](https://github.com/TanSta build: (entries) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:36](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L36) +Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L58) #### Parameters @@ -65,7 +65,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:36](https://github.com/TanSta clear: () => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:37](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L37) +Defined in: [packages/db/src/indexes/base-index.ts:59](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L59) #### Returns @@ -79,7 +79,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:37](https://github.com/TanSta equalityLookup: (value) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:41](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L41) +Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L63) #### Parameters @@ -99,7 +99,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:41](https://github.com/TanSta getStats: () => IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L85) +Defined in: [packages/db/src/indexes/base-index.ts:107](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L107) #### Returns @@ -113,7 +113,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanSta inArrayLookup: (values) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:42](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L42) +Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L64) #### Parameters @@ -133,7 +133,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:42](https://github.com/TanSta lookup: (operation, value) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:39](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L39) +Defined in: [packages/db/src/indexes/base-index.ts:61](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L61) #### Parameters @@ -157,7 +157,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:39](https://github.com/TanSta matchesCompareOptions: (compareOptions) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:82](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L82) +Defined in: [packages/db/src/indexes/base-index.ts:104](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L104) #### Parameters @@ -177,7 +177,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:82](https://github.com/TanSta matchesDirection: (direction) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:83](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L83) +Defined in: [packages/db/src/indexes/base-index.ts:105](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L105) #### Parameters @@ -197,7 +197,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:83](https://github.com/TanSta matchesField: (fieldPath) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:81](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L81) +Defined in: [packages/db/src/indexes/base-index.ts:103](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L103) #### Parameters @@ -217,7 +217,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:81](https://github.com/TanSta rangeQuery: (options) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L44) +Defined in: [packages/db/src/indexes/base-index.ts:66](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L66) #### Parameters @@ -237,7 +237,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanSta rangeQueryReversed: (options) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L45) +Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L67) #### Parameters @@ -257,7 +257,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanSta remove: (key, item) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:33](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L33) +Defined in: [packages/db/src/indexes/base-index.ts:55](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L55) #### Parameters @@ -281,7 +281,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:33](https://github.com/TanSta supports: (operation) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:70](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L70) +Defined in: [packages/db/src/indexes/base-index.ts:92](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L92) #### Parameters @@ -301,7 +301,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:70](https://github.com/TanSta take: (n, from, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L47) +Defined in: [packages/db/src/indexes/base-index.ts:69](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L69) #### Parameters @@ -329,7 +329,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanSta takeFromStart: (n, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:52](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L52) +Defined in: [packages/db/src/indexes/base-index.ts:74](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L74) #### Parameters @@ -353,7 +353,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:52](https://github.com/TanSta takeReversed: (n, from, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:53](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L53) +Defined in: [packages/db/src/indexes/base-index.ts:75](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L75) #### Parameters @@ -381,7 +381,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:53](https://github.com/TanSta takeReversedFromEnd: (n, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L58) +Defined in: [packages/db/src/indexes/base-index.ts:80](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L80) #### Parameters @@ -405,7 +405,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanSta update: (key, oldItem, newItem) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) +Defined in: [packages/db/src/indexes/base-index.ts:56](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L56) #### Parameters @@ -435,7 +435,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanSta get indexedKeysSet(): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L67) +Defined in: [packages/db/src/indexes/base-index.ts:89](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L89) ##### Returns @@ -451,7 +451,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanSta get keyCount(): number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L63) +Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L85) ##### Returns @@ -467,7 +467,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanSta get orderedEntriesArray(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L64) +Defined in: [packages/db/src/indexes/base-index.ts:86](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L86) ##### Returns @@ -483,7 +483,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanSta get orderedEntriesArrayReversed(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:65](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L65) +Defined in: [packages/db/src/indexes/base-index.ts:87](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L87) ##### Returns @@ -499,7 +499,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:65](https://github.com/TanSta get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:79](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L79) +Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -521,7 +521,7 @@ a full scan when this is `false`. get valueMapData(): Map>; ``` -Defined in: [packages/db/src/indexes/base-index.ts:68](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L68) +Defined in: [packages/db/src/indexes/base-index.ts:90](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L90) ##### Returns diff --git a/docs/reference/interfaces/IndexStats.md b/docs/reference/interfaces/IndexStats.md index 7852397f0..2fec3b770 100644 --- a/docs/reference/interfaces/IndexStats.md +++ b/docs/reference/interfaces/IndexStats.md @@ -5,7 +5,7 @@ title: IndexStats # Interface: IndexStats -Defined in: [packages/db/src/indexes/base-index.ts:22](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L22) +Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L44) Statistics about index usage and performance @@ -17,7 +17,7 @@ Statistics about index usage and performance readonly averageLookupTime: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:25](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L25) +Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L47) *** @@ -27,7 +27,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:25](https://github.com/TanSta readonly entryCount: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:23](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L23) +Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L45) *** @@ -37,7 +37,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:23](https://github.com/TanSta readonly lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:26](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L26) +Defined in: [packages/db/src/indexes/base-index.ts:48](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L48) *** @@ -47,4 +47,4 @@ Defined in: [packages/db/src/indexes/base-index.ts:26](https://github.com/TanSta readonly lookupCount: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:24](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L24) +Defined in: [packages/db/src/indexes/base-index.ts:46](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L46) diff --git a/docs/reference/type-aliases/IndexConstructor.md b/docs/reference/type-aliases/IndexConstructor.md index e3ec6a293..4e0b4bbda 100644 --- a/docs/reference/type-aliases/IndexConstructor.md +++ b/docs/reference/type-aliases/IndexConstructor.md @@ -9,7 +9,7 @@ title: IndexConstructor type IndexConstructor = (id, expression, name?, options?) => BaseIndex; ``` -Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) +Defined in: [packages/db/src/indexes/base-index.ts:266](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L266) Type for index constructor diff --git a/docs/reference/type-aliases/IndexOperation-1.md b/docs/reference/type-aliases/IndexOperation-1.md index 28b8a867c..025823309 100644 --- a/docs/reference/type-aliases/IndexOperation-1.md +++ b/docs/reference/type-aliases/IndexOperation-1.md @@ -9,6 +9,6 @@ title: IndexOperation type IndexOperation = typeof comparisonFunctions[number]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:12](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L12) +Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) Type for index operation values diff --git a/docs/reference/type-aliases/IndexOperation.md b/docs/reference/type-aliases/IndexOperation.md index 6816e9788..42c757f78 100644 --- a/docs/reference/type-aliases/IndexOperation.md +++ b/docs/reference/type-aliases/IndexOperation.md @@ -9,6 +9,6 @@ title: IndexOperation type IndexOperation = readonly ["eq", "gt", "gte", "lt", "lte", "in", "like", "ilike"]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:12](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L12) +Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) Operations that indexes can support, imported from available comparison functions From 657e4a10f95e883a4801bdafc585aa393123a1f3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 16:49:10 -0600 Subject: [PATCH 220/327] test(db): enforce exhaustive layout cells --- ...ncludes-collection-oracle.property.test.ts | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index c9f4da490..afe638f73 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -4334,16 +4334,29 @@ describe(`Collection-valued includes oracle`, () => { }, ) - fcTest.prop( - [layoutSwapScenarioArbitrary], - { - ...oraclePropertyOptions(20, `includes-collection.layout-swap`), - examples: exhaustiveLayoutSwapScenarios.map( - (scenario) => [scenario] as [LayoutSwapScenario], - ), + fcTest( + `publishes every bounded internal order-only swap through root and facade producers`, + async () => { + const observedCells: Array = [] + for (let length = 4; length <= 12; length++) { + for (let swapIndex = 1; swapIndex <= length - 3; swapIndex++) { + await expectRootAndFacadeLayoutSwap({ length, swapIndex }) + observedCells.push(`${length}:${swapIndex}`) + } + } + + expect(observedCells).toEqual( + exhaustiveLayoutSwapScenarios.map( + ({ length, swapIndex }) => `${length}:${swapIndex}`, + ), + ) }, - )( - `publishes every internal order-only swap through root and facade producers`, + ) + + fcTest.prop([layoutSwapScenarioArbitrary], { + ...oraclePropertyOptions(20, `includes-collection.layout-swap`), + })( + `publishes replayable random internal order-only swaps through root and facade producers`, expectRootAndFacadeLayoutSwap, ) From d71c80a8f6790f83b3b539bfb3ba98e6e60b08c8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 16:59:01 -0600 Subject: [PATCH 221/327] test(db): scan all facade layout candidates --- ...ncludes-collection-oracle.property.test.ts | 135 ++++++++++++++++++ 1 file changed, 135 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index afe638f73..a5a9af300 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -43,6 +43,11 @@ type LayoutSwapScenario = { swapIndex: number } +type FacadeCandidateScanScenario = { + candidatePosition: `first` | `last` + finalLayout: `moved` | `restored` +} + const exhaustiveLayoutSwapScenarios: Array = Array.from( { length: 9 }, (_, offset) => offset + 4, @@ -62,6 +67,14 @@ const layoutSwapScenarioArbitrary: fc.Arbitrary = fc })), ) +const facadeCandidateScanScenarios: ReadonlyArray = + [ + { candidatePosition: `first`, finalLayout: `moved` }, + { candidatePosition: `first`, finalLayout: `restored` }, + { candidatePosition: `last`, finalLayout: `moved` }, + { candidatePosition: `last`, finalLayout: `restored` }, + ] + type ProjectedChildChange = { type: `insert` | `update` | `delete` key: number @@ -259,6 +272,108 @@ async function expectRootAndFacadeLayoutSwap({ } } +async function expectFacadeCandidateScan({ + candidatePosition, + finalLayout, +}: FacadeCandidateScanScenario): Promise { + type OrderedChild = ChildRow & { position: number } + const parents = createControlledCollection(`candidate-scan-parents`, [ + { id: 1, group: 1 }, + ]) + const initialRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + { id: 30, parentGroup: 1, value: 30, position: 2 }, + ] + const children = createControlledCollection( + `candidate-scan-children`, + initialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + let subscription: { unsubscribe: () => void } | undefined + + try { + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const callbackValues: Array> = [] + subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const revision = facade._layoutRevision + const valueUpdate = { + type: `update` as const, + value: { ...initialRows[0]!, value: 11 }, + } + const orderUpdates = [ + { + type: `update` as const, + value: { ...initialRows[1]!, position: 3 }, + }, + ...(finalLayout === `restored` + ? [ + { + type: `update` as const, + value: initialRows[1]!, + }, + ] + : []), + ] + + children.writeBatch( + candidatePosition === `first` + ? [...orderUpdates, valueUpdate] + : [valueUpdate, ...orderUpdates], + ) + + const expectedKeys = + finalLayout === `moved` ? [10, 30, 20] : [10, 20, 30] + const expectedValues = + finalLayout === `moved` ? [11, 30, 20] : [11, 20, 30] + expect(keys()).toEqual(expectedKeys) + expect(values()).toEqual(expectedValues) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: 11, + previousValue: 10, + }, + ], + ]) + expect(callbackKeys).toEqual([expectedKeys]) + expect(callbackValues).toEqual([expectedValues]) + expect(facade._layoutRevision).toBe( + revision + (finalLayout === `moved` ? 1 : 0), + ) + } finally { + subscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } +} + function createCollectionQuery( parents: Collection, children: Collection, @@ -4360,6 +4475,26 @@ describe(`Collection-valued includes oracle`, () => { expectRootAndFacadeLayoutSwap, ) + fcTest( + `scans every changed facade key before deciding whether layout may differ`, + async () => { + const observedScenarios: Array = [] + for (const candidatePosition of [`first`, `last`] as const) { + for (const finalLayout of [`moved`, `restored`] as const) { + await expectFacadeCandidateScan({ candidatePosition, finalLayout }) + observedScenarios.push(`${candidatePosition}:${finalLayout}`) + } + } + + expect(observedScenarios).toEqual( + facadeCandidateScanScenarios.map( + ({ candidatePosition, finalLayout }) => + `${candidatePosition}:${finalLayout}`, + ), + ) + }, + ) + fcTest( `reconstructs nested conditional includes through guard transitions`, async () => { From f8dfa40ae7247502a702a05521780888845b7ae9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Mon, 31 Aug 2026 17:11:18 -0600 Subject: [PATCH 222/327] test(db): prove facade candidate positions --- ...ncludes-collection-oracle.property.test.ts | 51 +++++++++++++++---- 1 file changed, 42 insertions(+), 9 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index a5a9af300..b09f4122e 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -300,6 +300,7 @@ async function expectFacadeCandidateScan({ })), ) let subscription: { unsubscribe: () => void } | undefined + let restoreFacadeGetKey: (() => void) | undefined try { await live.preload() @@ -318,20 +319,37 @@ async function expectFacadeCandidateScan({ { includeInitialState: false }, ) const revision = facade._layoutRevision + const candidateRow = initialRows[candidatePosition === `first` ? 0 : 1]! + const valueRow = initialRows[candidatePosition === `first` ? 1 : 0]! + const changedKeyOrder: Array = [] + const originalGetKey = facade.config.getKey + facade.config.getKey = (row) => { + const key = Number(originalGetKey(row)) + if ( + (key === candidateRow.id || key === valueRow.id) && + !changedKeyOrder.includes(key) + ) { + changedKeyOrder.push(key) + } + return key + } + restoreFacadeGetKey = () => { + facade.config.getKey = originalGetKey + } const valueUpdate = { type: `update` as const, - value: { ...initialRows[0]!, value: 11 }, + value: { ...valueRow, value: valueRow.value + 1 }, } const orderUpdates = [ { type: `update` as const, - value: { ...initialRows[1]!, position: 3 }, + value: { ...candidateRow, position: 3 }, }, ...(finalLayout === `restored` ? [ { type: `update` as const, - value: initialRows[1]!, + value: candidateRow, }, ] : []), @@ -344,18 +362,32 @@ async function expectFacadeCandidateScan({ ) const expectedKeys = - finalLayout === `moved` ? [10, 30, 20] : [10, 20, 30] - const expectedValues = - finalLayout === `moved` ? [11, 30, 20] : [11, 20, 30] + finalLayout === `moved` + ? initialRows + .filter(({ id }) => id !== candidateRow.id) + .map(({ id }) => id) + .concat(candidateRow.id) + : initialRows.map(({ id }) => id) + const expectedValues = expectedKeys.map((id) => + id === valueRow.id ? valueRow.value + 1 : id, + ) + if (finalLayout === `moved`) { + expect(changedKeyOrder).toEqual([10, 20]) + expect( + changedKeyOrder[candidatePosition === `first` ? 0 : 1], + ).toBe(candidateRow.id) + } else { + expect(changedKeyOrder).toEqual([valueRow.id]) + } expect(keys()).toEqual(expectedKeys) expect(values()).toEqual(expectedValues) expect(publications).toEqual([ [ { type: `update`, - key: 10, - value: 11, - previousValue: 10, + key: valueRow.id, + value: valueRow.value + 1, + previousValue: valueRow.value, }, ], ]) @@ -365,6 +397,7 @@ async function expectFacadeCandidateScan({ revision + (finalLayout === `moved` ? 1 : 0), ) } finally { + restoreFacadeGetKey?.() subscription?.unsubscribe() await Promise.all([ live.cleanup(), From 847dae755040bbb4482beb9d9c8ecbfcad3063b4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 08:55:39 -0600 Subject: [PATCH 223/327] fix(db): retire confirmed direct mutations --- packages/db/src/collection/state.ts | 15 ++ .../query/includes-publication-oracle.test.ts | 138 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 15d55503c..7e55a30fa 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -527,6 +527,21 @@ export class CollectionStateManager< if (!this.isThisCollection(mutation.collection)) { continue } + + // Direct mutation handlers may publish their authoritative echo + // before the transaction completes. That sync commit consumes the + // pending-origin marker. Do not recreate optimistic state after the + // same mutation has already been confirmed. + const wasConfirmedDuringDirectMutation = + isDirectTransaction && !this.pendingLocalOrigins.has(mutation.key) + if (wasConfirmedDuringDirectMutation) { + this.pendingOptimisticUpserts.delete(mutation.key) + this.pendingOptimisticDeletes.delete(mutation.key) + this.pendingOptimisticDirectUpserts.delete(mutation.key) + this.pendingOptimisticDirectDeletes.delete(mutation.key) + continue + } + this.pendingLocalOrigins.add(mutation.key) if (!mutation.optimistic) { continue diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 38f4499f2..135b0d320 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -1,5 +1,6 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createOptimisticAction } from '../../src/optimistic-action.js' @@ -13,6 +14,7 @@ import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, withExpectedRejection } from '../utils.js' import { createControlledCollection } from './includes-oracle-helpers.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { SyncConfig } from '../../src/types.js' type ParentRow = { id: number @@ -41,6 +43,7 @@ type Q2Shape = `passThrough` | `where` | `orderBy` | `select` type Q1Shape = `direct` | `joined` type PendingPublicationOperation = `insert` | `update` | `delete` type PendingPublicationDepth = `direct` | `layered` +type SourceConfirmationOperation = `insert` | `update` | `delete` type PendingPublicationRow = { id: number @@ -744,6 +747,141 @@ describe(`layered-query publication oracle`, () => { }) describe(`source publication across pending derived mutations`, () => { + async function expectSourceConfirmationPreservesGraphIntegrity( + operation: SourceConfirmationOperation, + depth: PendingPublicationDepth, + ) { + type Row = { id: number; value: number } + let sync!: Parameters[`sync`]>[0] + + const commitSync = async () => { + const receipt = sync.commit() + if (receipt !== true) await receipt + } + + const source = createCollection({ + id: `same-key-source-confirmation-${nextCollectionId++}`, + getKey: (row) => row.id, + sync: { + sync: (config) => { + sync = config + config.markReady() + }, + }, + onInsert: async ({ transaction }) => { + sync.begin() + sync.write({ + type: `insert`, + value: transaction.mutations[0].modified, + }) + await commitSync() + }, + onUpdate: async ({ transaction }) => { + sync.begin() + sync.write({ + type: `update`, + value: transaction.mutations[0].modified, + }) + await commitSync() + }, + onDelete: async ({ transaction }) => { + sync.begin() + sync.write({ + type: `delete`, + key: transaction.mutations[0].key, + }) + await commitSync() + }, + }) + await source.preload() + + if (operation !== `insert`) { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 0 } }) + await commitSync() + } + + const q1 = createLiveQueryCollection({ + id: `same-key-source-confirmation-query-${nextCollectionId++}`, + query: (q) => + q.from({ row: source }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + const q2 = + depth === `layered` + ? createLiveQueryCollection({ + id: `same-key-source-confirmation-layer-${nextCollectionId++}`, + query: (q) => + q.from({ row: q1 }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + : undefined + const query = q2 ?? q1 + const subscription = query.subscribeChanges(() => {}) + + try { + await query.preload() + + const firstTransaction = (() => { + switch (operation) { + case `insert`: + return source.insert({ id: 1, value: 1 }) + case `update`: + return source.update(1, (draft) => { + draft.value = 1 + }) + case `delete`: + return source.delete(1) + } + })() + await firstTransaction.isPersisted.promise + + if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(1) + } + + const probeTransaction = + operation === `delete` + ? source.insert({ id: 1, value: 2 }) + : source.update(1, (draft) => { + draft.value = 2 + }) + await probeTransaction.isPersisted.promise + + expect(source.get(1)?.value).toBe(2) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(2) + } finally { + subscription.unsubscribe() + if (q2) await q2.cleanup() + await q1.cleanup() + await source.cleanup() + } + } + + for (const depth of pendingPublicationDepths) { + for (const operation of [ + `insert`, + `update`, + `delete`, + ] as const satisfies ReadonlyArray) { + it(`preserves ${depth} graph integrity after sync confirms a same-key optimistic ${operation}`, async () => { + await expectSourceConfirmationPreservesGraphIntegrity(operation, depth) + }) + } + } + for (const depth of pendingPublicationDepths) { for (const optimisticOperation of pendingPublicationOperations) { for (const sourceOperation of pendingPublicationOperations) { From 96f304d09bd75a51c4e746acf1441cd0ff69f800 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 09:22:58 -0600 Subject: [PATCH 224/327] fix(db): preserve mutations across replacement sync --- packages/db/src/collection/state.ts | 124 +++++++++----- .../query/includes-publication-oracle.test.ts | 153 ++++++++++++++++-- 2 files changed, 222 insertions(+), 55 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 7e55a30fa..32b02f5ac 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1103,6 +1103,12 @@ export class CollectionStateManager< this.hydrationSeedKeys.clear() this.hydratedKeys.clear() this.clearOriginTrackingState() + for (const key of truncatePendingLocalChanges) { + this.pendingLocalChanges.add(key) + } + for (const key of truncatePendingLocalOrigins) { + this.pendingLocalOrigins.add(key) + } // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations // are compared against the post-truncate state (undefined) rather than pre-truncate state @@ -1137,13 +1143,17 @@ export class CollectionStateManager< case `insert`: this.syncedData.set(key, operation.value) this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + // Ordinary same-key sync confirms pending local work. A full + // replacement has no causal link to that mutation and must + // preserve its optimistic overlay until a later echo arrives. + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break case `update`: { if (rowUpdateMode === `partial`) { @@ -1157,26 +1167,28 @@ export class CollectionStateManager< this.syncedData.set(key, operation.value) } this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break } case `delete`: this.syncedData.delete(key) this.syncedMetadata.delete(key) - // Clean up origin and pending tracking for deleted rows this.rowOrigins.delete(key) - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break } recordRequestProvenance(key, transaction.requestSignal) @@ -1292,6 +1304,23 @@ export class CollectionStateManager< } } + // An ordinary sync for another key must not create a hidden interval + // where a completed direct mutation disappears. Same-key confirmation + // removed these pending entries above; everything left is still an + // authoritative optimistic overlay for this publication turn. + for (const [key, value] of this.pendingOptimisticUpserts) { + if (this.pendingOptimisticDirectUpserts.has(key)) { + this.optimisticUpserts.set(key, value) + this.optimisticDeletes.delete(key) + } + } + for (const key of this.pendingOptimisticDeletes) { + if (this.pendingOptimisticDirectDeletes.has(key)) { + this.optimisticUpserts.delete(key) + this.optimisticDeletes.add(key) + } + } + // Always overlay any still-active optimistic transactions so mutations that started // after the truncate snapshot are preserved. for (const transaction of this.transactions.values()) { @@ -1321,34 +1350,41 @@ export class CollectionStateManager< } } - // A completed optimistic insert may have used a temporary client key while - // the sync confirmation used a different server-generated key. Once a - // sync commit has been applied, stop retaining completed optimistic keys - // that were not confirmed by this commit so the temporary row is removed. - for (const key of this.pendingOptimisticDirectUpserts) { - if (!changedKeys.has(key)) { - changedKeys.add(key) - if (!currentVisibleState.has(key)) { - const previousValue = previousOptimisticUpserts.get(key) - if (previousValue !== undefined) { - currentVisibleState.set(key, previousValue) + if ( + committedSyncedTransactions.some((transaction) => !transaction.truncate) + ) { + // A completed optimistic insert may have used a temporary client key while + // the sync confirmation used a different server-generated key. Once an + // ordinary sync commit has been applied, stop retaining completed + // optimistic keys that were not confirmed by this commit so the temporary + // row is removed. Truncate replacement is not confirmation. + for (const key of this.pendingOptimisticDirectUpserts) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + if (!currentVisibleState.has(key)) { + const previousValue = previousOptimisticUpserts.get(key) + if (previousValue !== undefined) { + currentVisibleState.set(key, previousValue) + } } + this.pendingOptimisticUpserts.delete(key) + this.pendingLocalOrigins.delete(key) + this.optimisticUpserts.delete(key) } - this.pendingOptimisticUpserts.delete(key) - this.pendingLocalOrigins.delete(key) + requestProvenanceByKey.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) } - requestProvenanceByKey.delete(key) - } - for (const key of this.pendingOptimisticDirectDeletes) { - if (!changedKeys.has(key)) { - changedKeys.add(key) + for (const key of this.pendingOptimisticDirectDeletes) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + } + this.pendingOptimisticDeletes.delete(key) + this.pendingLocalOrigins.delete(key) + this.optimisticDeletes.delete(key) + requestProvenanceByKey.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) } - this.pendingOptimisticDeletes.delete(key) - this.pendingLocalOrigins.delete(key) - requestProvenanceByKey.delete(key) } - this.pendingOptimisticDirectUpserts.clear() - this.pendingOptimisticDirectDeletes.clear() // Now check what actually changed in the final visible state for (const key of changedKeys) { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 135b0d320..fe91a6d08 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -44,6 +44,11 @@ type Q1Shape = `direct` | `joined` type PendingPublicationOperation = `insert` | `update` | `delete` type PendingPublicationDepth = `direct` | `layered` type SourceConfirmationOperation = `insert` | `update` | `delete` +type SourceConfirmationInterleaving = + | `handlerEcho` + | `replacementWhilePending` + | `replacementAfterSuccess` +type SourceConfirmationSettlement = `succeeds` | `rejects` type PendingPublicationRow = { id: number @@ -750,9 +755,17 @@ describe(`source publication across pending derived mutations`, () => { async function expectSourceConfirmationPreservesGraphIntegrity( operation: SourceConfirmationOperation, depth: PendingPublicationDepth, + interleaving: SourceConfirmationInterleaving, + settlement: SourceConfirmationSettlement, ) { type Row = { id: number; value: number } let sync!: Parameters[`sync`]>[0] + const handlerCanFinish = createDeferred() + let echoFromHandler = interleaving === `handlerEcho` + let handlerFailure = + settlement === `rejects` + ? new Error(`source confirmation handler rejection`) + : undefined const commitSync = async () => { const receipt = sync.commit() @@ -769,28 +782,52 @@ describe(`source publication across pending derived mutations`, () => { }, }, onInsert: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } sync.begin() sync.write({ type: `insert`, value: transaction.mutations[0].modified, }) await commitSync() + if (handlerFailure) throw handlerFailure }, onUpdate: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } sync.begin() sync.write({ type: `update`, value: transaction.mutations[0].modified, }) await commitSync() + if (handlerFailure) throw handlerFailure }, onDelete: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } sync.begin() sync.write({ type: `delete`, key: transaction.mutations[0].key, }) await commitSync() + if (handlerFailure) throw handlerFailure }, }) await source.preload() @@ -840,9 +877,78 @@ describe(`source publication across pending derived mutations`, () => { return source.delete(1) } })() - await firstTransaction.isPersisted.promise - if (operation === `delete`) { + if (interleaving === `replacementWhilePending`) { + sync.begin() + sync.truncate() + sync.write({ type: `insert`, value: { id: 1, value: 99 } }) + await commitSync() + + if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(false) + expect(query.get(1)?.value).toBe(1) + } + + handlerCanFinish.resolve() + } + if (handlerFailure) { + await expect(firstTransaction.isPersisted.promise).rejects.toBe( + handlerFailure, + ) + handlerFailure = undefined + } else { + await firstTransaction.isPersisted.promise + } + + if (interleaving === `replacementAfterSuccess`) { + sync.begin() + sync.truncate() + sync.write({ type: `insert`, value: { id: 1, value: 99 } }) + await commitSync() + } + + if (interleaving !== `handlerEcho` && settlement === `succeeds`) { + if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(false) + expect(query.get(1)?.value).toBe(1) + } + + echoFromHandler = true + await source.insert({ id: 2, value: 2 }).isPersisted.promise + + // A replacement is not confirmation, so the optimistic value survives + // it. Once persistence has succeeded, however, the next ordinary sync + // drain retires an unconfirmed direct overlay and reveals the base. + expect(source.get(1)?.value).toBe(99) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(99) + + sync.begin() + if (operation === `delete`) { + sync.write({ type: `delete`, key: 1 }) + } else { + sync.write({ type: `update`, value: { id: 1, value: 1 } }) + } + await commitSync() + } + + if ( + interleaving === `replacementWhilePending` && + settlement === `rejects` + ) { + expect(source.get(1)?.value).toBe(99) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(99) + echoFromHandler = true + } else if (operation === `delete`) { expect(source.get(1)).toBeUndefined() expect(query.get(1)).toBeUndefined() } else { @@ -852,7 +958,10 @@ describe(`source publication across pending derived mutations`, () => { } const probeTransaction = - operation === `delete` + operation === `delete` && + !( + interleaving === `replacementWhilePending` && settlement === `rejects` + ) ? source.insert({ id: 1, value: 2 }) : source.update(1, (draft) => { draft.value = 2 @@ -871,14 +980,36 @@ describe(`source publication across pending derived mutations`, () => { } for (const depth of pendingPublicationDepths) { - for (const operation of [ - `insert`, - `update`, - `delete`, - ] as const satisfies ReadonlyArray) { - it(`preserves ${depth} graph integrity after sync confirms a same-key optimistic ${operation}`, async () => { - await expectSourceConfirmationPreservesGraphIntegrity(operation, depth) - }) + for (const interleaving of [ + `handlerEcho`, + `replacementWhilePending`, + `replacementAfterSuccess`, + ] as const satisfies ReadonlyArray) { + for (const settlement of [ + `succeeds`, + `rejects`, + ] as const satisfies ReadonlyArray) { + if ( + interleaving === `replacementAfterSuccess` && + settlement === `rejects` + ) { + continue + } + for (const operation of [ + `insert`, + `update`, + `delete`, + ] as const satisfies ReadonlyArray) { + it(`preserves ${depth} graph integrity after a same-key optimistic ${operation} with ${interleaving} that ${settlement}`, async () => { + await expectSourceConfirmationPreservesGraphIntegrity( + operation, + depth, + interleaving, + settlement, + ) + }) + } + } } } From 4a17f68d0f9cadd23292a183426307c316e7a13a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 09:42:48 -0600 Subject: [PATCH 225/327] test(db): pin replacement publication trace --- .../query/includes-publication-oracle.test.ts | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index fe91a6d08..108ca18da 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -860,7 +860,26 @@ describe(`source publication across pending derived mutations`, () => { }) : undefined const query = q2 ?? q1 - const subscription = query.subscribeChanges(() => {}) + const sourceEvents: Array<{ type: string; key: number; value?: number }> = [] + const queryEvents: Array<{ type: string; key: number; value?: number }> = [] + const sourceSubscription = source.subscribeChanges((changes) => { + sourceEvents.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + value: change.value.value, + })), + ) + }) + const subscription = query.subscribeChanges((changes) => { + queryEvents.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + value: change.value.value, + })), + ) + }) try { await query.preload() @@ -877,6 +896,8 @@ describe(`source publication across pending derived mutations`, () => { return source.delete(1) } })() + sourceEvents.length = 0 + queryEvents.length = 0 if (interleaving === `replacementWhilePending`) { sync.begin() @@ -892,6 +913,15 @@ describe(`source publication across pending derived mutations`, () => { expect(source.get(1)?.$synced).toBe(false) expect(query.get(1)?.value).toBe(1) } + expect(sourceEvents).toEqual( + operation === `delete` + ? [] + : [ + { type: `delete`, key: 1, value: 1 }, + { type: `insert`, key: 1, value: 1 }, + ], + ) + expect(queryEvents).toEqual([]) handlerCanFinish.resolve() } @@ -973,6 +1003,7 @@ describe(`source publication across pending derived mutations`, () => { expect(query.get(1)?.value).toBe(2) } finally { subscription.unsubscribe() + sourceSubscription.unsubscribe() if (q2) await q2.cleanup() await q1.cleanup() await source.cleanup() From 7e7b6c131b0439d2bebc59257d24616e26d1017d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 09:54:29 -0600 Subject: [PATCH 226/327] fix(db): roll back failed root publications --- packages/db/src/collection/index.ts | 19 ++ packages/db/src/collection/indexes.ts | 7 + packages/db/src/collection/state.ts | 176 ++++++++++++++++++ .../query/live/collection-config-builder.ts | 10 + ...ncludes-collection-oracle.property.test.ts | 33 +++- .../query/includes-publication-oracle.test.ts | 12 +- 6 files changed, 247 insertions(+), 10 deletions(-) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 7b61a1350..319f7daea 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -14,6 +14,7 @@ import { CollectionIndexesManager } from './indexes' import { CollectionMutationsManager } from './mutations' import { CollectionEventsManager } from './events.js' import type { PublicationDeferral } from './changes' +import type { CollectionPublicationStateSnapshot } from './state' import type { CollectionSubscription } from './subscription' import type { AllCollectionEvents, @@ -454,6 +455,24 @@ export class CollectionImpl< return this._changes.deferPublication() } + /** Capture mutable state before a coherent graph publication is installed. */ + public _snapshotPublicationState( + keys: Iterable, + ): CollectionPublicationStateSnapshot< + TOutput, + TKey + > { + return this._state.snapshotPublicationState(keys) + } + + /** Restore a failed coherent graph publication and rebuild its indexes. */ + public _restorePublicationState( + snapshot: CollectionPublicationStateSnapshot, + ): void { + this._state.restorePublicationState(snapshot) + this._indexes.rebuildIndexes() + } + /** * Register a callback to be executed when the collection first becomes ready * Useful for preloading collections diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index 84e45d6fc..e6bdc1e2e 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -369,6 +369,13 @@ export class CollectionIndexesManager< } } + /** Rebuild every retained index from the Collection's current state. */ + public rebuildIndexes(): void { + for (const index of this.indexes.values()) { + index.build(this.state.entries()) + } + } + /** * Clean up indexes */ diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 32b02f5ac..6cd7c88b8 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -28,6 +28,33 @@ import type { CollectionIndexesManager } from './indexes' import type { CollectionEventsManager } from './events' import type { Deferred } from '../deferred' +function replaceMap( + target: { clear: () => void; set: (key: K, value: V) => unknown }, + entries: Iterable, +): void { + target.clear() + for (const [key, value] of entries) target.set(key, value) +} + +function replaceSet(target: Set, values: Iterable): void { + target.clear() + for (const value of values) target.add(value) +} + +function restoreMapEntry( + target: { set: (key: K, value: V) => unknown; delete: (key: K) => unknown }, + key: K, + entry: { present: boolean; value: V | undefined }, +): void { + if (entry.present) target.set(key, entry.value as V) + else target.delete(key) +} + +function restoreSetEntry(target: Set, value: T, present: boolean): void { + if (present) target.add(value) + else target.delete(value) +} + interface PendingSyncedTransaction< T extends object = Record, TKey extends string | number = string | number, @@ -57,6 +84,39 @@ interface PendingSyncedTransaction< immediate?: boolean } +export type CollectionPublicationStateSnapshot< + TOutput extends object, + TKey extends string | number, +> = { + pendingSyncedTransactions: Array> + applicationStarted: Map, boolean> + keys: Map< + TKey, + { + syncedData: { present: boolean; value: TOutput | undefined } + syncedMetadata: { present: boolean; value: unknown } + rowOrigin: { present: boolean; value: VirtualOrigin | undefined } + hydrationSeed: boolean + hydrated: boolean + synced: boolean + } + > + syncedCollectionMetadata: Array<[string, unknown]> + optimisticUpserts: Map + optimisticDeletes: Set + pendingOptimisticUpserts: Map + pendingOptimisticDeletes: Set + pendingOptimisticDirectUpserts: Set + pendingOptimisticDirectDeletes: Set + pendingLocalChanges: Set + pendingLocalOrigins: Set + size: number + preSyncVisibleState: Map + recentlySyncedKeys: Set + hasReceivedFirstCommit: boolean + isCommittingSyncTransactions: boolean +} + type PendingMetadataWrite = { type: `set`; value: unknown } | { type: `delete` } type InternalChangeMessage< @@ -171,6 +231,122 @@ export class CollectionStateManager< this._events = deps.events } + public snapshotPublicationState( + keys: Iterable, + ): CollectionPublicationStateSnapshot< + TOutput, + TKey + > { + const affectedKeys = new Set(keys) + for (const transaction of this.pendingSyncedTransactions) { + for (const operation of transaction.operations) { + affectedKeys.add(operation.key as TKey) + } + for (const key of transaction.rowMetadataWrites.keys()) { + affectedKeys.add(key) + } + } + + return { + pendingSyncedTransactions: [...this.pendingSyncedTransactions], + applicationStarted: new Map( + this.pendingSyncedTransactions.map((transaction) => [ + transaction, + transaction.applicationStarted, + ]), + ), + keys: new Map( + [...affectedKeys].map((key) => [ + key, + { + syncedData: { + present: this.syncedData.has(key), + value: this.syncedData.get(key), + }, + syncedMetadata: { + present: this.syncedMetadata.has(key), + value: this.syncedMetadata.get(key), + }, + rowOrigin: { + present: this.rowOrigins.has(key), + value: this.rowOrigins.get(key), + }, + hydrationSeed: this.hydrationSeedKeys.has(key), + hydrated: this.hydratedKeys.has(key), + synced: this.syncedKeys.has(key), + }, + ]), + ), + syncedCollectionMetadata: [ + ...this.syncedCollectionMetadata.entries(), + ], + optimisticUpserts: new Map(this.optimisticUpserts), + optimisticDeletes: new Set(this.optimisticDeletes), + pendingOptimisticUpserts: new Map(this.pendingOptimisticUpserts), + pendingOptimisticDeletes: new Set(this.pendingOptimisticDeletes), + pendingOptimisticDirectUpserts: new Set( + this.pendingOptimisticDirectUpserts, + ), + pendingOptimisticDirectDeletes: new Set( + this.pendingOptimisticDirectDeletes, + ), + pendingLocalChanges: new Set(this.pendingLocalChanges), + pendingLocalOrigins: new Set(this.pendingLocalOrigins), + size: this.size, + preSyncVisibleState: new Map(this.preSyncVisibleState), + recentlySyncedKeys: new Set(this.recentlySyncedKeys), + hasReceivedFirstCommit: this.hasReceivedFirstCommit, + isCommittingSyncTransactions: this.isCommittingSyncTransactions, + } + } + + public restorePublicationState( + snapshot: CollectionPublicationStateSnapshot, + ): void { + this.pendingSyncedTransactions = [...snapshot.pendingSyncedTransactions] + for (const [transaction, applicationStarted] of snapshot.applicationStarted) { + transaction.applicationStarted = applicationStarted + } + for (const [key, state] of snapshot.keys) { + restoreMapEntry(this.syncedData, key, state.syncedData) + restoreMapEntry(this.syncedMetadata, key, state.syncedMetadata) + restoreMapEntry(this.rowOrigins, key, state.rowOrigin) + restoreSetEntry(this.hydrationSeedKeys, key, state.hydrationSeed) + restoreSetEntry(this.hydratedKeys, key, state.hydrated) + restoreSetEntry(this.syncedKeys, key, state.synced) + } + replaceMap( + this.syncedCollectionMetadata, + snapshot.syncedCollectionMetadata, + ) + replaceMap(this.optimisticUpserts, snapshot.optimisticUpserts) + replaceSet(this.optimisticDeletes, snapshot.optimisticDeletes) + replaceMap( + this.pendingOptimisticUpserts, + snapshot.pendingOptimisticUpserts, + ) + replaceSet( + this.pendingOptimisticDeletes, + snapshot.pendingOptimisticDeletes, + ) + replaceSet( + this.pendingOptimisticDirectUpserts, + snapshot.pendingOptimisticDirectUpserts, + ) + replaceSet( + this.pendingOptimisticDirectDeletes, + snapshot.pendingOptimisticDirectDeletes, + ) + replaceSet(this.pendingLocalChanges, snapshot.pendingLocalChanges) + replaceSet(this.pendingLocalOrigins, snapshot.pendingLocalOrigins) + this.size = snapshot.size + replaceMap(this.preSyncVisibleState, snapshot.preSyncVisibleState) + replaceSet(this.recentlySyncedKeys, snapshot.recentlySyncedKeys) + this.hasReceivedFirstCommit = snapshot.hasReceivedFirstCommit + this.isCommittingSyncTransactions = snapshot.isCommittingSyncTransactions + this.virtualPropsCache = new WeakMap() + } + /** * Checks whether this row currently has no pending local optimistic writes. * diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 4e3dba739..a92a15b98 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -28,6 +28,7 @@ import type { LiveQueryInternalUtils } from './internal.js' import type { WindowOptions } from '../compiler/index.js' import type { SchedulerContextId } from '../../scheduler.js' import type { CollectionSubscription } from '../../collection/subscription.js' +import type { CollectionPublicationStateSnapshot } from '../../collection/state.js' import type { RootStreamBuilder } from '@tanstack/db-ivm' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' import type { Collection } from '../../collection/index.js' @@ -1072,6 +1073,9 @@ export class CollectionConfigBuilder< let rootPublication: | ReturnType | undefined + let rootStateSnapshot: + | CollectionPublicationStateSnapshot + | undefined try { facadePublication = bucketFacades.flush() rootPublication = hasParentChanges @@ -1093,6 +1097,9 @@ export class CollectionConfigBuilder< ) if (hasParentChanges) { + rootStateSnapshot = config.collection._snapshotPublicationState( + changesToApply.keys() as Iterable, + ) // The graph has already reached quiescence, so this is one complete // derived publication. Apply it beneath any pending optimistic // overlay instead of parking source progress behind that mutation. @@ -1106,6 +1113,9 @@ export class CollectionConfigBuilder< } catch (error) { pendingChanges = new Map() rootPublication?.discard() + if (rootStateSnapshot) { + config.collection._restorePublicationState(rootStateSnapshot) + } facadePublication?.rollback() throw error } diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index b09f4122e..b7b908c7b 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,6 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect } from 'vitest' import { createDeferred } from '../../src/deferred.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { createOptimisticAction } from '../../src/optimistic-action.js' import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' @@ -48,6 +49,15 @@ type FacadeCandidateScanScenario = { finalLayout: `moved` | `restored` } +class ThrowingUpdateIndex extends BasicIndex { + throwAfterUpdate = false + + override update(key: number, oldItem: unknown, newItem: unknown): void { + super.update(key, oldItem, newItem) + if (this.throwAfterUpdate) throw new Error(`root index failed`) + } +} + const exhaustiveLayoutSwapScenarios: Array = Array.from( { length: 9 }, (_, offset) => offset + 4, @@ -1095,6 +1105,9 @@ describe(`Collection-valued includes oracle`, () => { await live.preload() const facade = live.get(1)!.children + const rootIndex = live.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex const rootPublications: Array = [] const childPublications: Array = [] const rootCallbackFacadeSnapshots: Array<{ @@ -1122,13 +1135,11 @@ describe(`Collection-valued includes oracle`, () => { childObserver.subscribe(() => observerNotifications++) observerNotifications = 0 const observerBeforeFailure = childObserver.getSnapshot() + const rootStateRevisionBeforeFailure = live._stateRevision + const rootLayoutRevisionBeforeFailure = live._layoutRevision const childStateRevisionBeforeFailure = facade._stateRevision const childLayoutRevisionBeforeFailure = facade._layoutRevision - const originalGetKey = live.config.getKey - live.config.getKey = (row) => { - if (row.value === 2) throw new Error(`root key failed`) - return originalGetKey(row) - } + rootIndex.throwAfterUpdate = true try { expect(() => @@ -1146,8 +1157,10 @@ describe(`Collection-valued includes oracle`, () => { value: { ...initialSibling, value: 0 }, }, ]), - ).toThrow(`root key failed`) + ).toThrow(`root index failed`) expect(live.get(1)!.value).toBe(1) + expect([...rootIndex.equalityLookup(1)]).toEqual([1]) + expect([...rootIndex.equalityLookup(2)]).toEqual([]) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 10, value: 1 }, { id: 20, value: 2 }, @@ -1155,12 +1168,14 @@ describe(`Collection-valued includes oracle`, () => { expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) expect(rootCallbackFacadeSnapshots).toEqual([]) + expect(live._stateRevision).toBe(rootStateRevisionBeforeFailure) + expect(live._layoutRevision).toBe(rootLayoutRevisionBeforeFailure) expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure) expect(facade._layoutRevision).toBe(childLayoutRevisionBeforeFailure) expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) expect(observerNotifications).toBe(0) - live.config.getKey = originalGetKey + rootIndex.throwAfterUpdate = false nodes.writeBatch([ { type: `update`, @@ -1176,6 +1191,8 @@ describe(`Collection-valued includes oracle`, () => { }, ]) expect(live.get(1)!.value).toBe(3) + expect([...rootIndex.equalityLookup(1)]).toEqual([]) + expect([...rootIndex.equalityLookup(3)]).toEqual([1]) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 20, value: 3 }, { id: 10, value: 4 }, @@ -1199,7 +1216,7 @@ describe(`Collection-valued includes oracle`, () => { expect(childObserver.getSnapshot()).not.toBe(observerBeforeFailure) expect(observerNotifications).toBe(1) } finally { - live.config.getKey = originalGetKey + rootIndex.throwAfterUpdate = false childObserver.dispose() rootSubscription.unsubscribe() childSubscription.unsubscribe() diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 108ca18da..c44c06270 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -860,8 +860,16 @@ describe(`source publication across pending derived mutations`, () => { }) : undefined const query = q2 ?? q1 - const sourceEvents: Array<{ type: string; key: number; value?: number }> = [] - const queryEvents: Array<{ type: string; key: number; value?: number }> = [] + const sourceEvents: Array<{ + type: string + key: string | number + value?: number + }> = [] + const queryEvents: Array<{ + type: string + key: string | number + value?: number + }> = [] const sourceSubscription = source.subscribeChanges((changes) => { sourceEvents.push( ...changes.map((change) => ({ From 9ec63844cc0f7484d638f25ec5829e4f2275ae49 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 10:06:42 -0600 Subject: [PATCH 227/327] fix(db): retry reactivated subset demand --- .../query/live/subset-demand-controller.ts | 20 +++-- packages/db/tests/oracle-config.ts | 1 + .../query/includes-temporal-oracle.test.ts | 80 +++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts index a753086a9..e08d79164 100644 --- a/packages/db/src/query/live/subset-demand-controller.ts +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -57,6 +57,7 @@ export class SubsetDemandController { } const segments: Array = [] + let releaseFailure: { error: unknown } | undefined for (const segment of previous?.segments ?? []) { if (segment.state !== `failed` && intersects(segment.keys, nextKeys)) { @@ -65,10 +66,14 @@ export class SubsetDemandController { } segment.abortController.abort() - subscription.releaseSnapshot( - segment.where, - segment.abortController.signal, - ) + try { + subscription.releaseSnapshot( + segment.where, + segment.abortController.signal, + ) + } catch (error) { + releaseFailure ??= { error } + } } const coveredKeys = new Set( @@ -100,11 +105,16 @@ export class SubsetDemandController { (ready): ready is Promise => ready instanceof Promise, ) - return { + const update: DemandUpdate = { changed: true, empty: nextKeys.size === 0, ready: pending.length > 0 ? Promise.all(pending) : true, } + // A failed physical release remains cleanup debt, but it cannot leave an + // aborted segment representing current logical demand. Commit the demand + // transition first so a later incarnation acquires fresh coverage. + if (releaseFailure) throw releaseFailure.error + return update } clear(): void { diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 5505535c6..e4afea3f7 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -22,6 +22,7 @@ const staticOracleProperties = [ `includes-publication.child-scalar`, `includes-publication.optimistic-rollback`, `includes-publication.parent-route`, + `includes-temporal.release-reentry`, `includes-temporal.demand-scheduling`, `includes.alpha-renaming`, `includes.incremental-history`, diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 87ec2ffa4..bfb36140a 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -839,6 +839,68 @@ async function expectFailedDemandRetriesSameCoverage(): Promise { } } +async function expectDemandReactivationRetriesAfterReleaseFailure( + keys: ReadonlyArray, +): Promise { + let loadCount = 0 + let allowUnload = false + const releaseError = new Error(`child release failed`) + const comments = createCollection({ + id: nextCollectionId(`temporal-release-retry-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + markReady() + return true + }, + unloadSubset: () => { + if (!allowUnload) throw releaseError + }, + }), + }, + }) + comments.createIndex((comment) => comment.postId) + const subscription = comments.subscribeChanges(() => {}, { + includeInitialState: false, + }) + const controller = new SubsetDemandController() + const plan: LazyDemandPlan = { + id: `release-failure-retry`, + path: [`postId`], + collectionId: comments.id, + initialKeys: new Set(), + } + + try { + expect(controller.setDemand(subscription, plan, new Set(keys))).toMatchObject( + { changed: true, empty: false }, + ) + expect(loadCount).toBe(1) + + expect(() => controller.setDemand(subscription, plan, new Set())).toThrow( + releaseError, + ) + + const reactivated = controller.setDemand( + subscription, + plan, + new Set(keys), + ) + expect(reactivated).toMatchObject({ changed: true, empty: false }) + expect(loadCount).toBe(2) + } finally { + allowUnload = true + controller.clear() + subscription.unsubscribe() + await comments.cleanup() + } +} + async function expectSynchronousEmptyDemandIsReady(): Promise { const posts = createMutablePosts([ { id: 1, authorId: `selected`, title: `one` }, @@ -1221,6 +1283,24 @@ describe(`includes temporal oracle`, () => { expectFailedDemandRetriesSameCoverage, ) + it( + `reactivated demand retries after its prior release fails`, + () => expectDemandReactivationRetriesAfterReleaseFailure([1]), + ) + + fcTest.prop( + [ + fc.uniqueArray(fc.integer({ min: -3, max: 3 }), { + minLength: 1, + maxLength: 5, + }), + ], + oraclePropertyOptions(20, `includes-temporal.release-reentry`), + )( + `failed release never suppresses a later demand incarnation`, + expectDemandReactivationRetriesAfterReleaseFailure, + ) + it( `a synchronous empty demand can establish ready coverage`, expectSynchronousEmptyDemandIsReady, From d0b53939464629322f9c0d113be2806d655c817e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 10:25:51 -0600 Subject: [PATCH 228/327] fix(db): complete demand retirement after release failure --- packages/db/src/query/effect.ts | 4 + packages/db/src/query/live/ARCHITECTURE.md | 10 +++ .../src/query/live/collection-subscriber.ts | 5 ++ .../query/live/subset-demand-controller.ts | 3 +- packages/db/tests/effect.test.ts | 73 +++++++++++++++++++ .../query/includes-temporal-oracle.test.ts | 62 +++++++++++++++- .../tests/query/subset-error-matrix.test.ts | 23 ++++-- 7 files changed, 170 insertions(+), 10 deletions(-) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 40d58a50c..895b05939 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -753,6 +753,10 @@ class EffectPipelineRunner { if (this.starting) throw error return } + if (update.releaseFailure) { + this.onSourceError(normaliseError(update.releaseFailure.error)) + return + } if (update.ready instanceof Promise) { // Each segment reports its own failure through the subscription. Consume // the aggregate rejection so Promise.all does not create a second, diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 715ae3291..706d98664 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -869,6 +869,16 @@ logical owner's evaluator across source changes and truncate acquisition replacement. A released owner cannot supply a predicate, and a later logical demand compiles its own evaluator even when it reuses the same expression object. + +Logical demand state advances even when physical release fails. The failed +acquisition remains retryable cleanup debt in the Collection subscription, but +an aborted segment cannot remain the current demand or suppress a later +incarnation. The demand controller therefore returns the release failure with +the completed logical transition. A live query records that failure and +retires an empty demand without entering a fatal query state; an Effect reports +the same failure through its source-error policy and disposes. Reactivating the +route starts a fresh acquisition. + Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies coverage, or scans local rows. A callback may release or unsubscribe; obsolete diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 86c6bc0b8..2124c1965 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -254,6 +254,11 @@ export class CollectionSubscriber< if (isInitialSync) throw error return } + if (update.releaseFailure) { + this.collectionConfigBuilder.recordSubsetError( + update.releaseFailure.error, + ) + } if (!update.changed) return if (update.empty) { diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts index e08d79164..27c4e0f5d 100644 --- a/packages/db/src/query/live/subset-demand-controller.ts +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -26,6 +26,7 @@ export type DemandUpdate = { changed: boolean empty: boolean ready: Promise> | true + releaseFailure?: { error: unknown } } /** @@ -109,11 +110,11 @@ export class SubsetDemandController { changed: true, empty: nextKeys.size === 0, ready: pending.length > 0 ? Promise.all(pending) : true, + ...(releaseFailure && { releaseFailure }), } // A failed physical release remains cleanup debt, but it cannot leave an // aborted segment representing current logical demand. Commit the demand // transition first so a later incarnation acquires fresh coverage. - if (releaseFailure) throw releaseFailure.error return update } diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 4c26505ac..139f4ff3f 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -2067,6 +2067,79 @@ describe(`createEffect`, () => { } }) + it(`reports failed obsolete-demand release without failing the source commit`, async () => { + const failure = new Error(`obsolete effect demand release failed`) + const users = createCollection( + mockSyncCollectionOptions({ + id: `effect-obsolete-release-users`, + getKey: (user) => user.id, + initialData: [sampleUsers[0]!], + }), + ) + let loadCount = 0 + let unloadCount = 0 + const issues = createCollection({ + id: `effect-obsolete-release-issues`, + getKey: (issue) => issue.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + loadCount++ + return true + }, + unloadSubset: () => { + unloadCount++ + if (unloadCount === 1) throw failure + }, + } + }, + }, + }) + const sourceErrors: Array = [] + const effect = createEffect({ + query: (q) => + q + .from({ user: users }) + .leftJoin({ issue: issues }, ({ user, issue }) => + eq(user.id, issue.userId), + ) + .select(({ user, issue }) => ({ + id: user.id, + issueId: issue.id, + })), + onBatch: () => {}, + onSourceError: (error) => sourceErrors.push(error), + }) + + try { + await flushPromises() + expect(loadCount).toBe(1) + + let commitError: unknown + try { + users.utils.begin() + users.utils.write({ type: `delete`, value: sampleUsers[0]! }) + users.utils.commit() + } catch (error) { + commitError = error + } + await flushPromises() + + expect(commitError).toBeUndefined() + expect(sourceErrors).toEqual([failure]) + expect(effect.disposed).toBe(true) + expect(unloadCount).toBe(2) + } finally { + await effect.dispose() + await Promise.all([users.cleanup(), issues.cleanup()]) + } + }) + it(`reports a rejected ordered subset load and disposes the effect`, async () => { const failure = new Error(`ordered subset failed`) let loadCount = 0 diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index bfb36140a..a4892697c 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -882,9 +882,9 @@ async function expectDemandReactivationRetriesAfterReleaseFailure( ) expect(loadCount).toBe(1) - expect(() => controller.setDemand(subscription, plan, new Set())).toThrow( - releaseError, - ) + const retired = controller.setDemand(subscription, plan, new Set()) + expect(retired).toMatchObject({ changed: true, empty: true }) + expect(retired.releaseFailure?.error).toBe(releaseError) const reactivated = controller.setDemand( subscription, @@ -901,6 +901,57 @@ async function expectDemandReactivationRetriesAfterReleaseFailure( } } +async function expectRetiredDemandStaysNonfatalAfterReleaseFailure(): Promise { + const post = { id: 1, authorId: `selected`, title: `one` } + const posts = createMutablePosts([post]) + let loadCount = 0 + let allowUnload = false + const releaseError = new Error(`child release failed`) + const comments = createCollection({ + id: nextCollectionId(`temporal-retired-release-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + markReady() + return true + }, + unloadSubset: () => { + if (!allowUnload) throw releaseError + }, + }), + }, + }) + const live = createPostsWithCommentsLive(posts.collection, comments) + const consoleError = vi.spyOn(console, `error`).mockImplementation(() => {}) + + try { + await live.preload() + expect(loadCount).toBe(1) + expect(live.status).toBe(`ready`) + + posts.write(`delete`, post) + await flushPromises() + expect(live.size).toBe(0) + expect(live.status).toBe(`ready`) + expect(live.utils.lastSubsetError).toBe(releaseError) + + posts.write(`insert`, post) + await flushPromises() + expect(loadCount).toBe(2) + expect(live.status).toBe(`ready`) + } finally { + allowUnload = true + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + consoleError.mockRestore() + } +} + async function expectSynchronousEmptyDemandIsReady(): Promise { const posts = createMutablePosts([ { id: 1, authorId: `selected`, title: `one` }, @@ -1301,6 +1352,11 @@ describe(`includes temporal oracle`, () => { expectDemandReactivationRetriesAfterReleaseFailure, ) + it( + `failed release retires an empty live-query demand without poisoning reentry`, + expectRetiredDemandStaysNonfatalAfterReleaseFailure, + ) + it( `a synchronous empty demand can establish ready coverage`, expectSynchronousEmptyDemandIsReady, diff --git a/packages/db/tests/query/subset-error-matrix.test.ts b/packages/db/tests/query/subset-error-matrix.test.ts index a7fbe8764..c35fa3ba6 100644 --- a/packages/db/tests/query/subset-error-matrix.test.ts +++ b/packages/db/tests/query/subset-error-matrix.test.ts @@ -362,7 +362,7 @@ describe(`loadSubset failure matrix`, () => { ) it.each(cleanupFailureCases)( - `does not mistake an unreported cleanup failure for a source error: $name`, + `reports obsolete-demand cleanup failure without failing the source commit: $name`, async ({ consumer, failure }) => { const suffix = `${consumer}-${ failure === undefined @@ -394,7 +394,7 @@ describe(`loadSubset failure matrix`, () => { }, }, }) - const sourceErrors: Array = [] + const sourceErrors: Array = [] const effect = consumer === `effect` ? createEffect({ @@ -434,10 +434,21 @@ describe(`loadSubset failure matrix`, () => { thrown = error } - expect(didThrow).toBe(true) - expect(Object.is(thrown, failure)).toBe(true) - expect(sourceErrors).toEqual([]) - if (live) expect(live.utils.lastSubsetError).toBeUndefined() + await flushFailures() + + expect(didThrow).toBe(false) + expect(thrown).toBeUndefined() + if (effect) { + expect(sourceErrors).toHaveLength(1) + expect(sourceErrors[0]?.message).toBe(String(failure)) + expect(effect.disposed).toBe(true) + } else { + expect(sourceErrors).toEqual([]) + } + if (live) { + expect(Object.is(live.utils.lastSubsetError, failure)).toBe(true) + expect(live.status).toBe(`ready`) + } } finally { if (effect) await effect.dispose() if (live) await live.cleanup() From f16735578baf166e686ec8a7b931b42a22500727 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 10:38:09 -0600 Subject: [PATCH 229/327] fix(db): retain failed subset cleanup state --- docs/guides/error-handling.md | 9 ++++--- .../type-aliases/LiveQueryCollectionUtils.md | 8 +++++++ packages/db/src/query/effect.ts | 24 +++++++++++-------- packages/db/src/query/live/ARCHITECTURE.md | 5 +++- .../query/live/collection-config-builder.ts | 8 +++++++ packages/db/tests/effect.test.ts | 12 +++++++--- .../tests/query/subset-error-matrix.test.ts | 1 + 7 files changed, 50 insertions(+), 17 deletions(-) diff --git a/docs/guides/error-handling.md b/docs/guides/error-handling.md index bff185c1a..53634c863 100644 --- a/docs/guides/error-handling.md +++ b/docs/guides/error-handling.md @@ -141,14 +141,17 @@ console.log(subscription.lastError) ``` For ordered live queries, `utils.setWindow()` rejects with the same error. The -last failure is also available as `utils.lastSubsetError`, while the last -successful snapshot remains readable: +last failure is also available as `utils.lastSubsetError`, while +`utils.hasSubsetError` distinguishes a thrown `undefined` from no observed +failure. The last successful snapshot remains readable: ```ts try { await liveTodos.utils.setWindow({ offset: 0, limit: 100 }) } catch (error) { - console.error(liveTodos.utils.lastSubsetError) + if (liveTodos.utils.hasSubsetError) { + console.error(liveTodos.utils.lastSubsetError) + } } ``` diff --git a/docs/reference/type-aliases/LiveQueryCollectionUtils.md b/docs/reference/type-aliases/LiveQueryCollectionUtils.md index 620333d20..9fc4e7b62 100644 --- a/docs/reference/type-aliases/LiveQueryCollectionUtils.md +++ b/docs/reference/type-aliases/LiveQueryCollectionUtils.md @@ -52,6 +52,14 @@ Gets the current window (offset and limit) for an ordered query. The current window settings, or `undefined` if the query is not windowed +### hasSubsetError + +```ts +readonly hasSubsetError: boolean; +``` + +Whether this live query has observed a subset-load failure. + ### lastSubsetError ```ts diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 895b05939..219ec92bb 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -264,13 +264,13 @@ export function createEffect< // Abort signal for in-flight handlers abortController.abort() - disposalPromise = (async () => { + const attempt = (async () => { // Tear down the pipeline (unsubscribe from sources, etc.) - let cleanupError: unknown + let cleanupFailure: { error: unknown } | undefined try { runner.dispose() } catch (error) { - cleanupError = error + cleanupFailure = { error } } // Wait for any in-flight async handlers to settle @@ -278,9 +278,13 @@ export function createEffect< await Promise.allSettled([...inFlightHandlers]) } - if (cleanupError !== undefined) throw cleanupError + if (cleanupFailure) throw cleanupFailure.error })() - return disposalPromise + disposalPromise = attempt + void attempt.catch(() => { + if (disposalPromise === attempt) disposalPromise = undefined + }) + return attempt } // Create and start the pipeline @@ -1142,20 +1146,20 @@ class EffectPipelineRunner { /** Tear down subscriptions and clear state */ dispose(): void { - if (this.disposed) return + if (this.disposed && this.unsubscribeCallbacks.size === 0) return this.disposed = true this.subscribedToAllCollections = false // Immediately unsubscribe from every source, even if one release fails. - let firstCleanupError: unknown + let firstCleanupFailure: { error: unknown } | undefined for (const unsubscribe of this.unsubscribeCallbacks) { try { unsubscribe() + this.unsubscribeCallbacks.delete(unsubscribe) } catch (error) { - firstCleanupError ??= error + firstCleanupFailure ??= { error } } } - this.unsubscribeCallbacks.clear() this.sentToD2KeysBySource.clear() this.pendingChanges.clear() this.lazySources.clear() @@ -1184,7 +1188,7 @@ class EffectPipelineRunner { this.finalCleanup() } - if (firstCleanupError !== undefined) throw firstCleanupError + if (firstCleanupFailure) throw firstCleanupFailure.error } /** Clear graph references — called after graph run completes or immediately from dispose */ diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 706d98664..dd9dbc0e2 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -877,7 +877,10 @@ incarnation. The demand controller therefore returns the release failure with the completed logical transition. A live query records that failure and retires an empty demand without entering a fatal query state; an Effect reports the same failure through its source-error policy and disposes. Reactivating the -route starts a fresh acquisition. +route starts a fresh acquisition. Live-query diagnostics track failure presence +separately from its value so a thrown `undefined` remains observable. Effect +disposal retains failed unsubscribe callbacks and lets a later `dispose()` +retry them instead of caching a terminal rejected cleanup attempt. Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index a92a15b98..d4a005b10 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -54,6 +54,8 @@ import type { AllCollectionEvents } from '../../collection/events.js' export type LiveQueryCollectionUtils = UtilsRecord & { getRunCount: () => number + /** Whether this live query has observed a subset-load failure. */ + readonly hasSubsetError: boolean /** Most recent subset-load failure observed by this live query. */ readonly lastSubsetError: unknown | undefined /** @@ -121,6 +123,7 @@ export class CollectionConfigBuilder< private isInErrorState = false private fatalQueryError = false private readonly erroredSourceIds = new Set() + private hasSubsetError = false private lastSubsetError: unknown | undefined // Reference to the live query collection for error state transitions @@ -284,6 +287,9 @@ export class CollectionConfigBuilder< singleResult: this.query.singleResult, utils: { getRunCount: this.getRunCount.bind(this), + get hasSubsetError() { + return builder.hasSubsetError + }, get lastSubsetError() { return builder.lastSubsetError }, @@ -487,6 +493,7 @@ export class CollectionConfigBuilder< } recordSubsetError(error: unknown, fatalBeforeReady = false): void { + this.hasSubsetError = true this.lastSubsetError = error if (this.activeWindowOperation) { this.activeWindowOperation.failed = true @@ -840,6 +847,7 @@ export class CollectionConfigBuilder< this.isInErrorState = false this.fatalQueryError = false this.erroredSourceIds.clear() + this.hasSubsetError = false this.lastSubsetError = undefined this.latestSubsetOutcomes.clear() this.lastWindowOutcomes = [] diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 139f4ff3f..8e822cf7e 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -2078,6 +2078,7 @@ describe(`createEffect`, () => { ) let loadCount = 0 let unloadCount = 0 + const consoleError = vi.spyOn(console, `error`).mockImplementation(() => {}) const issues = createCollection({ id: `effect-obsolete-release-issues`, getKey: (issue) => issue.id, @@ -2094,7 +2095,7 @@ describe(`createEffect`, () => { }, unloadSubset: () => { unloadCount++ - if (unloadCount === 1) throw failure + if (unloadCount <= 2) throw failure }, } }, @@ -2134,9 +2135,13 @@ describe(`createEffect`, () => { expect(sourceErrors).toEqual([failure]) expect(effect.disposed).toBe(true) expect(unloadCount).toBe(2) + + await effect.dispose() + expect(unloadCount).toBe(3) } finally { await effect.dispose() await Promise.all([users.cleanup(), issues.cleanup()]) + consoleError.mockRestore() } }) @@ -2232,7 +2237,7 @@ describe(`createEffect`, () => { }, unloadSubset: () => { unloadCount++ - throw cleanupFailure + if (unloadCount <= 2) throw cleanupFailure }, } }, @@ -2268,7 +2273,8 @@ describe(`createEffect`, () => { cleanupFailure, cleanupFailure, ]) - await expect(effect.dispose()).rejects.toBe(cleanupError) + await effect.dispose() + expect(unloadCount).toBe(4) } finally { consoleErrorSpy.mockRestore() await users.cleanup() diff --git a/packages/db/tests/query/subset-error-matrix.test.ts b/packages/db/tests/query/subset-error-matrix.test.ts index c35fa3ba6..61bb1b997 100644 --- a/packages/db/tests/query/subset-error-matrix.test.ts +++ b/packages/db/tests/query/subset-error-matrix.test.ts @@ -446,6 +446,7 @@ describe(`loadSubset failure matrix`, () => { expect(sourceErrors).toEqual([]) } if (live) { + expect(live.utils.hasSubsetError).toBe(true) expect(Object.is(live.utils.lastSubsetError, failure)).toBe(true) expect(live.status).toBe(`ready`) } From 3a3cdc695a54170b28224f9994df48f056a7765d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 10:47:58 -0600 Subject: [PATCH 230/327] fix(db): retry falsy subset cleanup failures --- docs/guides/error-handling.md | 3 + .../type-aliases/LiveQueryCollectionUtils.md | 3 +- packages/db/src/query/live/ARCHITECTURE.md | 7 +- .../query/live/collection-config-builder.ts | 8 +-- packages/db/tests/effect.test.ts | 39 +++++++++-- .../tests/query/subset-error-matrix.test.ts | 67 +++++++++++++++++++ 6 files changed, 114 insertions(+), 13 deletions(-) diff --git a/docs/guides/error-handling.md b/docs/guides/error-handling.md index 53634c863..8de4468d6 100644 --- a/docs/guides/error-handling.md +++ b/docs/guides/error-handling.md @@ -155,6 +155,9 @@ try { } ``` +Both diagnostic values reset together when the live query starts a new sync +session. + Effects report subset failures through `onSourceError` and dispose because their incremental result can no longer be kept complete. diff --git a/docs/reference/type-aliases/LiveQueryCollectionUtils.md b/docs/reference/type-aliases/LiveQueryCollectionUtils.md index 9fc4e7b62..acdaf4473 100644 --- a/docs/reference/type-aliases/LiveQueryCollectionUtils.md +++ b/docs/reference/type-aliases/LiveQueryCollectionUtils.md @@ -58,7 +58,8 @@ The current window settings, or `undefined` if the query is not windowed readonly hasSubsetError: boolean; ``` -Whether this live query has observed a subset-load failure. +Whether this live query has observed a subset-load failure in its current sync +session. ### lastSubsetError diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index dd9dbc0e2..cf475aaa4 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -878,9 +878,10 @@ the completed logical transition. A live query records that failure and retires an empty demand without entering a fatal query state; an Effect reports the same failure through its source-error policy and disposes. Reactivating the route starts a fresh acquisition. Live-query diagnostics track failure presence -separately from its value so a thrown `undefined` remains observable. Effect -disposal retains failed unsubscribe callbacks and lets a later `dispose()` -retry them instead of caching a terminal rejected cleanup attempt. +separately from its value so a thrown `undefined` remains observable, and reset +both observations when a new sync session starts. Effect disposal retains +failed unsubscribe callbacks and lets a later `dispose()` retry them instead of +caching a terminal rejected cleanup attempt. Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index d4a005b10..79ca943d3 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -54,7 +54,7 @@ import type { AllCollectionEvents } from '../../collection/events.js' export type LiveQueryCollectionUtils = UtilsRecord & { getRunCount: () => number - /** Whether this live query has observed a subset-load failure. */ + /** Whether this live query has observed a subset-load failure in its current sync session. */ readonly hasSubsetError: boolean /** Most recent subset-load failure observed by this live query. */ readonly lastSubsetError: unknown | undefined @@ -870,13 +870,13 @@ export class CollectionConfigBuilder< if (this.syncSession === syncSession) this.syncSession++ } - let firstCleanupError: unknown + let firstCleanupFailure: { error: unknown } | undefined for (const unsubscribe of syncState.unsubscribeCallbacks) { try { unsubscribe() syncState.unsubscribeCallbacks.delete(unsubscribe) } catch (error) { - firstCleanupError ??= error + firstCleanupFailure ??= { error } } } @@ -908,7 +908,7 @@ export class CollectionConfigBuilder< this.compiledAliasToCollectionId = {} } - if (firstCleanupError !== undefined) throw firstCleanupError + if (firstCleanupFailure) throw firstCleanupFailure.error tornDown = true } diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 8e822cf7e..44a836d80 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -678,6 +678,8 @@ describe(`createEffect`, () => { it(`reports one in-progress cleanup failure to every disposer`, async () => { const failure = new Error(`source release failed`) + let unloadCount = 0 + let shouldFail = true let resolveHandler!: () => void const handlerPending = new Promise((resolve) => { resolveHandler = resolve @@ -697,7 +699,8 @@ describe(`createEffect`, () => { return true }, unloadSubset: () => { - throw failure + unloadCount++ + if (shouldFail) throw failure }, } }, @@ -711,10 +714,18 @@ describe(`createEffect`, () => { await flushPromises() const firstDispose = effect.dispose() const secondDispose = effect.dispose() + expect(secondDispose).toBe(firstDispose) resolveHandler() await expect(firstDispose).rejects.toBe(failure) await expect(secondDispose).rejects.toBe(failure) + expect(unloadCount).toBe(1) + + shouldFail = false + const retry = effect.dispose() + expect(retry).not.toBe(firstDispose) + await retry + expect(unloadCount).toBe(2) await source.cleanup() }) }) @@ -1795,25 +1806,36 @@ describe(`createEffect`, () => { it(`releases every source when one unsubscriber throws`, async () => { const failure = new Error(`first source unload failed`) + let leftShouldFail = true + let leftUnloadCount = 0 + let rightUnloadCount = 0 const createSource = (id: string, unloadSubset: () => void) => createCollection<{ id: number }>({ id, getKey: (row) => row.id, syncMode: `on-demand`, sync: { - sync: ({ markReady }) => { + sync: ({ begin, write, commit, markReady }) => { markReady() return { - loadSubset: () => true, + loadSubset: () => { + begin() + write({ type: `insert`, value: { id: 1 } }) + commit() + return true + }, unloadSubset, } }, }, }) const left = createSource(`effect-cleanup-left`, () => { - throw failure + leftUnloadCount++ + if (leftShouldFail) throw failure + }) + const right = createSource(`effect-cleanup-right`, () => { + rightUnloadCount++ }) - const right = createSource(`effect-cleanup-right`, () => {}) const effect = createEffect({ query: (q) => q @@ -1830,6 +1852,13 @@ describe(`createEffect`, () => { await expect(effect.dispose()).rejects.toBe(failure) expect(left.subscriberCount).toBe(0) expect(right.subscriberCount).toBe(0) + expect(leftUnloadCount).toBe(1) + expect(rightUnloadCount).toBe(1) + + leftShouldFail = false + await effect.dispose() + expect(leftUnloadCount).toBe(2) + expect(rightUnloadCount).toBe(1) await Promise.all([left.cleanup(), right.cleanup()]) }) diff --git a/packages/db/tests/query/subset-error-matrix.test.ts b/packages/db/tests/query/subset-error-matrix.test.ts index 61bb1b997..d6dc3a800 100644 --- a/packages/db/tests/query/subset-error-matrix.test.ts +++ b/packages/db/tests/query/subset-error-matrix.test.ts @@ -458,4 +458,71 @@ describe(`loadSubset failure matrix`, () => { } }, ) + + it(`retries live cleanup after an undefined failure survives demand retirement`, async () => { + const parent = createStaticSource(`undefined-cleanup-retry-parent`, [row]) + let unloadCount = 0 + const child = createCollection({ + id: `undefined-cleanup-retry-child`, + getKey: (item) => item.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => { + unloadCount++ + if (unloadCount <= 2) throw undefined + }, + } + }, + }, + }) + const live = createLiveQueryCollection((q) => + q + .from({ item: parent }) + .leftJoin({ child }, ({ item, child: childRow }) => + eq(item.id, childRow.parentId), + ), + ) + const originalQueueMicrotask = globalThis.queueMicrotask + const queuedMicrotasks: Array<() => void> = [] + + try { + await live.preload() + + parent.utils.begin() + parent.utils.write({ type: `delete`, value: row }) + parent.utils.commit() + await flushFailures() + + expect(unloadCount).toBe(1) + expect(live.utils.hasSubsetError).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() + + globalThis.queueMicrotask = (callback) => { + queuedMicrotasks.push(callback) + } + await live.cleanup() + expect(unloadCount).toBe(2) + expect(queuedMicrotasks).toHaveLength(1) + + let cleanupSurfaced = false + try { + queuedMicrotasks[0]!() + } catch { + cleanupSurfaced = true + } + expect(cleanupSurfaced).toBe(true) + + await live.cleanup() + expect(unloadCount).toBe(3) + } finally { + globalThis.queueMicrotask = originalQueueMicrotask + await Promise.all([live.cleanup(), parent.cleanup(), child.cleanup()]) + } + }) }) From 70a1ddc0d6ee96c3d81a547ae6c3600e8a6390d3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 10:57:51 -0600 Subject: [PATCH 231/327] fix(db): preserve falsy graph loader errors --- packages/db/src/query/live/ARCHITECTURE.md | 4 +- .../query/live/collection-config-builder.ts | 8 +-- packages/db/tests/effect.test.ts | 53 +++++++++++++++ ...d-subset-full-flow-oracle.property.test.ts | 2 + packages/db/tests/query/scheduler.test.ts | 68 +++++++++++++++++++ .../tests/query/subset-error-matrix.test.ts | 10 +-- 6 files changed, 136 insertions(+), 9 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index cf475aaa4..f356aadcf 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -732,7 +732,9 @@ window change reaches the pass with no graph work, core calls the loader first. It then drains every graph step created by a synchronous adapter commit before publishing. Async settlement schedules another pass under the same rule. A successful retry therefore cannot commit source rows while leaving the live -result stale until an unrelated later window change. +result stale until an unrelated later window change. When one pass has several +load callbacks, it attempts all of them and then rethrows the first failure +unchanged, including falsy values such as `undefined`, `false`, `0`, or `NaN`. Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 79ca943d3..d5300f325 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -796,17 +796,17 @@ export class CollectionConfigBuilder< const combinedLoader = () => { let allDone = true - let firstError: unknown + let firstFailure: { error: unknown } | undefined pending.loadCallbacks.forEach((loader) => { try { allDone = loader() && allDone } catch (error) { allDone = false - firstError ??= error + firstFailure ??= { error } } }) - if (firstError) { - throw firstError + if (firstFailure) { + throw firstFailure.error } // Returning false signals that callers should schedule another pass. return allDone diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 44a836d80..fb5eb3ae9 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -728,6 +728,59 @@ describe(`createEffect`, () => { expect(unloadCount).toBe(2) await source.cleanup() }) + + it.each([ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, + ])(`retries a falsy cleanup failure: $name`, async ({ name, failure }) => { + let unloadCount = 0 + const source = createCollection<{ id: number }>({ + id: `effect-falsy-cleanup-${name}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => { + unloadCount++ + if (unloadCount === 1) throw failure + }, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => q.from({ source }), + onBatch: () => {}, + }) + + try { + await flushPromises() + let didReject = false + let rejection: unknown + try { + await effect.dispose() + } catch (error) { + didReject = true + rejection = error + } + expect(didReject).toBe(true) + expect(Object.is(rejection, failure)).toBe(true) + expect(unloadCount).toBe(1) + + await effect.dispose() + expect(unloadCount).toBe(2) + } finally { + await effect.dispose() + await source.cleanup() + } + }) }) describe(`auto-generated IDs`, () => { diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 0efc04aa2..1de937821 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -1585,6 +1585,7 @@ it(`fences an unindexed fallback settlement from a cleaned query session`, async await flushPromises() expect(live.status).toBe(`ready`) expect(live.isLoadingSubset).toBe(false) + expect(live.utils.hasSubsetError).toBe(true) expect(live.utils.lastSubsetError).toBe(visibleFailure) firstWindow = live.utils.setWindow({ offset: 0, limit: 1 }) @@ -1596,6 +1597,7 @@ it(`fences an unindexed fallback settlement from a cleaned query session`, async await live.preload() expect(live.status).toBe(`ready`) expect(live.isLoadingSubset).toBe(false) + expect(live.utils.hasSubsetError).toBe(false) expect(live.utils.lastSubsetError).toBeUndefined() secondWindow = live.utils.setWindow({ offset: 0, limit: 1 }) expect(pending).toHaveLength(3) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 3faf36164..b947b8233 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -596,6 +596,74 @@ describe(`live query scheduler`, () => { maybeRunGraphSpy.mockRestore() }) + it.each([ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, + ])(`preserves the first falsy graph-loader failure: $name`, ({ failure }) => { + const baseCollection = createCollection({ + id: `falsy-loader-users-${String(failure)}`, + getKey: (user) => user.id, + sync: { + sync: () => () => {}, + }, + }) + const builder = new CollectionConfigBuilder({ + id: `falsy-loader-builder-${String(failure)}`, + query: (q) => q.from({ user: baseCollection }), + }) + const contextId = Symbol(`falsy-loader-context`) + const laterLoader = vi.fn(() => true) + const config = { + begin: vi.fn(), + write: vi.fn(), + commit: vi.fn(), + markReady: vi.fn(), + truncate: vi.fn(), + } as unknown as Parameters[`sync`]>[0] + const syncState = { + messagesCount: 0, + subscribedToAllCollections: true, + unsubscribeCallbacks: new Set<() => void>(), + graph: { + pendingWork: () => false, + run: vi.fn(), + }, + inputs: {}, + pipeline: {}, + } as unknown as FullSyncState + const maybeRunGraphSpy = vi + .spyOn(builder, `maybeRunGraph`) + .mockImplementation((combinedLoader) => { + combinedLoader?.() + }) + + builder.currentSyncConfig = config + builder.currentSyncState = syncState + builder.scheduleGraphRun(() => { + throw failure + }, { contextId }) + builder.scheduleGraphRun(laterLoader, { contextId }) + + let didThrow = false + let thrown: unknown + try { + transactionScopedScheduler.flush(contextId) + } catch (error) { + didThrow = true + thrown = error + } finally { + maybeRunGraphSpy.mockRestore() + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + expect(laterLoader).toHaveBeenCalledOnce() + }) + it(`should handle optimistic mutations with nested left joins without scheduler errors`, async () => { // This test verifies that optimistic mutations on collections with nested live query // collections using left joins complete successfully without scheduler errors. diff --git a/packages/db/tests/query/subset-error-matrix.test.ts b/packages/db/tests/query/subset-error-matrix.test.ts index d6dc3a800..5e8cbd393 100644 --- a/packages/db/tests/query/subset-error-matrix.test.ts +++ b/packages/db/tests/query/subset-error-matrix.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { SyncCleanupError } from '../../src/errors.js' import { createEffect, createLiveQueryCollection, eq } from '../../src/index.js' import { mockSyncCollectionOptions } from '../utils.js' @@ -510,13 +511,14 @@ describe(`loadSubset failure matrix`, () => { expect(unloadCount).toBe(2) expect(queuedMicrotasks).toHaveLength(1) - let cleanupSurfaced = false + let cleanupError: unknown try { queuedMicrotasks[0]!() - } catch { - cleanupSurfaced = true + } catch (error) { + cleanupError = error } - expect(cleanupSurfaced).toBe(true) + expect(cleanupError).toBeInstanceOf(SyncCleanupError) + expect((cleanupError as Error).message).toContain(`error: undefined`) await live.cleanup() expect(unloadCount).toBe(3) From ab3d32682c689d23f7667d5f7f41edc12949207e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 11:14:09 -0600 Subject: [PATCH 232/327] fix(db): attempt every source loader --- packages/db/src/query/live/ARCHITECTURE.md | 2 + .../query/live/collection-config-builder.ts | 39 +++---- packages/db/tests/query/scheduler.test.ts | 107 ++++++++++++++++++ 3 files changed, 127 insertions(+), 21 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index f356aadcf..caba5ae01 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -735,6 +735,8 @@ successful retry therefore cannot commit source rows while leaving the live result stale until an unrelated later window change. When one pass has several load callbacks, it attempts all of them and then rethrows the first failure unchanged, including falsy values such as `undefined`, `false`, `0`, or `NaN`. +This rule applies both to lexical source loaders nested in one graph callback +and to graph callbacks coalesced by the scheduler. Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index d5300f325..50ea91f0a 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -78,6 +78,21 @@ type PendingGraphRun = { loadCallbacks: Set<() => boolean> } +function runLoadCallbacks(callbacks: Iterable<() => boolean>): boolean { + let allDone = true + let firstFailure: { error: unknown } | undefined + for (const callback of callbacks) { + try { + allDone = callback() && allDone + } catch (error) { + allDone = false + firstFailure ??= { error } + } + } + if (firstFailure) throw firstFailure.error + return allDone +} + // Global counter for auto-generated collection IDs let liveQueryCollectionCounter = 0 @@ -794,23 +809,8 @@ export class CollectionConfigBuilder< this.incrementRunCount() - const combinedLoader = () => { - let allDone = true - let firstFailure: { error: unknown } | undefined - pending.loadCallbacks.forEach((loader) => { - try { - allDone = loader() && allDone - } catch (error) { - allDone = false - firstFailure ??= { error } - } - }) - if (firstFailure) { - throw firstFailure.error - } - // Returning false signals that callers should schedule another pass. - return allDone - } + // Returning false signals that callers should schedule another pass. + const combinedLoader = () => runLoadCallbacks(pending.loadCallbacks) this.maybeRunGraph(combinedLoader) } @@ -1399,10 +1399,7 @@ export class CollectionConfigBuilder< // Combine all loaders into a single callback that initiates loading more data // from any source that needs it. Returns true once all loaders have been called, // but the actual async loading may still be in progress. - const loadSubsetDataCallbacks = () => { - loaders.map((loader) => loader()) - return true - } + const loadSubsetDataCallbacks = () => runLoadCallbacks(loaders) // Mark as subscribed so the graph can start running // (graph only runs when all collections are subscribed) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index b947b8233..863cf995d 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -9,6 +9,7 @@ import { withPublicationContext, } from '../../src/scheduler.js' import { CollectionConfigBuilder } from '../../src/query/live/collection-config-builder.js' +import { CollectionSubscriber } from '../../src/query/live/collection-subscriber.js' import { mockSyncCollectionOptions, stripVirtualProps } from '../utils.js' import type { OutputWithVirtual } from '../utils.js' import type { FullSyncState } from '../../src/query/live/types.js' @@ -664,6 +665,112 @@ describe(`live query scheduler`, () => { expect(laterLoader).toHaveBeenCalledOnce() }) + it(`attempts every lexical source loader and preserves the first failure`, async () => { + const createSource = (name: string) => + createCollection({ + id: `source-loader-${name}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return () => {} + }, + }, + }) + const firstSource = createSource(`first`) + const secondSource = createSource(`second`) + const thirdSource = createSource(`third`) + const builder = new CollectionConfigBuilder({ + id: `source-loader-builder`, + query: (q) => + q + .from({ first: firstSource }) + .join( + { second: secondSource }, + ({ first, second }) => eq(first.id, second.id), + ) + .join( + { third: thirdSource }, + ({ first, third }) => eq(first.id, third.id), + ), + }) + type BuilderSyncConfig = Parameters< + ReturnType[`sync`][`sync`] + >[0] + const config = { + begin: vi.fn(), + write: vi.fn(), + commit: vi.fn(), + markReady: vi.fn(), + truncate: vi.fn(), + } as unknown as BuilderSyncConfig + const builderInternals = builder as unknown as { + graphCache: FullSyncState[`graph`] + inputsCache: FullSyncState[`inputs`] + pipelineCache: FullSyncState[`pipeline`] + subscribeToAllCollections: ( + syncConfig: typeof config, + state: FullSyncState, + ) => () => boolean + } + const syncState = { + messagesCount: 0, + unsubscribeCallbacks: new Set<() => void>(), + subscribedToAllCollections: false, + graph: builderInternals.graphCache, + inputs: builderInternals.inputsCache, + pipeline: builderInternals.pipelineCache, + } as unknown as FullSyncState + const laterFailure = new Error(`later source failed`) + const loaderCalls: Array = [] + const loadMoreSpy = vi + .spyOn(CollectionSubscriber.prototype, `loadMoreIfNeeded`) + .mockImplementationOnce(() => { + loaderCalls.push(1) + throw undefined + }) + .mockImplementationOnce(() => { + loaderCalls.push(2) + throw laterFailure + }) + .mockImplementationOnce(() => { + loaderCalls.push(3) + return true + }) + + try { + builder.currentSyncConfig = config + builder.currentSyncState = syncState + const loadAllSources = builderInternals.subscribeToAllCollections( + config, + syncState, + ) + + let didThrow = false + let thrown: unknown + try { + loadAllSources() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, undefined)).toBe(true) + expect(loaderCalls).toEqual([1, 2, 3]) + expect(loadMoreSpy).toHaveBeenCalledTimes(3) + } finally { + for (const unsubscribe of syncState.unsubscribeCallbacks) unsubscribe() + loadMoreSpy.mockRestore() + await Promise.all([ + firstSource.cleanup(), + secondSource.cleanup(), + thirdSource.cleanup(), + ]) + } + }) + it(`should handle optimistic mutations with nested left joins without scheduler errors`, async () => { // This test verifies that optimistic mutations on collections with nested live query // collections using left joins complete successfully without scheduler errors. From 7792b69311bbf351a6d947bb6963e800cce0cb92 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 11:23:40 -0600 Subject: [PATCH 233/327] test(db): preserve source loader identity --- packages/db/tests/query/scheduler.test.ts | 29 +++++++++++++---------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 863cf995d..2c66496c2 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -723,20 +723,18 @@ describe(`live query scheduler`, () => { pipeline: builderInternals.pipelineCache, } as unknown as FullSyncState const laterFailure = new Error(`later source failed`) - const loaderCalls: Array = [] + const loaderCalls: Array = [] + const loaderCallCounts = new Map() const loadMoreSpy = vi .spyOn(CollectionSubscriber.prototype, `loadMoreIfNeeded`) - .mockImplementationOnce(() => { - loaderCalls.push(1) - throw undefined - }) - .mockImplementationOnce(() => { - loaderCalls.push(2) - throw laterFailure - }) - .mockImplementationOnce(() => { - loaderCalls.push(3) - return true + .mockImplementation(function (this: unknown) { + const { alias } = this as { alias: string } + loaderCalls.push(alias) + loaderCallCounts.set(alias, (loaderCallCounts.get(alias) ?? 0) + 1) + if (alias === `first`) throw undefined + if (alias === `second`) throw laterFailure + if (alias === `third`) return true + throw new Error(`Unexpected source alias: ${alias}`) }) try { @@ -758,7 +756,12 @@ describe(`live query scheduler`, () => { expect(didThrow).toBe(true) expect(Object.is(thrown, undefined)).toBe(true) - expect(loaderCalls).toEqual([1, 2, 3]) + expect(loaderCalls).toEqual([`first`, `second`, `third`]) + expect(Object.fromEntries(loaderCallCounts)).toEqual({ + first: 1, + second: 1, + third: 1, + }) expect(loadMoreSpy).toHaveBeenCalledTimes(3) } finally { for (const unsubscribe of syncState.unsubscribeCallbacks) unsubscribe() From d1e3d3f752eabbb1b745e75d5051fca002c3f0c1 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 11:31:54 -0600 Subject: [PATCH 234/327] test(db): distinguish repeated source aliases --- packages/db/tests/query/scheduler.test.ts | 75 +++++++++++++++-------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 2c66496c2..fd9d3e9de 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -665,7 +665,7 @@ describe(`live query scheduler`, () => { expect(laterLoader).toHaveBeenCalledOnce() }) - it(`attempts every lexical source loader and preserves the first failure`, async () => { + it(`attempts every repeated-alias source loader and preserves the first failure`, async () => { const createSource = (name: string) => createCollection({ id: `source-loader-${name}`, @@ -684,16 +684,15 @@ describe(`live query scheduler`, () => { const builder = new CollectionConfigBuilder({ id: `source-loader-builder`, query: (q) => - q - .from({ first: firstSource }) - .join( - { second: secondSource }, - ({ first, second }) => eq(first.id, second.id), - ) - .join( - { third: thirdSource }, - ({ first, third }) => eq(first.id, third.id), - ), + q.from({ root: firstSource }).select(({ root }) => ({ + id: root.id, + second: q + .from({ item: secondSource }) + .where(({ item }) => eq(item.id, root.id)), + third: q + .from({ item: thirdSource }) + .where(({ item }) => eq(item.id, root.id)), + })), }) type BuilderSyncConfig = Parameters< ReturnType[`sync`][`sync`] @@ -709,6 +708,11 @@ describe(`live query scheduler`, () => { graphCache: FullSyncState[`graph`] inputsCache: FullSyncState[`inputs`] pipelineCache: FullSyncState[`pipeline`] + collectionSources: Array<{ + sourceId: string + alias: string + collection: object + }> subscribeToAllCollections: ( syncConfig: typeof config, state: FullSyncState, @@ -722,19 +726,36 @@ describe(`live query scheduler`, () => { inputs: builderInternals.inputsCache, pipeline: builderInternals.pipelineCache, } as unknown as FullSyncState + const sourceIdFor = (collection: object): string => { + const source = builderInternals.collectionSources.find( + (candidate) => candidate.collection === collection, + ) + if (!source) throw new Error(`Expected a lexical source`) + return source.sourceId + } + const firstSourceId = sourceIdFor(firstSource) + const secondSourceId = sourceIdFor(secondSource) + const thirdSourceId = sourceIdFor(thirdSource) + expect( + builderInternals.collectionSources.map(({ alias }) => alias), + ).toEqual([`root`, `item`, `item`]) + expect(new Set([firstSourceId, secondSourceId, thirdSourceId]).size).toBe(3) const laterFailure = new Error(`later source failed`) const loaderCalls: Array = [] const loaderCallCounts = new Map() const loadMoreSpy = vi .spyOn(CollectionSubscriber.prototype, `loadMoreIfNeeded`) .mockImplementation(function (this: unknown) { - const { alias } = this as { alias: string } - loaderCalls.push(alias) - loaderCallCounts.set(alias, (loaderCallCounts.get(alias) ?? 0) + 1) - if (alias === `first`) throw undefined - if (alias === `second`) throw laterFailure - if (alias === `third`) return true - throw new Error(`Unexpected source alias: ${alias}`) + const { sourceId } = this as { sourceId: string } + loaderCalls.push(sourceId) + loaderCallCounts.set( + sourceId, + (loaderCallCounts.get(sourceId) ?? 0) + 1, + ) + if (sourceId === firstSourceId) throw undefined + if (sourceId === secondSourceId) throw laterFailure + if (sourceId === thirdSourceId) return true + throw new Error(`Unexpected source: ${sourceId}`) }) try { @@ -756,12 +777,18 @@ describe(`live query scheduler`, () => { expect(didThrow).toBe(true) expect(Object.is(thrown, undefined)).toBe(true) - expect(loaderCalls).toEqual([`first`, `second`, `third`]) - expect(Object.fromEntries(loaderCallCounts)).toEqual({ - first: 1, - second: 1, - third: 1, - }) + expect(loaderCalls).toEqual([ + firstSourceId, + secondSourceId, + thirdSourceId, + ]) + expect(loaderCallCounts).toEqual( + new Map([ + [firstSourceId, 1], + [secondSourceId, 1], + [thirdSourceId, 1], + ]), + ) expect(loadMoreSpy).toHaveBeenCalledTimes(3) } finally { for (const unsubscribe of syncState.unsubscribeCallbacks) unsubscribe() From 3ec8cdfae4ede88487ddda411edf8e82a9bdbe8d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 11:46:14 -0600 Subject: [PATCH 235/327] fix(db): restore facade publications silently --- packages/db/src/query/live/ARCHITECTURE.md | 3 + .../src/query/live/bucket-facade-adapter.ts | 73 +++++++++++-------- .../tests/query/bucket-facade-adapter.test.ts | 43 ++++++++--- 3 files changed, 76 insertions(+), 43 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index caba5ae01..4da2e4eb6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1154,6 +1154,9 @@ read another participating Collection without seeing new rows behind an old revision. If a later root or containing-facade application fails before that release, rollback restores the installed state and discards both the held events and their revision advances. Routing and identity remain inside D2. +Facade rollback restores the Collection's internal publication snapshot. It +must not use a public sync transaction or emit change, layout, readiness, or +truncate lifecycle events for state that never committed. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 2d080071d..7f48ff246 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -5,6 +5,7 @@ import { BUCKET_FACADE_REF } from './materialized-pipeline.js' import type { Collection } from '../../collection/index.js' import type { SyncConfig } from '../../types.js' import type { PublicationDeferral } from '../../collection/changes.js' +import type { CollectionPublicationStateSnapshot } from '../../collection/state.js' import type { BucketFacadeCompilation, BucketFacadeRef, @@ -30,17 +31,23 @@ type FacadeEntry = { currentOrder: Map } +type FacadeEntrySnapshot = { + publicationState: CollectionPublicationStateSnapshot< + Record, + string | number + > + currentOrder: Map + rows: Array<{ + key: string | number + value: object + order: string | undefined + }> +} + type FacadeSnapshot = { activeBuckets: Map> entries: Map> - rows: Map< - FacadeEntry, - Array<{ - key: string | number - value: object - order: string | undefined - }> - > + entryStates: Map } export type FacadePublication = { @@ -256,24 +263,29 @@ export class BucketFacadeAdapter { } private snapshot(): FacadeSnapshot { - const rows = new Map< - FacadeEntry, - Array<{ - key: string | number - value: object - order: string | undefined - }> - >() - for (const byBucket of this.entries.values()) { - for (const entry of byBucket.values()) { - rows.set( - entry, - [...entry.collection._state.syncedData].map(([key, value]) => ({ + const entryStates = new Map() + for (const [edgeId, byBucket] of this.entries) { + for (const [bucketKey, entry] of byBucket) { + const affectedKeys = new Set(entry.collection._state.syncedData.keys()) + for (const change of this.pending + .get(edgeId) + ?.get(bucketKey) + ?.values() ?? []) { + const key = change.value.publicKey + if (typeof key === `string` || typeof key === `number`) { + affectedKeys.add(key) + } + } + entryStates.set(entry, { + publicationState: + entry.collection._snapshotPublicationState(affectedKeys), + currentOrder: new Map(entry.currentOrder), + rows: [...entry.collection._state.syncedData].map(([key, value]) => ({ key, value, order: entry.currentOrder.get(key), })), - ) + }) } } return { @@ -289,7 +301,7 @@ export class BucketFacadeAdapter { new Map(byBucket), ]), ), - rows, + entryStates, } } @@ -308,18 +320,17 @@ export class BucketFacadeAdapter { for (const entry of changedEntries) { if (!previousEntries.has(entry)) continue - const sync = entry.sync - if (!sync) continue - sync.begin() - sync.truncate() + const entryState = snapshot.entryStates.get(entry) + if (!entryState) continue + entry.collection._restorePublicationState(entryState.publicationState) entry.currentOrder.clear() - for (const row of snapshot.rows.get(entry) ?? []) { + for (const [key, order] of entryState.currentOrder) { + entry.currentOrder.set(key, order) + } + for (const row of entryState.rows) { entry.keys.set(row.value, row.key) if (row.order !== undefined) entry.order.set(row.value, row.order) - entry.currentOrder.set(row.key, row.order) - sync.write({ type: `insert`, value: row.value }) } - sync.commit() } this.entries.clear() diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 1122671d1..7311e5db7 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -64,13 +64,13 @@ describe(`BucketFacadeAdapter`, () => { await adapter.cleanup() }) - it(`restores facade state when a flush fails after writing`, async () => { + it(`restores facade state without public effects when a flush fails`, async () => { const graph = new D2() const rows = graph.newInput<[string, BucketRow]>() const activeBuckets = graph.newInput<[string, true]>() const adapter = new BucketFacadeAdapter( `facade-rollback-parent`, - [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: false }], + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: true }], () => {}, ) graph.finalize() @@ -81,10 +81,7 @@ describe(`BucketFacadeAdapter`, () => { rows.sendData( new MultiSet([ [ - [ - bucketKey, - { publicKey: original.id, value: original, order: undefined }, - ], + [bucketKey, { publicKey: original.id, value: original, order: `0` }], 1, ], ]), @@ -104,6 +101,20 @@ describe(`BucketFacadeAdapter`, () => { const subscription = facade.subscribeChanges((changes) => { publications.push(changes) }) + let layoutPublications = 0 + const unsubscribeLayout = facade._subscribeLayoutChanges(() => { + layoutPublications++ + }) + let statusChanges = 0 + const unsubscribeStatus = facade.on(`status:change`, () => { + statusChanges++ + }) + let truncates = 0 + const unsubscribeTruncate = facade.on(`truncate`, () => { + truncates++ + }) + const stateRevision = facade._stateRevision + const layoutRevision = facade._layoutRevision const entries = ( adapter as unknown as { @@ -124,13 +135,11 @@ describe(`BucketFacadeAdapter`, () => { } const replacement = { id: 1, value: `replacement` } + const added = { id: 2, value: `added` } rows.sendData( new MultiSet([ [ - [ - bucketKey, - { publicKey: original.id, value: original, order: undefined }, - ], + [bucketKey, { publicKey: original.id, value: original, order: `0` }], -1, ], [ @@ -139,11 +148,12 @@ describe(`BucketFacadeAdapter`, () => { { publicKey: replacement.id, value: replacement, - order: undefined, + order: `2`, }, ], 1, ], + [[bucketKey, { publicKey: added.id, value: added, order: `1` }], 1], ]), ) graph.run() @@ -151,7 +161,16 @@ describe(`BucketFacadeAdapter`, () => { expect(() => adapter.flush()).toThrow(`facade flush failed`) expect(facade.toArray.map(stripVirtualProps)).toEqual([original]) expect(publications).toEqual([]) - + expect(layoutPublications).toBe(0) + expect(statusChanges).toBe(0) + expect(truncates).toBe(0) + expect(facade._stateRevision).toBe(stateRevision) + expect(facade._layoutRevision).toBe(layoutRevision) + expect(facade.status).toBe(`ready`) + + unsubscribeTruncate() + unsubscribeStatus() + unsubscribeLayout() subscription.unsubscribe() await adapter.cleanup() }) From a848a38aeabc22ce3a4f60d89e4fbc77e89ba7d8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 12:10:32 -0600 Subject: [PATCH 236/327] fix(db): publish facade readiness after install --- packages/db/src/query/live/ARCHITECTURE.md | 5 + .../src/query/live/bucket-facade-adapter.ts | 92 +++++-- .../query/live/collection-config-builder.ts | 30 +- packages/db/src/query/live/utils.ts | 13 + .../tests/query/bucket-facade-adapter.test.ts | 256 +++++++++++++++++- 5 files changed, 344 insertions(+), 52 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 4da2e4eb6..bf427ecdd 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1157,6 +1157,11 @@ events and their revision advances. Routing and identity remain inside D2. Facade rollback restores the Collection's internal publication snapshot. It must not use a public sync transaction or emit change, layout, readiness, or truncate lifecycle events for state that never committed. +Fresh-facade readiness joins the prepared publication release only after every +facade and root install has succeeded, and it precedes root callbacks. A +recovery failure attempts every remaining restore and publication discard, +preserves the original graph-install error, and marks the affected facade as +errored so a later successful publication can recover it. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 7f48ff246..cd435d7e4 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -2,6 +2,7 @@ import { output, serializeValue } from '@tanstack/db-ivm' import { createCollection } from '../../collection/index.js' import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' import { BUCKET_FACADE_REF } from './materialized-pipeline.js' +import { runAllCallbacks } from './utils.js' import type { Collection } from '../../collection/index.js' import type { SyncConfig } from '../../types.js' import type { PublicationDeferral } from '../../collection/changes.js' @@ -107,6 +108,7 @@ export class BucketFacadeAdapter { const snapshot = this.snapshot() const deferredEntries = new Set() const publications: Array = [] + const readyEntries = new Set() const deferPublication = (entry: FacadeEntry) => { if (deferredEntries.has(entry)) return deferredEntries.add(entry) @@ -155,8 +157,9 @@ export class BucketFacadeAdapter { sync.collection._markLayoutChange() } sync.commit() + if (entry.collection.status !== `ready`) readyEntries.add(entry) } - for (const entry of newBaselines) entry.sync?.markReady() + for (const entry of newBaselines) readyEntries.add(entry) for (const [bucketKey, multiplicity] of activity ?? []) { if (multiplicity >= 0) continue @@ -165,9 +168,13 @@ export class BucketFacadeAdapter { } } } catch (error) { - this.restore(snapshot, deferredEntries) - this.retiredEntries.clear() - for (const publication of publications) publication.discard() + try { + this.rollbackInstallation(snapshot, deferredEntries, publications) + } catch { + // Preserve the graph-install failure. A failed state restore marks its + // facade as recoverably errored before this rollback closes every + // publication handle. + } throw error } this.pending.clear() @@ -178,37 +185,38 @@ export class BucketFacadeAdapter { const prepare = () => { if (prepared || closed) return prepared = true - for (const publication of publications) publication.prepare() + runAllCallbacks([ + ...publications.map((publication) => publication.prepare), + ...[...readyEntries].map((entry) => () => entry.sync?.markReady()), + ]) } return { prepare, publish: () => { if (closed) return - prepare() + let firstFailure: { error: unknown } | undefined + try { + prepare() + } catch (error) { + firstFailure = { error } + } closed = true - let hasPublicationError = false - let publicationError: unknown - for (const publication of publications) { - try { - publication.publish() - } catch (error) { - if (!hasPublicationError) { - hasPublicationError = true - publicationError = error - } - } + try { + runAllCallbacks( + publications.map((publication) => publication.publish), + ) + } catch (error) { + firstFailure ??= { error } } // Drop only the adapter's strong reference. External holders keep an // empty, ready facade; a later active interval receives a new one. this.retiredEntries.clear() - if (hasPublicationError) throw publicationError + if (firstFailure) throw firstFailure.error }, rollback: () => { if (closed || prepared) return closed = true - this.restore(snapshot, deferredEntries) - this.retiredEntries.clear() - for (const publication of publications) publication.discard() + this.rollbackInstallation(snapshot, deferredEntries, publications) }, } } @@ -309,6 +317,7 @@ export class BucketFacadeAdapter { snapshot: FacadeSnapshot, changedEntries: Set, ): void { + let firstFailure: { error: unknown } | undefined const previousEntries = new Set( [...snapshot.entries.values()].flatMap((byBucket) => [ ...byBucket.values(), @@ -322,14 +331,26 @@ export class BucketFacadeAdapter { if (!previousEntries.has(entry)) continue const entryState = snapshot.entryStates.get(entry) if (!entryState) continue - entry.collection._restorePublicationState(entryState.publicationState) - entry.currentOrder.clear() - for (const [key, order] of entryState.currentOrder) { - entry.currentOrder.set(key, order) - } - for (const row of entryState.rows) { - entry.keys.set(row.value, row.key) - if (row.order !== undefined) entry.order.set(row.value, row.order) + try { + entry.collection._restorePublicationState(entryState.publicationState) + } catch (error) { + firstFailure ??= { error } + if (entry.collection.status !== `error`) { + try { + entry.sync?.markError(error) + } catch (markError) { + firstFailure ??= { error: markError } + } + } + } finally { + entry.currentOrder.clear() + for (const [key, order] of entryState.currentOrder) { + entry.currentOrder.set(key, order) + } + for (const row of entryState.rows) { + entry.keys.set(row.value, row.key) + if (row.order !== undefined) entry.order.set(row.value, row.order) + } } } @@ -346,6 +367,19 @@ export class BucketFacadeAdapter { for (const entry of currentEntries) { if (!previousEntries.has(entry)) void entry.collection.cleanup() } + if (firstFailure) throw firstFailure.error + } + + private rollbackInstallation( + snapshot: FacadeSnapshot, + changedEntries: Set, + publications: Array, + ): void { + runAllCallbacks([ + () => this.restore(snapshot, changedEntries), + () => this.retiredEntries.clear(), + ...publications.map((publication) => publication.discard), + ]) } private accumulateActivity( diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 50ea91f0a..f013efcc7 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -23,6 +23,7 @@ import { extractCollectionFromSource, extractCollectionSources, extractCollectionsFromQuery, + runAllCallbacks, } from './utils.js' import type { LiveQueryInternalUtils } from './internal.js' import type { WindowOptions } from '../compiler/index.js' @@ -1129,28 +1130,15 @@ export class CollectionConfigBuilder< } pendingChanges = new Map() - // Advance every participating Collection's public clocks before the - // first callback can inspect another Collection from this graph turn. - rootPublication?.prepare() - facadePublication.prepare() - - let hasPublicationError = false - let publicationError: unknown - for (const publish of [ - rootPublication?.publish, + // Advance every participating Collection's public clocks and facade + // readiness before the first root callback. A callback failure cannot + // suppress another prepared participant's release. + runAllCallbacks([ + ...(rootPublication ? [rootPublication.prepare] : []), + facadePublication.prepare, + ...(rootPublication ? [rootPublication.publish] : []), facadePublication.publish, - ]) { - if (!publish) continue - try { - publish() - } catch (error) { - if (!hasPublicationError) { - hasPublicationError = true - publicationError = error - } - } - } - if (hasPublicationError) throw publicationError + ]) } graph.finalize() diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index b70842c39..27ab49f54 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -11,6 +11,19 @@ import type { Context } from '../builder/types.js' import type { OrderBy, QueryIR } from '../ir.js' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' +/** Attempt every callback, then rethrow the first exact failure value. */ +export function runAllCallbacks(callbacks: Iterable<() => void>): void { + let firstFailure: { error: unknown } | undefined + for (const callback of callbacks) { + try { + callback() + } catch (error) { + firstFailure ??= { error } + } + } + if (firstFailure) throw firstFailure.error +} + /** * Helper function to extract collections from a compiled query. * Traverses the query IR to find all collection references. diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 7311e5db7..46ab5333c 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' import { eq } from '../../src/query/builder/functions.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' import { BucketFacadeAdapter } from '../../src/query/live/bucket-facade-adapter.js' import { CollectionConfigBuilder } from '../../src/query/live/collection-config-builder.js' import { BUCKET_FACADE_REF } from '../../src/query/live/materialized-pipeline.js' @@ -17,6 +18,17 @@ import type { Context } from '../../src/query/builder/types.js' type FacadeSync = Parameters>[`sync`]>[0] +class ThrowingBuildIndex extends BasicIndex { + throwOnBuild = false + + override build(entries: Iterable<[number, unknown]>): void { + super.build(entries) + if (this.throwOnBuild) { + throw new Error(`facade index rebuild failed`) + } + } +} + describe(`BucketFacadeAdapter`, () => { it(`moves a row when the graph reuses its object for a new order`, async () => { const graph = new D2() @@ -77,6 +89,7 @@ describe(`BucketFacadeAdapter`, () => { const bucketKey = `group-1` const original = { id: 1, value: `original` } + const fixed = { id: 3, value: `fixed` } activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) rows.sendData( new MultiSet([ @@ -84,6 +97,7 @@ describe(`BucketFacadeAdapter`, () => { [bucketKey, { publicKey: original.id, value: original, order: `0` }], 1, ], + [[bucketKey, { publicKey: fixed.id, value: fixed, order: `1` }], 1], ]), ) graph.run() @@ -96,7 +110,7 @@ describe(`BucketFacadeAdapter`, () => { typeof original, number > - expect(facade.toArray.map(stripVirtualProps)).toEqual([original]) + expect(facade.toArray.map(stripVirtualProps)).toEqual([original, fixed]) const publications: Array = [] const subscription = facade.subscribeChanges((changes) => { publications.push(changes) @@ -159,7 +173,17 @@ describe(`BucketFacadeAdapter`, () => { graph.run() expect(() => adapter.flush()).toThrow(`facade flush failed`) - expect(facade.toArray.map(stripVirtualProps)).toEqual([original]) + expect(facade.toArray.map(stripVirtualProps)).toEqual([original, fixed]) + expect([ + ...( + entries.get(`children`)?.get(bucketKey) as unknown as { + currentOrder: Map + } + ).currentOrder, + ]).toEqual([ + [original.id, `0`], + [fixed.id, `1`], + ]) expect(publications).toEqual([]) expect(layoutPublications).toBe(0) expect(statusChanges).toBe(0) @@ -175,6 +199,234 @@ describe(`BucketFacadeAdapter`, () => { await adapter.cleanup() }) + it(`publishes fresh facade readiness only after every install succeeds`, async () => { + const graph = new D2() + const firstRows = graph.newInput<[string, BucketRow]>() + const firstActiveBuckets = graph.newInput<[string, true]>() + const secondRows = graph.newInput<[string, BucketRow]>() + const secondActiveBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-ready-parent`, + [ + { + edgeId: `first`, + rows: firstRows, + activeBuckets: firstActiveBuckets, + hasOrderBy: false, + }, + { + edgeId: `second`, + rows: secondRows, + activeBuckets: secondActiveBuckets, + hasOrderBy: false, + }, + ], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const firstFacade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `first`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + { id: number; value: string }, + number + > + const secondFacade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `second`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + { id: number; value: string }, + number + > + const firstStatuses: Array = [] + const secondStatuses: Array = [] + const unsubscribeFirst = firstFacade.on(`status:change`, ({ status }) => { + firstStatuses.push(status) + }) + const unsubscribeSecond = secondFacade.on(`status:change`, ({ status }) => { + secondStatuses.push(status) + }) + + const first = { id: 1, value: `first` } + const second = { id: 2, value: `second` } + firstActiveBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + secondActiveBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + firstRows.sendData( + new MultiSet([ + [ + [bucketKey, { publicKey: first.id, value: first, order: undefined }], + 1, + ], + ]), + ) + secondRows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { publicKey: second.id, value: second, order: undefined }, + ], + 1, + ], + ]), + ) + graph.run() + + const entries = ( + adapter as unknown as { + entries: Map> + } + ).entries + const secondSync = entries.get(`second`)?.get(bucketKey)?.sync + if (!secondSync) throw new Error(`Missing second facade sync`) + const commit = secondSync.commit + let shouldThrow = true + secondSync.commit = () => { + const applied = commit() + if (shouldThrow) { + shouldThrow = false + throw new Error(`second facade failed`) + } + return applied + } + + expect(() => adapter.flush()).toThrow(`second facade failed`) + expect(firstFacade.status).toBe(`loading`) + expect(secondFacade.status).toBe(`loading`) + expect(firstStatuses).toEqual([]) + expect(secondStatuses).toEqual([]) + expect(firstFacade.toArray).toEqual([]) + expect(secondFacade.toArray).toEqual([]) + + const retry = adapter.flush() + expect(firstFacade.status).toBe(`loading`) + expect(secondFacade.status).toBe(`loading`) + retry.prepare() + expect(firstFacade.status).toBe(`ready`) + expect(secondFacade.status).toBe(`ready`) + retry.publish() + expect(firstFacade.toArray.map(stripVirtualProps)).toEqual([first]) + expect(secondFacade.toArray.map(stripVirtualProps)).toEqual([second]) + expect(firstStatuses).toEqual([`ready`]) + expect(secondStatuses).toEqual([`ready`]) + + unsubscribeFirst() + unsubscribeSecond() + await adapter.cleanup() + }) + + it(`closes publication state when facade index restore fails`, async () => { + const graph = new D2() + const rows = graph.newInput<[string, BucketRow]>() + const activeBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-index-rollback-parent`, + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: false }], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const original = { id: 1, value: `original` } + activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { publicKey: original.id, value: original, order: undefined }, + ], + 1, + ], + ]), + ) + graph.run() + adapter.flush().publish() + + const facade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `children`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + typeof original, + number + > + const index = facade.createIndex((row) => row.value, { + indexType: ThrowingBuildIndex, + }) as ThrowingBuildIndex + const publications: Array = [] + const subscription = facade.subscribeChanges( + (changes) => { + publications.push(changes) + }, + { includeInitialState: false }, + ) + const revision = facade._stateRevision + + const entry = ( + adapter as unknown as { + entries: Map> + } + ).entries + .get(`children`) + ?.get(bucketKey) + const sync = entry?.sync + if (!sync) throw new Error(`Missing facade sync`) + const commit = sync.commit + let shouldThrow = true + sync.commit = () => { + const applied = commit() + if (shouldThrow) { + shouldThrow = false + throw new Error(`facade flush failed`) + } + return applied + } + + const replacement = { id: 1, value: `replacement` } + index.throwOnBuild = true + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { publicKey: original.id, value: original, order: undefined }, + ], + -1, + ], + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: undefined, + }, + ], + 1, + ], + ]), + ) + graph.run() + + expect(() => adapter.flush()).toThrow(`facade flush failed`) + expect(facade.status).toBe(`error`) + expect(facade._state.syncedData.get(original.id)).toMatchObject(original) + expect(publications).toEqual([]) + expect(facade._stateRevision).toBe(revision) + + index.throwOnBuild = false + adapter.flush().publish() + expect(facade.status).toBe(`ready`) + expect(facade.toArray.map(stripVirtualProps)).toEqual([replacement]) + expect(publications).toHaveLength(2) + expect(publications[0]).toEqual([]) + expect(publications[1]).toHaveLength(1) + expect(facade._stateRevision).toBe(revision + 1) + expect(index.lookup(`eq`, `replacement`)).toEqual(new Set([original.id])) + + subscription.unsubscribe() + await adapter.cleanup() + }) + it(`drops pending parent changes when facade flushing fails`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } From 00c1671a30185ea5fd360ace0529cfb9c3596006 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 12:32:58 -0600 Subject: [PATCH 237/327] fix(db): retry failed graph publications --- packages/db/src/query/live/ARCHITECTURE.md | 9 +- .../src/query/live/bucket-facade-adapter.ts | 104 +++++++++-- .../query/live/collection-config-builder.ts | 43 ++++- .../tests/query/bucket-facade-adapter.test.ts | 4 +- ...ncludes-collection-oracle.property.test.ts | 167 ++++++++++++++---- 5 files changed, 270 insertions(+), 57 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index bf427ecdd..0a80e836a 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1154,14 +1154,19 @@ read another participating Collection without seeing new rows behind an old revision. If a later root or containing-facade application fails before that release, rollback restores the installed state and discards both the held events and their revision advances. Routing and identity remain inside D2. +The root and facade adapters retain the graph deltas consumed by that failed +attempt. A later graph turn retries the whole uncommitted relation even when +the source emits only an unrelated root delta; D2 does not replay a delta that +an adapter has already consumed. Facade rollback restores the Collection's internal publication snapshot. It must not use a public sync transaction or emit change, layout, readiness, or truncate lifecycle events for state that never committed. Fresh-facade readiness joins the prepared publication release only after every facade and root install has succeeded, and it precedes root callbacks. A recovery failure attempts every remaining restore and publication discard, -preserves the original graph-install error, and marks the affected facade as -errored so a later successful publication can recover it. +preserves the original graph-install error, and marks the affected root or +facade as errored so a later successful publication can recover it and restore +readiness. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index cd435d7e4..ea05b5c8a 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -63,11 +63,8 @@ export type FacadePublication = { * graph's canonical bucket-row deltas to those facades. */ export class BucketFacadeAdapter { - private readonly pending = new Map< - string, - Map> - >() - private readonly pendingActivity = new Map>() + private pending = new Map>>() + private pendingActivity = new Map>() private readonly activeBuckets = new Map>() private readonly entries = new Map>() private readonly retiredEntries = new Map>() @@ -106,6 +103,8 @@ export class BucketFacadeAdapter { flush(): FacadePublication { const snapshot = this.snapshot() + const installedPending = this.pending + const installedActivity = this.pendingActivity const deferredEntries = new Set() const publications: Array = [] const readyEntries = new Set() @@ -119,7 +118,7 @@ export class BucketFacadeAdapter { // their containing rows are written to the next facade. try { for (const compilation of this.compilations) { - const activity = this.pendingActivity.get(compilation.edgeId) + const activity = installedActivity.get(compilation.edgeId) const active = this.getActiveBuckets(compilation.edgeId) const newBaselines: Array = [] for (const [bucketKey, multiplicity] of activity ?? []) { @@ -129,7 +128,7 @@ export class BucketFacadeAdapter { } } - const buckets = this.pending.get(compilation.edgeId) + const buckets = installedPending.get(compilation.edgeId) for (const [bucketKey, changes] of buckets ?? []) { const existing = this.entries.get(compilation.edgeId)?.get(bucketKey) if (!active.has(bucketKey) && !existing) continue @@ -177,8 +176,11 @@ export class BucketFacadeAdapter { } throw error } - this.pending.clear() - this.pendingActivity.clear() + // Detach, rather than clear, the deltas installed by this attempt. The + // containing root publication decides whether they commit or must be + // replayed with the next graph turn. + this.pending = new Map() + this.pendingActivity = new Map() let prepared = false let closed = false @@ -216,7 +218,17 @@ export class BucketFacadeAdapter { rollback: () => { if (closed || prepared) return closed = true - this.rollbackInstallation(snapshot, deferredEntries, publications) + runAllCallbacks([ + () => { + this.pending = mergePendingRows(installedPending, this.pending) + this.pendingActivity = mergePendingActivity( + installedActivity, + this.pendingActivity, + ) + }, + () => + this.rollbackInstallation(snapshot, deferredEntries, publications), + ]) }, } } @@ -622,3 +634,75 @@ function isPlainObject(value: unknown): value is Record { const prototype = Object.getPrototypeOf(value) return prototype === Object.prototype || prototype === null } + +function mergePendingRows( + earlier: Map>>, + later: Map>>, +): Map>> { + const merged = new Map>>() + + for (const [edgeId, buckets] of earlier) { + const mergedBuckets = new Map>() + merged.set(edgeId, mergedBuckets) + for (const [bucketKey, rows] of buckets) { + mergedBuckets.set( + bucketKey, + new Map( + [...rows].map(([key, change]) => [key, { ...change }] as const), + ), + ) + } + } + + for (const [edgeId, buckets] of later) { + let mergedBuckets = merged.get(edgeId) + if (!mergedBuckets) { + mergedBuckets = new Map() + merged.set(edgeId, mergedBuckets) + } + for (const [bucketKey, rows] of buckets) { + let mergedRows = mergedBuckets.get(bucketKey) + if (!mergedRows) { + mergedRows = new Map() + mergedBuckets.set(bucketKey, mergedRows) + } + for (const [key, laterChange] of rows) { + const earlierChange = mergedRows.get(key) + if (!earlierChange) { + mergedRows.set(key, { ...laterChange }) + continue + } + earlierChange.deletes += laterChange.deletes + earlierChange.inserts += laterChange.inserts + if (laterChange.inserts > 0) earlierChange.value = laterChange.value + } + } + } + + return merged +} + +function mergePendingActivity( + earlier: Map>, + later: Map>, +): Map> { + const merged = new Map( + [...earlier].map( + ([edgeId, buckets]) => [edgeId, new Map(buckets)] as const, + ), + ) + for (const [edgeId, buckets] of later) { + let mergedBuckets = merged.get(edgeId) + if (!mergedBuckets) { + mergedBuckets = new Map() + merged.set(edgeId, mergedBuckets) + } + for (const [bucketKey, multiplicity] of buckets) { + mergedBuckets.set( + bucketKey, + (mergedBuckets.get(bucketKey) ?? 0) + multiplicity, + ) + } + } + return merged +} diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index f013efcc7..bd5684c99 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1045,6 +1045,7 @@ export class CollectionConfigBuilder< // transaction, avoiding duplicate key errors when joins produce multiple outputs // for the same key (e.g., first output with null, then output with joined data). let pendingChanges: Map> = new Map() + let rootNeedsReady = false pipeline.pipe( output((data) => { @@ -1120,12 +1121,36 @@ export class CollectionConfigBuilder< commit() } } catch (error) { - pendingChanges = new Map() - rootPublication?.discard() - if (rootStateSnapshot) { - config.collection._restorePublicationState(rootStateSnapshot) + const failedRootState = rootStateSnapshot + try { + runAllCallbacks([ + ...(rootPublication ? [rootPublication.discard] : []), + ...(failedRootState + ? [ + () => { + try { + config.collection._restorePublicationState( + failedRootState, + ) + } catch (restoreError) { + rootNeedsReady = true + try { + config.markError(restoreError) + } catch { + // The install failure remains authoritative. Recovery + // still continues through every facade participant. + } + throw restoreError + } + }, + ] + : []), + ...(facadePublication ? [facadePublication.rollback] : []), + ]) + } catch { + // Preserve the graph-install failure after attempting every recovery + // step. A failed root restore remains retryable from staged deltas. } - facadePublication?.rollback() throw error } pendingChanges = new Map() @@ -1136,6 +1161,14 @@ export class CollectionConfigBuilder< runAllCallbacks([ ...(rootPublication ? [rootPublication.prepare] : []), facadePublication.prepare, + ...(rootNeedsReady + ? [ + () => { + rootNeedsReady = false + config.markReady() + }, + ] + : []), ...(rootPublication ? [rootPublication.publish] : []), facadePublication.publish, ]) diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 46ab5333c..8ea284880 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -427,7 +427,7 @@ describe(`BucketFacadeAdapter`, () => { await adapter.cleanup() }) - it(`drops pending parent changes when facade flushing fails`, async () => { + it(`retries pending parent changes when facade flushing fails`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } const parents = createCollection( @@ -486,7 +486,7 @@ describe(`BucketFacadeAdapter`, () => { throw new Error(`Missing live query sync state`) } syncState.flushPendingChanges() - expect(live.has(2)).toBe(false) + expect(live.has(2)).toBe(true) } finally { CollectionConfigBuilder.prototype.getConfig = originalGetConfig vi.restoreAllMocks() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index b7b908c7b..f85038dcd 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -51,11 +51,17 @@ type FacadeCandidateScanScenario = { class ThrowingUpdateIndex extends BasicIndex { throwAfterUpdate = false + throwAfterBuild = false override update(key: number, oldItem: unknown, newItem: unknown): void { super.update(key, oldItem, newItem) if (this.throwAfterUpdate) throw new Error(`root index failed`) } + + override build(entries: Iterable<[number, unknown]>): void { + super.build(entries) + if (this.throwAfterBuild) throw new Error(`root index rebuild failed`) + } } const exhaustiveLayoutSwapScenarios: Array = Array.from( @@ -383,9 +389,9 @@ async function expectFacadeCandidateScan({ ) if (finalLayout === `moved`) { expect(changedKeyOrder).toEqual([10, 20]) - expect( - changedKeyOrder[candidatePosition === `first` ? 0 : 1], - ).toBe(candidateRow.id) + expect(changedKeyOrder[candidatePosition === `first` ? 0 : 1]).toBe( + candidateRow.id, + ) } else { expect(changedKeyOrder).toEqual([valueRow.id]) } @@ -1176,34 +1182,24 @@ describe(`Collection-valued includes oracle`, () => { expect(observerNotifications).toBe(0) rootIndex.throwAfterUpdate = false - nodes.writeBatch([ - { - type: `update`, - value: { ...initialParent, value: 3 }, - }, - { - type: `update`, - value: { ...initialChild, value: 4 }, - }, - { - type: `update`, - value: { ...initialSibling, value: 3 }, - }, - ]) + // Only the root changes on retry. The child deltas consumed by the + // failed graph turn must remain staged until the whole publication + // commits; the source will not emit them again. + nodes.write(`update`, { ...initialParent, value: 3 }) expect(live.get(1)!.value).toBe(3) expect([...rootIndex.equalityLookup(1)]).toEqual([]) expect([...rootIndex.equalityLookup(3)]).toEqual([1]) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ - { id: 20, value: 3 }, - { id: 10, value: 4 }, + { id: 20, value: 0 }, + { id: 10, value: 3 }, ]) expect(rootPublications).toHaveLength(1) expect(childPublications).toHaveLength(2) expect(rootCallbackFacadeSnapshots).toEqual([ { rows: [ - { id: 20, value: 3 }, - { id: 10, value: 4 }, + { id: 20, value: 0 }, + { id: 10, value: 3 }, ], stateRevision: childStateRevisionBeforeFailure + 1, layoutRevision: childLayoutRevisionBeforeFailure + 1, @@ -1225,6 +1221,111 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `root restore failure preserves the install error and releases facade rollback`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` + group: number + value: number + } + const initialParent: NodeRow = { + id: 1, + kind: `parent`, + group: 1, + value: 1, + } + const initialChild: NodeRow = { + id: 10, + kind: `child`, + group: 1, + value: 1, + } + const nodes = createControlledCollection( + `root-restore-failure-nodes`, + [initialParent, initialChild], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootIndex = live.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex + const rootPublications: Array = [] + const childPublications: Array = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(...batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => childPublications.push(...batch), + { includeInitialState: false }, + ) + const rootRevision = live._stateRevision + const childRevision = facade._stateRevision + rootIndex.throwAfterUpdate = true + rootIndex.throwAfterBuild = true + + try { + expect(() => + nodes.writeBatch([ + { + type: `update`, + value: { ...initialParent, value: 2 }, + }, + { + type: `update`, + value: { ...initialChild, value: 2 }, + }, + ]), + ).toThrow(`root index failed`) + expect(live.status).toBe(`error`) + expect(live.get(1)!.value).toBe(1) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(live._stateRevision).toBe(rootRevision) + expect(facade._stateRevision).toBe(childRevision) + + rootIndex.throwAfterUpdate = false + rootIndex.throwAfterBuild = false + nodes.write(`update`, { ...initialParent, value: 3 }) + + expect(live.status).toBe(`ready`) + expect(live.get(1)!.value).toBe(3) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 2 }, + ]) + expect(rootPublications).toHaveLength(1) + expect(childPublications).toHaveLength(1) + expect(live._stateRevision).toBe(rootRevision + 1) + expect(facade._stateRevision).toBe(childRevision + 1) + } finally { + rootIndex.throwAfterUpdate = false + rootIndex.throwAfterBuild = false + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + fcTest( `child-only changes flush the facade without republishing the parent`, async () => { @@ -1285,11 +1386,7 @@ describe(`Collection-valued includes oracle`, () => { }, ) - for (const { - throwingParentId, - position, - failure, - } of [ + for (const { throwingParentId, position, failure } of [ { throwingParentId: 1, position: `first`, failure: `error` }, { throwingParentId: 2, position: `middle`, failure: `undefined` }, { throwingParentId: 3, position: `last`, failure: `null` }, @@ -1315,9 +1412,7 @@ describe(`Collection-valued includes oracle`, () => { id: parent.id, children: q .from({ child: children.collection }) - .where(({ child }) => - eq(child.parentGroup, parent.group), - ), + .where(({ child }) => eq(child.parentGroup, parent.group)), })), ) @@ -1583,9 +1678,7 @@ describe(`Collection-valued includes oracle`, () => { value: child.value, grandchildren: q .from({ grandchild: nodes.collection }) - .where(({ grandchild }) => - eq(grandchild.kind, `grandchild`), - ) + .where(({ grandchild }) => eq(grandchild.kind, `grandchild`)) .where(({ grandchild }) => eq(grandchild.parentGroup, child.group), ) @@ -1802,9 +1895,7 @@ describe(`Collection-valued includes oracle`, () => { childRevision = facade._stateRevision childSubscription = facade.subscribeChanges( (batch) => { - childBatches.push( - batch.map((change) => change.value.value), - ) + childBatches.push(batch.map((change) => change.value.value)) }, { includeInitialState: false }, ) @@ -2280,9 +2371,9 @@ describe(`Collection-valued includes oracle`, () => { }) await parentReady expect( - live.utils[LIVE_QUERY_INTERNAL] - .getLastWindowOutcomes() - .map(({ sourceId }) => sourceId), + live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes().map( + ({ sourceId }) => sourceId, + ), ).toEqual([`initial`, `before-nested`, `rollback`, `after-catch`]) } finally { subscription.unsubscribe() From 479a3832b360069977b5d5466b36d34ed7b145b8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 13:21:33 -0600 Subject: [PATCH 238/327] fix(db): retry failed index restoration --- packages/db/src/query/live/ARCHITECTURE.md | 6 ++ .../src/query/live/bucket-facade-adapter.ts | 80 ++++++++++++++----- .../query/live/collection-config-builder.ts | 35 ++++++-- .../tests/query/bucket-facade-adapter.test.ts | 40 +++++++++- ...ncludes-collection-oracle.property.test.ts | 37 +++++++-- 5 files changed, 165 insertions(+), 33 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 0a80e836a..2df1b5188 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1167,6 +1167,12 @@ recovery failure attempts every remaining restore and publication discard, preserves the original graph-install error, and marks the affected root or facade as errored so a later successful publication can recover it and restore readiness. +An index-rebuild failure remains explicit recovery debt. The next graph turn +must finish that restore before applying retained deltas or marking the +Collection ready; an ordinary row update cannot repair an unknown partial +index rebuild. Error status is published only after every root and facade has +restored its reader-visible state and closed its held publication, so a +synchronous status observer cannot see a mixed graph snapshot. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index ea05b5c8a..759492d78 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -68,6 +68,7 @@ export class BucketFacadeAdapter { private readonly activeBuckets = new Map>() private readonly entries = new Map>() private readonly retiredEntries = new Map>() + private readonly recoveryStates = new Map() private resolvedValues = new WeakMap() constructor( @@ -102,6 +103,7 @@ export class BucketFacadeAdapter { } flush(): FacadePublication { + this.recoverEntries() const snapshot = this.snapshot() const installedPending = this.pending const installedActivity = this.pendingActivity @@ -248,6 +250,7 @@ export class BucketFacadeAdapter { this.pending.clear() this.pendingActivity.clear() this.activeBuckets.clear() + this.recoveryStates.clear() } private accumulate( @@ -328,6 +331,7 @@ export class BucketFacadeAdapter { private restore( snapshot: FacadeSnapshot, changedEntries: Set, + failedEntries: Map, ): void { let firstFailure: { error: unknown } | undefined const previousEntries = new Set( @@ -344,25 +348,12 @@ export class BucketFacadeAdapter { const entryState = snapshot.entryStates.get(entry) if (!entryState) continue try { - entry.collection._restorePublicationState(entryState.publicationState) + this.restoreEntryState(entry, entryState) + this.recoveryStates.delete(entry) } catch (error) { firstFailure ??= { error } - if (entry.collection.status !== `error`) { - try { - entry.sync?.markError(error) - } catch (markError) { - firstFailure ??= { error: markError } - } - } - } finally { - entry.currentOrder.clear() - for (const [key, order] of entryState.currentOrder) { - entry.currentOrder.set(key, order) - } - for (const row of entryState.rows) { - entry.keys.set(row.value, row.key) - if (row.order !== undefined) entry.order.set(row.value, row.order) - } + failedEntries.set(entry, error) + this.recoveryStates.set(entry, entryState) } } @@ -382,15 +373,68 @@ export class BucketFacadeAdapter { if (firstFailure) throw firstFailure.error } + private restoreEntryState( + entry: FacadeEntry, + entryState: FacadeEntrySnapshot, + ): void { + try { + entry.collection._restorePublicationState(entryState.publicationState) + } finally { + entry.currentOrder.clear() + for (const [key, order] of entryState.currentOrder) { + entry.currentOrder.set(key, order) + } + for (const row of entryState.rows) { + entry.keys.set(row.value, row.key) + if (row.order !== undefined) entry.order.set(row.value, row.order) + } + } + } + + private recoverEntries(): void { + let firstFailure: { error: unknown } | undefined + const failedEntries = new Map() + for (const [entry, entryState] of this.recoveryStates) { + try { + this.restoreEntryState(entry, entryState) + this.recoveryStates.delete(entry) + } catch (error) { + firstFailure ??= { error } + failedEntries.set(entry, error) + } + } + if (!firstFailure) return + + try { + runAllCallbacks( + [...failedEntries].map(([entry, error]) => () => { + if (entry.collection.status !== `error`) entry.sync?.markError(error) + }), + ) + } catch { + // The index recovery failure remains authoritative and retryable. + } + throw firstFailure.error + } + private rollbackInstallation( snapshot: FacadeSnapshot, changedEntries: Set, publications: Array, ): void { + const failedEntries = new Map() runAllCallbacks([ - () => this.restore(snapshot, changedEntries), + () => this.restore(snapshot, changedEntries, failedEntries), () => this.retiredEntries.clear(), ...publications.map((publication) => publication.discard), + () => + runAllCallbacks( + [...failedEntries].map(([entry, error]) => () => { + if (entry.collection.status !== `error`) { + entry.sync?.markError(error) + } + }), + ), ]) } diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index bd5684c99..f01bca4a2 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1046,6 +1046,9 @@ export class CollectionConfigBuilder< // for the same key (e.g., first output with null, then output with joined data). let pendingChanges: Map> = new Map() let rootNeedsReady = false + let rootRecoveryState: + | CollectionPublicationStateSnapshot + | undefined pipeline.pipe( output((data) => { @@ -1077,6 +1080,21 @@ export class CollectionConfigBuilder< return } + if (rootRecoveryState) { + const stateToRecover = rootRecoveryState + try { + config.collection._restorePublicationState(stateToRecover) + rootRecoveryState = undefined + } catch (error) { + try { + config.markError(error) + } catch { + // Keep the restore failure authoritative and its state retryable. + } + throw error + } + } + let facadePublication: | ReturnType | undefined @@ -1122,6 +1140,7 @@ export class CollectionConfigBuilder< } } catch (error) { const failedRootState = rootStateSnapshot + let rootRestoreFailure: { error: unknown } | undefined try { runAllCallbacks([ ...(rootPublication ? [rootPublication.discard] : []), @@ -1134,12 +1153,8 @@ export class CollectionConfigBuilder< ) } catch (restoreError) { rootNeedsReady = true - try { - config.markError(restoreError) - } catch { - // The install failure remains authoritative. Recovery - // still continues through every facade participant. - } + rootRecoveryState = failedRootState + rootRestoreFailure = { error: restoreError } throw restoreError } }, @@ -1151,6 +1166,14 @@ export class CollectionConfigBuilder< // Preserve the graph-install failure after attempting every recovery // step. A failed root restore remains retryable from staged deltas. } + if (rootRestoreFailure) { + try { + config.markError(rootRestoreFailure.error) + } catch { + // The install failure remains authoritative after every graph + // participant has restored its public state. + } + } throw error } pendingChanges = new Map() diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 8ea284880..da527dc1d 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -19,9 +19,13 @@ import type { Context } from '../../src/query/builder/types.js' type FacadeSync = Parameters>[`sync`]>[0] class ThrowingBuildIndex extends BasicIndex { + throwBeforeBuild = false throwOnBuild = false override build(entries: Iterable<[number, unknown]>): void { + if (this.throwBeforeBuild) { + throw new Error(`facade index rebuild failed`) + } super.build(entries) if (this.throwOnBuild) { throw new Error(`facade index rebuild failed`) @@ -382,7 +386,7 @@ describe(`BucketFacadeAdapter`, () => { } const replacement = { id: 1, value: `replacement` } - index.throwOnBuild = true + index.throwBeforeBuild = true rows.sendData( new MultiSet([ [ @@ -413,15 +417,43 @@ describe(`BucketFacadeAdapter`, () => { expect(publications).toEqual([]) expect(facade._stateRevision).toBe(revision) - index.throwOnBuild = false + const final = { id: 1, value: `final` } + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: undefined, + }, + ], + -1, + ], + [ + [bucketKey, { publicKey: final.id, value: final, order: undefined }], + 1, + ], + ]), + ) + graph.run() + expect(() => adapter.flush()).toThrow(`facade index rebuild failed`) + expect(facade.status).toBe(`error`) + expect(facade._state.syncedData.get(original.id)).toMatchObject(original) + expect(publications).toEqual([]) + + index.throwBeforeBuild = false adapter.flush().publish() expect(facade.status).toBe(`ready`) - expect(facade.toArray.map(stripVirtualProps)).toEqual([replacement]) + expect(facade.toArray.map(stripVirtualProps)).toEqual([final]) expect(publications).toHaveLength(2) expect(publications[0]).toEqual([]) expect(publications[1]).toHaveLength(1) expect(facade._stateRevision).toBe(revision + 1) - expect(index.lookup(`eq`, `replacement`)).toEqual(new Set([original.id])) + expect(index.lookup(`eq`, `original`)).toEqual(new Set()) + expect(index.lookup(`eq`, `replacement`)).toEqual(new Set()) + expect(index.lookup(`eq`, `final`)).toEqual(new Set([original.id])) subscription.unsubscribe() await adapter.cleanup() diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index f85038dcd..9f3153572 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -51,6 +51,7 @@ type FacadeCandidateScanScenario = { class ThrowingUpdateIndex extends BasicIndex { throwAfterUpdate = false + throwBeforeBuild = false throwAfterBuild = false override update(key: number, oldItem: unknown, newItem: unknown): void { @@ -59,6 +60,7 @@ class ThrowingUpdateIndex extends BasicIndex { } override build(entries: Iterable<[number, unknown]>): void { + if (this.throwBeforeBuild) throw new Error(`root index rebuild failed`) super.build(entries) if (this.throwAfterBuild) throw new Error(`root index rebuild failed`) } @@ -1275,10 +1277,17 @@ describe(`Collection-valued includes oracle`, () => { (batch) => childPublications.push(...batch), { includeInitialState: false }, ) + const errorSnapshots: Array<{ root: number; child: number }> = [] + const unsubscribeError = live.on(`status:error`, () => { + errorSnapshots.push({ + root: live.get(1)!.value, + child: facade.get(10)!.value, + }) + }) const rootRevision = live._stateRevision const childRevision = facade._stateRevision rootIndex.throwAfterUpdate = true - rootIndex.throwAfterBuild = true + rootIndex.throwBeforeBuild = true try { expect(() => @@ -1302,13 +1311,27 @@ describe(`Collection-valued includes oracle`, () => { expect(childPublications).toEqual([]) expect(live._stateRevision).toBe(rootRevision) expect(facade._stateRevision).toBe(childRevision) + expect(errorSnapshots).toEqual([{ root: 1, child: 1 }]) rootIndex.throwAfterUpdate = false - rootIndex.throwAfterBuild = false - nodes.write(`update`, { ...initialParent, value: 3 }) + expect(() => + nodes.write(`update`, { ...initialParent, value: 3 }), + ).toThrow(`root index rebuild failed`) + expect(live.status).toBe(`error`) + expect(live.get(1)!.value).toBe(1) + expect(facade.get(10)!.value).toBe(1) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(errorSnapshots).toEqual([ + { root: 1, child: 1 }, + { root: 1, child: 1 }, + ]) + + rootIndex.throwBeforeBuild = false + nodes.write(`update`, { ...initialParent, value: 4 }) expect(live.status).toBe(`ready`) - expect(live.get(1)!.value).toBe(3) + expect(live.get(1)!.value).toBe(4) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 10, value: 2 }, ]) @@ -1316,9 +1339,13 @@ describe(`Collection-valued includes oracle`, () => { expect(childPublications).toHaveLength(1) expect(live._stateRevision).toBe(rootRevision + 1) expect(facade._stateRevision).toBe(childRevision + 1) + expect([...rootIndex.equalityLookup(2)]).toEqual([]) + expect([...rootIndex.equalityLookup(3)]).toEqual([]) + expect([...rootIndex.equalityLookup(4)]).toEqual([1]) } finally { rootIndex.throwAfterUpdate = false - rootIndex.throwAfterBuild = false + rootIndex.throwBeforeBuild = false + unsubscribeError() rootSubscription.unsubscribe() childSubscription.unsubscribe() await Promise.all([live.cleanup(), nodes.collection.cleanup()]) From a1b74e9204554f7c7f4af7a9e607e7dcc4645aef Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 13:38:21 -0600 Subject: [PATCH 239/327] fix(db): recover graph publications together --- packages/db/src/query/live/ARCHITECTURE.md | 11 ++++--- .../src/query/live/bucket-facade-adapter.ts | 6 +++- .../query/live/collection-config-builder.ts | 30 ++++++++++++----- ...ncludes-collection-oracle.property.test.ts | 33 ++++++++++++++++--- 4 files changed, 60 insertions(+), 20 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 2df1b5188..b81e72fa9 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1168,11 +1168,12 @@ preserves the original graph-install error, and marks the affected root or facade as errored so a later successful publication can recover it and restore readiness. An index-rebuild failure remains explicit recovery debt. The next graph turn -must finish that restore before applying retained deltas or marking the -Collection ready; an ordinary row update cannot repair an unknown partial -index rebuild. Error status is published only after every root and facade has -restored its reader-visible state and closed its held publication, so a -synchronous status observer cannot see a mixed graph snapshot. +must attempt every root and facade restore as one recovery preflight, then +finish them all before applying retained deltas or marking any Collection +ready; an ordinary row update cannot repair an unknown partial index rebuild. +Error status is published only after every root and facade recovery attempt and +after each held publication has closed, so a synchronous status observer cannot +see avoidable stale sibling state from a skipped restore. Once release begins, one subscriber callback failure cannot suppress another prepared root or facade publication. Release attempts every participant, then rethrows the first callback failure unchanged, including `null` or `undefined`. diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 759492d78..3ce132cf9 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -102,8 +102,12 @@ export class BucketFacadeAdapter { return this.pending.size > 0 || this.pendingActivity.size > 0 } - flush(): FacadePublication { + recover(): void { this.recoverEntries() + } + + flush(): FacadePublication { + this.recover() const snapshot = this.snapshot() const installedPending = this.pending const installedActivity = this.pendingActivity diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index f01bca4a2..18281013e 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1080,19 +1080,31 @@ export class CollectionConfigBuilder< return } - if (rootRecoveryState) { - const stateToRecover = rootRecoveryState - try { - config.collection._restorePublicationState(stateToRecover) - rootRecoveryState = undefined - } catch (error) { + let rootRecoveryFailure: { error: unknown } | undefined + try { + runAllCallbacks([ + () => { + if (!rootRecoveryState) return + const stateToRecover = rootRecoveryState + try { + config.collection._restorePublicationState(stateToRecover) + rootRecoveryState = undefined + } catch (error) { + rootRecoveryFailure = { error } + throw error + } + }, + () => bucketFacades.recover(), + ]) + } catch (error) { + if (rootRecoveryFailure) { try { - config.markError(error) + config.markError(rootRecoveryFailure.error) } catch { - // Keep the restore failure authoritative and its state retryable. + // Keep the first recovery failure authoritative and retryable. } - throw error } + throw error } let facadePublication: diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 9f3153572..709fb5b75 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -53,6 +53,7 @@ class ThrowingUpdateIndex extends BasicIndex { throwAfterUpdate = false throwBeforeBuild = false throwAfterBuild = false + buildCalls = 0 override update(key: number, oldItem: unknown, newItem: unknown): void { super.update(key, oldItem, newItem) @@ -60,6 +61,7 @@ class ThrowingUpdateIndex extends BasicIndex { } override build(entries: Iterable<[number, unknown]>): void { + this.buildCalls += 1 if (this.throwBeforeBuild) throw new Error(`root index rebuild failed`) super.build(entries) if (this.throwAfterBuild) throw new Error(`root index rebuild failed`) @@ -1224,7 +1226,7 @@ describe(`Collection-valued includes oracle`, () => { ) fcTest( - `root restore failure preserves the install error and releases facade rollback`, + `root and facade recovery retry together after a failed graph install`, async () => { type NodeRow = { id: number @@ -1267,6 +1269,9 @@ describe(`Collection-valued includes oracle`, () => { const rootIndex = live.createIndex((row) => row.value, { indexType: ThrowingUpdateIndex, }) as ThrowingUpdateIndex + const facadeIndex = facade.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex const rootPublications: Array = [] const childPublications: Array = [] const rootSubscription = live.subscribeChanges( @@ -1277,17 +1282,25 @@ describe(`Collection-valued includes oracle`, () => { (batch) => childPublications.push(...batch), { includeInitialState: false }, ) - const errorSnapshots: Array<{ root: number; child: number }> = [] + const errorSnapshots: Array<{ + root: number + child: number + facadeHasFailedValue: boolean + }> = [] const unsubscribeError = live.on(`status:error`, () => { errorSnapshots.push({ root: live.get(1)!.value, child: facade.get(10)!.value, + facadeHasFailedValue: [ + ...facadeIndex.equalityLookup(2), + ].includes(10), }) }) const rootRevision = live._stateRevision const childRevision = facade._stateRevision rootIndex.throwAfterUpdate = true rootIndex.throwBeforeBuild = true + facadeIndex.throwBeforeBuild = true try { expect(() => @@ -1311,21 +1324,30 @@ describe(`Collection-valued includes oracle`, () => { expect(childPublications).toEqual([]) expect(live._stateRevision).toBe(rootRevision) expect(facade._stateRevision).toBe(childRevision) - expect(errorSnapshots).toEqual([{ root: 1, child: 1 }]) + expect(errorSnapshots).toEqual([ + { root: 1, child: 1, facadeHasFailedValue: true }, + ]) rootIndex.throwAfterUpdate = false + facadeIndex.throwBeforeBuild = false + const rootBuildCalls = rootIndex.buildCalls + const facadeBuildCalls = facadeIndex.buildCalls expect(() => nodes.write(`update`, { ...initialParent, value: 3 }), ).toThrow(`root index rebuild failed`) + expect(rootIndex.buildCalls).toBe(rootBuildCalls + 1) + expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 1) expect(live.status).toBe(`error`) expect(live.get(1)!.value).toBe(1) expect(facade.get(10)!.value).toBe(1) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) expect(errorSnapshots).toEqual([ - { root: 1, child: 1 }, - { root: 1, child: 1 }, + { root: 1, child: 1, facadeHasFailedValue: true }, + { root: 1, child: 1, facadeHasFailedValue: false }, ]) + expect([...facadeIndex.equalityLookup(1)]).toEqual([10]) + expect([...facadeIndex.equalityLookup(2)]).toEqual([]) rootIndex.throwBeforeBuild = false nodes.write(`update`, { ...initialParent, value: 4 }) @@ -1345,6 +1367,7 @@ describe(`Collection-valued includes oracle`, () => { } finally { rootIndex.throwAfterUpdate = false rootIndex.throwBeforeBuild = false + facadeIndex.throwBeforeBuild = false unsubscribeError() rootSubscription.unsubscribe() childSubscription.unsubscribe() From d2b00ac38d4cabb5516a3791a4b97dd84b95056d Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 13:50:53 -0600 Subject: [PATCH 240/327] test(db): strengthen graph recovery oracle --- ...ncludes-collection-oracle.property.test.ts | 99 ++++++++++++++----- 1 file changed, 74 insertions(+), 25 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 709fb5b75..086d6fa09 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -50,21 +50,35 @@ type FacadeCandidateScanScenario = { } class ThrowingUpdateIndex extends BasicIndex { - throwAfterUpdate = false - throwBeforeBuild = false - throwAfterBuild = false + updateFailure: { error: unknown } | undefined + buildFailure: + | { error: unknown; stage: `before` | `after` } + | undefined buildCalls = 0 override update(key: number, oldItem: unknown, newItem: unknown): void { super.update(key, oldItem, newItem) - if (this.throwAfterUpdate) throw new Error(`root index failed`) + if (this.updateFailure) throw this.updateFailure.error } override build(entries: Iterable<[number, unknown]>): void { this.buildCalls += 1 - if (this.throwBeforeBuild) throw new Error(`root index rebuild failed`) + if (this.buildFailure?.stage === `before`) { + throw this.buildFailure.error + } super.build(entries) - if (this.throwAfterBuild) throw new Error(`root index rebuild failed`) + if (this.buildFailure?.stage === `after`) { + throw this.buildFailure.error + } + } +} + +function captureFailure(callback: () => void): { error: unknown } | undefined { + try { + callback() + return undefined + } catch (error) { + return { error } } } @@ -1149,7 +1163,7 @@ describe(`Collection-valued includes oracle`, () => { const rootLayoutRevisionBeforeFailure = live._layoutRevision const childStateRevisionBeforeFailure = facade._stateRevision const childLayoutRevisionBeforeFailure = facade._layoutRevision - rootIndex.throwAfterUpdate = true + rootIndex.updateFailure = { error: new Error(`root index failed`) } try { expect(() => @@ -1185,7 +1199,7 @@ describe(`Collection-valued includes oracle`, () => { expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) expect(observerNotifications).toBe(0) - rootIndex.throwAfterUpdate = false + rootIndex.updateFailure = undefined // Only the root changes on retry. The child deltas consumed by the // failed graph turn must remain staged until the whole publication // commits; the source will not emit them again. @@ -1216,7 +1230,7 @@ describe(`Collection-valued includes oracle`, () => { expect(childObserver.getSnapshot()).not.toBe(observerBeforeFailure) expect(observerNotifications).toBe(1) } finally { - rootIndex.throwAfterUpdate = false + rootIndex.updateFailure = undefined childObserver.dispose() rootSubscription.unsubscribe() childSubscription.unsubscribe() @@ -1296,14 +1310,22 @@ describe(`Collection-valued includes oracle`, () => { ].includes(10), }) }) + const readinessOrder: Array<`facade` | `root`> = [] + const unsubscribeRootReady = live.on(`status:ready`, () => { + readinessOrder.push(`root`) + }) + const unsubscribeFacadeReady = facade.on(`status:ready`, () => { + readinessOrder.push(`facade`) + }) const rootRevision = live._stateRevision const childRevision = facade._stateRevision - rootIndex.throwAfterUpdate = true - rootIndex.throwBeforeBuild = true - facadeIndex.throwBeforeBuild = true + const installFailure = new Error(`root index failed`) + rootIndex.updateFailure = { error: installFailure } + rootIndex.buildFailure = { error: false, stage: `before` } + facadeIndex.buildFailure = { error: undefined, stage: `before` } try { - expect(() => + const failedInstall = captureFailure(() => nodes.writeBatch([ { type: `update`, @@ -1314,7 +1336,8 @@ describe(`Collection-valued includes oracle`, () => { value: { ...initialChild, value: 2 }, }, ]), - ).toThrow(`root index failed`) + ) + expect(failedInstall?.error).toBe(installFailure) expect(live.status).toBe(`error`) expect(live.get(1)!.value).toBe(1) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ @@ -1328,13 +1351,13 @@ describe(`Collection-valued includes oracle`, () => { { root: 1, child: 1, facadeHasFailedValue: true }, ]) - rootIndex.throwAfterUpdate = false - facadeIndex.throwBeforeBuild = false + rootIndex.updateFailure = undefined const rootBuildCalls = rootIndex.buildCalls const facadeBuildCalls = facadeIndex.buildCalls - expect(() => + const simultaneousRecoveryFailure = captureFailure(() => nodes.write(`update`, { ...initialParent, value: 3 }), - ).toThrow(`root index rebuild failed`) + ) + expect(simultaneousRecoveryFailure).toEqual({ error: false }) expect(rootIndex.buildCalls).toBe(rootBuildCalls + 1) expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 1) expect(live.status).toBe(`error`) @@ -1343,17 +1366,32 @@ describe(`Collection-valued includes oracle`, () => { expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) expect(errorSnapshots).toEqual([ + { root: 1, child: 1, facadeHasFailedValue: true }, + { root: 1, child: 1, facadeHasFailedValue: true }, + ]) + + facadeIndex.buildFailure = undefined + const facadeRecoveryFailure = captureFailure(() => + nodes.write(`update`, { ...initialParent, value: 4 }), + ) + expect(facadeRecoveryFailure).toEqual({ error: false }) + expect(rootIndex.buildCalls).toBe(rootBuildCalls + 2) + expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 2) + expect(errorSnapshots).toEqual([ + { root: 1, child: 1, facadeHasFailedValue: true }, { root: 1, child: 1, facadeHasFailedValue: true }, { root: 1, child: 1, facadeHasFailedValue: false }, ]) expect([...facadeIndex.equalityLookup(1)]).toEqual([10]) expect([...facadeIndex.equalityLookup(2)]).toEqual([]) - rootIndex.throwBeforeBuild = false - nodes.write(`update`, { ...initialParent, value: 4 }) + rootIndex.buildFailure = undefined + nodes.write(`update`, { ...initialParent, value: 5 }) expect(live.status).toBe(`ready`) - expect(live.get(1)!.value).toBe(4) + expect(facade.status).toBe(`ready`) + expect(readinessOrder).toEqual([`facade`, `root`]) + expect(live.get(1)!.value).toBe(5) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 10, value: 2 }, ]) @@ -1363,12 +1401,23 @@ describe(`Collection-valued includes oracle`, () => { expect(facade._stateRevision).toBe(childRevision + 1) expect([...rootIndex.equalityLookup(2)]).toEqual([]) expect([...rootIndex.equalityLookup(3)]).toEqual([]) - expect([...rootIndex.equalityLookup(4)]).toEqual([1]) + expect([...rootIndex.equalityLookup(4)]).toEqual([]) + expect([...rootIndex.equalityLookup(5)]).toEqual([1]) + + const facadeRevisionAfterRecovery = facade._stateRevision + nodes.write(`update`, { ...initialParent, value: 6 }) + expect(live.get(1)!.value).toBe(6) + expect(rootPublications).toHaveLength(2) + expect(childPublications).toHaveLength(1) + expect(facade._stateRevision).toBe(facadeRevisionAfterRecovery) + expect(readinessOrder).toEqual([`facade`, `root`]) } finally { - rootIndex.throwAfterUpdate = false - rootIndex.throwBeforeBuild = false - facadeIndex.throwBeforeBuild = false + rootIndex.updateFailure = undefined + rootIndex.buildFailure = undefined + facadeIndex.buildFailure = undefined unsubscribeError() + unsubscribeRootReady() + unsubscribeFacadeReady() rootSubscription.unsubscribe() childSubscription.unsubscribe() await Promise.all([live.cleanup(), nodes.collection.cleanup()]) From 692f782a9e7390acf591deb232f5f65bfb6d7bb8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 14:01:56 -0600 Subject: [PATCH 241/327] test(db): close graph recovery proof gaps --- ...ncludes-collection-oracle.property.test.ts | 56 ++++++++++++++++--- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 086d6fa09..e435821ed 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1299,12 +1299,16 @@ describe(`Collection-valued includes oracle`, () => { const errorSnapshots: Array<{ root: number child: number + facadeHasRestoredValue: boolean facadeHasFailedValue: boolean }> = [] const unsubscribeError = live.on(`status:error`, () => { errorSnapshots.push({ root: live.get(1)!.value, child: facade.get(10)!.value, + facadeHasRestoredValue: [ + ...facadeIndex.equalityLookup(1), + ].includes(10), facadeHasFailedValue: [ ...facadeIndex.equalityLookup(2), ].includes(10), @@ -1348,7 +1352,12 @@ describe(`Collection-valued includes oracle`, () => { expect(live._stateRevision).toBe(rootRevision) expect(facade._stateRevision).toBe(childRevision) expect(errorSnapshots).toEqual([ - { root: 1, child: 1, facadeHasFailedValue: true }, + { + root: 1, + child: 1, + facadeHasRestoredValue: false, + facadeHasFailedValue: true, + }, ]) rootIndex.updateFailure = undefined @@ -1366,8 +1375,18 @@ describe(`Collection-valued includes oracle`, () => { expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) expect(errorSnapshots).toEqual([ - { root: 1, child: 1, facadeHasFailedValue: true }, - { root: 1, child: 1, facadeHasFailedValue: true }, + { + root: 1, + child: 1, + facadeHasRestoredValue: false, + facadeHasFailedValue: true, + }, + { + root: 1, + child: 1, + facadeHasRestoredValue: false, + facadeHasFailedValue: true, + }, ]) facadeIndex.buildFailure = undefined @@ -1378,9 +1397,24 @@ describe(`Collection-valued includes oracle`, () => { expect(rootIndex.buildCalls).toBe(rootBuildCalls + 2) expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 2) expect(errorSnapshots).toEqual([ - { root: 1, child: 1, facadeHasFailedValue: true }, - { root: 1, child: 1, facadeHasFailedValue: true }, - { root: 1, child: 1, facadeHasFailedValue: false }, + { + root: 1, + child: 1, + facadeHasRestoredValue: false, + facadeHasFailedValue: true, + }, + { + root: 1, + child: 1, + facadeHasRestoredValue: false, + facadeHasFailedValue: true, + }, + { + root: 1, + child: 1, + facadeHasRestoredValue: true, + facadeHasFailedValue: false, + }, ]) expect([...facadeIndex.equalityLookup(1)]).toEqual([10]) expect([...facadeIndex.equalityLookup(2)]).toEqual([]) @@ -1405,7 +1439,14 @@ describe(`Collection-valued includes oracle`, () => { expect([...rootIndex.equalityLookup(5)]).toEqual([1]) const facadeRevisionAfterRecovery = facade._stateRevision - nodes.write(`update`, { ...initialParent, value: 6 }) + facadeIndex.updateFailure = { + error: new Error(`retained facade delta replayed`), + } + const postRecoveryFailure = captureFailure(() => { + nodes.write(`update`, { ...initialParent, value: 6 }) + }) + expect(postRecoveryFailure).toBeUndefined() + facadeIndex.updateFailure = undefined expect(live.get(1)!.value).toBe(6) expect(rootPublications).toHaveLength(2) expect(childPublications).toHaveLength(1) @@ -1414,6 +1455,7 @@ describe(`Collection-valued includes oracle`, () => { } finally { rootIndex.updateFailure = undefined rootIndex.buildFailure = undefined + facadeIndex.updateFailure = undefined facadeIndex.buildFailure = undefined unsubscribeError() unsubscribeRootReady() From fc093061aecda635926ab79370b1f393a92dbcb7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 14:14:01 -0600 Subject: [PATCH 242/327] test(db): observe graph recovery internals --- ...ncludes-collection-oracle.property.test.ts | 122 ++++++++++++------ 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index e435821ed..73d43b490 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,9 +1,10 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect } from 'vitest' +import { describe, expect, vi } from 'vitest' import { createDeferred } from '../../src/deferred.js' import { BasicIndex } from '../../src/indexes/basic-index.js' import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { createOptimisticAction } from '../../src/optimistic-action.js' +import { BucketFacadeAdapter } from '../../src/query/live/bucket-facade-adapter.js' import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' import { add, @@ -1260,9 +1261,15 @@ describe(`Collection-valued includes oracle`, () => { group: 1, value: 1, } + const initialSibling: NodeRow = { + id: 11, + kind: `child`, + group: 1, + value: 10, + } const nodes = createControlledCollection( `root-restore-failure-nodes`, - [initialParent, initialChild], + [initialParent, initialChild, initialSibling], ) const live = createLiveQueryCollection((q) => q @@ -1298,20 +1305,24 @@ describe(`Collection-valued includes oracle`, () => { ) const errorSnapshots: Array<{ root: number - child: number - facadeHasRestoredValue: boolean - facadeHasFailedValue: boolean + children: Array<{ id: number; value: number }> + indexKeys: { + one: Array + two: Array + ten: Array + twenty: Array + } }> = [] const unsubscribeError = live.on(`status:error`, () => { errorSnapshots.push({ root: live.get(1)!.value, - child: facade.get(10)!.value, - facadeHasRestoredValue: [ - ...facadeIndex.equalityLookup(1), - ].includes(10), - facadeHasFailedValue: [ - ...facadeIndex.equalityLookup(2), - ].includes(10), + children: facade.toArray.map(({ id, value }) => ({ id, value })), + indexKeys: { + one: [...facadeIndex.equalityLookup(1)], + two: [...facadeIndex.equalityLookup(2)], + ten: [...facadeIndex.equalityLookup(10)], + twenty: [...facadeIndex.equalityLookup(20)], + }, }) }) const readinessOrder: Array<`facade` | `root`> = [] @@ -1339,6 +1350,10 @@ describe(`Collection-valued includes oracle`, () => { type: `update`, value: { ...initialChild, value: 2 }, }, + { + type: `update`, + value: { ...initialSibling, value: 20 }, + }, ]), ) expect(failedInstall?.error).toBe(installFailure) @@ -1346,6 +1361,7 @@ describe(`Collection-valued includes oracle`, () => { expect(live.get(1)!.value).toBe(1) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 10, value: 1 }, + { id: 11, value: 10 }, ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) @@ -1354,9 +1370,11 @@ describe(`Collection-valued includes oracle`, () => { expect(errorSnapshots).toEqual([ { root: 1, - child: 1, - facadeHasRestoredValue: false, - facadeHasFailedValue: true, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, }, ]) @@ -1371,21 +1389,28 @@ describe(`Collection-valued includes oracle`, () => { expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 1) expect(live.status).toBe(`error`) expect(live.get(1)!.value).toBe(1) - expect(facade.get(10)!.value).toBe(1) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) expect(errorSnapshots).toEqual([ { root: 1, - child: 1, - facadeHasRestoredValue: false, - facadeHasFailedValue: true, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, }, { root: 1, - child: 1, - facadeHasRestoredValue: false, - facadeHasFailedValue: true, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, }, ]) @@ -1399,25 +1424,33 @@ describe(`Collection-valued includes oracle`, () => { expect(errorSnapshots).toEqual([ { root: 1, - child: 1, - facadeHasRestoredValue: false, - facadeHasFailedValue: true, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, }, { root: 1, - child: 1, - facadeHasRestoredValue: false, - facadeHasFailedValue: true, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, }, { root: 1, - child: 1, - facadeHasRestoredValue: true, - facadeHasFailedValue: false, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [10], two: [], ten: [11], twenty: [] }, }, ]) expect([...facadeIndex.equalityLookup(1)]).toEqual([10]) expect([...facadeIndex.equalityLookup(2)]).toEqual([]) + expect([...facadeIndex.equalityLookup(10)]).toEqual([11]) + expect([...facadeIndex.equalityLookup(20)]).toEqual([]) rootIndex.buildFailure = undefined nodes.write(`update`, { ...initialParent, value: 5 }) @@ -1428,9 +1461,10 @@ describe(`Collection-valued includes oracle`, () => { expect(live.get(1)!.value).toBe(5) expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ { id: 10, value: 2 }, + { id: 11, value: 20 }, ]) expect(rootPublications).toHaveLength(1) - expect(childPublications).toHaveLength(1) + expect(childPublications).toHaveLength(2) expect(live._stateRevision).toBe(rootRevision + 1) expect(facade._stateRevision).toBe(childRevision + 1) expect([...rootIndex.equalityLookup(2)]).toEqual([]) @@ -1439,17 +1473,25 @@ describe(`Collection-valued includes oracle`, () => { expect([...rootIndex.equalityLookup(5)]).toEqual([1]) const facadeRevisionAfterRecovery = facade._stateRevision - facadeIndex.updateFailure = { - error: new Error(`retained facade delta replayed`), - } - const postRecoveryFailure = captureFailure(() => { + const pendingChecks: Array = [] + const hasPendingChanges = + BucketFacadeAdapter.prototype.hasPendingChanges + const pendingSpy = vi + .spyOn(BucketFacadeAdapter.prototype, `hasPendingChanges`) + .mockImplementation(function (this: BucketFacadeAdapter) { + const result = hasPendingChanges.call(this) + pendingChecks.push(result) + return result + }) + try { nodes.write(`update`, { ...initialParent, value: 6 }) - }) - expect(postRecoveryFailure).toBeUndefined() - facadeIndex.updateFailure = undefined + } finally { + pendingSpy.mockRestore() + } + expect(pendingChecks).toEqual([false]) expect(live.get(1)!.value).toBe(6) expect(rootPublications).toHaveLength(2) - expect(childPublications).toHaveLength(1) + expect(childPublications).toHaveLength(2) expect(facade._stateRevision).toBe(facadeRevisionAfterRecovery) expect(readinessOrder).toEqual([`facade`, `root`]) } finally { From dc448ff853455e04c9981d16526307e027e9fca6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 14:56:53 -0600 Subject: [PATCH 243/327] test(db): preserve graph recovery traces --- ...ncludes-collection-oracle.property.test.ts | 121 ++++++++++++++++-- 1 file changed, 113 insertions(+), 8 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 73d43b490..fcff8aac4 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1293,14 +1293,69 @@ describe(`Collection-valued includes oracle`, () => { const facadeIndex = facade.createIndex((row) => row.value, { indexType: ThrowingUpdateIndex, }) as ThrowingUpdateIndex - const rootPublications: Array = [] - const childPublications: Array = [] + type ProjectedNodeChange = { + type: `insert` | `update` | `delete` + key: number + value: Pick + previousValue?: Pick + } + const projectNodeRow = ({ id, kind, group, value }: NodeRow) => ({ + id, + kind, + group, + value, + }) + const projectNodeChange = ( + change: ChangeMessage, + ): ProjectedNodeChange => ({ + type: change.type, + key: Number(change.key), + value: projectNodeRow(change.value), + ...(change.previousValue + ? { previousValue: projectNodeRow(change.previousValue) } + : {}), + }) + type ProjectedRootChange = { + type: `insert` | `update` | `delete` + key: number + value: { id: number; value: number; preservesFacade: boolean } + previousValue?: { + id: number + value: number + preservesFacade: boolean + } + } + const projectRootChange = ( + change: ChangeMessage< + { id: number; value: number; children: typeof facade }, + string | number + >, + ): ProjectedRootChange => ({ + type: change.type, + key: Number(change.key), + value: { + id: change.value.id, + value: change.value.value, + preservesFacade: change.value.children === facade, + }, + ...(change.previousValue + ? { + previousValue: { + id: change.previousValue.id, + value: change.previousValue.value, + preservesFacade: change.previousValue.children === facade, + }, + } + : {}), + }) + const rootPublications: Array> = [] + const childPublications: Array> = [] const rootSubscription = live.subscribeChanges( - (batch) => rootPublications.push(...batch), + (batch) => rootPublications.push(batch.map(projectRootChange)), { includeInitialState: false }, ) const childSubscription = facade.subscribeChanges( - (batch) => childPublications.push(...batch), + (batch) => childPublications.push(batch.map(projectNodeChange)), { includeInitialState: false }, ) const errorSnapshots: Array<{ @@ -1463,8 +1518,40 @@ describe(`Collection-valued includes oracle`, () => { { id: 10, value: 2 }, { id: 11, value: 20 }, ]) - expect(rootPublications).toHaveLength(1) - expect(childPublications).toHaveLength(2) + const rootPublicationsAfterRecovery: Array< + Array + > = [ + [], + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 5, preservesFacade: true }, + previousValue: { id: 1, value: 1, preservesFacade: true }, + }, + ], + ] + const childPublicationsAfterRecovery: Array< + Array + > = [ + [], + [ + { + type: `update`, + key: 10, + value: { ...initialChild, value: 2 }, + previousValue: initialChild, + }, + { + type: `update`, + key: 11, + value: { ...initialSibling, value: 20 }, + previousValue: initialSibling, + }, + ], + ] + expect(rootPublications).toEqual(rootPublicationsAfterRecovery) + expect(childPublications).toEqual(childPublicationsAfterRecovery) expect(live._stateRevision).toBe(rootRevision + 1) expect(facade._stateRevision).toBe(childRevision + 1) expect([...rootIndex.equalityLookup(2)]).toEqual([]) @@ -1473,6 +1560,7 @@ describe(`Collection-valued includes oracle`, () => { expect([...rootIndex.equalityLookup(5)]).toEqual([1]) const facadeRevisionAfterRecovery = facade._stateRevision + const facadeBuildCallsAfterRecovery = facadeIndex.buildCalls const pendingChecks: Array = [] const hasPendingChanges = BucketFacadeAdapter.prototype.hasPendingChanges @@ -1489,9 +1577,26 @@ describe(`Collection-valued includes oracle`, () => { pendingSpy.mockRestore() } expect(pendingChecks).toEqual([false]) + expect(facadeIndex.buildCalls).toBe(facadeBuildCallsAfterRecovery) expect(live.get(1)!.value).toBe(6) - expect(rootPublications).toHaveLength(2) - expect(childPublications).toHaveLength(2) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 2 }, + { id: 11, value: 20 }, + ]) + expect([...facadeIndex.equalityLookup(2)]).toEqual([10]) + expect([...facadeIndex.equalityLookup(20)]).toEqual([11]) + expect(rootPublications).toEqual([ + ...rootPublicationsAfterRecovery, + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 6, preservesFacade: true }, + previousValue: { id: 1, value: 5, preservesFacade: true }, + }, + ], + ]) + expect(childPublications).toEqual(childPublicationsAfterRecovery) expect(facade._stateRevision).toBe(facadeRevisionAfterRecovery) expect(readinessOrder).toEqual([`facade`, `root`]) } finally { From 0c0a9f29313a587c9c7cd155d1f1c939356868cb Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 15:06:15 -0600 Subject: [PATCH 244/327] fix(db): complete first-ready fan-out --- packages/db/src/collection/lifecycle.ts | 8 +-- packages/db/src/query/live/ARCHITECTURE.md | 5 ++ .../src/query/live/bucket-facade-adapter.ts | 2 +- .../query/live/collection-config-builder.ts | 2 +- packages/db/src/query/live/utils.ts | 13 ----- packages/db/src/utils/callbacks.ts | 12 +++++ .../db/tests/collection-lifecycle.test.ts | 52 +++++++++++++++++++ 7 files changed, 76 insertions(+), 18 deletions(-) create mode 100644 packages/db/src/utils/callbacks.ts diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 661e8410d..d16ecd598 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -7,6 +7,7 @@ import { safeCancelIdleCallback, safeRequestIdleCallback, } from '../utils/browser-polyfills' +import { runAllCallbacks } from '../utils/callbacks' import { CleanupQueue } from './cleanup-queue' import type { IdleCallbackDeadline } from '../utils/browser-polyfills' import type { StandardSchemaV1 } from '@standard-schema/spec' @@ -138,6 +139,7 @@ export class CollectionLifecycleManager< if (this.status === `loading` || this.status === `error`) { this.syncError = undefined this.setStatus(`ready`, true) + const readyEffects: Array<() => void> = [] // Call any registered first ready callbacks (only on first time becoming ready) if (!this.hasBeenReady) { @@ -148,15 +150,15 @@ export class CollectionLifecycleManager< this.hasReceivedFirstCommit = true } - const callbacks = [...this.onFirstReadyCallbacks] + readyEffects.push(...this.onFirstReadyCallbacks) this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) } // Notify dependents when markReady is called, after status is set // This ensures live queries get notified when their dependencies become ready if (this.changes.changeSubscriptions.size > 0) { - this.changes.emitEmptyReadyEvent() + readyEffects.push(() => this.changes.emitEmptyReadyEvent()) } + runAllCallbacks(readyEffects) } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index b81e72fa9..562c1b545 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1107,6 +1107,11 @@ child demand, but its root demand must still settle. Later readiness transitions follow the existing Collection contract until an executable test defines another public behavior. +The first-ready transition is an attempt-all fan-out. One callback failure +cannot suppress later first-ready callbacks, preload settlement, or the empty +ready event that wakes dependent Collections. Core completes every effect, then +rethrows the first failure unchanged, including a falsy value. + Pending demand does not hide the parent row. An active empty bucket gives it the current canonical bucket value, and available partial source rows produce the current partial materialization when the source supports progressive diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 3ce132cf9..6ecf2909d 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -1,8 +1,8 @@ import { output, serializeValue } from '@tanstack/db-ivm' import { createCollection } from '../../collection/index.js' +import { runAllCallbacks } from '../../utils/callbacks.js' import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' import { BUCKET_FACADE_REF } from './materialized-pipeline.js' -import { runAllCallbacks } from './utils.js' import type { Collection } from '../../collection/index.js' import type { SyncConfig } from '../../types.js' import type { PublicationDeferral } from '../../collection/changes.js' diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 18281013e..3f052cfaf 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -11,6 +11,7 @@ import { } from '../../scheduler.js' import { getActiveTransaction } from '../../transactions.js' import { deepEquals } from '../../utils.js' +import { runAllCallbacks } from '../../utils/callbacks.js' import { getLoadSubsetDemandKey } from '../ir-stable-identity.js' import { isAppliedLoadSubsetOutcome } from '../load-subset-outcome.js' import { CollectionSubscriber } from './collection-subscriber.js' @@ -23,7 +24,6 @@ import { extractCollectionFromSource, extractCollectionSources, extractCollectionsFromQuery, - runAllCallbacks, } from './utils.js' import type { LiveQueryInternalUtils } from './internal.js' import type { WindowOptions } from '../compiler/index.js' diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index 27ab49f54..b70842c39 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -11,19 +11,6 @@ import type { Context } from '../builder/types.js' import type { OrderBy, QueryIR } from '../ir.js' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' -/** Attempt every callback, then rethrow the first exact failure value. */ -export function runAllCallbacks(callbacks: Iterable<() => void>): void { - let firstFailure: { error: unknown } | undefined - for (const callback of callbacks) { - try { - callback() - } catch (error) { - firstFailure ??= { error } - } - } - if (firstFailure) throw firstFailure.error -} - /** * Helper function to extract collections from a compiled query. * Traverses the query IR to find all collection references. diff --git a/packages/db/src/utils/callbacks.ts b/packages/db/src/utils/callbacks.ts new file mode 100644 index 000000000..1b0aba485 --- /dev/null +++ b/packages/db/src/utils/callbacks.ts @@ -0,0 +1,12 @@ +/** Attempt every callback, then rethrow the first exact failure value. */ +export function runAllCallbacks(callbacks: Iterable<() => void>): void { + let firstFailure: { error: unknown } | undefined + for (const callback of callbacks) { + try { + callback() + } catch (error) { + firstFailure ??= { error } + } + } + if (firstFailure) throw firstFailure.error +} diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index a5cf03f19..10cdaa2a6 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -511,6 +511,58 @@ describe(`Collection Lifecycle Management`, () => { subscription.unsubscribe() }) + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { + let markReadyCallback: (() => void) | undefined + const readyBatches: Array> = [] + const laterFailure = new Error(`later first-ready failure`) + const laterCallback = vi.fn(() => { + throw laterFailure + }) + let preloadSettled = false + + const collection = createCollection<{ id: string; name: string }>({ + id: `first-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + const subscription = collection.subscribeChanges((batch) => { + readyBatches.push(batch) + }) + collection.onFirstReady(() => { + throw undefined + }) + collection.onFirstReady(laterCallback) + void collection.preload().then(() => { + preloadSettled = true + }) + + try { + let didThrow = false + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + didThrow = true + thrown = error + } + await Promise.resolve() + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(laterCallback).toHaveBeenCalledOnce() + expect(preloadSettled).toBe(true) + expect(readyBatches).toEqual([[]]) + expect(collection.status).toBe(`ready`) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`should fire status:change event with 'cleaned-up' status before clearing event handlers`, () => { const collection = createCollection<{ id: string; name: string }>({ id: `cleanup-event-test`, From 0385154a7b17f0afdfd93db70fb4a8f2eeff3899 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 15:22:05 -0600 Subject: [PATCH 245/327] fix(db): isolate ready effect failures --- packages/db/src/collection/changes.ts | 9 +- packages/db/src/collection/lifecycle.ts | 19 ++- packages/db/src/collection/sync.ts | 13 +- packages/db/src/query/live/ARCHITECTURE.md | 11 +- .../db/tests/collection-lifecycle.test.ts | 134 ++++++++++++++++++ 5 files changed, 178 insertions(+), 8 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 5267a9118..ef0969acc 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,5 +1,6 @@ import { NegativeActiveSubscribersError } from '../errors' import { withPublicationContext } from '../scheduler.js' +import { runAllCallbacks } from '../utils/callbacks.js' import { createSingleRowRefProxy, toExpression, @@ -108,9 +109,11 @@ export class CollectionChangesManager< */ public emitEmptyReadyEvent(): void { withPublicationContext(() => { - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents([]) - } + runAllCallbacks( + [...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents([]), + ), + ) }) } diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index d16ecd598..579cdc718 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -134,6 +134,16 @@ export class CollectionLifecycleManager< * @private - Should only be called by sync implementations */ public markReady(): void { + const failure = this.applyReadyTransition() + if (failure) throw failure.error + } + + /** @internal Capture ready-effect failures while the sync entry completes. */ + public markReadyDuringSyncStart(): { error: unknown } | undefined { + return this.applyReadyTransition() + } + + private applyReadyTransition(): { error: unknown } | undefined { this.validateStatusTransition(this.status, `ready`) // A successful initial sync or recovery establishes a ready snapshot. if (this.status === `loading` || this.status === `error`) { @@ -155,11 +165,14 @@ export class CollectionLifecycleManager< } // Notify dependents when markReady is called, after status is set // This ensures live queries get notified when their dependencies become ready - if (this.changes.changeSubscriptions.size > 0) { - readyEffects.push(() => this.changes.emitEmptyReadyEvent()) + readyEffects.push(() => this.changes.emitEmptyReadyEvent()) + try { + runAllCallbacks(readyEffects) + } catch (error) { + return { error } } - runAllCallbacks(readyEffects) } + return undefined } /** Mark an asynchronous sync failure after sync has started. */ diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index 791998715..675bc42ad 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -222,6 +222,8 @@ export class CollectionSyncManager< const syncEpoch = ++this.syncEpoch const isCurrentSync = () => syncEpoch === this.syncEpoch this.lifecycle.setStatus(`loading`) + let syncEntryActive = true + let readyEffectFailure: { error: unknown } | undefined try { const syncRes = normalizeSyncFnResult( @@ -383,7 +385,13 @@ export class CollectionSyncManager< return receipt }, markReady: () => { - if (isCurrentSync()) this.lifecycle.markReady() + if (!isCurrentSync()) return + if (syncEntryActive) { + readyEffectFailure ??= + this.lifecycle.markReadyDuringSyncStart() + } else { + this.lifecycle.markReady() + } }, markError: (error?: unknown) => { if (isCurrentSync()) this.lifecycle.markError(error) @@ -427,6 +435,7 @@ export class CollectionSyncManager< metadata: this.createSyncMetadataApi(isCurrentSync), }), ) + syncEntryActive = false // Store cleanup function if provided this.syncCleanupFn = syncRes?.cleanup ?? null @@ -445,9 +454,11 @@ export class CollectionSyncManager< ) } } catch (error) { + syncEntryActive = false this.lifecycle.markError(error) throw error } + if (readyEffectFailure) throw readyEffectFailure.error } public deferStart(): boolean { diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 562c1b545..9f53a8076 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1110,7 +1110,16 @@ another public behavior. The first-ready transition is an attempt-all fan-out. One callback failure cannot suppress later first-ready callbacks, preload settlement, or the empty ready event that wakes dependent Collections. Core completes every effect, then -rethrows the first failure unchanged, including a falsy value. +rethrows the first failure unchanged, including a falsy value. Status is ready +before these effects run; first-ready callbacks keep registration order, and +the dependent-ready event runs after them. That event snapshots the dependents +present at delivery and attempts every one even if an earlier listener fails. + +When `markReady()` runs during the synchronous adapter-entry call, core retains +any ready-effect failure until the adapter finishes its own setup. It then +propagates the exact failure without reclassifying it as a sync failure or +moving the Collection to `error`. A later asynchronous `markReady()` call keeps +the ordinary synchronous throw boundary. Pending demand does not hide the parent row. An active empty bucket gives it the current canonical bucket value, and available partial source rows produce diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 10cdaa2a6..9c0902071 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -514,8 +514,10 @@ describe(`Collection Lifecycle Management`, () => { it(`attempts every first-ready effect before rethrowing the first failure`, async () => { let markReadyCallback: (() => void) | undefined const readyBatches: Array> = [] + const readyTrace: Array = [] const laterFailure = new Error(`later first-ready failure`) const laterCallback = vi.fn(() => { + readyTrace.push(`later:${collection.status}`) throw laterFailure }) let preloadSettled = false @@ -530,9 +532,11 @@ describe(`Collection Lifecycle Management`, () => { }, }) const subscription = collection.subscribeChanges((batch) => { + readyTrace.push(`dependent:${collection.status}`) readyBatches.push(batch) }) collection.onFirstReady(() => { + readyTrace.push(`first:${collection.status}`) throw undefined }) collection.onFirstReady(laterCallback) @@ -556,13 +560,143 @@ describe(`Collection Lifecycle Management`, () => { expect(laterCallback).toHaveBeenCalledOnce() expect(preloadSettled).toBe(true) expect(readyBatches).toEqual([[]]) + expect(readyTrace).toEqual([ + `first:ready`, + `later:ready`, + `dependent:ready`, + ]) expect(collection.status).toBe(`ready`) + + expect(() => markReadyCallback!()).not.toThrow() + expect(laterCallback).toHaveBeenCalledOnce() + expect(readyBatches).toEqual([[]]) + expect(readyTrace).toEqual([ + `first:ready`, + `later:ready`, + `dependent:ready`, + ]) } finally { subscription.unsubscribe() await collection.cleanup() } }) + it(`does not classify synchronous first-ready callback failures as sync failures`, async () => { + const laterFailure = new Error(`later synchronous first-ready failure`) + const callbackTrace: Array = [] + let syncContinued = false + + const collection = createCollection<{ id: string; name: string }>({ + id: `synchronous-first-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + syncContinued = true + }, + }, + }) + collection.onFirstReady(() => { + callbackTrace.push(`first`) + throw undefined + }) + collection.onFirstReady(() => { + callbackTrace.push(`later`) + throw laterFailure + }) + + try { + let didThrow = false + let thrown: unknown + try { + collection._sync.startSync() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(syncContinued).toBe(true) + expect(callbackTrace).toEqual([`first`, `later`]) + expect(collection.status).toBe(`ready`) + await expect(collection.preload()).resolves.toBeUndefined() + } finally { + await collection.cleanup() + } + }) + + it(`attempts every dependent ready listener before rethrowing`, async () => { + let markReadyCallback: (() => void) | undefined + const firstFailure = new Error(`first dependent failed`) + const firstBatches: Array> = [] + const secondBatches: Array> = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + const first = collection.subscribeChanges((batch) => { + firstBatches.push(batch) + throw firstFailure + }) + const second = collection.subscribeChanges((batch) => { + secondBatches.push(batch) + }) + + try { + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(firstBatches).toEqual([[]]) + expect(secondBatches).toEqual([[]]) + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`notifies a dependent added during the first-ready fan-out`, async () => { + let markReadyCallback: (() => void) | undefined + let dependent: { unsubscribe: () => void } | undefined + const readyBatches: Array> = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-dependent-ready-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + collection.onFirstReady(() => { + dependent = collection.subscribeChanges((batch) => { + readyBatches.push(batch) + }) + }) + const preload = collection.preload() + + try { + markReadyCallback!() + await preload + expect(readyBatches).toEqual([[]]) + } finally { + dependent?.unsubscribe() + await collection.cleanup() + } + }) + it(`should fire status:change event with 'cleaned-up' status before clearing event handlers`, () => { const collection = createCollection<{ id: string; name: string }>({ id: `cleanup-event-test`, From 8eba678228c763551df3c10010b38a453f4e9fbd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 15:38:18 -0600 Subject: [PATCH 246/327] fix(db): preserve ready propagation outcomes --- packages/db/src/collection/changes.ts | 28 +++- packages/db/src/collection/sync.ts | 26 ++-- packages/db/src/query/live/ARCHITECTURE.md | 8 +- .../db/tests/collection-lifecycle.test.ts | 133 ++++++++++++++++++ 4 files changed, 179 insertions(+), 16 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index ef0969acc..85917c355 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -108,13 +108,27 @@ export class CollectionChangesManager< * This bypasses the normal empty array check in emitEvents */ public emitEmptyReadyEvent(): void { - withPublicationContext(() => { - runAllCallbacks( - [...this.changeSubscriptions].map( - (subscription) => () => subscription.emitEvents([]), - ), - ) - }) + let deliveryFailure: { error: unknown } | undefined + let graphFailure: { error: unknown } | undefined + try { + withPublicationContext(() => { + try { + runAllCallbacks( + [...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents([]), + ), + ) + } catch (error) { + // The ready snapshot is already public. Keep the callback failure, + // but let work queued by earlier dependents reach the graph flush. + deliveryFailure = { error } + } + }) + } catch (error) { + graphFailure = { error } + } + if (deliveryFailure) throw deliveryFailure.error + if (graphFailure) throw graphFailure.error } /** diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index 675bc42ad..c5e3f78d9 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -387,8 +387,7 @@ export class CollectionSyncManager< markReady: () => { if (!isCurrentSync()) return if (syncEntryActive) { - readyEffectFailure ??= - this.lifecycle.markReadyDuringSyncStart() + readyEffectFailure ??= this.lifecycle.markReadyDuringSyncStart() } else { this.lifecycle.markReady() } @@ -729,10 +728,14 @@ export class CollectionSyncManager< } let settled = false - let startingSync = false + const syncStartState = { active: false, ready: false } let unsubscribeError = () => {} let unsubscribeReady = () => {} const resolveReady = () => { + if (syncStartState.active) { + syncStartState.ready = true + return + } if (settled) return settled = true unsubscribeError() @@ -750,7 +753,7 @@ export class CollectionSyncManager< // Register callback BEFORE starting sync to avoid race condition unsubscribeReady = this.lifecycle.onFirstReady(resolveReady) unsubscribeError = this.collection.on(`status:error`, () => { - if (startingSync) { + if (syncStartState.active) { return } rejectError(this.getPreloadError()) @@ -761,17 +764,24 @@ export class CollectionSyncManager< this.lifecycle.status === `idle` || this.lifecycle.status === `cleaned-up` ) { - startingSync = true + syncStartState.active = true + let startFailure: { error: unknown } | undefined try { this.startSync() } catch (error) { - rejectError(error) - return + startFailure = { error } } finally { - startingSync = false + syncStartState.active = false } if (this.collection.status === `error`) { rejectError(this.getPreloadError()) + } else if (syncStartState.ready) { + // A first-ready listener can throw after readiness is established. + // That failure still escapes direct startSync(), but preload follows + // the final collection state after synchronous adapter entry. + resolveReady() + } else if (startFailure) { + rejectError(startFailure.error) } } }) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 9f53a8076..c61a656d7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1114,12 +1114,18 @@ rethrows the first failure unchanged, including a falsy value. Status is ready before these effects run; first-ready callbacks keep registration order, and the dependent-ready event runs after them. That event snapshots the dependents present at delivery and attempts every one even if an earlier listener fails. +Because the ready snapshot is already public, a listener failure also cannot +discard graph work queued by an earlier listener. Core flushes that work before +it rethrows the first listener failure. When `markReady()` runs during the synchronous adapter-entry call, core retains any ready-effect failure until the adapter finishes its own setup. It then propagates the exact failure without reclassifying it as a sync failure or moving the Collection to `error`. A later asynchronous `markReady()` call keeps -the ordinary synchronous throw boundary. +the ordinary synchronous throw boundary. A preload already pending across this +entry waits for the adapter's final synchronous outcome: a ready-effect failure +alone leaves it resolved, while a later adapter failure rejects it and leaves +the Collection in `error`. Pending demand does not hide the parent row. An active empty bucket gives it the current canonical bucket value, and available partial source rows produce diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 9c0902071..930b81878 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' import { CleanupQueue } from '../src/collection/cleanup-queue.js' +import { + getActivePublicationContext, + transactionScopedScheduler, +} from '../src/scheduler.js' // Mock setTimeout and clearTimeout for testing GC behavior const originalSetTimeout = global.setTimeout @@ -626,6 +630,68 @@ describe(`Collection Lifecycle Management`, () => { } }) + it(`rejects a pending preload when the adapter fails after marking ready`, async () => { + const adapterFailure = new Error(`adapter failed after ready`) + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-then-adapter-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + throw adapterFailure + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + await expect(collection.preload()).rejects.toBe(adapterFailure) + expect(collection.status).toBe(`error`) + } finally { + await collection.cleanup() + } + }) + + it(`ends the synchronous sync-entry boundary after an adapter failure`, async () => { + const adapterFailure = new Error(`adapter entry failed`) + let markReadyCallback: (() => void) | undefined + const collection = createCollection<{ id: string; name: string }>({ + id: `failed-sync-entry-boundary-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + throw adapterFailure + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + expect(() => collection._sync.startSync()).toThrow(adapterFailure) + expect(collection.status).toBe(`error`) + + let didThrow = false + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(collection.status).toBe(`ready`) + } finally { + await collection.cleanup() + } + }) + it(`attempts every dependent ready listener before rethrowing`, async () => { let markReadyCallback: (() => void) | undefined const firstFailure = new Error(`first dependent failed`) @@ -667,6 +733,73 @@ describe(`Collection Lifecycle Management`, () => { } }) + it(`flushes work queued by a ready listener when a sibling throws`, async () => { + let markReadyCallback: (() => void) | undefined + const firstFailure = new Error(`dependent failed after sibling queued`) + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-scheduler-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + expect(contextId).toBeDefined() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw firstFailure + }) + + try { + expect(() => markReadyCallback!()).toThrow(firstFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`delivers ready to the subscription snapshot when one listener unsubscribes another`, async () => { + let markReadyCallback: (() => void) | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-membership-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + calls.push(`first`) + second.unsubscribe() + }) + const second = collection.subscribeChanges(() => { + calls.push(`second`) + }) + + try { + markReadyCallback!() + expect(calls).toEqual([`first`, `second`]) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + it(`notifies a dependent added during the first-ready fan-out`, async () => { let markReadyCallback: (() => void) | undefined let dependent: { unsubscribe: () => void } | undefined From 37e6dd16eb6ed9ff4b819f3b71456787b06778d9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 15:46:41 -0600 Subject: [PATCH 247/327] fix(db): release deleted synced keys --- packages/db/package.json | 2 +- packages/db/src/collection/state.ts | 30 ++-- ...on-state-retention-oracle.property.test.ts | 145 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 4 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 packages/db/tests/collection-state-retention-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 4be51dfb6..3e7965590 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 6cd7c88b8..bedd32e2e 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -233,10 +233,7 @@ export class CollectionStateManager< public snapshotPublicationState( keys: Iterable, - ): CollectionPublicationStateSnapshot< - TOutput, - TKey - > { + ): CollectionPublicationStateSnapshot { const affectedKeys = new Set(keys) for (const transaction of this.pendingSyncedTransactions) { for (const operation of transaction.operations) { @@ -277,9 +274,7 @@ export class CollectionStateManager< }, ]), ), - syncedCollectionMetadata: [ - ...this.syncedCollectionMetadata.entries(), - ], + syncedCollectionMetadata: [...this.syncedCollectionMetadata.entries()], optimisticUpserts: new Map(this.optimisticUpserts), optimisticDeletes: new Set(this.optimisticDeletes), pendingOptimisticUpserts: new Map(this.pendingOptimisticUpserts), @@ -304,7 +299,10 @@ export class CollectionStateManager< snapshot: CollectionPublicationStateSnapshot, ): void { this.pendingSyncedTransactions = [...snapshot.pendingSyncedTransactions] - for (const [transaction, applicationStarted] of snapshot.applicationStarted) { + for (const [ + transaction, + applicationStarted, + ] of snapshot.applicationStarted) { transaction.applicationStarted = applicationStarted } for (const [key, state] of snapshot.keys) { @@ -315,20 +313,11 @@ export class CollectionStateManager< restoreSetEntry(this.hydratedKeys, key, state.hydrated) restoreSetEntry(this.syncedKeys, key, state.synced) } - replaceMap( - this.syncedCollectionMetadata, - snapshot.syncedCollectionMetadata, - ) + replaceMap(this.syncedCollectionMetadata, snapshot.syncedCollectionMetadata) replaceMap(this.optimisticUpserts, snapshot.optimisticUpserts) replaceSet(this.optimisticDeletes, snapshot.optimisticDeletes) - replaceMap( - this.pendingOptimisticUpserts, - snapshot.pendingOptimisticUpserts, - ) - replaceSet( - this.pendingOptimisticDeletes, - snapshot.pendingOptimisticDeletes, - ) + replaceMap(this.pendingOptimisticUpserts, snapshot.pendingOptimisticUpserts) + replaceSet(this.pendingOptimisticDeletes, snapshot.pendingOptimisticDeletes) replaceSet( this.pendingOptimisticDirectUpserts, snapshot.pendingOptimisticDirectUpserts, @@ -1355,6 +1344,7 @@ export class CollectionStateManager< } case `delete`: this.syncedData.delete(key) + this.syncedKeys.delete(key) this.syncedMetadata.delete(key) this.rowOrigins.delete(key) if (!transaction.truncate) { diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts new file mode 100644 index 000000000..8ae78cd03 --- /dev/null +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -0,0 +1,145 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { oraclePropertyOptions } from './oracle-config.js' +import type { Collection } from '../src/collection/index.js' +import type { SyncConfig } from '../src/types.js' + +type RetainedRow = { + id: number + value: number +} + +type SyncActions = Parameters[`sync`]>[0] + +type RetentionAction = + | { type: `put`; row: RetainedRow } + | { type: `delete`; key: number } + | { type: `replace`; rows: ReadonlyArray } + +type RetentionHarness = { + collection: Collection + sync: SyncActions +} + +const retainedRowArbitrary = fc.record({ + id: fc.integer({ min: 0, max: 3 }), + value: fc.integer({ min: -2, max: 2 }), +}) + +const retentionActionArbitrary: fc.Arbitrary = fc.oneof( + retainedRowArbitrary.map((row) => ({ type: `put` as const, row })), + fc + .integer({ min: 0, max: 3 }) + .map((key) => ({ type: `delete` as const, key })), + fc + .uniqueArray(retainedRowArbitrary, { + selector: (row) => row.id, + maxLength: 4, + }) + .map((rows) => ({ type: `replace` as const, rows })), +) + +function createRetentionHarness(): RetentionHarness { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + return { collection, sync } +} + +function applyAction( + action: RetentionAction, + model: Map, + sync: SyncActions, +): void { + sync.begin() + switch (action.type) { + case `put`: { + sync.write({ + type: model.has(action.row.id) ? `update` : `insert`, + value: action.row, + }) + model.set(action.row.id, action.row) + break + } + case `delete`: + sync.write({ type: `delete`, key: action.key }) + model.delete(action.key) + break + case `replace`: + sync.truncate() + model.clear() + for (const row of action.rows) { + sync.write({ type: `insert`, value: row }) + model.set(row.id, row) + } + break + } + expect(sync.commit()).toBe(true) +} + +function expectRetainedState( + collection: Collection, + model: ReadonlyMap, +): void { + const expectedRows = [...model.entries()].sort(([a], [b]) => a - b) + const retainedRows = [...collection._state.syncedData.entries()].sort( + ([a], [b]) => a - b, + ) + + expect(retainedRows).toEqual(expectedRows) + expect([...collection._state.syncedKeys].sort((a, b) => a - b)).toEqual( + expectedRows.map(([key]) => key), + ) + expect( + [...collection.state.entries()] + .map(([key, row]) => [key, { id: row.id, value: row.value }] as const) + .sort(([a], [b]) => a - b), + ).toEqual(expectedRows) +} + +async function runRetentionHistory( + actions: ReadonlyArray, +): Promise { + const { collection, sync } = createRetentionHarness() + const model = new Map() + try { + expectRetainedState(collection, model) + for (const action of actions) { + applyAction(action, model, sync) + expectRetainedState(collection, model) + } + } finally { + await collection.cleanup() + } +} + +it(`retains only keys in the authoritative synced state`, async () => { + await runRetentionHistory([ + { type: `put`, row: { id: 1, value: 1 } }, + { type: `put`, row: { id: 2, value: 2 } }, + { type: `delete`, key: 1 }, + { type: `put`, row: { id: 1, value: -1 } }, + { type: `replace`, rows: [{ id: 3, value: 0 }] }, + { type: `delete`, key: 3 }, + ]) +}) + +fcTest.prop( + [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], + oraclePropertyOptions(100, `collection-state.retention`), +)( + `matches retained authoritative state after every committed sync history`, + async (actions) => { + await runRetentionHistory(actions) + }, +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index e4afea3f7..c15d168b8 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -2,6 +2,7 @@ type OracleEnvironment = Record const staticOracleProperties = [ `collection-sync.reentrant-drain`, + `collection-state.retention`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, `includes-collection.layout-swap`, From 935112c04189a9b84395c7144260a090501e7817 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 15:57:53 -0600 Subject: [PATCH 248/327] fix(db): defer nested ready failures --- packages/db/src/collection/changes.ts | 40 ++-- packages/db/src/query/live/ARCHITECTURE.md | 8 +- packages/db/src/scheduler.ts | 39 +++- .../db/tests/collection-lifecycle.test.ts | 188 ++++++++++++++++++ packages/db/tests/query/scheduler.test.ts | 52 ++++- 5 files changed, 295 insertions(+), 32 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 85917c355..a392df67a 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,5 +1,8 @@ import { NegativeActiveSubscribersError } from '../errors' -import { withPublicationContext } from '../scheduler.js' +import { + deferPublicationFailure, + withPublicationContext, +} from '../scheduler.js' import { runAllCallbacks } from '../utils/callbacks.js' import { createSingleRowRefProxy, @@ -108,27 +111,20 @@ export class CollectionChangesManager< * This bypasses the normal empty array check in emitEvents */ public emitEmptyReadyEvent(): void { - let deliveryFailure: { error: unknown } | undefined - let graphFailure: { error: unknown } | undefined - try { - withPublicationContext(() => { - try { - runAllCallbacks( - [...this.changeSubscriptions].map( - (subscription) => () => subscription.emitEvents([]), - ), - ) - } catch (error) { - // The ready snapshot is already public. Keep the callback failure, - // but let work queued by earlier dependents reach the graph flush. - deliveryFailure = { error } - } - }) - } catch (error) { - graphFailure = { error } - } - if (deliveryFailure) throw deliveryFailure.error - if (graphFailure) throw graphFailure.error + withPublicationContext(() => { + try { + runAllCallbacks( + [...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents([]), + ), + ) + } catch (error) { + // The ready snapshot is already public. Keep the callback failure on + // the shared publication so nested graph work can drain before the + // outer boundary rethrows it. + deferPublicationFailure(error) + } + }) } /** diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index c61a656d7..1f7569534 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1114,9 +1114,15 @@ rethrows the first failure unchanged, including a falsy value. Status is ready before these effects run; first-ready callbacks keep registration order, and the dependent-ready event runs after them. That event snapshots the dependents present at delivery and attempts every one even if an earlier listener fails. +Removing or adding a dependent during delivery does not change that frozen +batch; an added dependent starts with the next publication. Because the ready snapshot is already public, a listener failure also cannot discard graph work queued by an earlier listener. Core flushes that work before -it rethrows the first listener failure. +it rethrows the first listener failure. If readiness is nested inside an +existing publication, core retains the exact listener failure on that shared +context and the outer boundary rethrows it only after the queued graph work +drains. When both the listener and that queued graph work fail, the first ready +listener failure remains the reported error. When `markReady()` runs during the synchronous adapter-entry call, core retains any ready-effect failure until the adapter finishes its own setup. It then diff --git a/packages/db/src/scheduler.ts b/packages/db/src/scheduler.ts index d87ac0531..ea17dfc05 100644 --- a/packages/db/src/scheduler.ts +++ b/packages/db/src/scheduler.ts @@ -221,7 +221,12 @@ export class Scheduler { export const transactionScopedScheduler = new Scheduler() -let activePublicationContext: SchedulerContextId | undefined +type ActivePublication = { + contextId: SchedulerContextId + failure?: { error: unknown } +} + +let activePublication: ActivePublication | undefined /** * Returns the Collection publication that currently owns synchronous change @@ -229,7 +234,19 @@ let activePublicationContext: SchedulerContextId | undefined * observe one committed batch. */ export function getActivePublicationContext(): SchedulerContextId | undefined { - return activePublicationContext + return activePublication?.contextId +} + +/** + * Retains the first failure produced by a nested publication effect. The + * outer publication surfaces it only after all work already queued in the + * shared context has run. + */ +export function deferPublicationFailure(error: unknown): void { + if (!activePublication) { + throw new Error(`Cannot defer a failure outside a publication context`) + } + activePublication.failure ??= { error } } /** @@ -238,18 +255,28 @@ export function getActivePublicationContext(): SchedulerContextId | undefined { * only after every subscriber to the original committed batch has observed it. */ export function withPublicationContext(publish: () => T): T { - if (activePublicationContext !== undefined) return publish() + if (activePublication) return publish() const contextId = Symbol(`collection-publication`) - activePublicationContext = contextId + const publication: ActivePublication = { contextId } + activePublication = publication try { const result = publish() - transactionScopedScheduler.flush(contextId) + let graphFailure: { error: unknown } | undefined + try { + transactionScopedScheduler.flush(contextId) + } catch (error) { + graphFailure = { error } + } + if (publication.failure) { + throw publication.failure.error + } + if (graphFailure) throw graphFailure.error return result } catch (error) { transactionScopedScheduler.clear(contextId) throw error } finally { - activePublicationContext = undefined + activePublication = undefined } } diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 930b81878..8a09af20c 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -4,6 +4,7 @@ import { CleanupQueue } from '../src/collection/cleanup-queue.js' import { getActivePublicationContext, transactionScopedScheduler, + withPublicationContext, } from '../src/scheduler.js' // Mock setTimeout and clearTimeout for testing GC behavior @@ -770,6 +771,142 @@ describe(`Collection Lifecycle Management`, () => { } }) + it(`flushes ready work before rethrowing at an outer publication boundary`, async () => { + let markReadyCallback: (() => void) | undefined + const listenerFailure = new Error(`nested dependent failed`) + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-dependent-ready-scheduler-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(() => + withPublicationContext(() => markReadyCallback!()), + ).toThrow(listenerFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`surfaces a ready graph failure after running its job`, async () => { + let markReadyCallback: (() => void) | undefined + const graphFailure = new Error(`ready graph failed`) + const scheduledJob = vi.fn(() => { + throw graphFailure + }) + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-graph-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const subscription = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + + try { + expect(() => markReadyCallback!()).toThrow(graphFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`keeps the ready listener failure when its queued graph job also fails`, async () => { + let markReadyCallback: (() => void) | undefined + const listenerFailure = new Error(`ready listener failed first`) + const graphFailure = new Error(`ready graph also failed`) + const scheduledJob = vi.fn(() => { + throw graphFailure + }) + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-failure-priority-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(() => markReadyCallback!()).toThrow(listenerFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`resolves a pending preload after a ready callback failure alone`, async () => { + let syncContinued = false + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-callback-preload-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + syncContinued = true + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + await expect(collection.preload()).resolves.toBeUndefined() + expect(syncContinued).toBe(true) + expect(collection.status).toBe(`ready`) + } finally { + await collection.cleanup() + } + }) + it(`delivers ready to the subscription snapshot when one listener unsubscribes another`, async () => { let markReadyCallback: (() => void) | undefined const calls: Array = [] @@ -800,6 +937,57 @@ describe(`Collection Lifecycle Management`, () => { } }) + it(`excludes a dependent added during ready delivery until the next batch`, async () => { + let beginCallback: (() => void) | undefined + let writeCallback: + | ((message: { + type: `insert` + value: { id: string; name: string } + }) => void) + | undefined + let commitCallback: (() => void) | undefined + let markReadyCallback: (() => void) | undefined + let added: { unsubscribe: () => void } | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-addition-test`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + beginCallback = begin + writeCallback = write + commitCallback = () => { + commit() + } + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + calls.push(`first`) + added ??= collection.subscribeChanges(() => calls.push(`added`)) + }) + const second = collection.subscribeChanges(() => calls.push(`second`)) + + try { + markReadyCallback!() + expect(calls).toEqual([`first`, `second`]) + + beginCallback!() + writeCallback!({ + type: `insert`, + value: { id: `one`, name: `One` }, + }) + commitCallback!() + expect(calls).toEqual([`first`, `second`, `first`, `second`, `added`]) + } finally { + first.unsubscribe() + second.unsubscribe() + added?.unsubscribe() + await collection.cleanup() + } + }) + it(`notifies a dependent added during the first-ready fan-out`, async () => { let markReadyCallback: (() => void) | undefined let dependent: { unsubscribe: () => void } | undefined diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index fd9d3e9de..c4eb8f8cd 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -145,6 +145,49 @@ describe(`Collection publication scheduler context`, () => { }) describe(`live query scheduler`, () => { + it(`settles a dependent live query before a nested ready failure escapes`, async () => { + let markSourceReady: (() => void) | undefined + const listenerFailure = new Error(`source ready listener failed`) + const source = createCollection({ + id: `nested-ready-live-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: ({ begin, commit, markReady }) => { + begin() + commit() + markSourceReady = markReady + }, + }, + }) + const live = createLiveQueryCollection({ + id: `nested-ready-live-dependent`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + const preload = live.preload() + const throwingSubscription = source.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(live.status).toBe(`loading`) + expect(() => withPublicationContext(() => markSourceReady!())).toThrow( + listenerFailure, + ) + await expect(preload).resolves.toBeUndefined() + expect(source.status).toBe(`ready`) + expect(live.status).toBe(`ready`) + } finally { + throwingSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }) + it(`runs the live query graph once per transaction that touches multiple collections`, async () => { const { users, tasks, assignments } = setupLiveQueryCollections(`single-batch`) @@ -644,9 +687,12 @@ describe(`live query scheduler`, () => { builder.currentSyncConfig = config builder.currentSyncState = syncState - builder.scheduleGraphRun(() => { - throw failure - }, { contextId }) + builder.scheduleGraphRun( + () => { + throw failure + }, + { contextId }, + ) builder.scheduleGraphRun(laterLoader, { contextId }) let didThrow = false From 3c2e63a174d40aa9cf3021a0979ced0654389063 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:03:49 -0600 Subject: [PATCH 249/327] test(db): strengthen retained state oracle --- ...on-state-retention-oracle.property.test.ts | 49 ++++++++++++++----- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 8ae78cd03..4fc567bf9 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -1,6 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' +import { DuplicateKeySyncError } from '../src/errors.js' import { oraclePropertyOptions } from './oracle-config.js' import type { Collection } from '../src/collection/index.js' import type { SyncConfig } from '../src/types.js' @@ -13,7 +14,8 @@ type RetainedRow = { type SyncActions = Parameters[`sync`]>[0] type RetentionAction = - | { type: `put`; row: RetainedRow } + | { type: `insert`; row: RetainedRow } + | { type: `update`; row: RetainedRow } | { type: `delete`; key: number } | { type: `replace`; rows: ReadonlyArray } @@ -28,7 +30,8 @@ const retainedRowArbitrary = fc.record({ }) const retentionActionArbitrary: fc.Arbitrary = fc.oneof( - retainedRowArbitrary.map((row) => ({ type: `put` as const, row })), + retainedRowArbitrary.map((row) => ({ type: `insert` as const, row })), + retainedRowArbitrary.map((row) => ({ type: `update` as const, row })), fc .integer({ min: 0, max: 3 }) .map((key) => ({ type: `delete` as const, key })), @@ -63,11 +66,20 @@ function applyAction( ): void { sync.begin() switch (action.type) { - case `put`: { - sync.write({ - type: model.has(action.row.id) ? `update` : `insert`, - value: action.row, - }) + case `insert`: { + const previous = model.get(action.row.id) + if (previous !== undefined && previous.value !== action.row.value) { + expect(() => sync.write({ type: `insert`, value: action.row })).toThrow( + DuplicateKeySyncError, + ) + break + } + sync.write({ type: `insert`, value: action.row }) + model.set(action.row.id, action.row) + break + } + case `update`: { + sync.write({ type: action.type, value: action.row }) model.set(action.row.id, action.row) break } @@ -125,20 +137,35 @@ async function runRetentionHistory( it(`retains only keys in the authoritative synced state`, async () => { await runRetentionHistory([ - { type: `put`, row: { id: 1, value: 1 } }, - { type: `put`, row: { id: 2, value: 2 } }, + { type: `insert`, row: { id: 1, value: 1 } }, + { type: `insert`, row: { id: 2, value: 2 } }, { type: `delete`, key: 1 }, - { type: `put`, row: { id: 1, value: -1 } }, + { type: `update`, row: { id: 1, value: -1 } }, { type: `replace`, rows: [{ id: 3, value: 0 }] }, { type: `delete`, key: 3 }, ]) }) +it(`retains a missing row introduced by a sync update`, async () => { + await runRetentionHistory([{ type: `update`, row: { id: 1, value: 1 } }]) +}) + +it(`releases retained keys after long unique-key churn`, async () => { + const keyCount = 1_000 + const actions: Array = [] + for (let key = 0; key < keyCount; key++) { + actions.push({ type: `insert`, row: { id: key, value: key } }) + actions.push({ type: `delete`, key }) + } + + await runRetentionHistory(actions) +}) + fcTest.prop( [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], oraclePropertyOptions(100, `collection-state.retention`), )( - `matches retained authoritative state after every committed sync history`, + `matches retained authoritative state without optimistic overlays after every committed sync history`, async (actions) => { await runRetentionHistory(actions) }, From fe1e83675d7d84471b812afb45b40e837e6ba5bc Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:09:50 -0600 Subject: [PATCH 250/327] fix(db): reset retained publication state --- packages/db/src/collection/state.ts | 2 + packages/db/src/query/live/ARCHITECTURE.md | 5 ++ ...on-state-retention-oracle.property.test.ts | 62 +++++++++++++++++++ 3 files changed, 69 insertions(+) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index bedd32e2e..f2f516f73 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1884,6 +1884,8 @@ export class CollectionStateManager< this.size = 0 this.pendingSyncedTransactions = [] this.syncedKeys.clear() + this.preSyncVisibleState.clear() + this.recentlySyncedKeys.clear() this.hasReceivedFirstCommit = false } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 1f7569534..a474ea0e6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1388,6 +1388,11 @@ window progress; only a complete publication snapshot reaches readers. Release, truncate, replacement, restart, and cleanup change the relevant identity or generation without changing this sequence. +Cleanup also ends the current publication history. It must discard both the +pre-sync visible snapshot and the recently-synced suppression set before a new +sync session starts. Otherwise stale rows can classify a fresh insert as an +update, or stale keys can suppress the new session's first event. + An imperative load operation is a separate caller boundary around this flow. It owns the future requests caused while it is current, retains the promises it already acquired after a newer operation supersedes it, and settles only after diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 4fc567bf9..3586d19ad 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -161,6 +161,68 @@ it(`releases retained keys after long unique-key churn`, async () => { await runRetentionHistory(actions) }) +it(`starts a new sync session without retained publication state`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + const events: Array<{ type: string; key: number }> = [] + let subscription: ReturnType | undefined + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + subscription = collection.subscribeChanges( + (changes) => { + events.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + })), + ) + }, + { includeInitialState: false }, + ) + + sync.begin() + sync.write({ type: `update`, value: { id: 1, value: 2 } }) + collection._state.capturePreSyncVisibleState() + expect(collection._state.preSyncVisibleState.size).toBe(1) + expect(collection._state.recentlySyncedKeys).toEqual(new Set([1])) + + const cleanup = collection.cleanup() + const retainedAfterCleanup = { + visibleRows: collection._state.preSyncVisibleState.size, + recentKeys: collection._state.recentlySyncedKeys.size, + } + await cleanup + + events.length = 0 + collection.startSyncImmediate() + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 3 } }) + expect(sync.commit()).toBe(true) + + expect({ retainedAfterCleanup, events }).toEqual({ + retainedAfterCleanup: { visibleRows: 0, recentKeys: 0 }, + events: [{ type: `insert`, key: 1 }], + }) + } finally { + subscription?.unsubscribe() + await collection.cleanup() + } +}) + fcTest.prop( [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], oraclePropertyOptions(100, `collection-state.retention`), From bc0961155834a96933f189d0ced1b51e5cd542a8 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:14:56 -0600 Subject: [PATCH 251/327] fix(db): preserve publication callback failures --- packages/db/src/collection/changes.ts | 27 ++-- packages/db/src/query/live/ARCHITECTURE.md | 8 ++ packages/db/tests/query/scheduler.test.ts | 160 +++++++++++++++++++++ 3 files changed, 186 insertions(+), 9 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index a392df67a..4d37f16df 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -280,15 +280,24 @@ export class CollectionChangesManager< // Every subscriber sees one committed source batch before dependent query // graphs run. This keeps repeated aliases and sibling subqueries coherent. withPublicationContext(() => { - // Notify both internal layout consumers and the public subscription API. - // Public subscribers historically receive an empty batch for order-only - // moves because there is no row-value ChangeMessage to publish. - if (rawEvents.length === 0) { - for (const listener of this.layoutChangeListeners) listener() - } - - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents(enrichedEvents) + const callbacks = [ + // Notify both internal layout consumers and the public subscription API. + // Public subscribers historically receive an empty batch for order-only + // moves because there is no row-value ChangeMessage to publish. + ...(rawEvents.length === 0 + ? [...this.layoutChangeListeners].map((listener) => () => listener()) + : []), + ...[...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents(enrichedEvents), + ), + ] + try { + runAllCallbacks(callbacks) + } catch (error) { + // The committed batch is already public. Keep the first callback + // failure on the publication while later subscribers and graph work + // finish observing the same snapshot. + deferPublicationFailure(error) } }) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a474ea0e6..47dd99e80 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1219,6 +1219,14 @@ joins that context and runs only after the current root and facade callbacks finish; it cannot start a second graph turn inside the first one or disappear through the graph's reentrancy guard. +Each ordinary Collection publication freezes its layout listeners and public +subscribers, then attempts every callback in registration order. Adding or +removing a listener during delivery does not change that batch. The first exact +callback failure stays on the shared publication context, including when it +came from a nested readiness transition. Later callback failures cannot replace +it. Core runs the dependent graph work queued by the batch before rethrowing the +retained failure. + Window metadata follows the same causal order as the published rows. If a publication callback starts a newer window operation, that newer generation owns the final public window and the older caller cannot overwrite it when it diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index c4eb8f8cd..d64f16346 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -145,6 +145,166 @@ describe(`Collection publication scheduler context`, () => { }) describe(`live query scheduler`, () => { + it(`delivers an ordinary source batch to its frozen listener snapshot`, async () => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const calls: Array = [] + const source = createCollection({ + id: `ordinary-listener-membership-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + let added: { unsubscribe: () => void } | undefined + const first = source.subscribeChanges(() => { + calls.push(`first`) + second.unsubscribe() + added ??= source.subscribeChanges(() => calls.push(`added`), { + includeInitialState: false, + }) + }) + const second = source.subscribeChanges(() => calls.push(`second`)) + + try { + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + commit() + expect(calls).toEqual([`first`, `second`]) + + begin() + write({ type: `insert`, value: { id: 2, name: `Grace` } }) + commit() + expect(calls).toEqual([`first`, `second`, `first`, `added`]) + } finally { + first.unsubscribe() + second.unsubscribe() + added?.unsubscribe() + await source.cleanup() + } + }) + + it(`settles a dependent live query when an earlier source listener throws`, async () => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const listenerFailure = new Error(`source listener failed`) + const source = createCollection({ + id: `throwing-listener-live-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + () => { + throw listenerFailure + }, + { includeInitialState: false }, + ) + const live = createLiveQueryCollection({ + id: `throwing-listener-live-dependent`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + expect(() => commit()).toThrow(listenerFailure) + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + } finally { + throwingSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }) + + it(`keeps a nested ready failure when a later outer listener throws`, async () => { + let markInnerReady!: () => void + const readyFailure = new Error(`nested ready listener failed`) + const laterFailure = new Error(`later outer listener failed`) + const scheduledJob = vi.fn() + const inner = createCollection({ + id: `nested-ready-collision-inner`, + getKey: (user) => user.id, + sync: { + sync: ({ markReady }) => { + markInnerReady = markReady + }, + }, + }) + const innerFirst = inner.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const innerSecond = inner.subscribeChanges(() => { + throw readyFailure + }) + + let beginOuter!: () => void + let writeOuter!: (message: { type: `insert`; value: User }) => void + let commitOuter!: () => void + const outer = createCollection({ + id: `nested-ready-collision-outer`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + beginOuter = actions.begin + writeOuter = actions.write + commitOuter = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const outerFirst = outer.subscribeChanges(() => markInnerReady()) + const outerSecond = outer.subscribeChanges(() => { + throw laterFailure + }) + + try { + beginOuter() + writeOuter({ type: `insert`, value: { id: 1, name: `Ada` } }) + expect(() => commitOuter()).toThrow(readyFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + } finally { + outerFirst.unsubscribe() + outerSecond.unsubscribe() + innerFirst.unsubscribe() + innerSecond.unsubscribe() + await outer.cleanup() + await inner.cleanup() + } + }) + it(`settles a dependent live query before a nested ready failure escapes`, async () => { let markSourceReady: (() => void) | undefined const listenerFailure = new Error(`source ready listener failed`) From 8edfa4f7037a970487ae5b160c57921d6981948b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:21:43 -0600 Subject: [PATCH 252/327] test(db): type retained restart events --- .../db/tests/collection-state-retention-oracle.property.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 3586d19ad..219e8dfc3 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -174,7 +174,7 @@ it(`starts a new sync session without retained publication state`, async () => { }, }, }) - const events: Array<{ type: string; key: number }> = [] + const events: Array<{ type: string; key: string | number }> = [] let subscription: ReturnType | undefined try { From 2c1c61455eb31e4088d99da9155c5cdbae1361ee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:23:57 -0600 Subject: [PATCH 253/327] test(db): pin publication failure boundaries --- packages/db/src/query/live/ARCHITECTURE.md | 11 +++-- .../db/tests/collection-lifecycle.test.ts | 44 +++++++++++++++++++ packages/db/tests/query/scheduler.test.ts | 24 ++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 47dd99e80..4a8361253 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1121,8 +1121,9 @@ discard graph work queued by an earlier listener. Core flushes that work before it rethrows the first listener failure. If readiness is nested inside an existing publication, core retains the exact listener failure on that shared context and the outer boundary rethrows it only after the queued graph work -drains. When both the listener and that queued graph work fail, the first ready -listener failure remains the reported error. +finishes or the first graph failure stops that turn. When both the listener and +queued graph work fail, the first ready listener failure remains the reported +error and the scheduler clears the turn's remaining work. When `markReady()` runs during the synchronous adapter-entry call, core retains any ready-effect failure until the adapter finishes its own setup. It then @@ -1224,8 +1225,10 @@ subscribers, then attempts every callback in registration order. Adding or removing a listener during delivery does not change that batch. The first exact callback failure stays on the shared publication context, including when it came from a nested readiness transition. Later callback failures cannot replace -it. Core runs the dependent graph work queued by the batch before rethrowing the -retained failure. +it. Core runs the dependent graph turn queued by the batch before rethrowing the +retained failure. A graph failure stops that turn and clears its remaining +work; scheduler dependencies are not a complete proof that two jobs can commit +or roll back independently. Window metadata follows the same causal order as the published rows. If a publication callback starts a newer window operation, that newer generation diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 8a09af20c..9c71fa00a 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -809,6 +809,50 @@ describe(`Collection Lifecycle Management`, () => { } }) + it(`preserves a falsy ready failure through a nested publication`, async () => { + let markReadyCallback: (() => void) | undefined + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-falsy-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw undefined + }) + + try { + let didThrow = false + let thrown: unknown + try { + withPublicationContext(() => markReadyCallback!()) + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(scheduledJob).toHaveBeenCalledOnce() + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + it(`surfaces a ready graph failure after running its job`, async () => { let markReadyCallback: (() => void) | undefined const graphFailure = new Error(`ready graph failed`) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index d64f16346..f461c57f5 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -142,6 +142,30 @@ describe(`Collection publication scheduler context`, () => { expect(getActivePublicationContext()).toBeUndefined() expect(transactionScopedScheduler.hasPendingJobs(contextId!)).toBe(false) }) + + it(`preserves a falsy graph failure through a publication boundary`, () => { + let didThrow = false + let thrown: unknown + + try { + withPublicationContext(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: `failing`, + run: () => { + throw undefined + }, + }) + }) + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + }) }) describe(`live query scheduler`, () => { From f561f955d3c5e64723aaf9138800b3d6754eceab Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:29:11 -0600 Subject: [PATCH 254/327] fix(db): fence publication state by sync session --- packages/db/src/collection/state.ts | 26 +++--- packages/db/src/query/live/ARCHITECTURE.md | 6 +- ...on-state-retention-oracle.property.test.ts | 80 +++++++++++++++++++ 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index f2f516f73..4a4257c5c 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -201,6 +201,7 @@ export class CollectionStateManager< public hasReceivedFirstCommit = false public isCommittingSyncTransactions = false private isDrainingSyncTransactions = false + private syncSessionGeneration = 0 public isLocalOnly = false /** @@ -1042,6 +1043,8 @@ export class CollectionStateManager< processed: boolean publicationError?: { error: unknown } } { + const syncSessionGeneration = this.syncSessionGeneration + // Check if there are any persisting transaction let hasPersistingTransaction = false for (const transaction of this.transactions.values()) { @@ -1695,17 +1698,21 @@ export class CollectionStateManager< publicationError = { error } } - // Clear the pre-sync state since sync operations are complete - this.preSyncVisibleState.clear() + if (this.syncSessionGeneration === syncSessionGeneration) { + // Clear the pre-sync state since sync operations are complete + this.preSyncVisibleState.clear() - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - this.recentlySyncedKeys.clear() - }) + // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them + Promise.resolve().then(() => { + if (this.syncSessionGeneration === syncSessionGeneration) { + this.recentlySyncedKeys.clear() + } + }) - // Mark that we've received the first commit (for tracking purposes) - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true + // Mark that we've received the first commit (for tracking purposes) + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } } for (const transaction of committedSyncedTransactions) { @@ -1865,6 +1872,7 @@ export class CollectionStateManager< * This can be called manually or automatically by garbage collection */ public cleanup(): void { + this.syncSessionGeneration++ for (const transaction of this.pendingSyncedTransactions) { transaction.applied.reject(new SyncTransactionAbortedError()) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 4a8361253..e640d5c7e 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1401,8 +1401,10 @@ identity or generation without changing this sequence. Cleanup also ends the current publication history. It must discard both the pre-sync visible snapshot and the recently-synced suppression set before a new -sync session starts. Otherwise stale rows can classify a fresh insert as an -update, or stale keys can suppress the new session's first event. +sync session starts. Synchronous publication tails and queued microtasks remain +scoped to the session that created them; after cleanup they cannot clear or mark +state owned by a restarted session. Otherwise stale rows can classify a fresh +insert as an update, or stale keys can suppress the new session's first event. An imperative load operation is a separate caller boundary around this flow. It owns the future requests caused while it is current, retains the promises it diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 219e8dfc3..4b610db13 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -223,6 +223,86 @@ it(`starts a new sync session without retained publication state`, async () => { } }) +it(`keeps a restarted session's publication state after the old listener returns`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + let cleanup: Promise | undefined + let restarted = false + const subscription = collection.subscribeChanges( + () => { + if (restarted) return + restarted = true + cleanup = collection.cleanup() + collection.startSyncImmediate() + collection._state.preSyncVisibleState.set(2, { id: 2, value: 2 }) + collection._state.recentlySyncedKeys.add(2) + }, + { includeInitialState: false }, + ) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + expect(restarted).toBe(true) + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[2, { id: 2, value: 2 }]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + await cleanup + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`does not let an old publication microtask clear restarted sync state`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + const cleanup = collection.cleanup() + collection.startSyncImmediate() + sync.begin() + sync.write({ type: `insert`, value: { id: 2, value: 2 } }) + collection._state.capturePreSyncVisibleState() + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + + await Promise.resolve() + + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + await cleanup + } finally { + await collection.cleanup() + } +}) + fcTest.prop( [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], oraclePropertyOptions(100, `collection-state.retention`), From ff8d847a7d58470dbb0cb09604f5f4cd6c659178 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:47:42 -0600 Subject: [PATCH 255/327] test(db): widen sync session retention oracle --- ...on-state-retention-oracle.property.test.ts | 194 ++++++++++++++++-- 1 file changed, 180 insertions(+), 14 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 4b610db13..1985f891f 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -2,6 +2,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { DuplicateKeySyncError } from '../src/errors.js' +import { createTransaction } from '../src/transactions.js' import { oraclePropertyOptions } from './oracle-config.js' import type { Collection } from '../src/collection/index.js' import type { SyncConfig } from '../src/types.js' @@ -18,6 +19,8 @@ type RetentionAction = | { type: `update`; row: RetainedRow } | { type: `delete`; key: number } | { type: `replace`; rows: ReadonlyArray } + | { type: `restart` } + | { type: `reentrantRestart`; row: RetainedRow } type RetentionHarness = { collection: Collection @@ -30,17 +33,43 @@ const retainedRowArbitrary = fc.record({ }) const retentionActionArbitrary: fc.Arbitrary = fc.oneof( - retainedRowArbitrary.map((row) => ({ type: `insert` as const, row })), - retainedRowArbitrary.map((row) => ({ type: `update` as const, row })), - fc - .integer({ min: 0, max: 3 }) - .map((key) => ({ type: `delete` as const, key })), - fc - .uniqueArray(retainedRowArbitrary, { - selector: (row) => row.id, - maxLength: 4, - }) - .map((rows) => ({ type: `replace` as const, rows })), + { + weight: 4, + arbitrary: retainedRowArbitrary.map((row) => ({ + type: `insert` as const, + row, + })), + }, + { + weight: 4, + arbitrary: retainedRowArbitrary.map((row) => ({ + type: `update` as const, + row, + })), + }, + { + weight: 4, + arbitrary: fc + .integer({ min: 0, max: 3 }) + .map((key) => ({ type: `delete` as const, key })), + }, + { + weight: 2, + arbitrary: fc + .uniqueArray(retainedRowArbitrary, { + selector: (row) => row.id, + maxLength: 4, + }) + .map((rows) => ({ type: `replace` as const, rows })), + }, + { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, + { + weight: 1, + arbitrary: retainedRowArbitrary.map((row) => ({ + type: `reentrantRestart` as const, + row, + })), + }, ) function createRetentionHarness(): RetentionHarness { @@ -56,7 +85,12 @@ function createRetentionHarness(): RetentionHarness { }, }, }) - return { collection, sync } + return { + collection, + get sync() { + return sync + }, + } } function applyAction( @@ -95,6 +129,9 @@ function applyAction( model.set(row.id, row) } break + case `restart`: + case `reentrantRestart`: + throw new Error(`Restart actions require the lifecycle driver`) } expect(sync.commit()).toBe(true) } @@ -122,12 +159,54 @@ function expectRetainedState( async function runRetentionHistory( actions: ReadonlyArray, ): Promise { - const { collection, sync } = createRetentionHarness() + const harness = createRetentionHarness() + const { collection } = harness const model = new Map() try { expectRetainedState(collection, model) for (const action of actions) { - applyAction(action, model, sync) + if (action.type === `restart`) { + await collection.cleanup() + collection.startSyncImmediate() + model.clear() + } else if (action.type === `reentrantRestart`) { + const oldSync = harness.sync + const triggerRow = { + id: action.row.id, + value: (model.get(action.row.id)?.value ?? action.row.value) + 1, + } + const restartedRow = { + id: (action.row.id + 1) % 4, + value: action.row.value + 1, + } + let cleanup: Promise | undefined + let restarted = false + const subscription = collection.subscribeChanges( + () => { + if (restarted) return + restarted = true + cleanup = collection.cleanup() + collection.startSyncImmediate() + harness.sync.begin() + harness.sync.write({ type: `insert`, value: restartedRow }) + collection._state.recentlySyncedKeys.add(restartedRow.id) + harness.sync.commit() + }, + { includeInitialState: false }, + ) + + oldSync.begin() + oldSync.write({ type: `update`, value: triggerRow }) + expect(oldSync.commit()).toBe(true) + subscription.unsubscribe() + expect(restarted).toBe(true) + await Promise.resolve() + await cleanup + model.clear() + model.set(restartedRow.id, restartedRow) + } else { + applyAction(action, model, harness.sync) + } expectRetainedState(collection, model) } } finally { @@ -261,6 +340,14 @@ it(`keeps a restarted session's publication state after the old listener returns ) expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) expect(collection._state.hasReceivedFirstCommit).toBe(false) + + sync.begin() + sync.write({ type: `insert`, value: { id: 3, value: 3 } }) + expect(sync.commit()).toBe(true) + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.recentlySyncedKeys.size).toBe(0) await cleanup } finally { subscription.unsubscribe() @@ -297,12 +384,91 @@ it(`does not let an old publication microtask clear restarted sync state`, async await Promise.resolve() expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + + expect(sync.commit()).toBe(true) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.recentlySyncedKeys.size).toBe(0) await cleanup } finally { await collection.cleanup() } }) +it(`publishes one insert when a restarted optimistic row is confirmed and rolled back`, async () => { + let sync!: SyncActions + let syncSession = 0 + let releaseMutation!: () => void + const mutationHold = new Promise((resolve) => { + releaseMutation = resolve + }) + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + syncSession++ + if (syncSession === 1) actions.markReady() + }, + }, + }) + const events: Array<{ type: string; key: string | number }> = [] + const restartStatuses: Array = [] + let restarted = false + let mutationCommit: Promise | undefined + const subscription = collection.subscribeChanges( + (changes) => { + events.push(...changes.map(({ type, key }) => ({ type, key }))) + if (restarted || !changes.some(({ key }) => key === 1)) return + + restarted = true + restartStatuses.push(collection.status) + void collection.cleanup() + restartStatuses.push(collection.status) + collection.startSyncImmediate() + restartStatuses.push(collection.status) + sync.markReady() + restartStatuses.push(collection.status) + + const transaction = createTransaction({ + autoCommit: false, + mutationFn: () => mutationHold, + }) + void transaction.isPersisted.promise.catch(() => undefined) + transaction.mutate(() => collection.insert({ id: 2, value: 2 })) + mutationCommit = transaction.commit().catch(() => undefined) + + sync.begin() + sync.write({ type: `insert`, value: { id: 2, value: 2 } }) + sync.commit() + transaction.rollback() + }, + { includeInitialState: false }, + ) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + expect(events).toEqual([ + { type: `insert`, key: 1 }, + { type: `insert`, key: 2 }, + ]) + expect([...collection.state.keys()]).toEqual([2]) + expect(restartStatuses).toEqual([`ready`, `cleaned-up`, `loading`, `ready`]) + expect(collection.status).toBe(`ready`) + } finally { + releaseMutation() + await mutationCommit + subscription.unsubscribe() + await collection.cleanup() + } +}) + fcTest.prop( [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], oraclePropertyOptions(100, `collection-state.retention`), From 55d5d309db6a589e7207791e89dc4d233382d898 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 16:55:11 -0600 Subject: [PATCH 256/327] fix(db): preserve exact D2 source rows --- packages/db/package.json | 2 +- packages/db/src/query/effect.ts | 26 ++- packages/db/src/query/live/ARCHITECTURE.md | 8 + .../src/query/live/collection-subscriber.ts | 19 +- packages/db/src/query/live/utils.ts | 60 ++++--- ...ction-subscriber-duplicate-inserts.test.ts | 36 +++- ...rce-reconciliation-oracle.property.test.ts | 170 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 8 files changed, 275 insertions(+), 47 deletions(-) create mode 100644 packages/db/tests/d2-source-reconciliation-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 3e7965590..4c13d2fdf 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 219ec92bb..c91cff12d 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -17,7 +17,7 @@ import { computeSubscriptionOrderByHints, extractCollectionSources, extractCollectionsFromQuery, - filterDuplicateInserts, + reconcileChangesForD2, sendChangesToInput, splitUpdates, trackBiggestSentValue, @@ -400,10 +400,10 @@ class EffectPipelineRunner { // Subscription management private readonly unsubscribeCallbacks = new Set<() => void>() - // Duplicate insert prevention per lexical source - private readonly sentToD2KeysBySource = new Map< + // Exact D2 contributions per lexical source + private readonly sentToD2RowsBySource = new Map< string, - Set + Map> >() // Output accumulator @@ -525,8 +525,7 @@ class EffectPipelineRunner { const { sourceId, alias, collection } = source const collectionId = collection.id - // Initialise per-source duplicate tracking - this.sentToD2KeysBySource.set(sourceId, new Set()) + this.sentToD2RowsBySource.set(sourceId, new Map()) // Discover dependencies: if source collection is itself a live query // collection, its builder must run first during transaction flushes. @@ -635,7 +634,7 @@ class EffectPipelineRunner { const truncateUnsubscribe = collection.on(`truncate`, () => { this.lastLoadRequestKey.delete(sourceId) this.biggestSentValue.delete(sourceId) - this.sentToD2KeysBySource.get(sourceId)?.clear() + this.sentToD2RowsBySource.get(sourceId)?.clear() this.pendingOrderedLoadPromise = undefined }) this.unsubscribeCallbacks.add(truncateUnsubscribe) @@ -839,11 +838,10 @@ class EffectPipelineRunner { const input = this.inputs[sourceId] if (!input) return 0 - // Filter duplicates per lexical source - const sentKeys = this.sentToD2KeysBySource.get(sourceId)! - const filtered = filterDuplicateInserts(changes, sentKeys) + const sentRows = this.sentToD2RowsBySource.get(sourceId)! + const reconciled = reconcileChangesForD2(changes, sentRows) - return sendChangesToInput(input, filtered) + return sendChangesToInput(input, reconciled) } /** @@ -1131,11 +1129,11 @@ class EffectPipelineRunner { changes: Array>, comparator: (a: any, b: any) => number, ): void { - const sentKeys = this.sentToD2KeysBySource.get(sourceId) ?? new Set() + const sentRows = this.sentToD2RowsBySource.get(sourceId) ?? new Map() const result = trackBiggestSentValue( changes, this.biggestSentValue.get(sourceId), - sentKeys, + sentRows, comparator, ) this.biggestSentValue.set(sourceId, result.biggest) @@ -1160,7 +1158,7 @@ class EffectPipelineRunner { firstCleanupFailure ??= { error } } } - this.sentToD2KeysBySource.clear() + this.sentToD2RowsBySource.clear() this.pendingChanges.clear() this.lazySources.clear() this.demand.clear() diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index e640d5c7e..9b9e7ccf7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -217,6 +217,14 @@ inside D2 do not need their own lifecycle objects or generations. D2 multisets are the source of truth. A row with positive weight contributes; a row with negative weight retracts the same contribution. +Each lexical source boundary retains the exact row last contributed for every +source key. A replayed insert for an existing key adds no second contribution. +An update or delete retracts the retained row, rather than trusting event +metadata that may describe a newer value, then replaces or removes that entry. +Truncate and graph teardown clear this boundary state. Thus every source key +has multiplicity zero or one and every negative weight cancels the exact +positive row that entered D2. + Internal contribution identity is independent of the user-visible Collection key. When several internal rows collapse to one public key, a keyed D2 reduction retains all contributors and derives at most one canonical row: diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 2124c1965..b8353541e 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -6,7 +6,7 @@ import { import { computeOrderedLoadCursor, computeSubscriptionOrderByHints, - filterDuplicateInserts, + reconcileChangesForD2, sendChangesToInput, splitUpdates, } from './utils.js' @@ -47,10 +47,9 @@ export class CollectionSubscriber< { resolve: () => void } >() - // Track keys that have been sent to the D2 pipeline to prevent duplicate inserts - // This is necessary because different code paths (initial load, change events) - // can potentially send the same item to D2 multiple times. - private sentToD2Keys = new Set() + // Track the exact row contributed for each source key. D2 retractions must + // use that row, even when the incoming event reports a changed previous row. + private sentToD2Rows = new Map>() // Direct load tracking callback for ordered path (set during subscribeToOrderedChanges, // used by loadNextItems for subsequent requestLimitedSnapshot calls) @@ -298,16 +297,16 @@ export class CollectionSubscriber< callback?: () => boolean, ) { const changesArray = Array.isArray(changes) ? changes : [...changes] - const filteredChanges = filterDuplicateInserts( + const reconciledChanges = reconcileChangesForD2( changesArray, - this.sentToD2Keys, + this.sentToD2Rows, ) // currentSyncState and input are always defined when this method is called // (only called from active subscriptions during a sync session) const input = this.collectionConfigBuilder.currentSyncState!.inputs[this.sourceId]! - const sentChanges = sendChangesToInput(input, filteredChanges) + const sentChanges = sendChangesToInput(input, reconciledChanges) // Do not provide the callback that loads more data // if there's no more data to load @@ -424,14 +423,14 @@ export class CollectionSubscriber< subscriptionHolder.current = subscription this.registerSubscriptionCleanup(subscription) - // Listen for truncate events to reset cursor tracking state and sentToD2Keys + // Listen for truncate events to reset cursor and D2 source tracking state. // This ensures that after a must-refetch/truncate, we don't use stale cursor data // and allow re-inserts of previously sent keys const truncateUnsubscribe = this.collection.on(`truncate`, () => { this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined this.pendingOrderedLoadPromise = undefined - this.sentToD2Keys.clear() + this.sentToD2Rows.clear() }) // Clean up truncate listener when subscription is unsubscribed diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index b70842c39..ee8d33966 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -140,29 +140,45 @@ export function* splitUpdates< } /** - * Filter changes to prevent duplicate inserts to a D2 pipeline. - * Maintains D2 multiplicity at 1 for visible items so that deletes - * properly reduce multiplicity to 0. - * - * Mutates `sentKeys` in place: adds keys on insert, removes on delete. + * Reconcile source changes with the exact rows previously contributed to D2. + * This keeps each source key at multiplicity one and makes every retraction + * match the row identity that D2 originally received. */ -export function filterDuplicateInserts( - changes: Array>, - sentKeys: Set, -): Array> { - const filtered: Array> = [] +export function reconcileChangesForD2< + T extends object, + TKey extends string | number, +>( + changes: Array>, + sentRows: Map, +): Array> { + const reconciled: Array> = [] for (const change of changes) { if (change.type === `insert`) { - if (sentKeys.has(change.key)) { - continue // Skip duplicate - } - sentKeys.add(change.key) - } else if (change.type === `delete`) { - sentKeys.delete(change.key) + if (sentRows.has(change.key)) continue + sentRows.set(change.key, change.value) + reconciled.push(change) + continue + } + + const previousValue = sentRows.get(change.key) + if (change.type === `delete`) { + sentRows.delete(change.key) + reconciled.push( + previousValue === undefined || previousValue === change.value + ? change + : { ...change, value: previousValue }, + ) + continue } - filtered.push(change) + + sentRows.set(change.key, change.value) + reconciled.push( + previousValue === undefined || previousValue === change.previousValue + ? change + : { ...change, previousValue }, + ) } - return filtered + return reconciled } /** @@ -172,15 +188,19 @@ export function filterDuplicateInserts( * * @param changes - changes to process (deletes are skipped) * @param current - the current biggest value (or undefined if none) - * @param sentKeys - set of keys already sent to D2 (for new-key detection) + * @param sentKeys - lookup of keys already sent to D2 (for new-key detection) * @param comparator - orderBy comparator * @returns `{ biggest, shouldResetLoadKey }` — the new biggest value and * whether the caller should clear its last-load-request-key */ +interface SentKeyLookup { + has: (key: string | number) => boolean +} + export function trackBiggestSentValue( changes: Array>, current: unknown | undefined, - sentKeys: Set, + sentKeys: SentKeyLookup, comparator: (a: any, b: any) => number, ): { biggest: unknown; shouldResetLoadKey: boolean } { let biggest = current diff --git a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts index 8b9d6be57..6627383e2 100644 --- a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts +++ b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { BTreeIndex } from '../src/indexes/btree-index.js' import { createLiveQueryCollection, eq } from '../src/query/index.js' +import { reconcileChangesForD2 } from '../src/query/live/utils.js' import { mockSyncCollectionOptions } from './utils.js' import type { ChangeMessage } from '../src/types.js' @@ -15,8 +16,8 @@ import type { ChangeMessage } from '../src/types.js' * If duplicate inserts reach D2, multiplicity becomes > 1, and deletes won't * properly remove items (multiplicity goes from 2 to 1, not triggering removal). * - * The fix: CollectionSubscriber tracks keys sent to D2 (sentToD2Keys) and - * filters out duplicate inserts before they reach the pipeline. + * The source boundary tracks the exact row sent for each key. It filters + * duplicate inserts and uses the stored row for later D2 retractions. * * Additionally, for JOIN queries with lazy sources: * - The includeInitialState fix ensures internal lazy-loading subscriptions @@ -40,6 +41,37 @@ type Order = { } describe(`CollectionSubscriber duplicate insert prevention`, () => { + it(`retracts the exact row previously contributed for a source key`, () => { + const sentRows = new Map>() + const inserted = { id: `1`, status: `draft` } + const changed = { id: `1`, status: `published` } + + reconcileChangesForD2( + [{ type: `insert`, key: `1`, value: inserted }], + sentRows, + ) + const reconciled = reconcileChangesForD2( + [ + { + type: `update`, + key: `1`, + value: changed, + previousValue: changed, + }, + ], + sentRows, + ) + + expect(reconciled).toEqual([ + { + type: `update`, + key: `1`, + value: changed, + previousValue: inserted, + }, + ]) + }) + it(`should properly delete items from live query with orderBy + limit`, async () => { // This test verifies that items can be properly deleted from a live query // with orderBy + limit. If duplicate inserts reach D2, the delete won't work. diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts new file mode 100644 index 000000000..2322c98b4 --- /dev/null +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -0,0 +1,170 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect } from 'vitest' +import { reconcileChangesForD2 } from '../src/query/live/utils.js' +import { oraclePropertyOptions } from './oracle-config.js' +import type { ChangeMessage } from '../src/types.js' + +type SourceRow = { + id: number + revision: number + value: number +} + +type SourceOperation = + | { + type: `upsert` + row: SourceRow + reportedPreviousValue: SourceRow + } + | { type: `replay`; key: number } + | { type: `delete`; key: number; reportedValue: SourceRow } + +type ReconciliationStep = + | { type: `batch`; operations: ReadonlyArray } + | { type: `reset` } + +const sourceRowArbitrary = fc.record({ + id: fc.integer({ min: 0, max: 3 }), + revision: fc.integer({ min: 0, max: 4 }), + value: fc.integer({ min: -2, max: 2 }), +}) + +const sourceOperationArbitrary: fc.Arbitrary = fc.oneof( + fc + .record({ + row: sourceRowArbitrary, + reportedPreviousValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `upsert` as const, ...operation })), + fc + .integer({ min: 0, max: 3 }) + .map((key) => ({ type: `replay` as const, key })), + fc + .record({ + key: fc.integer({ min: 0, max: 3 }), + reportedValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `delete` as const, ...operation })), +) + +const reconciliationStepArbitrary: fc.Arbitrary = fc.oneof( + { + weight: 8, + arbitrary: fc + .array(sourceOperationArbitrary, { minLength: 1, maxLength: 5 }) + .map((operations) => ({ type: `batch` as const, operations })), + }, + { weight: 1, arbitrary: fc.constant({ type: `reset` as const }) }, +) + +function rowIdentity(row: SourceRow): string { + return `${row.id}:${row.revision}:${row.value}` +} + +function addWeight( + relation: Map, + row: SourceRow, + weight: 1 | -1, +): void { + const identity = rowIdentity(row) + const nextWeight = (relation.get(identity) ?? 0) + weight + if (nextWeight === 0) relation.delete(identity) + else relation.set(identity, nextWeight) +} + +function applyToRelation( + relation: Map, + changes: ReadonlyArray>, +): void { + for (const change of changes) { + if (change.type === `insert`) { + addWeight(relation, change.value, 1) + } else if (change.type === `update`) { + addWeight(relation, change.previousValue!, -1) + addWeight(relation, change.value, 1) + } else { + addWeight(relation, change.value, -1) + } + } +} + +function sourceChangesFor( + operations: ReadonlyArray, + sourceRows: Map, +): Array> { + const changes: Array> = [] + for (const operation of operations) { + if (operation.type === `upsert`) { + const previousValue = sourceRows.get(operation.row.id) + changes.push( + previousValue === undefined + ? { type: `insert`, key: operation.row.id, value: operation.row } + : { + type: `update`, + key: operation.row.id, + value: operation.row, + previousValue: operation.reportedPreviousValue, + }, + ) + sourceRows.set(operation.row.id, operation.row) + } else if (operation.type === `replay`) { + const row = sourceRows.get(operation.key) + if (row !== undefined) { + changes.push({ type: `insert`, key: operation.key, value: row }) + } + } else { + const row = sourceRows.get(operation.key) + if (row !== undefined) { + changes.push({ + type: `delete`, + key: operation.key, + value: operation.reportedValue, + }) + sourceRows.delete(operation.key) + } + } + } + return changes +} + +function expectExactSourceRelation( + sourceRows: ReadonlyMap, + sentRows: ReadonlyMap, + relation: ReadonlyMap, +): void { + expect([...sentRows.entries()].sort(([a], [b]) => a - b)).toEqual( + [...sourceRows.entries()].sort(([a], [b]) => a - b), + ) + expect( + [...relation.entries()].sort(([a], [b]) => a.localeCompare(b)), + ).toEqual( + [...sourceRows.values()] + .map((row) => [rowIdentity(row), 1] as const) + .sort(([a], [b]) => a.localeCompare(b)), + ) +} + +fcTest.prop( + [fc.array(reconciliationStepArbitrary, { minLength: 1, maxLength: 30 })], + oraclePropertyOptions(200, `d2-source.exact-retractions`), +)( + `keeps one exact D2 contribution per source key across batched histories`, + (steps) => { + const sourceRows = new Map() + const sentRows = new Map() + const relation = new Map() + + for (const step of steps) { + if (step.type === `reset`) { + sourceRows.clear() + sentRows.clear() + relation.clear() + } else { + const changes = sourceChangesFor(step.operations, sourceRows) + const reconciled = reconcileChangesForD2(changes, sentRows) + applyToRelation(relation, reconciled) + } + expectExactSourceRelation(sourceRows, sentRows, relation) + } + }, +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index c15d168b8..958127d2f 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -5,6 +5,7 @@ const staticOracleProperties = [ `collection-state.retention`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, + `d2-source.exact-retractions`, `includes-collection.layout-swap`, `includes-collection.optimistic-child-history`, `includes-collection.public-key-order`, From d0b7fc2e871d5e041f7accaffdc3a37fabcdb84c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:06:02 -0600 Subject: [PATCH 257/327] fix(db): unify sync affected key tracking --- packages/db/package.json | 2 +- packages/db/src/collection/state.ts | 59 ++--- packages/db/src/query/live/ARCHITECTURE.md | 50 ++-- ...tadata-publication-oracle.property.test.ts | 240 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 5 files changed, 297 insertions(+), 55 deletions(-) create mode 100644 packages/db/tests/collection-metadata-publication-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 4c13d2fdf..d504fd2a6 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 4a4257c5c..ad003c9c8 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -232,18 +232,31 @@ export class CollectionStateManager< this._events = deps.events } - public snapshotPublicationState( - keys: Iterable, - ): CollectionPublicationStateSnapshot { - const affectedKeys = new Set(keys) - for (const transaction of this.pendingSyncedTransactions) { + /** Collect every row key whose visible state a sync transaction can change. */ + private collectAffectedKeys( + transactions: Iterable>, + ): Set { + const keys = new Set() + for (const transaction of transactions) { for (const operation of transaction.operations) { - affectedKeys.add(operation.key as TKey) + keys.add(operation.key as TKey) } for (const key of transaction.rowMetadataWrites.keys()) { - affectedKeys.add(key) + keys.add(key) } } + return keys + } + + public snapshotPublicationState( + keys: Iterable, + ): CollectionPublicationStateSnapshot { + const affectedKeys = new Set(keys) + for (const key of this.collectAffectedKeys( + this.pendingSyncedTransactions, + )) { + affectedKeys.add(key) + } return { pendingSyncedTransactions: [...this.pendingSyncedTransactions], @@ -1132,17 +1145,7 @@ export class CollectionStateManager< let truncatePendingLocalChanges: Set | undefined let truncatePendingLocalOrigins: Set | undefined - // First collect all keys that will be affected by sync operations - const changedKeys = new Set() - for (const transaction of committedSyncedTransactions) { - for (const operation of transaction.operations) { - const key = operation.key as TKey - changedKeys.add(key) - } - for (const [key] of transaction.rowMetadataWrites) { - changedKeys.add(key) - } - } + const changedKeys = this.collectAffectedKeys(committedSyncedTransactions) type AppliedRequestProvenance = { version: { @@ -1771,14 +1774,10 @@ export class CollectionStateManager< this.pendingSyncedTransactions.splice(index, 1) transaction.applied.reject(new SyncTransactionAbortedError()) - const remainingPendingKeys = new Set() - for (const pending of this.pendingSyncedTransactions) { - for (const operation of pending.operations) { - remainingPendingKeys.add(operation.key as TKey) - } - } - for (const operation of transaction.operations) { - const key = operation.key as TKey + const remainingPendingKeys = this.collectAffectedKeys( + this.pendingSyncedTransactions, + ) + for (const key of this.collectAffectedKeys([transaction])) { if (!remainingPendingKeys.has(key)) { this.recentlySyncedKeys.delete(key) this.preSyncVisibleState.delete(key) @@ -1826,13 +1825,7 @@ export class CollectionStateManager< public capturePreSyncVisibleState(): void { if (this.pendingSyncedTransactions.length === 0) return - // Get all keys that will be affected by sync operations - const syncedKeys = new Set() - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - syncedKeys.add(operation.key as TKey) - } - } + const syncedKeys = this.collectAffectedKeys(this.pendingSyncedTransactions) // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState for (const key of syncedKeys) { diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 9b9e7ccf7..762be9c84 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1238,6 +1238,13 @@ retained failure. A graph failure stops that turn and clears its remaining work; scheduler dependencies are not a complete proof that two jobs can commit or roll back independently. +Each ordinary Collection batch is a keyed diff and names an affected row key at +most once. Row operations and row-metadata writes use one shared affected-key +derivation for pre-sync capture, commit, cancellation, and rollback snapshots. +A metadata-only transaction can retire optimistic state and change virtual row +properties, so omitting its key from any of those phases can publish the same +transition twice or leave stale suppression state after cancellation. + Window metadata follows the same causal order as the published rows. If a publication callback starts a newer window operation, that newer generation owns the final public window and the older caller cannot overwrite it when it @@ -1346,27 +1353,28 @@ create recursive Collection machinery. ## Executable contracts -| Contract | Test suite | -| ---------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | -| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | -| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | -| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | -| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | -| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | -| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | -| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | -| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | -| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | -| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | -| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | -| End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | -| Shared subset acquisition, readiness, receipt, and replay interpreter | `packages/db/tests/query/load-subset-refinement-model.property.test.ts` | -| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.test.ts` | -| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | -| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | -| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | +| Contract | Test suite | +| ---------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | +| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | +| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | +| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | +| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | +| Keyed Collection diffs across row-metadata settlement and cancellation | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | +| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | +| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | +| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | +| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | +| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | +| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | +| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | +| End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | +| Shared subset acquisition, readiness, receipt, and replay interpreter | `packages/db/tests/query/load-subset-refinement-model.property.test.ts` | +| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.test.ts` | +| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | +| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | +| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | ### Oracle family boundary diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts new file mode 100644 index 000000000..8ed10a59b --- /dev/null +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -0,0 +1,240 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { SyncTransactionAbortedError } from '../src/errors.js' +import { createLiveQueryCollection } from '../src/query/index.js' +import { createTransaction } from '../src/transactions.js' +import { oraclePropertyOptions } from './oracle-config.js' +import type { Collection } from '../src/collection/index.js' +import type { ChangeMessage, SyncConfig } from '../src/types.js' + +type PublicationRow = { + id: number + position: number +} + +type SyncActions = Parameters[`sync`]>[0] + +type MetadataWrite = { + key: number + type: `set` | `delete` +} + +type PublicationRound = { + key: number + delta: number + metadata: ReadonlyArray + outcome: `commit` | `abort` +} + +type ReadablePublicationCollection = { + values: () => IterableIterator + cleanup: () => Promise +} + +type PublicationHarness = { + rows: Collection + liveRows: ReadablePublicationCollection + batches: Array>> + unsubscribe: () => void + getSync: () => SyncActions +} + +const metadataWriteArbitrary = fc.record({ + key: fc.integer({ min: 0, max: 2 }), + type: fc.constantFrom(`set` as const, `delete` as const), +}) + +const publicationRoundArbitrary: fc.Arbitrary = fc + .record({ + key: fc.integer({ min: 0, max: 2 }), + delta: fc.constantFrom(-2, -1, 1, 2), + extraMetadata: fc.array(metadataWriteArbitrary, { maxLength: 2 }), + outcome: fc.constantFrom(`commit` as const, `abort` as const), + primaryMetadataType: fc.constantFrom(`set` as const, `delete` as const), + }) + .map(({ key, delta, extraMetadata, outcome, primaryMetadataType }) => ({ + key, + delta, + outcome, + metadata: [{ key, type: primaryMetadataType }, ...extraMetadata], + })) + +async function createPublicationHarness(): Promise { + let sync!: SyncActions + const rows = createCollection({ + id: `metadata-publication-source`, + getKey: (row) => row.id, + startSync: true, + sync: { + sync: (actions) => { + sync = actions + actions.begin() + for (let id = 0; id < 3; id++) { + actions.write({ type: `insert`, value: { id, position: id } }) + } + actions.commit() + actions.markReady() + }, + }, + }) + const liveRows = createLiveQueryCollection((query) => + query.from({ row: rows }), + ) + await liveRows.preload() + + const batches: Array< + Array> + > = [] + const subscription = rows.subscribeChanges((changes) => { + batches.push(changes) + }) + return { + rows, + liveRows, + batches, + unsubscribe: () => subscription.unsubscribe(), + getSync: () => sync, + } +} + +function expectUniqueBatchKeys( + batches: ReadonlyArray< + ReadonlyArray> + >, +): void { + for (const batch of batches) { + const keys = batch.map((change) => change.key) + expect(keys).toEqual([...new Set(keys)]) + } +} + +function expectPublishedRows( + harness: PublicationHarness, + model: ReadonlyMap, +): void { + const expected = [...model.values()].sort((a, b) => a.id - b.id) + const selectBaseRows = (collection: ReadablePublicationCollection) => + [...collection.values()] + .map((row) => ({ id: row.id, position: row.position })) + .sort((a, b) => a.id - b.id) + + expect(selectBaseRows(harness.rows)).toEqual(expected) + expect(selectBaseRows(harness.liveRows)).toEqual(expected) +} + +async function applyRound( + harness: PublicationHarness, + round: PublicationRound, + roundIndex: number, + model: Map, + metadataModel: Map, +): Promise { + const previous = model.get(round.key)! + const next = { ...previous, position: previous.position + round.delta } + const sync = harness.getSync() + const transaction = createTransaction({ + mutationFn: async () => { + sync.begin({ immediate: true }) + sync.write({ type: `update`, value: next }) + sync.commit() + + sync.begin() + for (const write of round.metadata) { + if (write.type === `set`) { + const metadata = { round: roundIndex, owner: round.key } + sync.metadata!.row.set(write.key, metadata) + } else { + sync.metadata!.row.delete(write.key) + } + } + if (round.outcome === `commit`) { + sync.commit() + } else { + const controller = new AbortController() + const receipt = sync.commit(controller.signal) + controller.abort() + if (receipt !== true) { + await receipt.catch((error: unknown) => { + if (!(error instanceof SyncTransactionAbortedError)) throw error + }) + } + } + }, + }) + transaction.mutate(() => { + harness.rows.update(round.key, (draft) => { + draft.position = next.position + }) + }) + await transaction.isPersisted.promise + + model.set(round.key, next) + if (round.outcome === `commit`) { + for (const write of round.metadata) { + if (write.type === `set`) { + metadataModel.set(write.key, { + round: roundIndex, + owner: round.key, + }) + } else { + metadataModel.delete(write.key) + } + } + } + await Promise.resolve() + expectUniqueBatchKeys(harness.batches) + expectPublishedRows(harness, model) + const byKey = ( + [a]: readonly [number, unknown], + [b]: readonly [number, unknown], + ) => a - b + expect([...harness.rows._state.syncedMetadata.entries()].sort(byKey)).toEqual( + [...metadataModel.entries()].sort(byKey), + ) + expect(harness.rows._state.preSyncVisibleState.size).toBe(0) + expect(harness.rows._state.recentlySyncedKeys.size).toBe(0) +} + +async function runPublicationHistory( + rounds: ReadonlyArray, +): Promise { + const harness = await createPublicationHarness() + const model = new Map( + [0, 1, 2].map((id) => [id, { id, position: id }] as const), + ) + const metadataModel = new Map() + try { + for (const [index, round] of rounds.entries()) { + await applyRound(harness, round, index, model, metadataModel) + } + } finally { + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +} + +it(`publishes one event per key when metadata-only sync retires optimistic work`, async () => { + await runPublicationHistory([ + { + key: 1, + delta: 1, + metadata: [{ key: 1, type: `set` }], + outcome: `commit`, + }, + { + key: 1, + delta: 1, + metadata: [{ key: 1, type: `delete` }], + outcome: `commit`, + }, + ]) +}) + +fcTest.prop( + [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], + oraclePropertyOptions(50, `collection-publication.metadata-only`), +)( + `keeps metadata-only optimistic settlement a valid keyed diff across histories`, + runPublicationHistory, +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 958127d2f..0bdec6e45 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -3,6 +3,7 @@ type OracleEnvironment = Record const staticOracleProperties = [ `collection-sync.reentrant-drain`, `collection-state.retention`, + `collection-publication.metadata-only`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, `d2-source.exact-retractions`, From aedf280bbf37fb921484037883306b9579e37f3e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:11:19 -0600 Subject: [PATCH 258/327] test(db): observe restarted sync ownership --- ...on-state-retention-oracle.property.test.ts | 56 +++++++++++++++++-- 1 file changed, 51 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 1985f891f..1697c9962 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -171,6 +171,7 @@ async function runRetentionHistory( model.clear() } else if (action.type === `reentrantRestart`) { const oldSync = harness.sync + const triggerType = model.has(action.row.id) ? `update` : `insert` const triggerRow = { id: action.row.id, value: (model.get(action.row.id)?.value ?? action.row.value) + 1, @@ -179,18 +180,33 @@ async function runRetentionHistory( id: (action.row.id + 1) % 4, value: action.row.value + 1, } + const retainedMarker = { id: -1, value: action.row.value } let cleanup: Promise | undefined let restarted = false + let restartedSync: SyncActions | undefined + const events: Array<{ + type: string + key: string | number + row: RetainedRow + }> = [] const subscription = collection.subscribeChanges( - () => { + (changes) => { + events.push( + ...changes.map(({ type, key, value }) => ({ + type, + key, + row: { id: value.id, value: value.value }, + })), + ) if (restarted) return restarted = true cleanup = collection.cleanup() collection.startSyncImmediate() - harness.sync.begin() - harness.sync.write({ type: `insert`, value: restartedRow }) + restartedSync = harness.sync + restartedSync.begin() + restartedSync.write({ type: `insert`, value: restartedRow }) + collection._state.preSyncVisibleState.set(-1, retainedMarker) collection._state.recentlySyncedKeys.add(restartedRow.id) - harness.sync.commit() }, { includeInitialState: false }, ) @@ -198,9 +214,39 @@ async function runRetentionHistory( oldSync.begin() oldSync.write({ type: `update`, value: triggerRow }) expect(oldSync.commit()).toBe(true) - subscription.unsubscribe() expect(restarted).toBe(true) + expect(restartedSync).toBeDefined() + if (restartedSync === undefined) { + throw new Error(`restarted sync session was not captured`) + } + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, retainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([restartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + await Promise.resolve() + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, retainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([restartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + expect(restartedSync.commit()).toBe(true) + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.recentlySyncedKeys.size).toBe(0) + expect(events).toEqual([ + { type: triggerType, key: triggerRow.id, row: triggerRow }, + { type: `insert`, key: restartedRow.id, row: restartedRow }, + ]) + subscription.unsubscribe() + await cleanup model.clear() model.set(restartedRow.id, restartedRow) From 34228a5df2eac7ce5a912cb369d4b765d241d381 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:16:56 -0600 Subject: [PATCH 259/327] fix(db): keep rollback terminal during persistence --- packages/db/src/query/live/ARCHITECTURE.md | 3 ++ packages/db/src/transactions.ts | 8 ++++ ...on-state-retention-oracle.property.test.ts | 38 +++++++++++++++++-- packages/db/tests/transactions.test.ts | 32 ++++++++++++++++ 4 files changed, 78 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 762be9c84..332540b84 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -959,6 +959,9 @@ events are emitted, so an abort raised by a publication observer is already late. A successful `loadSubset` implementation must await or return every receipt for the transactions that establish its result. A source must not add priority merely to make a subset load settle. +A rollback that wins while an optimistic transaction's `mutationFn` is still +in flight is terminal. A later successful return from that function cannot +change the transaction from `failed` to `completed` or republish its overlay. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index c30ee78f0..db06b4403 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -636,6 +636,14 @@ class Transaction> { transaction: this as unknown as TransactionWithMutations, }) + // Rollback can win while mutationFn is in flight. Its failed state and + // rejected persistence receipt are terminal for this commit attempt. + // TypeScript keeps the entry-state narrowing across the await, although + // rollback may reenter and change it while mutationFn is pending. + if ((this.state as TransactionState) !== `persisting`) { + return this + } + this.setState(`completed`) this.touchCollection() diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 1697c9962..7265a9492 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -5,7 +5,7 @@ import { DuplicateKeySyncError } from '../src/errors.js' import { createTransaction } from '../src/transactions.js' import { oraclePropertyOptions } from './oracle-config.js' import type { Collection } from '../src/collection/index.js' -import type { SyncConfig } from '../src/types.js' +import type { SyncConfig, TransactionState } from '../src/types.js' type RetainedRow = { id: number @@ -464,7 +464,10 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled const events: Array<{ type: string; key: string | number }> = [] const restartStatuses: Array = [] let restarted = false + let readMutationState: (() => TransactionState) | undefined let mutationCommit: Promise | undefined + let syncReceipt: ReturnType | undefined + let syncReceiptSettled = false const subscription = collection.subscribeChanges( (changes) => { events.push(...changes.map(({ type, key }) => ({ type, key }))) @@ -483,13 +486,19 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled autoCommit: false, mutationFn: () => mutationHold, }) + readMutationState = () => transaction.state void transaction.isPersisted.promise.catch(() => undefined) transaction.mutate(() => collection.insert({ id: 2, value: 2 })) - mutationCommit = transaction.commit().catch(() => undefined) + mutationCommit = transaction.commit() sync.begin() sync.write({ type: `insert`, value: { id: 2, value: 2 } }) - sync.commit() + syncReceipt = sync.commit() + if (syncReceipt !== true) { + void syncReceipt.then(() => { + syncReceiptSettled = true + }) + } transaction.rollback() }, { includeInitialState: false }, @@ -507,6 +516,29 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled expect([...collection.state.keys()]).toEqual([2]) expect(restartStatuses).toEqual([`ready`, `cleaned-up`, `loading`, `ready`]) expect(collection.status).toBe(`ready`) + + expect(syncReceipt).toBeDefined() + expect(syncReceipt).not.toBe(true) + expect(syncReceiptSettled).toBe(false) + if (syncReceipt === undefined || syncReceipt === true) { + throw new Error(`restarted sync receipt was not parked`) + } + await syncReceipt + expect(syncReceiptSettled).toBe(true) + expect(events).toEqual([ + { type: `insert`, key: 1 }, + { type: `insert`, key: 2 }, + ]) + expect([...collection.state.keys()]).toEqual([2]) + + releaseMutation() + await mutationCommit + expect(readMutationState?.()).toBe(`failed`) + expect(events).toEqual([ + { type: `insert`, key: 1 }, + { type: `insert`, key: 2 }, + ]) + expect([...collection.state.keys()]).toEqual([2]) } finally { releaseMutation() await mutationCommit diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index d77e19600..40ce8f72e 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -216,6 +216,38 @@ describe(`Transactions`, () => { transaction.isPersisted.promise.catch(() => {}) expect(transaction.state).toBe(`failed`) }) + it(`keeps a persisting transaction failed when rollback wins`, async () => { + let releasePersistence!: () => void + const persistence = new Promise((resolve) => { + releasePersistence = resolve + }) + const collection = createCollection<{ id: number }>({ + id: `persisting-rollback-wins`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const transaction = createTransaction({ + autoCommit: false, + mutationFn: () => persistence, + }) + + try { + transaction.mutate(() => collection.insert({ id: 1 })) + void transaction.isPersisted.promise.catch(() => undefined) + const commit = transaction.commit() + expect(transaction.state).toBe(`persisting`) + + transaction.rollback() + expect(transaction.state).toBe(`failed`) + + releasePersistence() + await commit + expect(transaction.state).toBe(`failed`) + } finally { + releasePersistence() + await collection.cleanup() + } + }) it(`should rollback if the mutationFn throws an error`, async () => { const transaction = createTransaction({ mutationFn: async () => { From e66d7f646fcc1b88931094b6ff6e7c56cf870013 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:38:41 -0600 Subject: [PATCH 260/327] fix(db): retain D2 rows through truncate --- packages/db/src/query/effect.ts | 1 - packages/db/src/query/live/ARCHITECTURE.md | 9 +- .../src/query/live/collection-subscriber.ts | 7 +- packages/db/src/query/live/utils.ts | 9 +- ...rce-reconciliation-oracle.property.test.ts | 331 +++++++++++++++--- 5 files changed, 303 insertions(+), 54 deletions(-) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index c91cff12d..20cc28f9a 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -634,7 +634,6 @@ class EffectPipelineRunner { const truncateUnsubscribe = collection.on(`truncate`, () => { this.lastLoadRequestKey.delete(sourceId) this.biggestSentValue.delete(sourceId) - this.sentToD2RowsBySource.get(sourceId)?.clear() this.pendingOrderedLoadPromise = undefined }) this.unsubscribeCallbacks.add(truncateUnsubscribe) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 332540b84..db2b9de55 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -221,9 +221,11 @@ Each lexical source boundary retains the exact row last contributed for every source key. A replayed insert for an existing key adds no second contribution. An update or delete retracts the retained row, rather than trusting event metadata that may describe a newer value, then replaces or removes that entry. -Truncate and graph teardown clear this boundary state. Thus every source key -has multiplicity zero or one and every negative weight cancels the exact -positive row that entered D2. +An update for an unknown key becomes an insert, while a delete for an unknown +key contributes nothing. Truncate keeps this boundary state until its later +source batch retracts or replaces the retained rows. Graph teardown clears the +tracker and graph together. Thus every source key has multiplicity zero or one +and every negative weight cancels the exact positive row that entered D2. Internal contribution identity is independent of the user-visible Collection key. When several internal rows collapse to one public key, a keyed D2 @@ -1360,6 +1362,7 @@ create recursive Collection machinery. | ---------------------------------------------------------------------------- | --------------------------------------------------------------------------- | | State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | | Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | +| Exact D2 source retractions, replay suppression, and boundary lifecycle | `packages/db/tests/d2-source-reconciliation-oracle.property.test.ts` | | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index b8353541e..d3a363f3d 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -218,6 +218,7 @@ export class CollectionSubscriber< this.sourceId, subscription, ) + this.sentToD2Rows.clear() } // currentSyncState is always defined when subscribe() is called // (called during sync session setup) @@ -423,14 +424,12 @@ export class CollectionSubscriber< subscriptionHolder.current = subscription this.registerSubscriptionCleanup(subscription) - // Listen for truncate events to reset cursor and D2 source tracking state. - // This ensures that after a must-refetch/truncate, we don't use stale cursor data - // and allow re-inserts of previously sent keys + // Reset ordered-load state on truncate. Keep exact D2 source rows until + // their later delete/replacement batch retracts them from the live graph. const truncateUnsubscribe = this.collection.on(`truncate`, () => { this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined this.pendingOrderedLoadPromise = undefined - this.sentToD2Rows.clear() }) // Clean up truncate listener when subscription is unsubscribed diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index ee8d33966..5b27e8873 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -162,9 +162,10 @@ export function reconcileChangesForD2< const previousValue = sentRows.get(change.key) if (change.type === `delete`) { + if (previousValue === undefined) continue sentRows.delete(change.key) reconciled.push( - previousValue === undefined || previousValue === change.value + previousValue === change.value ? change : { ...change, value: previousValue }, ) @@ -172,8 +173,12 @@ export function reconcileChangesForD2< } sentRows.set(change.key, change.value) + if (previousValue === undefined) { + reconciled.push({ type: `insert`, key: change.key, value: change.value }) + continue + } reconciled.push( - previousValue === undefined || previousValue === change.previousValue + previousValue === change.previousValue ? change : { ...change, previousValue }, ) diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index 2322c98b4..43a07366b 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -1,8 +1,16 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { expect } from 'vitest' +import { expect, it } from 'vitest' +import { + createCollection, + createLiveQueryCollection, + eq, +} from '../src/index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' +import { createEffect } from '../src/query/effect.js' import { reconcileChangesForD2 } from '../src/query/live/utils.js' import { oraclePropertyOptions } from './oracle-config.js' -import type { ChangeMessage } from '../src/types.js' +import { flushPromises } from './utils.js' +import type { ChangeMessage, SyncConfig } from '../src/types.js' type SourceRow = { id: number @@ -10,18 +18,31 @@ type SourceRow = { value: number } +type SourceSyncActions = Parameters[`sync`]>[0] + +type SourceKey = string | number + type SourceOperation = | { type: `upsert` + key: SourceKey + row: SourceRow + reportedPreviousValue: SourceRow + } + | { + type: `rawUpdate` + key: SourceKey row: SourceRow reportedPreviousValue: SourceRow } - | { type: `replay`; key: number } - | { type: `delete`; key: number; reportedValue: SourceRow } + | { type: `replay`; key: SourceKey } + | { type: `delete`; key: SourceKey; reportedValue: SourceRow } type ReconciliationStep = | { type: `batch`; operations: ReadonlyArray } - | { type: `reset` } + | { type: `truncate` } + | { type: `teardown` } + | { type: `restart` } const sourceRowArbitrary = fc.record({ id: fc.integer({ min: 0, max: 3 }), @@ -29,19 +50,30 @@ const sourceRowArbitrary = fc.record({ value: fc.integer({ min: -2, max: 2 }), }) +const sourceKeyArbitrary: fc.Arbitrary = fc.oneof( + fc.integer({ min: 0, max: 2 }), + fc.constantFrom(`0`, `1`, `source`), +) + const sourceOperationArbitrary: fc.Arbitrary = fc.oneof( fc .record({ + key: sourceKeyArbitrary, row: sourceRowArbitrary, reportedPreviousValue: sourceRowArbitrary, }) .map((operation) => ({ type: `upsert` as const, ...operation })), fc - .integer({ min: 0, max: 3 }) - .map((key) => ({ type: `replay` as const, key })), + .record({ + key: sourceKeyArbitrary, + row: sourceRowArbitrary, + reportedPreviousValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `rawUpdate` as const, ...operation })), + sourceKeyArbitrary.map((key) => ({ type: `replay` as const, key })), fc .record({ - key: fc.integer({ min: 0, max: 3 }), + key: sourceKeyArbitrary, reportedValue: sourceRowArbitrary, }) .map((operation) => ({ type: `delete` as const, ...operation })), @@ -54,7 +86,9 @@ const reconciliationStepArbitrary: fc.Arbitrary = fc.oneof( .array(sourceOperationArbitrary, { minLength: 1, maxLength: 5 }) .map((operations) => ({ type: `batch` as const, operations })), }, - { weight: 1, arbitrary: fc.constant({ type: `reset` as const }) }, + { weight: 1, arbitrary: fc.constant({ type: `truncate` as const }) }, + { weight: 1, arbitrary: fc.constant({ type: `teardown` as const }) }, + { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, ) function rowIdentity(row: SourceRow): string { @@ -63,10 +97,11 @@ function rowIdentity(row: SourceRow): string { function addWeight( relation: Map, + key: SourceKey, row: SourceRow, weight: 1 | -1, ): void { - const identity = rowIdentity(row) + const identity = `${typeof key}:${String(key)}|${rowIdentity(row)}` const nextWeight = (relation.get(identity) ?? 0) + weight if (nextWeight === 0) relation.delete(identity) else relation.set(identity, nextWeight) @@ -74,97 +109,305 @@ function addWeight( function applyToRelation( relation: Map, - changes: ReadonlyArray>, + changes: ReadonlyArray>, ): void { for (const change of changes) { if (change.type === `insert`) { - addWeight(relation, change.value, 1) + addWeight(relation, change.key, change.value, 1) } else if (change.type === `update`) { - addWeight(relation, change.previousValue!, -1) - addWeight(relation, change.value, 1) + addWeight(relation, change.key, change.previousValue!, -1) + addWeight(relation, change.key, change.value, 1) } else { - addWeight(relation, change.value, -1) + addWeight(relation, change.key, change.value, -1) } } } function sourceChangesFor( operations: ReadonlyArray, - sourceRows: Map, -): Array> { - const changes: Array> = [] + sourceRows: Map, +): Array> { + const changes: Array> = [] for (const operation of operations) { if (operation.type === `upsert`) { - const previousValue = sourceRows.get(operation.row.id) + const previousValue = sourceRows.get(operation.key) changes.push( previousValue === undefined - ? { type: `insert`, key: operation.row.id, value: operation.row } + ? { type: `insert`, key: operation.key, value: operation.row } : { type: `update`, - key: operation.row.id, + key: operation.key, value: operation.row, previousValue: operation.reportedPreviousValue, }, ) - sourceRows.set(operation.row.id, operation.row) + sourceRows.set(operation.key, operation.row) + } else if (operation.type === `rawUpdate`) { + changes.push({ + type: `update`, + key: operation.key, + value: operation.row, + previousValue: operation.reportedPreviousValue, + }) + sourceRows.set(operation.key, operation.row) } else if (operation.type === `replay`) { const row = sourceRows.get(operation.key) if (row !== undefined) { changes.push({ type: `insert`, key: operation.key, value: row }) } } else { - const row = sourceRows.get(operation.key) - if (row !== undefined) { - changes.push({ - type: `delete`, - key: operation.key, - value: operation.reportedValue, - }) - sourceRows.delete(operation.key) - } + changes.push({ + type: `delete`, + key: operation.key, + value: operation.reportedValue, + }) + sourceRows.delete(operation.key) } } return changes } function expectExactSourceRelation( - sourceRows: ReadonlyMap, - sentRows: ReadonlyMap, + sourceRows: ReadonlyMap, + sentRows: ReadonlyMap, relation: ReadonlyMap, ): void { - expect([...sentRows.entries()].sort(([a], [b]) => a - b)).toEqual( - [...sourceRows.entries()].sort(([a], [b]) => a - b), + const compareEntries = ( + [a]: readonly [SourceKey, SourceRow], + [b]: readonly [SourceKey, SourceRow], + ) => `${typeof a}:${String(a)}`.localeCompare(`${typeof b}:${String(b)}`) + expect([...sentRows.entries()].sort(compareEntries)).toEqual( + [...sourceRows.entries()].sort(compareEntries), ) expect( [...relation.entries()].sort(([a], [b]) => a.localeCompare(b)), ).toEqual( - [...sourceRows.values()] - .map((row) => [rowIdentity(row), 1] as const) + [...sourceRows.entries()] + .map( + ([key, row]) => + [`${typeof key}:${String(key)}|${rowIdentity(row)}`, 1] as const, + ) .sort(([a], [b]) => a.localeCompare(b)), ) } +function createOrderedSourceHarness(id: string) { + let sync!: SourceSyncActions + let loadSubsetCalls = 0 + const contributed = { id: 1, revision: 1, value: 1 } + const staleDelete = { id: 1, revision: 2, value: 1 } + const source = createCollection({ + id, + getKey: (row) => row.id, + startSync: true, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (actions) => { + sync = actions + actions.markReady() + return { + loadSubset: async () => { + loadSubsetCalls++ + if (loadSubsetCalls > 1) await new Promise(() => {}) + return { + hasMore: false as const, + appliedRowKeys: [contributed.id], + } + }, + } + }, + }, + }) + sync.begin() + sync.write({ type: `insert`, value: contributed }) + expect(sync.commit()).toBe(true) + + let sourceCallback: Parameters[0] | undefined + let suppressSourceChanges = false + const subscribeChanges = source.subscribeChanges.bind(source) + source.subscribeChanges = ((callback, options) => { + sourceCallback = callback + return subscribeChanges((changes) => { + if (!suppressSourceChanges) callback(changes) + }, options) + }) as typeof source.subscribeChanges + + return { + contributed, + source, + staleDelete, + suppressSourceChanges: () => { + suppressSourceChanges = true + }, + publish: (changes: Array>) => { + if (sourceCallback === undefined) { + throw new Error(`Query did not subscribe to its source`) + } + const publish = sourceCallback as unknown as ( + messages: Array>, + ) => void + publish(changes) + }, + truncate: () => { + sync.begin() + sync.truncate() + expect(sync.commit()).toBe(true) + }, + } +} + +it(`ignores unknown deletes and inserts unknown updates at the D2 boundary`, () => { + const sentRows = new Map() + const stale = { id: 1, revision: 1, value: 1 } + const current = { id: 2, revision: 2, value: 2 } + + expect( + reconcileChangesForD2( + [{ type: `delete`, key: `row`, value: stale }], + sentRows, + ), + ).toEqual([]) + expect( + reconcileChangesForD2( + [ + { + type: `update`, + key: `row`, + previousValue: stale, + value: current, + }, + ], + sentRows, + ), + ).toEqual([{ type: `insert`, key: `row`, value: current }]) + expect(sentRows).toEqual(new Map([[`row`, current]])) +}) + +it(`retracts the exact Effect source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-effect-truncate-reconciliation`, + ) + const { contributed, source, staleDelete } = harness + const events: Array<{ + type: string + value: { id: number; revision: number; value: number } + }> = [] + const effect = createEffect({ + query: (query) => + query + .from({ row: source }) + .where(({ row }) => eq(row.revision, contributed.revision)) + .orderBy(({ row }) => row.value) + .limit(1), + onBatch: (batch) => { + events.push(...batch) + }, + }) + try { + await flushPromises() + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ + type: `enter`, + key: 1, + value: contributed, + }) + const publishedValue = events[0]!.value + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(events).toEqual([{ type: `enter`, key: 1, value: publishedValue }]) + + harness.publish([{ type: `delete`, key: 1, value: staleDelete }]) + await flushPromises() + expect(events).toEqual([ + { type: `enter`, key: 1, value: publishedValue }, + { type: `exit`, key: 1, value: publishedValue }, + ]) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + +it(`retracts the exact live-query source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-live-query-truncate-reconciliation`, + ) + const { contributed, source, staleDelete } = harness + const live = createLiveQueryCollection({ + id: `d2-live-query-truncate-result`, + query: (query) => + query + .from({ row: source }) + .where(({ row }) => eq(row.revision, contributed.revision)) + .orderBy(({ row }) => row.value) + .limit(1), + startSync: true, + }) + + try { + await live.preload() + expect(live.get(contributed.id)).toMatchObject(contributed) + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(live.get(contributed.id)).toMatchObject(contributed) + + harness.publish([{ type: `delete`, key: 1, value: staleDelete }]) + await flushPromises() + expect(live.get(contributed.id)).toBeUndefined() + } finally { + await live.cleanup() + await source.cleanup() + } +}) + fcTest.prop( [fc.array(reconciliationStepArbitrary, { minLength: 1, maxLength: 30 })], oraclePropertyOptions(200, `d2-source.exact-retractions`), )( `keeps one exact D2 contribution per source key across batched histories`, (steps) => { - const sourceRows = new Map() - const sentRows = new Map() + const sourceRows = new Map() + const sentRows = new Map() const relation = new Map() + let graphActive = true for (const step of steps) { - if (step.type === `reset`) { - sourceRows.clear() + if (step.type === `truncate`) { + // Truncate is only an early lifecycle signal. Its later source batch + // still needs the retained exact rows to retract the active graph. + } else if (step.type === `teardown`) { sentRows.clear() relation.clear() + graphActive = false + } else if (step.type === `restart`) { + if (!graphActive) { + const replay = [...sourceRows].map(([key, value]) => ({ + type: `insert` as const, + key, + value, + })) + applyToRelation(relation, reconcileChangesForD2(replay, sentRows)) + graphActive = true + } } else { const changes = sourceChangesFor(step.operations, sourceRows) - const reconciled = reconcileChangesForD2(changes, sentRows) - applyToRelation(relation, reconciled) + if (graphActive) { + const reconciled = reconcileChangesForD2(changes, sentRows) + applyToRelation(relation, reconciled) + } + } + if (graphActive) { + expectExactSourceRelation(sourceRows, sentRows, relation) + } else { + expect(sentRows.size).toBe(0) + expect(relation.size).toBe(0) } - expectExactSourceRelation(sourceRows, sentRows, relation) } }, ) From 9e31a27e6b73786b820904f8540fa93e15b5ddcd Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:43:30 -0600 Subject: [PATCH 261/327] test(db): require exact metadata settlement --- packages/db/src/query/live/ARCHITECTURE.md | 2 +- ...tadata-publication-oracle.property.test.ts | 80 ++++++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index db2b9de55..ef4cf91b8 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1366,7 +1366,7 @@ create recursive Collection machinery. | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Keyed Collection diffs across row-metadata settlement and cancellation | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | +| Exact row-metadata settlement publication and keyed Collection diffs | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | | Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 8ed10a59b..ef91779e8 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -40,6 +40,13 @@ type PublicationHarness = { getSync: () => SyncActions } +type PublishedPublicationRow = PublicationRow & { + $collectionId: string + $key: number + $origin: `local` | `remote` + $synced: boolean +} + const metadataWriteArbitrary = fc.record({ key: fc.integer({ min: 0, max: 2 }), type: fc.constantFrom(`set` as const, `delete` as const), @@ -83,9 +90,8 @@ async function createPublicationHarness(): Promise { ) await liveRows.preload() - const batches: Array< - Array> - > = [] + const batches: Array>> = + [] const subscription = rows.subscribeChanges((changes) => { batches.push(changes) }) @@ -109,6 +115,32 @@ function expectUniqueBatchKeys( } } +function selectPublishedRow( + row: PublicationRow | undefined, +): PublishedPublicationRow | undefined { + if (row === undefined) return undefined + const published = row as PublishedPublicationRow + return { + id: published.id, + position: published.position, + $collectionId: published.$collectionId, + $key: published.$key, + $origin: published.$origin, + $synced: published.$synced, + } +} + +function selectPublishedChange( + change: ChangeMessage, +) { + return { + type: change.type, + key: change.key, + value: selectPublishedRow(change.value), + previousValue: selectPublishedRow(change.previousValue), + } +} + function expectPublishedRows( harness: PublicationHarness, model: ReadonlyMap, @@ -132,6 +164,10 @@ async function applyRound( ): Promise { const previous = model.get(round.key)! const next = { ...previous, position: previous.position + round.delta } + const batchCountBefore = harness.batches.length + const keyWasPreviouslyPublished = harness.batches.some((batch) => + batch.some((change) => change.key === round.key), + ) const sync = harness.getSync() const transaction = createTransaction({ mutationFn: async () => { @@ -183,6 +219,44 @@ async function applyRound( } } await Promise.resolve() + const virtualRow = ( + row: PublicationRow, + synced: boolean, + ): PublishedPublicationRow => ({ + ...row, + $collectionId: harness.rows.id, + $key: row.id, + $origin: `local`, + $synced: synced, + }) + const expectedOptimisticChange = keyWasPreviouslyPublished + ? { + type: `update`, + key: round.key, + value: virtualRow(next, false), + previousValue: virtualRow(previous, true), + } + : { + type: `insert`, + key: round.key, + value: virtualRow(next, false), + previousValue: undefined, + } + expect( + harness.batches + .slice(batchCountBefore) + .map((batch) => batch.map(selectPublishedChange)), + ).toEqual([ + [expectedOptimisticChange], + [ + { + type: `update`, + key: round.key, + value: virtualRow(next, true), + previousValue: virtualRow(next, false), + }, + ], + ]) expectUniqueBatchKeys(harness.batches) expectPublishedRows(harness, model) const byKey = ( From 22f5174194805d10985e415b54bc94a81ec87c50 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:47:31 -0600 Subject: [PATCH 262/327] test(db): model metadata cancellation ownership --- packages/db/src/query/live/ARCHITECTURE.md | 2 +- ...tadata-publication-oracle.property.test.ts | 91 +++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index ef4cf91b8..8b90a8656 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1366,7 +1366,7 @@ create recursive Collection machinery. | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Exact row-metadata settlement publication and keyed Collection diffs | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | +| Exact metadata settlement, keyed diffs, and per-key cancellation ownership | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | | Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index ef91779e8..988560a12 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -1,6 +1,7 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' +import { createDeferred } from '../src/deferred.js' import { SyncTransactionAbortedError } from '../src/errors.js' import { createLiveQueryCollection } from '../src/query/index.js' import { createTransaction } from '../src/transactions.js' @@ -67,6 +68,17 @@ const publicationRoundArbitrary: fc.Arbitrary = fc metadata: [{ key, type: primaryMetadataType }, ...extraMetadata], })) +const metadataCancellationArbitrary = fc.record({ + canceledKeys: fc.uniqueArray(fc.integer({ min: 0, max: 2 }), { + minLength: 1, + maxLength: 3, + }), + retainedKeys: fc.uniqueArray(fc.integer({ min: 0, max: 2 }), { + minLength: 1, + maxLength: 3, + }), +}) + async function createPublicationHarness(): Promise { let sync!: SyncActions const rows = createCollection({ @@ -288,6 +300,72 @@ async function runPublicationHistory( } } +async function expectMetadataCancellationOwnership( + canceledKeys: ReadonlyArray, + retainedKeys: ReadonlyArray, +): Promise { + const harness = await createPublicationHarness() + const persistence = createDeferred() + const heldTransaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + heldTransaction.mutate(() => { + harness.rows.insert({ id: 99, position: 99 }) + }) + expect(heldTransaction.state).toBe(`persisting`) + + const stageMetadata = (keys: ReadonlyArray, owner: string) => { + const sync = harness.getSync() + sync.begin() + for (const key of keys) { + sync.metadata!.row.set(key, { owner }) + } + const receipt = sync.commit() + if (receipt === true) { + throw new Error(`Persisting optimistic work did not hold metadata sync`) + } + const transaction = harness.rows._state.pendingSyncedTransactions.at(-1)! + void receipt.catch(() => undefined) + return { receipt, transaction } + } + + const canceled = stageMetadata(canceledKeys, `canceled`) + const retained = stageMetadata(retainedKeys, `retained`) + + try { + harness.rows._state.capturePreSyncVisibleState() + const expectedBefore = new Set([...canceledKeys, ...retainedKeys]) + expect(harness.rows._state.recentlySyncedKeys).toEqual(expectedBefore) + expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( + expectedBefore, + ) + const batchCountBefore = harness.batches.length + + harness.rows._state.cancelPendingSyncedTransaction(canceled.transaction) + + const expectedAfter = new Set(retainedKeys) + expect(harness.rows._state.pendingSyncedTransactions).toEqual([ + retained.transaction, + ]) + expect(harness.rows._state.recentlySyncedKeys).toEqual(expectedAfter) + expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( + expectedAfter, + ) + expect(harness.batches).toHaveLength(batchCountBefore) + expect(harness.rows._state.syncedMetadata.size).toBe(0) + await expect(canceled.receipt).rejects.toBeInstanceOf( + SyncTransactionAbortedError, + ) + } finally { + harness.rows._state.cancelPendingSyncedTransaction(retained.transaction) + await retained.receipt.catch(() => undefined) + persistence.resolve() + await heldTransaction.isPersisted.promise.catch(() => undefined) + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +} + it(`publishes one event per key when metadata-only sync retires optimistic work`, async () => { await runPublicationHistory([ { @@ -305,6 +383,10 @@ it(`publishes one event per key when metadata-only sync retires optimistic work` ]) }) +it(`releases only canceled metadata keys while another sync remains pending`, async () => { + await expectMetadataCancellationOwnership([0, 1], [1, 2]) +}) + fcTest.prop( [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], oraclePropertyOptions(50, `collection-publication.metadata-only`), @@ -312,3 +394,12 @@ fcTest.prop( `keeps metadata-only optimistic settlement a valid keyed diff across histories`, runPublicationHistory, ) + +fcTest.prop( + [metadataCancellationArbitrary], + oraclePropertyOptions(50, `collection-publication.metadata-cancellation`), +)( + `keeps metadata suppression owned by the remaining pending transactions`, + ({ canceledKeys, retainedKeys }) => + expectMetadataCancellationOwnership(canceledKeys, retainedKeys), +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 0bdec6e45..50943ed9e 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -3,6 +3,7 @@ type OracleEnvironment = Record const staticOracleProperties = [ `collection-sync.reentrant-drain`, `collection-state.retention`, + `collection-publication.metadata-cancellation`, `collection-publication.metadata-only`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, From 9bbfaa565956549fb3516aeb62d2fb1166caf603 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:54:28 -0600 Subject: [PATCH 263/327] test(db): model metadata publication rollback --- packages/db/src/query/live/ARCHITECTURE.md | 5 +- ...tadata-publication-oracle.property.test.ts | 160 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 3 files changed, 164 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 8b90a8656..ac9d67e8e 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1248,7 +1248,8 @@ most once. Row operations and row-metadata writes use one shared affected-key derivation for pre-sync capture, commit, cancellation, and rollback snapshots. A metadata-only transaction can retire optimistic state and change virtual row properties, so omitting its key from any of those phases can publish the same -transition twice or leave stale suppression state after cancellation. +transition twice, leave stale suppression state after cancellation, or retain +applied metadata after a failed coherent publication rolls back. Window metadata follows the same causal order as the published rows. If a publication callback starts a newer window operation, that newer generation @@ -1366,7 +1367,7 @@ create recursive Collection machinery. | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Exact metadata settlement, keyed diffs, and per-key cancellation ownership | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | +| Exact metadata settlement, cancellation ownership, and rollback recovery | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | | Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | | Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 988560a12..58b29cace 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -79,6 +79,22 @@ const metadataCancellationArbitrary = fc.record({ }), }) +const metadataRollbackArbitrary = fc + .record({ + sourceKey: fc.integer({ min: 0, max: 2 }), + metadataKeyOffset: fc.constantFrom(1, 2), + sourceDelta: fc.integer({ min: 1, max: 10 }), + oldMetadataOwner: fc.integer(), + pendingMetadataOwner: fc.integer(), + }) + .map(({ sourceKey, metadataKeyOffset, ...scenario }) => ({ + ...scenario, + sourceKey, + metadataKey: (sourceKey + metadataKeyOffset) % 3, + })) + +let nextMetadataRollbackHarnessId = 0 + async function createPublicationHarness(): Promise { let sync!: SyncActions const rows = createCollection({ @@ -366,6 +382,133 @@ async function expectMetadataCancellationOwnership( } } +async function expectMetadataRollbackRecovery({ + sourceKey, + metadataKey, + sourceDelta, + oldMetadataOwner, + pendingMetadataOwner, +}: { + sourceKey: number + metadataKey: number + sourceDelta: number + oldMetadataOwner: number + pendingMetadataOwner: number +}): Promise { + const harnessId = nextMetadataRollbackHarnessId++ + const source = await createPublicationHarness() + const { rows, getSync } = source + const derived = createLiveQueryCollection({ + id: `metadata-rollback-derived-${harnessId}`, + query: (query) => + query.from({ row: rows }).select(({ row }) => ({ + id: row.id, + position: row.position, + })), + getKey: (row) => row.id, + }) + await derived.preload() + + const stageMetadata = (key: number, value: unknown) => { + const applied = createDeferred() + void applied.promise.catch(() => undefined) + const transaction = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map([[key, { type: `set` as const, value }]]), + collectionMetadataWrites: new Map(), + applied, + } + derived._state.pendingSyncedTransactions.push(transaction) + return transaction + } + + const oldMetadata = { owner: oldMetadataOwner } + const pendingMetadata = { owner: pendingMetadataOwner } + stageMetadata(metadataKey, oldMetadata) + derived._state.commitPendingTransactions() + + const pending = stageMetadata(metadataKey, pendingMetadata) + const sourceRowsBefore = [...rows.values()].map((row) => ({ ...row })) + const rowsBefore = [...derived.values()].map((row) => ({ ...row })) + const originBefore = new Map(derived._state.rowOrigins) + const hydrationSeedsBefore = new Set(derived._state.hydrationSeedKeys) + const hydratedBefore = new Set(derived._state.hydratedKeys) + const syncedBefore = new Set(derived._state.syncedKeys) + const preSyncBefore = new Map(derived._state.preSyncVisibleState) + const recentlySyncedBefore = new Set(derived._state.recentlySyncedKeys) + const published: Array< + ReadonlyArray> + > = [] + const subscription = derived.subscribeChanges((changes) => { + published.push(changes) + }) + + const publicationFailure = new Error(`metadata rollback publication failed`) + const commitPendingTransactions = derived._state.commitPendingTransactions + let shouldFail = true + derived._state.commitPendingTransactions = () => { + commitPendingTransactions() + if (shouldFail) { + shouldFail = false + throw publicationFailure + } + } + + try { + const previousSourceRow = rows.get(sourceKey)! + expect(() => { + getSync().begin() + getSync().write({ + type: `update`, + value: { + ...previousSourceRow, + position: previousSourceRow.position + sourceDelta, + }, + }) + getSync().commit() + }).toThrow(publicationFailure) + + expect(rows.get(sourceKey)?.position).toBe( + previousSourceRow.position + sourceDelta, + ) + expect([...rows.values()].map((row) => ({ ...row }))).toEqual( + sourceRowsBefore.map((row) => + row.id === sourceKey + ? { ...row, position: row.position + sourceDelta } + : row, + ), + ) + expect([...derived.values()].map((row) => ({ ...row }))).toEqual(rowsBefore) + expect(derived._state.syncedMetadata).toEqual( + new Map([[metadataKey, oldMetadata]]), + ) + expect(derived._state.pendingSyncedTransactions).toHaveLength(1) + expect(derived._state.pendingSyncedTransactions[0]).toBe(pending) + expect(pending.applicationStarted).toBe(false) + expect(derived._state.rowOrigins).toEqual(originBefore) + expect(derived._state.hydrationSeedKeys).toEqual(hydrationSeedsBefore) + expect(derived._state.hydratedKeys).toEqual(hydratedBefore) + expect(derived._state.syncedKeys).toEqual(syncedBefore) + expect(derived._state.preSyncVisibleState).toEqual(preSyncBefore) + expect(derived._state.recentlySyncedKeys).toEqual(recentlySyncedBefore) + expect(published).toEqual([]) + } finally { + derived._state.commitPendingTransactions = commitPendingTransactions + derived._state.cancelPendingSyncedTransaction(pending) + subscription.unsubscribe() + source.unsubscribe() + await Promise.all([ + derived.cleanup(), + source.liveRows.cleanup(), + rows.cleanup(), + ]) + } +} + it(`publishes one event per key when metadata-only sync retires optimistic work`, async () => { await runPublicationHistory([ { @@ -387,6 +530,16 @@ it(`releases only canceled metadata keys while another sync remains pending`, as await expectMetadataCancellationOwnership([0, 1], [1, 2]) }) +it(`restores pending metadata when a derived publication fails`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + oldMetadataOwner: 1, + pendingMetadataOwner: 2, + }) +}) + fcTest.prop( [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], oraclePropertyOptions(50, `collection-publication.metadata-only`), @@ -403,3 +556,10 @@ fcTest.prop( ({ canceledKeys, retainedKeys }) => expectMetadataCancellationOwnership(canceledKeys, retainedKeys), ) +fcTest.prop( + [metadataRollbackArbitrary], + oraclePropertyOptions(30, `collection-publication.metadata-rollback`), +)( + `restores metadata-only state after failed derived publications`, + expectMetadataRollbackRecovery, +) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 50943ed9e..c9c70ca3c 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -5,6 +5,7 @@ const staticOracleProperties = [ `collection-state.retention`, `collection-publication.metadata-cancellation`, `collection-publication.metadata-only`, + `collection-publication.metadata-rollback`, `coverage-registry.claim-churn`, `coverage-registry.state-machine`, `d2-source.exact-retractions`, From 04a0026e4b349b8219b39cc85a0f1b4891940af0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 17:59:19 -0600 Subject: [PATCH 264/327] test(db): vary metadata publication values --- packages/db/src/query/live/ARCHITECTURE.md | 2 + ...tadata-publication-oracle.property.test.ts | 155 ++++++++++++------ 2 files changed, 110 insertions(+), 47 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index ac9d67e8e..da55d5cec 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1250,6 +1250,8 @@ A metadata-only transaction can retire optimistic state and change virtual row properties, so omitting its key from any of those phases can publish the same transition twice, leave stale suppression state after cancellation, or retain applied metadata after a failed coherent publication rolls back. +Affectedness depends on key presence, not metadata value truthiness. In +particular, setting a key to `undefined` is distinct from deleting that key. Window metadata follows the same causal order as the published rows. If a publication callback starts a newer window operation, that newer generation diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 58b29cace..9eabba17a 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -16,10 +16,9 @@ type PublicationRow = { type SyncActions = Parameters[`sync`]>[0] -type MetadataWrite = { - key: number - type: `set` | `delete` -} +type MetadataOperation = { type: `set`; value: unknown } | { type: `delete` } + +type MetadataWrite = { key: number } & MetadataOperation type PublicationRound = { key: number @@ -48,10 +47,27 @@ type PublishedPublicationRow = PublicationRow & { $synced: boolean } -const metadataWriteArbitrary = fc.record({ - key: fc.integer({ min: 0, max: 2 }), - type: fc.constantFrom(`set` as const, `delete` as const), -}) +const metadataValueArbitrary = fc.oneof( + fc.constant(undefined), + fc.constant(null), + fc.constant(false), + fc.constant(true), + fc.constant(0), + fc.constant(Number.NaN), + fc.constant(``), + fc.integer(), + fc.string(), + fc.record({ nested: fc.integer() }), +) + +const metadataOperationArbitrary: fc.Arbitrary = fc.oneof( + metadataValueArbitrary.map((value) => ({ type: `set` as const, value })), + fc.constant({ type: `delete` as const }), +) + +const metadataWriteArbitrary = fc + .tuple(fc.integer({ min: 0, max: 2 }), metadataOperationArbitrary) + .map(([key, operation]) => ({ key, ...operation })) const publicationRoundArbitrary: fc.Arbitrary = fc .record({ @@ -59,13 +75,13 @@ const publicationRoundArbitrary: fc.Arbitrary = fc delta: fc.constantFrom(-2, -1, 1, 2), extraMetadata: fc.array(metadataWriteArbitrary, { maxLength: 2 }), outcome: fc.constantFrom(`commit` as const, `abort` as const), - primaryMetadataType: fc.constantFrom(`set` as const, `delete` as const), + primaryMetadata: metadataOperationArbitrary, }) - .map(({ key, delta, extraMetadata, outcome, primaryMetadataType }) => ({ + .map(({ key, delta, extraMetadata, outcome, primaryMetadata }) => ({ key, delta, outcome, - metadata: [{ key, type: primaryMetadataType }, ...extraMetadata], + metadata: [{ key, ...primaryMetadata }, ...extraMetadata], })) const metadataCancellationArbitrary = fc.record({ @@ -77,20 +93,33 @@ const metadataCancellationArbitrary = fc.record({ minLength: 1, maxLength: 3, }), + canceledOperation: metadataOperationArbitrary, + retainedOperation: metadataOperationArbitrary, }) +const metadataRollbackCaseArbitrary = fc.oneof( + metadataValueArbitrary.map((value) => ({ + initialMetadata: { present: false as const }, + pendingOperation: { type: `set` as const, value }, + })), + metadataValueArbitrary.map((value) => ({ + initialMetadata: { present: true as const, value }, + pendingOperation: { type: `delete` as const }, + })), +) + const metadataRollbackArbitrary = fc .record({ sourceKey: fc.integer({ min: 0, max: 2 }), metadataKeyOffset: fc.constantFrom(1, 2), sourceDelta: fc.integer({ min: 1, max: 10 }), - oldMetadataOwner: fc.integer(), - pendingMetadataOwner: fc.integer(), + metadataCase: metadataRollbackCaseArbitrary, }) - .map(({ sourceKey, metadataKeyOffset, ...scenario }) => ({ - ...scenario, + .map(({ sourceKey, metadataKeyOffset, sourceDelta, metadataCase }) => ({ + ...metadataCase, sourceKey, metadataKey: (sourceKey + metadataKeyOffset) % 3, + sourceDelta, })) let nextMetadataRollbackHarnessId = 0 @@ -186,7 +215,6 @@ function expectPublishedRows( async function applyRound( harness: PublicationHarness, round: PublicationRound, - roundIndex: number, model: Map, metadataModel: Map, ): Promise { @@ -206,8 +234,7 @@ async function applyRound( sync.begin() for (const write of round.metadata) { if (write.type === `set`) { - const metadata = { round: roundIndex, owner: round.key } - sync.metadata!.row.set(write.key, metadata) + sync.metadata!.row.set(write.key, write.value) } else { sync.metadata!.row.delete(write.key) } @@ -237,10 +264,7 @@ async function applyRound( if (round.outcome === `commit`) { for (const write of round.metadata) { if (write.type === `set`) { - metadataModel.set(write.key, { - round: roundIndex, - owner: round.key, - }) + metadataModel.set(write.key, write.value) } else { metadataModel.delete(write.key) } @@ -307,8 +331,8 @@ async function runPublicationHistory( ) const metadataModel = new Map() try { - for (const [index, round] of rounds.entries()) { - await applyRound(harness, round, index, model, metadataModel) + for (const round of rounds) { + await applyRound(harness, round, model, metadataModel) } } finally { harness.unsubscribe() @@ -319,8 +343,23 @@ async function runPublicationHistory( async function expectMetadataCancellationOwnership( canceledKeys: ReadonlyArray, retainedKeys: ReadonlyArray, + canceledOperation: MetadataOperation, + retainedOperation: MetadataOperation, ): Promise { const harness = await createPublicationHarness() + const initialMetadata = new Map([ + [0, undefined], + [1, false], + [2, null], + ]) + const initialSync = harness.getSync() + initialSync.begin() + for (const [key, value] of initialMetadata) { + initialSync.metadata!.row.set(key, value) + } + initialSync.commit() + await Promise.resolve() + const persistence = createDeferred() const heldTransaction = createTransaction({ mutationFn: () => persistence.promise, @@ -330,11 +369,18 @@ async function expectMetadataCancellationOwnership( }) expect(heldTransaction.state).toBe(`persisting`) - const stageMetadata = (keys: ReadonlyArray, owner: string) => { + const stageMetadata = ( + keys: ReadonlyArray, + operation: MetadataOperation, + ) => { const sync = harness.getSync() sync.begin() for (const key of keys) { - sync.metadata!.row.set(key, { owner }) + if (operation.type === `set`) { + sync.metadata!.row.set(key, operation.value) + } else { + sync.metadata!.row.delete(key) + } } const receipt = sync.commit() if (receipt === true) { @@ -345,8 +391,8 @@ async function expectMetadataCancellationOwnership( return { receipt, transaction } } - const canceled = stageMetadata(canceledKeys, `canceled`) - const retained = stageMetadata(retainedKeys, `retained`) + const canceled = stageMetadata(canceledKeys, canceledOperation) + const retained = stageMetadata(retainedKeys, retainedOperation) try { harness.rows._state.capturePreSyncVisibleState() @@ -368,7 +414,7 @@ async function expectMetadataCancellationOwnership( expectedAfter, ) expect(harness.batches).toHaveLength(batchCountBefore) - expect(harness.rows._state.syncedMetadata.size).toBe(0) + expect(harness.rows._state.syncedMetadata).toEqual(initialMetadata) await expect(canceled.receipt).rejects.toBeInstanceOf( SyncTransactionAbortedError, ) @@ -386,14 +432,14 @@ async function expectMetadataRollbackRecovery({ sourceKey, metadataKey, sourceDelta, - oldMetadataOwner, - pendingMetadataOwner, + initialMetadata, + pendingOperation, }: { sourceKey: number metadataKey: number sourceDelta: number - oldMetadataOwner: number - pendingMetadataOwner: number + initialMetadata: { present: false } | { present: true; value: unknown } + pendingOperation: MetadataOperation }): Promise { const harnessId = nextMetadataRollbackHarnessId++ const source = await createPublicationHarness() @@ -409,7 +455,7 @@ async function expectMetadataRollbackRecovery({ }) await derived.preload() - const stageMetadata = (key: number, value: unknown) => { + const stageMetadata = (key: number, operation: MetadataOperation) => { const applied = createDeferred() void applied.promise.catch(() => undefined) const transaction = { @@ -418,7 +464,7 @@ async function expectMetadataRollbackRecovery({ layoutChanged: false, operations: [], deletedKeys: new Set(), - rowMetadataWrites: new Map([[key, { type: `set` as const, value }]]), + rowMetadataWrites: new Map([[key, operation]]), collectionMetadataWrites: new Map(), applied, } @@ -426,12 +472,15 @@ async function expectMetadataRollbackRecovery({ return transaction } - const oldMetadata = { owner: oldMetadataOwner } - const pendingMetadata = { owner: pendingMetadataOwner } - stageMetadata(metadataKey, oldMetadata) - derived._state.commitPendingTransactions() + if (initialMetadata.present) { + stageMetadata(metadataKey, { + type: `set`, + value: initialMetadata.value, + }) + derived._state.commitPendingTransactions() + } - const pending = stageMetadata(metadataKey, pendingMetadata) + const pending = stageMetadata(metadataKey, pendingOperation) const sourceRowsBefore = [...rows.values()].map((row) => ({ ...row })) const rowsBefore = [...derived.values()].map((row) => ({ ...row })) const originBefore = new Map(derived._state.rowOrigins) @@ -484,7 +533,9 @@ async function expectMetadataRollbackRecovery({ ) expect([...derived.values()].map((row) => ({ ...row }))).toEqual(rowsBefore) expect(derived._state.syncedMetadata).toEqual( - new Map([[metadataKey, oldMetadata]]), + initialMetadata.present + ? new Map([[metadataKey, initialMetadata.value]]) + : new Map(), ) expect(derived._state.pendingSyncedTransactions).toHaveLength(1) expect(derived._state.pendingSyncedTransactions[0]).toBe(pending) @@ -514,7 +565,7 @@ it(`publishes one event per key when metadata-only sync retires optimistic work` { key: 1, delta: 1, - metadata: [{ key: 1, type: `set` }], + metadata: [{ key: 1, type: `set`, value: false }], outcome: `commit`, }, { @@ -527,7 +578,12 @@ it(`publishes one event per key when metadata-only sync retires optimistic work` }) it(`releases only canceled metadata keys while another sync remains pending`, async () => { - await expectMetadataCancellationOwnership([0, 1], [1, 2]) + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `delete` }, + { type: `set`, value: false }, + ) }) it(`restores pending metadata when a derived publication fails`, async () => { @@ -535,8 +591,8 @@ it(`restores pending metadata when a derived publication fails`, async () => { sourceKey: 0, metadataKey: 1, sourceDelta: 1, - oldMetadataOwner: 1, - pendingMetadataOwner: 2, + initialMetadata: { present: true, value: false }, + pendingOperation: { type: `delete` }, }) }) @@ -553,8 +609,13 @@ fcTest.prop( oraclePropertyOptions(50, `collection-publication.metadata-cancellation`), )( `keeps metadata suppression owned by the remaining pending transactions`, - ({ canceledKeys, retainedKeys }) => - expectMetadataCancellationOwnership(canceledKeys, retainedKeys), + ({ canceledKeys, retainedKeys, canceledOperation, retainedOperation }) => + expectMetadataCancellationOwnership( + canceledKeys, + retainedKeys, + canceledOperation, + retainedOperation, + ), ) fcTest.prop( [metadataRollbackArbitrary], From f114732e19588d47b890e232af052fd75543355b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:03:10 -0600 Subject: [PATCH 265/327] test(db): restore reentrant restart receipt coverage --- packages/db/src/query/live/ARCHITECTURE.md | 4 + ...on-state-retention-oracle.property.test.ts | 103 +++++++++++++----- 2 files changed, 78 insertions(+), 29 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index da55d5cec..eb5a498d7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1366,6 +1366,7 @@ create recursive Collection machinery. | State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | | Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | | Exact D2 source retractions, replay suppression, and boundary lifecycle | `packages/db/tests/d2-source-reconciliation-oracle.property.test.ts` | +| Sync-session retention, cleanup, reentrant restart, and applied receipts | `packages/db/tests/collection-state-retention-oracle.property.test.ts` | | Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | @@ -1430,6 +1431,9 @@ sync session starts. Synchronous publication tails and queued microtasks remain scoped to the session that created them; after cleanup they cannot clear or mark state owned by a restarted session. Otherwise stale rows can classify a fresh insert as an update, or stale keys can suppress the new session's first event. +A new-session transaction committed inside the old session's publication +callback remains in the causal queue. Its receipt stays pending until that new +row and its event are visible; the old drain cannot discard it. An imperative load operation is a separate caller boundary around this flow. It owns the future requests caused while it is current, retains the promises it diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 7265a9492..ee21d22b1 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -20,7 +20,11 @@ type RetentionAction = | { type: `delete`; key: number } | { type: `replace`; rows: ReadonlyArray } | { type: `restart` } - | { type: `reentrantRestart`; row: RetainedRow } + | { + type: `reentrantRestart` + row: RetainedRow + commitPhase: `insideListener` | `afterOldReturn` + } type RetentionHarness = { collection: Collection @@ -65,10 +69,16 @@ const retentionActionArbitrary: fc.Arbitrary = fc.oneof( { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, { weight: 1, - arbitrary: retainedRowArbitrary.map((row) => ({ - type: `reentrantRestart` as const, - row, - })), + arbitrary: fc + .tuple( + retainedRowArbitrary, + fc.constantFrom(`insideListener` as const, `afterOldReturn` as const), + ) + .map(([row, commitPhase]) => ({ + type: `reentrantRestart` as const, + row, + commitPhase, + })), }, ) @@ -184,6 +194,8 @@ async function runRetentionHistory( let cleanup: Promise | undefined let restarted = false let restartedSync: SyncActions | undefined + let restartedReceipt: true | Promise | undefined + let restartedReceiptSettled = false const events: Array<{ type: string key: string | number @@ -205,8 +217,17 @@ async function runRetentionHistory( restartedSync = harness.sync restartedSync.begin() restartedSync.write({ type: `insert`, value: restartedRow }) - collection._state.preSyncVisibleState.set(-1, retainedMarker) - collection._state.recentlySyncedKeys.add(restartedRow.id) + if (action.commitPhase === `insideListener`) { + restartedReceipt = restartedSync.commit() + if (restartedReceipt !== true) { + void restartedReceipt.then(() => { + restartedReceiptSettled = true + }) + } + } else { + collection._state.preSyncVisibleState.set(-1, retainedMarker) + collection._state.recentlySyncedKeys.add(restartedRow.id) + } }, { includeInitialState: false }, ) @@ -219,28 +240,39 @@ async function runRetentionHistory( if (restartedSync === undefined) { throw new Error(`restarted sync session was not captured`) } - expect(collection._state.preSyncVisibleState).toEqual( - new Map([[-1, retainedMarker]]), - ) - expect(collection._state.recentlySyncedKeys).toEqual( - new Set([restartedRow.id]), - ) - expect(collection._state.hasReceivedFirstCommit).toBe(false) - - await Promise.resolve() - expect(collection._state.preSyncVisibleState).toEqual( - new Map([[-1, retainedMarker]]), - ) - expect(collection._state.recentlySyncedKeys).toEqual( - new Set([restartedRow.id]), - ) - expect(collection._state.hasReceivedFirstCommit).toBe(false) - - expect(restartedSync.commit()).toBe(true) - expect(collection._state.preSyncVisibleState.size).toBe(0) - expect(collection._state.hasReceivedFirstCommit).toBe(true) - await Promise.resolve() - expect(collection._state.recentlySyncedKeys.size).toBe(0) + if (action.commitPhase === `insideListener`) { + expect(restartedReceipt).toBeDefined() + expect(restartedReceipt).not.toBe(true) + expect(restartedReceiptSettled).toBe(false) + if (restartedReceipt === undefined || restartedReceipt === true) { + throw new Error(`restarted sync receipt was not parked`) + } + await restartedReceipt + expect(restartedReceiptSettled).toBe(true) + } else { + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, retainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([restartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + await Promise.resolve() + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, retainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([restartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + expect(restartedSync.commit()).toBe(true) + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.recentlySyncedKeys.size).toBe(0) + } expect(events).toEqual([ { type: triggerType, key: triggerRow.id, row: triggerRow }, { type: `insert`, key: restartedRow.id, row: restartedRow }, @@ -275,6 +307,19 @@ it(`retains a missing row introduced by a sync update`, async () => { await runRetentionHistory([{ type: `update`, row: { id: 1, value: 1 } }]) }) +it.each([`insideListener`, `afterOldReturn`] as const)( + `retains a restarted row committed %s`, + async (commitPhase) => { + await runRetentionHistory([ + { + type: `reentrantRestart`, + row: { id: 1, value: 1 }, + commitPhase, + }, + ]) + }, +) + it(`releases retained keys after long unique-key churn`, async () => { const keyCount = 1_000 const actions: Array = [] From 5fd9cb6b009d52756d17fa68d3d857d5791943d7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:07:19 -0600 Subject: [PATCH 266/327] test(db): preserve restart publication traces --- ...on-state-retention-oracle.property.test.ts | 66 ++++++++++++++++--- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index ee21d22b1..d7385ce97 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -196,20 +196,34 @@ async function runRetentionHistory( let restartedSync: SyncActions | undefined let restartedReceipt: true | Promise | undefined let restartedReceiptSettled = false - const events: Array<{ - type: string - key: string | number - row: RetainedRow + const batches: Array<{ + changes: Array<{ + type: string + key: string | number + row: RetainedRow + previousRow: RetainedRow | undefined + }> + rows: Array }> = [] const subscription = collection.subscribeChanges( (changes) => { - events.push( - ...changes.map(({ type, key, value }) => ({ + batches.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ type, key, row: { id: value.id, value: value.value }, + previousRow: + previousValue === undefined + ? undefined + : { + id: previousValue.id, + value: previousValue.value, + }, })), - ) + rows: [...collection.values()] + .map(({ id, value }) => ({ id, value })) + .sort((left, right) => left.id - right.id), + }) if (restarted) return restarted = true cleanup = collection.cleanup() @@ -270,12 +284,43 @@ async function runRetentionHistory( expect(restartedSync.commit()).toBe(true) expect(collection._state.preSyncVisibleState.size).toBe(0) expect(collection._state.hasReceivedFirstCommit).toBe(true) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([restartedRow.id]), + ) await Promise.resolve() expect(collection._state.recentlySyncedKeys.size).toBe(0) } - expect(events).toEqual([ - { type: triggerType, key: triggerRow.id, row: triggerRow }, - { type: `insert`, key: restartedRow.id, row: restartedRow }, + const triggerRows = new Map(model) + triggerRows.set(triggerRow.id, triggerRow) + expect(batches).toEqual([ + { + changes: [ + { + type: triggerType, + key: triggerRow.id, + row: triggerRow, + previousRow: model.get(triggerRow.id), + }, + ], + rows: [...triggerRows.values()].sort( + (left, right) => left.id - right.id, + ), + }, + { + changes: [], + rows: [], + }, + { + changes: [ + { + type: `insert`, + key: restartedRow.id, + row: restartedRow, + previousRow: undefined, + }, + ], + rows: [restartedRow], + }, ]) subscription.unsubscribe() @@ -311,6 +356,7 @@ it.each([`insideListener`, `afterOldReturn`] as const)( `retains a restarted row committed %s`, async (commitPhase) => { await runRetentionHistory([ + { type: `insert`, row: { id: 1, value: 1 } }, { type: `reentrantRestart`, row: { id: 1, value: 1 }, From e027d1a80c4ff0b8efedecd16b3c1b8f2c5f9b20 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:11:48 -0600 Subject: [PATCH 267/327] fix(db): keep rollback terminal after rejection --- packages/db/src/query/live/ARCHITECTURE.md | 5 +- packages/db/src/transactions.ts | 7 ++ packages/db/tests/transactions.test.ts | 94 +++++++++++++++++++++- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index eb5a498d7..407ef16cf 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -962,8 +962,9 @@ late. A successful `loadSubset` implementation must await or return every receipt for the transactions that establish its result. A source must not add priority merely to make a subset load settle. A rollback that wins while an optimistic transaction's `mutationFn` is still -in flight is terminal. A later successful return from that function cannot -change the transaction from `failed` to `completed` or republish its overlay. +in flight is terminal. A later resolve or rejection from that function cannot +change its outcome, run rollback again, affect newer transactions, or republish +its overlay. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index db06b4403..13c2d2abc 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -649,6 +649,13 @@ class Transaction> { this.isPersisted.resolve(this) } catch (error) { + // A manual or cascading rollback can also win while mutationFn is in + // flight. Its terminal outcome owns this commit attempt, so a late + // rejection cannot run rollback again or affect newer transactions. + if ((this.state as TransactionState) !== `persisting`) { + return this + } + // Preserve the original error for rethrowing const originalError = error instanceof Error ? error : new Error(String(error)) diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index 40ce8f72e..0d519ea01 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -233,7 +233,10 @@ describe(`Transactions`, () => { try { transaction.mutate(() => collection.insert({ id: 1 })) - void transaction.isPersisted.promise.catch(() => undefined) + const persisted = transaction.isPersisted.promise.then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) const commit = transaction.commit() expect(transaction.state).toBe(`persisting`) @@ -241,13 +244,100 @@ describe(`Transactions`, () => { expect(transaction.state).toBe(`failed`) releasePersistence() - await commit + await expect(commit).resolves.toBe(transaction) + expect(await persisted).toEqual({ + status: `rejected`, + reason: undefined, + }) expect(transaction.state).toBe(`failed`) + expect(transaction.error).toBeUndefined() } finally { releasePersistence() await collection.cleanup() } }) + it(`ignores a late persistence rejection after rollback wins`, async () => { + type Row = { id: number; owner: string } + let rejectPersistence!: (reason: unknown) => void + const persistence = new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + const collection = createCollection({ + id: `late-persistence-rejection`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const batches: Array> = [] + const subscription = collection.subscribeChanges( + (changes) => { + batches.push( + changes.map(({ type, key }) => ({ + type, + key, + })), + ) + }, + { includeInitialState: false }, + ) + const first = createTransaction({ + autoCommit: false, + mutationFn: () => persistence, + }) + const second = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + + try { + const persisted = first.isPersisted.promise.then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + first.mutate(() => collection.insert({ id: 1, owner: `first` })) + const commit = first.commit().then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + first.rollback() + second.mutate(() => collection.insert({ id: 1, owner: `second` })) + const lateError = new Error(`late persistence rejection`) + rejectPersistence(lateError) + + const commitOutcome = await commit + expect(commitOutcome.status).toBe(`fulfilled`) + if (commitOutcome.status === `fulfilled`) { + expect(commitOutcome.value).toBe(first) + } + expect(await persisted).toEqual({ + status: `rejected`, + reason: undefined, + }) + expect(first.state).toBe(`failed`) + expect(first.error).toBeUndefined() + expect(second.state).toBe(`pending`) + expect(collection.get(1)).toEqual({ + id: 1, + owner: `second`, + $collectionId: collection.id, + $key: 1, + $origin: `local`, + $synced: false, + }) + expect(batches).toEqual([ + [{ type: `insert`, key: 1 }], + [{ type: `delete`, key: 1 }], + [{ type: `insert`, key: 1 }], + ]) + } finally { + rejectPersistence(new Error(`test cleanup`)) + if (second.state === `pending`) { + second.rollback({ isSecondaryRollback: true }) + } + subscription.unsubscribe() + await collection.cleanup() + } + }) it(`should rollback if the mutationFn throws an error`, async () => { const transaction = createTransaction({ mutationFn: async () => { From e66a601f341f9b997d6497dde5ae00e3159ddc29 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:18:15 -0600 Subject: [PATCH 268/327] fix(db): publish optimistic confirmation state --- packages/db/src/collection/state.ts | 26 +++-- packages/db/src/query/live/ARCHITECTURE.md | 17 ++-- ...tadata-publication-oracle.property.test.ts | 9 ++ ...on-state-retention-oracle.property.test.ts | 97 ++++++++++++++++--- .../query/load-subset-oracle.property.test.ts | 2 + 5 files changed, 123 insertions(+), 28 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index ad003c9c8..2682de9e9 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -112,6 +112,7 @@ export type CollectionPublicationStateSnapshot< pendingLocalOrigins: Set size: number preSyncVisibleState: Map + preSyncVirtualState: Map> recentlySyncedKeys: Set hasReceivedFirstCommit: boolean isCommittingSyncTransactions: boolean @@ -197,6 +198,7 @@ export class CollectionStateManager< // State used for computing the change events public syncedKeys = new Set() public preSyncVisibleState = new Map() + public preSyncVirtualState = new Map>() public recentlySyncedKeys = new Set() public hasReceivedFirstCommit = false public isCommittingSyncTransactions = false @@ -303,6 +305,7 @@ export class CollectionStateManager< pendingLocalOrigins: new Set(this.pendingLocalOrigins), size: this.size, preSyncVisibleState: new Map(this.preSyncVisibleState), + preSyncVirtualState: new Map(this.preSyncVirtualState), recentlySyncedKeys: new Set(this.recentlySyncedKeys), hasReceivedFirstCommit: this.hasReceivedFirstCommit, isCommittingSyncTransactions: this.isCommittingSyncTransactions, @@ -344,6 +347,7 @@ export class CollectionStateManager< replaceSet(this.pendingLocalOrigins, snapshot.pendingLocalOrigins) this.size = snapshot.size replaceMap(this.preSyncVisibleState, snapshot.preSyncVisibleState) + replaceMap(this.preSyncVirtualState, snapshot.preSyncVirtualState) replaceSet(this.recentlySyncedKeys, snapshot.recentlySyncedKeys) this.hasReceivedFirstCommit = snapshot.hasReceivedFirstCommit this.isCommittingSyncTransactions = snapshot.isCommittingSyncTransactions @@ -1562,12 +1566,14 @@ export class CollectionStateManager< for (const key of changedKeys) { const previousVisibleValue = currentVisibleState.get(key) const newVisibleValue = this.get(key) // This returns the new derived state - const previousVirtualProps = this.getVirtualPropsSnapshotForState(key, { - rowOrigins: previousRowOrigins, - optimisticUpserts: previousOptimisticUpserts, - optimisticDeletes: previousOptimisticDeletes, - completedOptimisticKeys: completedOptimisticOps, - }) + const previousVirtualProps = + this.preSyncVirtualState.get(key) ?? + this.getVirtualPropsSnapshotForState(key, { + rowOrigins: previousRowOrigins, + optimisticUpserts: previousOptimisticUpserts, + optimisticDeletes: previousOptimisticDeletes, + completedOptimisticKeys: completedOptimisticOps, + }) const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) const virtualChanged = previousVirtualProps.$synced !== nextVirtualProps.$synced || @@ -1704,6 +1710,7 @@ export class CollectionStateManager< if (this.syncSessionGeneration === syncSessionGeneration) { // Clear the pre-sync state since sync operations are complete this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them Promise.resolve().then(() => { @@ -1781,11 +1788,13 @@ export class CollectionStateManager< if (!remainingPendingKeys.has(key)) { this.recentlySyncedKeys.delete(key) this.preSyncVisibleState.delete(key) + this.preSyncVirtualState.delete(key) } } if (this.pendingSyncedTransactions.length === 0) { this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() this.recentlySyncedKeys.clear() this.changes.emitEvents([], true) } else { @@ -1840,6 +1849,10 @@ export class CollectionStateManager< const currentValue = this.get(key) if (currentValue !== undefined) { this.preSyncVisibleState.set(key, currentValue) + this.preSyncVirtualState.set( + key, + this.getVirtualPropsSnapshotForState(key), + ) } } } @@ -1886,6 +1899,7 @@ export class CollectionStateManager< this.pendingSyncedTransactions = [] this.syncedKeys.clear() this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() this.recentlySyncedKeys.clear() this.hasReceivedFirstCommit = false } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 407ef16cf..4e3e40f36 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1247,6 +1247,10 @@ or roll back independently. Each ordinary Collection batch is a keyed diff and names an affected row key at most once. Row operations and row-metadata writes use one shared affected-key derivation for pre-sync capture, commit, cancellation, and rollback snapshots. +A pre-sync capture retains both the exact visible row and its virtual row +properties. Confirmation must therefore publish a same-value update when only +`$origin` or `$synced` changes; its `previousValue` describes the state readers +actually saw before confirmation. A metadata-only transaction can retire optimistic state and change virtual row properties, so omitting its key from any of those phases can publish the same transition twice, leave stale suppression state after cancellation, or retain @@ -1426,12 +1430,13 @@ window progress; only a complete publication snapshot reaches readers. Release, truncate, replacement, restart, and cleanup change the relevant identity or generation without changing this sequence. -Cleanup also ends the current publication history. It must discard both the -pre-sync visible snapshot and the recently-synced suppression set before a new -sync session starts. Synchronous publication tails and queued microtasks remain -scoped to the session that created them; after cleanup they cannot clear or mark -state owned by a restarted session. Otherwise stale rows can classify a fresh -insert as an update, or stale keys can suppress the new session's first event. +Cleanup also ends the current publication history. It must discard the +pre-sync row and virtual-property snapshots plus the recently-synced +suppression set before a new sync session starts. Synchronous publication tails +and queued microtasks remain scoped to the session that created them; after +cleanup they cannot clear or mark state owned by a restarted session. Otherwise +stale rows can classify a fresh insert as an update, or stale keys can suppress +the new session's first event. A new-session transaction committed inside the old session's publication callback remains in the causal queue. Its receipt stays pending until that new row and its event are visible; the old drain cannot discard it. diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 9eabba17a..abea7069c 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -319,6 +319,7 @@ async function applyRound( [...metadataModel.entries()].sort(byKey), ) expect(harness.rows._state.preSyncVisibleState.size).toBe(0) + expect(harness.rows._state.preSyncVirtualState.size).toBe(0) expect(harness.rows._state.recentlySyncedKeys.size).toBe(0) } @@ -401,6 +402,9 @@ async function expectMetadataCancellationOwnership( expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( expectedBefore, ) + expect(new Set(harness.rows._state.preSyncVirtualState.keys())).toEqual( + expectedBefore, + ) const batchCountBefore = harness.batches.length harness.rows._state.cancelPendingSyncedTransaction(canceled.transaction) @@ -413,6 +417,9 @@ async function expectMetadataCancellationOwnership( expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( expectedAfter, ) + expect(new Set(harness.rows._state.preSyncVirtualState.keys())).toEqual( + expectedAfter, + ) expect(harness.batches).toHaveLength(batchCountBefore) expect(harness.rows._state.syncedMetadata).toEqual(initialMetadata) await expect(canceled.receipt).rejects.toBeInstanceOf( @@ -488,6 +495,7 @@ async function expectMetadataRollbackRecovery({ const hydratedBefore = new Set(derived._state.hydratedKeys) const syncedBefore = new Set(derived._state.syncedKeys) const preSyncBefore = new Map(derived._state.preSyncVisibleState) + const preSyncVirtualBefore = new Map(derived._state.preSyncVirtualState) const recentlySyncedBefore = new Set(derived._state.recentlySyncedKeys) const published: Array< ReadonlyArray> @@ -545,6 +553,7 @@ async function expectMetadataRollbackRecovery({ expect(derived._state.hydratedKeys).toEqual(hydratedBefore) expect(derived._state.syncedKeys).toEqual(syncedBefore) expect(derived._state.preSyncVisibleState).toEqual(preSyncBefore) + expect(derived._state.preSyncVirtualState).toEqual(preSyncVirtualBefore) expect(derived._state.recentlySyncedKeys).toEqual(recentlySyncedBefore) expect(published).toEqual([]) } finally { diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index d7385ce97..03da8433d 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -419,6 +419,7 @@ it(`starts a new sync session without retained publication state`, async () => { const cleanup = collection.cleanup() const retainedAfterCleanup = { visibleRows: collection._state.preSyncVisibleState.size, + virtualRows: collection._state.preSyncVirtualState.size, recentKeys: collection._state.recentlySyncedKeys.size, } await cleanup @@ -430,7 +431,7 @@ it(`starts a new sync session without retained publication state`, async () => { expect(sync.commit()).toBe(true) expect({ retainedAfterCleanup, events }).toEqual({ - retainedAfterCleanup: { visibleRows: 0, recentKeys: 0 }, + retainedAfterCleanup: { visibleRows: 0, virtualRows: 0, recentKeys: 0 }, events: [{ type: `insert`, key: 1 }], }) } finally { @@ -482,6 +483,7 @@ it(`keeps a restarted session's publication state after the old listener returns sync.write({ type: `insert`, value: { id: 3, value: 3 } }) expect(sync.commit()).toBe(true) expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.preSyncVirtualState.size).toBe(0) expect(collection._state.hasReceivedFirstCommit).toBe(true) await Promise.resolve() expect(collection._state.recentlySyncedKeys.size).toBe(0) @@ -526,6 +528,7 @@ it(`does not let an old publication microtask clear restarted sync state`, async expect(collection._state.hasReceivedFirstCommit).toBe(true) await Promise.resolve() expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.preSyncVirtualState.size).toBe(0) expect(collection._state.recentlySyncedKeys.size).toBe(0) await cleanup } finally { @@ -533,7 +536,7 @@ it(`does not let an old publication microtask clear restarted sync state`, async } }) -it(`publishes one insert when a restarted optimistic row is confirmed and rolled back`, async () => { +it(`publishes a virtual-state update when a restarted optimistic row is confirmed`, async () => { let sync!: SyncActions let syncSession = 0 let releaseMutation!: () => void @@ -552,7 +555,30 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled }, }, }) - const events: Array<{ type: string; key: string | number }> = [] + type ObservedRow = RetainedRow & { + $collectionId: string + $key: number + $origin: `local` | `remote` + $synced: boolean + } + type ObservedChange = { + type: string + key: string | number + value: ObservedRow + previousValue?: ObservedRow + } + const snapshotRow = (row: ObservedRow): ObservedRow => ({ + id: row.id, + value: row.value, + $collectionId: row.$collectionId, + $key: row.$key, + $origin: row.$origin, + $synced: row.$synced, + }) + const publications: Array<{ + changes: Array + rows: Array + }> = [] const restartStatuses: Array = [] let restarted = false let readMutationState: (() => TransactionState) | undefined @@ -561,7 +587,17 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled let syncReceiptSettled = false const subscription = collection.subscribeChanges( (changes) => { - events.push(...changes.map(({ type, key }) => ({ type, key }))) + publications.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotRow(value), + ...(previousValue === undefined + ? {} + : { previousValue: snapshotRow(previousValue) }), + })), + rows: [...collection.state.values()].map(snapshotRow), + }) if (restarted || !changes.some(({ key }) => key === 1)) return restarted = true @@ -600,10 +636,45 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled sync.write({ type: `insert`, value: { id: 1, value: 1 } }) expect(sync.commit()).toBe(true) - expect(events).toEqual([ - { type: `insert`, key: 1 }, - { type: `insert`, key: 2 }, - ]) + const remoteRow = (id: number): ObservedRow => ({ + id, + value: id, + $collectionId: collection.id, + $key: id, + $origin: `remote`, + $synced: true, + }) + const localRow = (id: number): ObservedRow => ({ + id, + value: id, + $collectionId: collection.id, + $key: id, + $origin: `local`, + $synced: false, + }) + const expectedPublications = [ + { + changes: [{ type: `insert`, key: 1, value: remoteRow(1) }], + rows: [remoteRow(1)], + }, + { changes: [], rows: [] }, + { + changes: [{ type: `insert`, key: 2, value: localRow(2) }], + rows: [localRow(2)], + }, + { + changes: [ + { + type: `update`, + key: 2, + value: remoteRow(2), + previousValue: localRow(2), + }, + ], + rows: [remoteRow(2)], + }, + ] + expect(publications).toEqual(expectedPublications) expect([...collection.state.keys()]).toEqual([2]) expect(restartStatuses).toEqual([`ready`, `cleaned-up`, `loading`, `ready`]) expect(collection.status).toBe(`ready`) @@ -616,19 +687,13 @@ it(`publishes one insert when a restarted optimistic row is confirmed and rolled } await syncReceipt expect(syncReceiptSettled).toBe(true) - expect(events).toEqual([ - { type: `insert`, key: 1 }, - { type: `insert`, key: 2 }, - ]) + expect(publications).toEqual(expectedPublications) expect([...collection.state.keys()]).toEqual([2]) releaseMutation() await mutationCommit expect(readMutationState?.()).toBe(`failed`) - expect(events).toEqual([ - { type: `insert`, key: 1 }, - { type: `insert`, key: 2 }, - ]) + expect(publications).toEqual(expectedPublications) expect([...collection.state.keys()]).toEqual([2]) } finally { releaseMutation() diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index 6dd830b0b..21c1b27e4 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -1781,6 +1781,8 @@ async function expectCanceledReceiptReleasesOnlyItsSuppression() { expect(source._state.recentlySyncedKeys).toEqual(new Set([`second`])) expect(source._state.preSyncVisibleState.has(`first`)).toBe(false) expect(source._state.preSyncVisibleState.has(`second`)).toBe(true) + expect(source._state.preSyncVirtualState.has(`first`)).toBe(false) + expect(source._state.preSyncVirtualState.has(`second`)).toBe(true) if (canceled !== true) { await expect(canceled).rejects.toMatchObject({ name: `AbortError` }) } From 2e65c4d8ec8682f46db844b7e4e28f1ee3ab5a43 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:20:59 -0600 Subject: [PATCH 269/327] test(db): order parked receipt settlement --- ...on-state-retention-oracle.property.test.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 03da8433d..4b6da233c 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -580,10 +580,13 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme rows: Array }> = [] const restartStatuses: Array = [] + const settlementTimeline: Array<`publication` | `receipt`> = [] let restarted = false let readMutationState: (() => TransactionState) | undefined + let rollbackMutation: (() => void) | undefined let mutationCommit: Promise | undefined let syncReceipt: ReturnType | undefined + let syncReceiptOutcome: Promise | undefined let syncReceiptSettled = false const subscription = collection.subscribeChanges( (changes) => { @@ -598,6 +601,9 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme })), rows: [...collection.state.values()].map(snapshotRow), }) + if (changes.some(({ type, key }) => type === `update` && key === 2)) { + queueMicrotask(() => settlementTimeline.push(`publication`)) + } if (restarted || !changes.some(({ key }) => key === 1)) return restarted = true @@ -614,6 +620,7 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme mutationFn: () => mutationHold, }) readMutationState = () => transaction.state + rollbackMutation = () => transaction.rollback() void transaction.isPersisted.promise.catch(() => undefined) transaction.mutate(() => collection.insert({ id: 2, value: 2 })) mutationCommit = transaction.commit() @@ -622,11 +629,12 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme sync.write({ type: `insert`, value: { id: 2, value: 2 } }) syncReceipt = sync.commit() if (syncReceipt !== true) { - void syncReceipt.then(() => { + syncReceiptOutcome = syncReceipt.then((value) => { + settlementTimeline.push(`receipt`) syncReceiptSettled = true + return value }) } - transaction.rollback() }, { includeInitialState: false }, ) @@ -674,7 +682,7 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme rows: [remoteRow(2)], }, ] - expect(publications).toEqual(expectedPublications) + expect(publications).toEqual(expectedPublications.slice(0, 3)) expect([...collection.state.keys()]).toEqual([2]) expect(restartStatuses).toEqual([`ready`, `cleaned-up`, `loading`, `ready`]) expect(collection.status).toBe(`ready`) @@ -685,8 +693,18 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme if (syncReceipt === undefined || syncReceipt === true) { throw new Error(`restarted sync receipt was not parked`) } - await syncReceipt + expect(syncReceiptOutcome).toBeDefined() + expect(rollbackMutation).toBeDefined() + await Promise.resolve() + expect(syncReceiptSettled).toBe(false) + expect(settlementTimeline).toEqual([]) + + rollbackMutation?.() + expect(publications).toEqual(expectedPublications) + expect(syncReceiptSettled).toBe(false) + await expect(syncReceiptOutcome).resolves.toBeUndefined() expect(syncReceiptSettled).toBe(true) + expect(settlementTimeline).toEqual([`publication`, `receipt`]) expect(publications).toEqual(expectedPublications) expect([...collection.state.keys()]).toEqual([2]) From 86171506b5459f55a6b72c13d20b019eef3ada12 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:23:57 -0600 Subject: [PATCH 270/327] test(db): preserve falsy row listener failures --- packages/db/tests/query/scheduler.test.ts | 73 +++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index f461c57f5..870aa8492 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -265,6 +265,79 @@ describe(`live query scheduler`, () => { } }) + it.each([ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, + ])( + `preserves an exact $name row-listener failure after later delivery`, + async ({ name, failure }) => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const laterListener = vi.fn() + const source = createCollection({ + id: `falsy-row-listener-${name.replaceAll(` `, `-`)}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + () => { + throw failure + }, + { includeInitialState: false }, + ) + const laterSubscription = source.subscribeChanges(laterListener, { + includeInitialState: false, + }) + const live = createLiveQueryCollection({ + id: `falsy-row-listener-dependent-${name.replaceAll(` `, `-`)}`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + let didThrow = false + let thrown: unknown + try { + commit() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + expect(laterListener).toHaveBeenCalledOnce() + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + } finally { + throwingSubscription.unsubscribe() + laterSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }, + ) + it(`keeps a nested ready failure when a later outer listener throws`, async () => { let markInnerReady!: () => void const readyFailure = new Error(`nested ready listener failed`) From 216125e65fcd871a806020b3aea6266fe8edb156 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:28:29 -0600 Subject: [PATCH 271/327] test(db): freeze layout publication listeners --- packages/db/tests/query/scheduler.test.ts | 121 ++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 870aa8492..00037de28 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -217,6 +217,127 @@ describe(`live query scheduler`, () => { } }) + it(`delivers a layout-only batch to its frozen listener snapshot`, async () => { + type RankedUser = User & { rank: number } + const calls: Array = [] + const firstFailure = new Error(`first layout listener failed`) + const laterFailure = new Error(`later public listener failed`) + const graphJob = vi.fn(() => calls.push(`graph`)) + const source = createCollection( + mockSyncCollectionOptions({ + id: `layout-listener-membership-source`, + getKey: (user) => user.id, + initialData: [ + { id: 1, name: `Ada`, rank: 1 }, + { id: 2, name: `Grace`, rank: 2 }, + ], + }), + ) + const ordered = createLiveQueryCollection({ + id: `layout-listener-membership-ordered`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .orderBy(({ user }) => user.rank, `asc`) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + await ordered.preload() + expect(ordered.toArray.map(({ id }) => id)).toEqual([1, 2]) + let firstPublication = true + let addedLayout: (() => void) | undefined + let addedPublic: { unsubscribe: () => void } | undefined + const unsubscribeFirstLayout = ordered._subscribeLayoutChanges(() => { + calls.push(`layout:first`) + if (!firstPublication) return + unsubscribeSecondLayout() + secondPublic.unsubscribe() + addedLayout ??= ordered._subscribeLayoutChanges(() => + calls.push(`layout:added`), + ) + addedPublic ??= ordered.subscribeChanges( + () => calls.push(`public:added`), + { includeInitialState: false }, + ) + throw firstFailure + }) + const unsubscribeSecondLayout = ordered._subscribeLayoutChanges(() => { + calls.push(`layout:second`) + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: graphJob, + run: graphJob, + }) + }) + const firstPublic = ordered.subscribeChanges( + () => { + calls.push(`public:first`) + if (firstPublication) throw laterFailure + }, + { includeInitialState: false }, + ) + const secondPublic = ordered.subscribeChanges( + () => calls.push(`public:second`), + { + includeInitialState: false, + }, + ) + + try { + let thrown: unknown + try { + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: 1, name: `Ada`, rank: 3 }, + }) + source.utils.commit() + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(calls).toEqual([ + `layout:first`, + `layout:second`, + `public:first`, + `public:second`, + `graph`, + ]) + expect(graphJob).toHaveBeenCalledOnce() + expect(ordered.toArray.map(({ id }) => id)).toEqual([2, 1]) + + firstPublication = false + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: 1, name: `Ada`, rank: 0 }, + }) + expect(() => source.utils.commit()).not.toThrow() + expect(calls).toEqual([ + `layout:first`, + `layout:second`, + `public:first`, + `public:second`, + `graph`, + `layout:first`, + `layout:added`, + `public:first`, + `public:added`, + ]) + } finally { + unsubscribeFirstLayout() + unsubscribeSecondLayout() + addedLayout?.() + firstPublic.unsubscribe() + secondPublic.unsubscribe() + addedPublic?.unsubscribe() + await ordered.cleanup() + await source.cleanup() + } + }) + it(`settles a dependent live query when an earlier source listener throws`, async () => { let begin!: () => void let write!: (message: { type: `insert`; value: User }) => void From 34d2d4f23a06c425a95a2dbb41177c43a8582fed Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:32:57 -0600 Subject: [PATCH 272/327] test(db): preserve filtered listener failures --- .../db/tests/collection-change-events.test.ts | 25 ++++++- packages/db/tests/query/scheduler.test.ts | 70 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/collection-change-events.test.ts b/packages/db/tests/collection-change-events.test.ts index 085af31f0..dee88c6d5 100644 --- a/packages/db/tests/collection-change-events.test.ts +++ b/packages/db/tests/collection-change-events.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' -import { currentStateAsChanges } from '../src/collection/change-events.js' +import { + createFilterFunctionFromExpression, + currentStateAsChanges, +} from '../src/collection/change-events.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' import { BTreeIndex } from '../src/indexes/btree-index.js' @@ -13,6 +16,26 @@ interface TestUser { status: `active` | `inactive` } +it(`treats predicate evaluation failures as nonmatches`, () => { + const filter = createFilterFunctionFromExpression( + new Func(`eq`, [new PropRef([`status`]), new Value(`active`)]), + ) + const row = { + id: `1`, + name: `Ada`, + age: 36, + score: 100, + status: `active`, + } as TestUser + Object.defineProperty(row, `status`, { + get: () => { + throw new Error(`predicate evaluation failed`) + }, + }) + + expect(filter(row)).toBe(false) +}) + describe(`currentStateAsChanges`, () => { let mockSync: ReturnType diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 00037de28..2231cf571 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -459,6 +459,76 @@ describe(`live query scheduler`, () => { }, ) + it(`preserves a filtered row-listener failure after later delivery`, async () => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const failure = new Error(`filtered source listener failed`) + const filteredCalls = vi.fn() + const laterListener = vi.fn() + const source = createCollection({ + id: `filtered-throwing-listener-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + (changes) => { + filteredCalls(changes) + throw failure + }, + { + includeInitialState: false, + where: (user) => eq(user.name, `Ada`), + }, + ) + const laterSubscription = source.subscribeChanges(laterListener, { + includeInitialState: false, + }) + const live = createLiveQueryCollection({ + id: `filtered-throwing-listener-dependent`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + expect(() => commit()).toThrow(failure) + expect(filteredCalls).toHaveBeenCalledOnce() + expect(filteredCalls.mock.calls[0]?.[0]).toEqual([ + expect.objectContaining({ type: `insert`, key: 1 }), + ]) + expect(laterListener).toHaveBeenCalledOnce() + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + + begin() + write({ type: `insert`, value: { id: 2, name: `Grace` } }) + expect(() => commit()).not.toThrow() + expect(filteredCalls).toHaveBeenCalledOnce() + expect(laterListener).toHaveBeenCalledTimes(2) + expect(live.get(2)).toEqual(expect.objectContaining({ name: `Grace` })) + } finally { + throwingSubscription.unsubscribe() + laterSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }) + it(`keeps a nested ready failure when a later outer listener throws`, async () => { let markInnerReady!: () => void const readyFailure = new Error(`nested ready listener failed`) From 049f311b547f938a16957e96420bfe2f6efc338a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:37:34 -0600 Subject: [PATCH 273/327] fix(db): preserve scheduler clear failures --- packages/db/src/query/live/ARCHITECTURE.md | 6 +- packages/db/src/scheduler.ts | 14 +++- packages/db/tests/query/scheduler.test.ts | 74 ++++++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 4e3e40f36..c5a56a992 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1241,8 +1241,10 @@ callback failure stays on the shared publication context, including when it came from a nested readiness transition. Later callback failures cannot replace it. Core runs the dependent graph turn queued by the batch before rethrowing the retained failure. A graph failure stops that turn and clears its remaining -work; scheduler dependencies are not a complete proof that two jobs can commit -or roll back independently. +work. Scheduler context cleanup attempts every clear listener, but a cleanup +failure cannot replace the publication or graph failure that caused the clear. +Scheduler dependencies are not a complete proof that two jobs can commit or +roll back independently. Each ordinary Collection batch is a keyed diff and names an affected row key at most once. Row operations and row-metadata writes use one shared affected-key diff --git a/packages/db/src/scheduler.ts b/packages/db/src/scheduler.ts index ea17dfc05..aaf8475ce 100644 --- a/packages/db/src/scheduler.ts +++ b/packages/db/src/scheduler.ts @@ -1,3 +1,5 @@ +import { runAllCallbacks } from './utils/callbacks.js' + /** * Identifier used to scope scheduled work. Maps to a transaction id for live queries. */ @@ -187,8 +189,9 @@ export class Scheduler { /** Clear all scheduled jobs for a context. */ clear(contextId: SchedulerContextId): void { this.contexts.delete(contextId) - // Notify listeners that this context was cleared - this.clearListeners.forEach((listener) => listener(contextId)) + runAllCallbacks( + [...this.clearListeners].map((listener) => () => listener(contextId)), + ) } /** Register a listener to be notified when a context is cleared. */ @@ -274,7 +277,12 @@ export function withPublicationContext(publish: () => T): T { if (graphFailure) throw graphFailure.error return result } catch (error) { - transactionScopedScheduler.clear(contextId) + try { + transactionScopedScheduler.clear(contextId) + } catch { + // Clearing is cleanup for an already failed publication. Its own failure + // cannot replace the exact publication or graph failure that caused it. + } throw error } finally { activePublication = undefined diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 2231cf571..1b709ff90 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -4,6 +4,7 @@ import { createLiveQueryCollection, eq, isNull } from '../../src/query/index.js' import { createTransaction } from '../../src/transactions.js' import { createOptimisticAction } from '../../src/optimistic-action.js' import { + Scheduler, getActivePublicationContext, transactionScopedScheduler, withPublicationContext, @@ -166,6 +167,79 @@ describe(`Collection publication scheduler context`, () => { expect(didThrow).toBe(true) expect(thrown).toBeUndefined() }) + + it(`attempts every clear listener and preserves its first failure`, () => { + const scheduler = new Scheduler() + const firstFailure = new Error(`first clear listener failed`) + const laterFailure = new Error(`later clear listener failed`) + const calls: Array = [] + let firstClear = true + let removeAdded: (() => void) | undefined + scheduler.onClear(() => { + calls.push(`first`) + if (!firstClear) return + removeSecond() + removeAdded ??= scheduler.onClear(() => calls.push(`added`)) + throw firstFailure + }) + const removeSecond = scheduler.onClear(() => { + calls.push(`second`) + if (firstClear) throw laterFailure + }) + + let thrown: unknown + try { + scheduler.clear(`context`) + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(calls).toEqual([`first`, `second`]) + + firstClear = false + expect(() => scheduler.clear(`next context`)).not.toThrow() + expect(calls).toEqual([`first`, `second`, `first`, `added`]) + removeAdded?.() + }) + + it.each([`publication`, `graph`] as const)( + `does not replace a $source failure with a clear-listener failure`, + (source) => { + const primaryFailure = new Error(`${source} failed`) + const clearFailure = new Error(`clear listener failed`) + const laterClear = vi.fn() + const removeThrowingClear = transactionScopedScheduler.onClear(() => { + throw clearFailure + }) + const removeLaterClear = transactionScopedScheduler.onClear(laterClear) + + try { + let thrown: unknown + try { + withPublicationContext(() => { + if (source === `publication`) throw primaryFailure + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: `failing graph`, + run: () => { + throw primaryFailure + }, + }) + }) + } catch (error) { + thrown = error + } + + expect(thrown).toBe(primaryFailure) + expect(laterClear).toHaveBeenCalledOnce() + } finally { + removeThrowingClear() + removeLaterClear() + } + }, + ) }) describe(`live query scheduler`, () => { From c84047e36504d437fb9b25b06c4c9cbc05c7400c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:40:58 -0600 Subject: [PATCH 274/327] test(db): freeze first-ready callback membership --- packages/db/src/query/live/ARCHITECTURE.md | 5 ++- .../db/tests/collection-lifecycle.test.ts | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index c5a56a992..a2a3131ed 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1126,7 +1126,10 @@ cannot suppress later first-ready callbacks, preload settlement, or the empty ready event that wakes dependent Collections. Core completes every effect, then rethrows the first failure unchanged, including a falsy value. Status is ready before these effects run; first-ready callbacks keep registration order, and -the dependent-ready event runs after them. That event snapshots the dependents +the callback set is frozen before delivery. Removing a copied callback during +delivery cannot skip it. Since readiness is already public, a callback added +during delivery runs immediately at its registration point. The dependent-ready +event runs after the frozen first-ready batch. That event snapshots the dependents present at delivery and attempts every one even if an earlier listener fails. Removing or adding a dependent during delivery does not change that frozen batch; an added dependent starts with the next publication. diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 9c71fa00a..fba34d182 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -516,6 +516,41 @@ describe(`Collection Lifecycle Management`, () => { subscription.unsubscribe() }) + it(`freezes first-ready callback membership before delivery`, async () => { + let markReadyCallback: (() => void) | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `first-ready-membership-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const subscription = collection.subscribeChanges(() => {}) + let removeLater = () => {} + + collection.onFirstReady(() => { + calls.push(`first`) + removeLater() + collection.onFirstReady(() => calls.push(`nested`)) + }) + removeLater = collection.onFirstReady(() => calls.push(`later`)) + + try { + markReadyCallback!() + + expect(calls).toEqual([`first`, `nested`, `later`]) + + collection.onFirstReady(() => calls.push(`after`)) + expect(calls).toEqual([`first`, `nested`, `later`, `after`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { let markReadyCallback: (() => void) | undefined const readyBatches: Array> = [] From 088da659313535316d59accb71c8e34ec03adf38 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:45:07 -0600 Subject: [PATCH 275/327] test(db): define mark-ready transitions --- packages/db/src/query/live/ARCHITECTURE.md | 6 ++ .../db/tests/collection-lifecycle.test.ts | 86 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a2a3131ed..b60b38cb0 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1133,6 +1133,12 @@ event runs after the frozen first-ready batch. That event snapshots the dependen present at delivery and attempts every one even if an earlier listener fails. Removing or adding a dependent during delivery does not change that frozen batch; an added dependent starts with the next publication. + +`markReady()` from `ready` is a no-op. Recovery from `error` clears the current +sync error and emits a dependent-ready event, but does not start a second +first-ready cycle. `idle` and `cleaned-up` cannot transition directly to +`ready`; sync must establish `loading` first. + Because the ready snapshot is already public, a listener failure also cannot discard graph work queued by an earlier listener. Core flushes that work before it rethrows the first listener failure. If readiness is nested inside an diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index fba34d182..694400b62 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -1,6 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' import { CleanupQueue } from '../src/collection/cleanup-queue.js' +import { InvalidCollectionStatusTransitionError } from '../src/errors.js' import { getActivePublicationContext, transactionScopedScheduler, @@ -551,6 +552,91 @@ describe(`Collection Lifecycle Management`, () => { } }) + it.each([ + { + from: `ready`, + expectedStatus: `ready`, + expectedFirstReadyCalls: 1, + expectedDependentReadyEvents: 1, + invalid: false, + }, + { + from: `error`, + expectedStatus: `ready`, + expectedFirstReadyCalls: 1, + expectedDependentReadyEvents: 2, + invalid: false, + }, + { + from: `idle`, + expectedStatus: `idle`, + expectedFirstReadyCalls: 0, + expectedDependentReadyEvents: 0, + invalid: true, + }, + { + from: `cleaned-up`, + expectedStatus: `cleaned-up`, + expectedFirstReadyCalls: 0, + expectedDependentReadyEvents: 0, + invalid: true, + }, + ] as const)( + `defines the $from -> ready transition`, + async ({ + from, + expectedStatus, + expectedFirstReadyCalls, + expectedDependentReadyEvents, + invalid, + }) => { + const syncFailure = new Error(`sync failed before recovery`) + let firstReadyCalls = 0 + const collection = createCollection<{ id: string; name: string }>({ + id: `mark-ready-from-${from}`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + collection.onFirstReady(() => { + firstReadyCalls++ + }) + + if (from === `ready` || from === `error`) { + collection._lifecycle.setStatus(`loading`) + collection._lifecycle.markReady() + } + if (from === `error`) { + collection._lifecycle.markError(syncFailure) + expect(collection._lifecycle.getSyncError()).toBe(syncFailure) + } else if (from === `cleaned-up`) { + collection._lifecycle.setStatus(`cleaned-up`) + } + expect(collection.status).toBe(from) + + let didThrow = false + let thrown: unknown + try { + collection._lifecycle.markReady() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(invalid) + if (invalid) { + expect(thrown).toBeInstanceOf(InvalidCollectionStatusTransitionError) + } + expect(collection.status).toBe(expectedStatus) + expect(firstReadyCalls).toBe(expectedFirstReadyCalls) + expect(readyEvent).toHaveBeenCalledTimes(expectedDependentReadyEvents) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + + await collection.cleanup() + }, + ) + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { let markReadyCallback: (() => void) | undefined const readyBatches: Array> = [] From 5b768731c776c1c9c310aa3dfa9977034f0e8634 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:48:25 -0600 Subject: [PATCH 276/327] fix(db): stop superseded ready effects --- packages/db/src/collection/lifecycle.ts | 6 ++++ packages/db/src/query/live/ARCHITECTURE.md | 5 ++++ .../db/tests/collection-lifecycle.test.ts | 30 +++++++++++++++++++ 3 files changed, 41 insertions(+) diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 579cdc718..be51f1bf4 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -149,6 +149,12 @@ export class CollectionLifecycleManager< if (this.status === `loading` || this.status === `error`) { this.syncError = undefined this.setStatus(`ready`, true) + + // A status listener can synchronously supersede this transition, for + // example by cleaning up the Collection. Do not publish ready effects + // for a snapshot that is no longer ready. + if ((this.status as CollectionStatus) !== `ready`) return undefined + const readyEffects: Array<() => void> = [] // Call any registered first ready callbacks (only on first time becoming ready) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index b60b38cb0..09bf0e0d6 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1139,6 +1139,11 @@ sync error and emits a dependent-ready event, but does not start a second first-ready cycle. `idle` and `cleaned-up` cannot transition directly to `ready`; sync must establish `loading` first. +The `status:ready` event precedes first-ready effects. If one of its listeners +synchronously moves the Collection away from `ready`, that newer lifecycle +transition supersedes the current one. Core does not resume first-ready effects +or emit a dependent-ready event for the superseded snapshot. + Because the ready snapshot is already public, a listener failure also cannot discard graph work queued by an earlier listener. Core flushes that work before it rethrows the first listener failure. If readiness is nested inside an diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 694400b62..4b537cf5c 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -637,6 +637,36 @@ describe(`Collection Lifecycle Management`, () => { }, ) + it(`does not resume ready effects after a status listener cleans up`, () => { + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-cleanup-test`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const firstReadyStatuses: Array = [] + collection.onFirstReady(() => { + firstReadyStatuses.push(collection.status) + }) + collection.on(`status:ready`, () => { + void collection.cleanup() + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + + expect(collection.status).toBe(`cleaned-up`) + expect(collection._lifecycle.hasBeenReady).toBe(false) + expect(firstReadyStatuses).toEqual([`ready`]) + expect(readyEvent).not.toHaveBeenCalled() + + const laterFirstReady = vi.fn() + const removeLater = collection.onFirstReady(laterFirstReady) + expect(laterFirstReady).not.toHaveBeenCalled() + removeLater() + }) + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { let markReadyCallback: (() => void) | undefined const readyBatches: Array> = [] From 73b81405d8db8899a83bf0dd9a5488defc102ea0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:51:03 -0600 Subject: [PATCH 277/327] test(db): define ready restart cycles --- packages/db/src/query/live/ARCHITECTURE.md | 5 ++ .../db/tests/collection-lifecycle.test.ts | 60 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 09bf0e0d6..432b010a1 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1144,6 +1144,11 @@ synchronously moves the Collection away from `ready`, that newer lifecycle transition supersedes the current one. Core does not resume first-ready effects or emit a dependent-ready event for the superseded snapshot. +A ready-effect failure does not undo effects already attempted in that cycle. +After cleanup, the next sync is a new first-ready cycle with a fresh preload +promise. It runs only callbacks registered for that new cycle; completed +callbacks from the prior cycle are not replayed. + Because the ready snapshot is already public, a listener failure also cannot discard graph work queued by an earlier listener. Core flushes that work before it rethrows the first listener failure. If readiness is nested inside an diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 4b537cf5c..bd9cba15b 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -667,6 +667,66 @@ describe(`Collection Lifecycle Management`, () => { removeLater() }) + it(`starts a fresh first-ready cycle after cleanup of a failed ready effect`, async () => { + const readyCallbacks: Array<() => void> = [] + const firstFailure = new Error(`first ready cycle failed exactly`) + const trace: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-effect-restart-test`, + getKey: (item) => item.id, + startSync: false, + sync: { + sync: ({ markReady }) => { + readyCallbacks.push(markReady) + }, + }, + }) + const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + collection.onFirstReady(() => { + trace.push(`first failure:${collection.status}`) + throw firstFailure + }) + collection.onFirstReady(() => { + trace.push(`first later:${collection.status}`) + }) + const firstPreload = collection.preload() + + let thrown: unknown + try { + readyCallbacks[0]!() + } catch (error) { + thrown = error + } + expect(thrown).toBe(firstFailure) + await expect(firstPreload).resolves.toBeUndefined() + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + expect(readyEvent).toHaveBeenCalledOnce() + + await collection.cleanup() + expect(collection.status).toBe(`cleaned-up`) + expect(collection._lifecycle.hasBeenReady).toBe(false) + + collection.onFirstReady(() => { + trace.push(`second:${collection.status}`) + }) + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + + const secondPreload = collection.preload() + expect(secondPreload).not.toBe(firstPreload) + expect(readyCallbacks).toHaveLength(2) + readyCallbacks[1]!() + await expect(secondPreload).resolves.toBeUndefined() + + expect(trace).toEqual([ + `first failure:ready`, + `first later:ready`, + `second:ready`, + ]) + expect(readyEvent).toHaveBeenCalledTimes(2) + + await collection.cleanup() + }) + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { let markReadyCallback: (() => void) | undefined const readyBatches: Array> = [] From 82488668092c8d47ea80d763335bc825e45c8ffe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 18:59:40 -0600 Subject: [PATCH 278/327] test(db): prove facade metadata recovery --- packages/db/src/query/live/ARCHITECTURE.md | 4 + .../src/query/live/bucket-facade-adapter.ts | 20 +---- .../tests/query/bucket-facade-adapter.test.ts | 81 ++++++++++++++++++- 3 files changed, 87 insertions(+), 18 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 432b010a1..33be96f95 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1221,6 +1221,10 @@ an adapter has already consumed. Facade rollback restores the Collection's internal publication snapshot. It must not use a public sync transaction or emit change, layout, readiness, or truncate lifecycle events for state that never committed. +It also restores the mutable key-to-order map used to classify later layout +changes. Per-object public-key and order WeakMaps are monotonic metadata written +before installation; failed installs do not clear them, so they are retained +rather than copied into rollback state. Fresh-facade readiness joins the prepared publication release only after every facade and root install has succeeded, and it precedes root callbacks. A recovery failure attempts every remaining restore and publication discard, diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index 6ecf2909d..2d3ba9171 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -27,9 +27,9 @@ type PendingRow = { type FacadeEntry = { collection: Collection sync: FacadeSync | undefined - keys: WeakMap - order: WeakMap - currentOrder: Map + readonly keys: WeakMap + readonly order: WeakMap + readonly currentOrder: Map } type FacadeEntrySnapshot = { @@ -38,11 +38,6 @@ type FacadeEntrySnapshot = { string | number > currentOrder: Map - rows: Array<{ - key: string | number - value: object - order: string | undefined - }> } type FacadeSnapshot = { @@ -307,11 +302,6 @@ export class BucketFacadeAdapter { publicationState: entry.collection._snapshotPublicationState(affectedKeys), currentOrder: new Map(entry.currentOrder), - rows: [...entry.collection._state.syncedData].map(([key, value]) => ({ - key, - value, - order: entry.currentOrder.get(key), - })), }) } } @@ -388,10 +378,6 @@ export class BucketFacadeAdapter { for (const [key, order] of entryState.currentOrder) { entry.currentOrder.set(key, order) } - for (const row of entryState.rows) { - entry.keys.set(row.value, row.key) - if (row.order !== undefined) entry.order.set(row.value, row.order) - } } } diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index da527dc1d..f5d223d86 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -171,7 +171,7 @@ describe(`BucketFacadeAdapter`, () => { ], 1, ], - [[bucketKey, { publicKey: added.id, value: added, order: `1` }], 1], + [[bucketKey, { publicKey: added.id, value: added, order: `3` }], 1], ]), ) graph.run() @@ -196,6 +196,85 @@ describe(`BucketFacadeAdapter`, () => { expect(facade._layoutRevision).toBe(layoutRevision) expect(facade.status).toBe(`ready`) + const restoredOriginal = facade.get(original.id) + const restoredFixed = facade.get(fixed.id) + if (!restoredOriginal || !restoredFixed) { + throw new Error(`Missing restored facade rows`) + } + expect(facade.getKeyFromItem(restoredOriginal)).toBe(original.id) + expect(facade.getKeyFromItem(restoredFixed)).toBe(fixed.id) + + adapter.flush().publish() + + expect(facade.toArray.map(stripVirtualProps)).toEqual([ + fixed, + replacement, + added, + ]) + expect(layoutPublications).toBe(0) + expect(publications).toHaveLength(1) + expect(publications[0]).toHaveLength(2) + expect(statusChanges).toBe(0) + expect(truncates).toBe(0) + expect(facade._stateRevision).toBe(stateRevision + 1) + expect(facade._layoutRevision).toBe(layoutRevision + 1) + expect(facade.toArray.map((row) => facade.getKeyFromItem(row))).toEqual([ + fixed.id, + replacement.id, + added.id, + ]) + expect([ + ...( + entries.get(`children`)?.get(bucketKey) as unknown as { + currentOrder: Map + } + ).currentOrder, + ]).toEqual([ + [original.id, `2`], + [fixed.id, `1`], + [added.id, `3`], + ]) + + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: `2`, + }, + ], + -1, + ], + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: `0`, + }, + ], + 1, + ], + ]), + ) + graph.run() + adapter.flush().publish() + + expect(facade.toArray.map(stripVirtualProps)).toEqual([ + replacement, + fixed, + added, + ]) + expect(layoutPublications).toBe(1) + expect(publications).toHaveLength(2) + expect(publications[1]).toEqual([]) + expect(facade._stateRevision).toBe(stateRevision + 1) + expect(facade._layoutRevision).toBe(layoutRevision + 2) + unsubscribeTruncate() unsubscribeStatus() unsubscribeLayout() From bfdf68cef04f2f882d85eed2cd40dc2bb1429114 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:07:07 -0600 Subject: [PATCH 279/327] test(db): prove post-truncate replacements --- packages/db/src/query/live/ARCHITECTURE.md | 9 +- ...rce-reconciliation-oracle.property.test.ts | 118 ++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 33be96f95..aa22ae3c8 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -223,9 +223,12 @@ An update or delete retracts the retained row, rather than trusting event metadata that may describe a newer value, then replaces or removes that entry. An update for an unknown key becomes an insert, while a delete for an unknown key contributes nothing. Truncate keeps this boundary state until its later -source batch retracts or replaces the retained rows. Graph teardown clears the -tracker and graph together. Thus every source key has multiplicity zero or one -and every negative weight cancels the exact positive row that entered D2. +source batch retracts or replaces the retained rows. A replacement's exact +retraction and new contribution enter the same graph turn, so the public +boundary observes one update rather than an intermediate removal. Graph +teardown clears the tracker and graph together. Thus every source key has +multiplicity zero or one and every negative weight cancels the exact positive +row that entered D2. Internal contribution identity is independent of the user-visible Collection key. When several internal rows collapse to one public key, a keyed D2 diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index 43a07366b..574f6c98a 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -196,6 +196,7 @@ function createOrderedSourceHarness(id: string) { let loadSubsetCalls = 0 const contributed = { id: 1, revision: 1, value: 1 } const staleDelete = { id: 1, revision: 2, value: 1 } + const replacement = { id: 1, revision: 3, value: 2 } const source = createCollection({ id, getKey: (row) => row.id, @@ -236,6 +237,7 @@ function createOrderedSourceHarness(id: string) { return { contributed, + replacement, source, staleDelete, suppressSourceChanges: () => { @@ -366,6 +368,122 @@ it(`retracts the exact live-query source row after an ordered truncate`, async ( } }) +it(`replaces the retained Effect source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness(`d2-effect-truncate-replacement`) + const { contributed, replacement, source, staleDelete } = harness + const batches: Array< + Array<{ + type: string + value: SourceRow + previousValue?: SourceRow + }> + > = [] + const effect = createEffect({ + query: (query) => + query + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + onBatch: (batch) => { + batches.push(batch) + }, + }) + + try { + await flushPromises() + expect(batches).toHaveLength(1) + expect(batches[0]).toHaveLength(1) + expect(batches[0]![0]).toMatchObject({ + type: `enter`, + key: 1, + value: contributed, + }) + const publishedValue = batches[0]![0]!.value + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(batches).toHaveLength(1) + + harness.publish([ + { + type: `update`, + key: 1, + previousValue: staleDelete, + value: replacement, + }, + ]) + await flushPromises() + expect(batches).toHaveLength(2) + expect(batches[1]).toHaveLength(1) + expect(batches[1]![0]).toMatchObject({ + type: `update`, + key: 1, + value: replacement, + }) + expect(batches[1]![0]!.previousValue).toBe(publishedValue) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + +it(`replaces the retained live-query source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-live-query-truncate-replacement`, + ) + const { contributed, replacement, source, staleDelete } = harness + const live = createLiveQueryCollection({ + id: `d2-live-query-truncate-replacement-result`, + query: (query) => + query + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + startSync: true, + }) + const batches: Array>> = [] + + try { + await live.preload() + expect(live.get(contributed.id)).toMatchObject(contributed) + const publishedValue = live.get(contributed.id) + const subscription = live.subscribeChanges( + (changes) => batches.push(changes), + { includeInitialState: false }, + ) + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(batches).toEqual([]) + expect(live.get(contributed.id)).toBe(publishedValue) + + harness.publish([ + { + type: `update`, + key: 1, + previousValue: staleDelete, + value: replacement, + }, + ]) + await flushPromises() + expect(batches).toHaveLength(1) + expect(batches[0]).toHaveLength(1) + expect(batches[0]![0]).toMatchObject({ + type: `update`, + key: 1, + value: replacement, + }) + expect(batches[0]![0]!.previousValue).toEqual(publishedValue) + expect(live.get(replacement.id)).toMatchObject(replacement) + subscription.unsubscribe() + } finally { + await live.cleanup() + await source.cleanup() + } +}) + fcTest.prop( [fc.array(reconciliationStepArbitrary, { minLength: 1, maxLength: 30 })], oraclePropertyOptions(200, `d2-source.exact-retractions`), From 856e3836602978a5267b24c6717a98c23feb3a36 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:11:29 -0600 Subject: [PATCH 280/327] test(db): separate source and graph lifecycle --- packages/db/src/query/live/ARCHITECTURE.md | 4 +- ...rce-reconciliation-oracle.property.test.ts | 180 ++++++++++++++---- 2 files changed, 147 insertions(+), 37 deletions(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index aa22ae3c8..f85a835e7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -228,7 +228,9 @@ retraction and new contribution enter the same graph turn, so the public boundary observes one update rather than an intermediate removal. Graph teardown clears the tracker and graph together. Thus every source key has multiplicity zero or one and every negative weight cancels the exact positive -row that entered D2. +row that entered D2. The source Collection owns its rows independently: graph +teardown does not erase them, source changes continue while the graph is down, +and a new graph replays the source's then-current rows into a fresh tracker. Internal contribution identity is independent of the user-visible Collection key. When several internal rows collapse to one public key, a keyed D2 diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index 574f6c98a..434d51003 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -44,6 +44,13 @@ type ReconciliationStep = | { type: `teardown` } | { type: `restart` } +type ReconciliationModel = { + sourceRows: Map + sentRows: Map + relation: Map + graphActive: boolean +} + const sourceRowArbitrary = fc.record({ id: fc.integer({ min: 0, max: 3 }), revision: fc.integer({ min: 0, max: 4 }), @@ -91,6 +98,11 @@ const reconciliationStepArbitrary: fc.Arbitrary = fc.oneof( { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, ) +const reconciliationHistoryArbitrary = fc.array(reconciliationStepArbitrary, { + minLength: 1, + maxLength: 30, +}) + function rowIdentity(row: SourceRow): string { return `${row.id}:${row.revision}:${row.value}` } @@ -191,6 +203,66 @@ function expectExactSourceRelation( ) } +function createReconciliationModel(): ReconciliationModel { + return { + sourceRows: new Map(), + sentRows: new Map(), + relation: new Map(), + graphActive: true, + } +} + +function applyReconciliationStep( + model: ReconciliationModel, + step: ReconciliationStep, +): void { + if (step.type === `truncate`) { + // Truncate is only an early lifecycle signal. Its later source batch + // still needs the retained exact rows to retract the active graph. + } else if (step.type === `teardown`) { + model.sentRows.clear() + model.relation.clear() + model.graphActive = false + } else if (step.type === `restart`) { + if (!model.graphActive) { + const replay = [...model.sourceRows].map(([key, value]) => ({ + type: `insert` as const, + key, + value, + })) + applyToRelation( + model.relation, + reconcileChangesForD2(replay, model.sentRows), + ) + model.graphActive = true + } + } else { + const changes = sourceChangesFor(step.operations, model.sourceRows) + if (model.graphActive) { + const reconciled = reconcileChangesForD2(changes, model.sentRows) + applyToRelation(model.relation, reconciled) + } + } + + if (model.graphActive) { + expectExactSourceRelation(model.sourceRows, model.sentRows, model.relation) + } else { + expect(model.sentRows.size).toBe(0) + expect(model.relation.size).toBe(0) + } +} + +function upsert( + key: SourceKey, + row: SourceRow, + reportedPreviousValue: SourceRow = row, +): ReconciliationStep { + return { + type: `batch`, + operations: [{ type: `upsert`, key, row, reportedPreviousValue }], + } +} + function createOrderedSourceHarness(id: string) { let sync!: SourceSyncActions let loadSubsetCalls = 0 @@ -484,48 +556,84 @@ it(`replaces the retained live-query source row after an ordered truncate`, asyn } }) +it(`preserves external source rows across graph teardown and restart`, () => { + const model = createReconciliationModel() + const row = { id: 1, revision: 1, value: 1 } + + applyReconciliationStep(model, upsert(`row`, row)) + applyReconciliationStep(model, { type: `teardown` }) + expect(model.graphActive).toBe(false) + expect(model.sourceRows).toEqual(new Map([[`row`, row]])) + expect(model.sentRows).toEqual(new Map()) + expect(model.relation).toEqual(new Map()) + + applyReconciliationStep(model, { type: `restart` }) + expect(model.graphActive).toBe(true) + expect(model.sourceRows).toEqual(new Map([[`row`, row]])) + expect(model.sentRows).toEqual(new Map([[`row`, row]])) + expect(model.relation).toEqual( + new Map([[`string:row|${rowIdentity(row)}`, 1]]), + ) +}) + +it(`replays external source changes made while the graph is down`, () => { + const model = createReconciliationModel() + const first = { id: 1, revision: 1, value: 1 } + const replacement = { id: 1, revision: 2, value: 2 } + + applyReconciliationStep(model, upsert(`row`, first)) + applyReconciliationStep(model, { type: `teardown` }) + applyReconciliationStep(model, upsert(`row`, replacement, first)) + expect(model.sourceRows).toEqual(new Map([[`row`, replacement]])) + expect(model.sentRows).toEqual(new Map()) + expect(model.relation).toEqual(new Map()) + + applyReconciliationStep(model, { type: `restart` }) + expect(model.sourceRows).toEqual(new Map([[`row`, replacement]])) + expect(model.sentRows).toEqual(new Map([[`row`, replacement]])) + expect(model.relation).toEqual( + new Map([[`string:row|${rowIdentity(replacement)}`, 1]]), + ) +}) + +it(`generates teardown, down-state source changes, and restart`, () => { + const histories = fc.sample(reconciliationHistoryArbitrary, { + seed: 1780, + numRuns: 500, + }) + + expect( + histories.some((steps) => { + let graphActive = true + let sawTeardown = false + let sawDownStateSourceChange = false + for (const step of steps) { + if (step.type === `teardown`) { + graphActive = false + sawTeardown = true + } else if (step.type === `restart`) { + if (!graphActive && sawTeardown && sawDownStateSourceChange) { + return true + } + graphActive = true + } else if (step.type === `batch` && !graphActive) { + sawDownStateSourceChange = true + } + } + return false + }), + ).toBe(true) +}) + fcTest.prop( - [fc.array(reconciliationStepArbitrary, { minLength: 1, maxLength: 30 })], + [reconciliationHistoryArbitrary], oraclePropertyOptions(200, `d2-source.exact-retractions`), )( `keeps one exact D2 contribution per source key across batched histories`, (steps) => { - const sourceRows = new Map() - const sentRows = new Map() - const relation = new Map() - let graphActive = true - + const model = createReconciliationModel() for (const step of steps) { - if (step.type === `truncate`) { - // Truncate is only an early lifecycle signal. Its later source batch - // still needs the retained exact rows to retract the active graph. - } else if (step.type === `teardown`) { - sentRows.clear() - relation.clear() - graphActive = false - } else if (step.type === `restart`) { - if (!graphActive) { - const replay = [...sourceRows].map(([key, value]) => ({ - type: `insert` as const, - key, - value, - })) - applyToRelation(relation, reconcileChangesForD2(replay, sentRows)) - graphActive = true - } - } else { - const changes = sourceChangesFor(step.operations, sourceRows) - if (graphActive) { - const reconciled = reconcileChangesForD2(changes, sentRows) - applyToRelation(relation, reconciled) - } - } - if (graphActive) { - expectExactSourceRelation(sourceRows, sentRows, relation) - } else { - expect(sentRows.size).toBe(0) - expect(relation.size).toBe(0) - } + applyReconciliationStep(model, step) } }, ) From 14231ec8affd5f13952f0560826664705bf3d6a3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:15:10 -0600 Subject: [PATCH 281/327] test(db): separate D2 expected identity --- ...rce-reconciliation-oracle.property.test.ts | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index 434d51003..5f7c99adc 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -107,6 +107,14 @@ function rowIdentity(row: SourceRow): string { return `${row.id}:${row.revision}:${row.value}` } +function expectedWeightedRowIdentity(key: SourceKey, row: SourceRow): string { + const sourceIdentity = [typeof key, String(key)].join(`:`) + const payloadIdentity = [row.id, row.revision, row.value] + .map(String) + .join(`:`) + return `${sourceIdentity}|${payloadIdentity}` +} + function addWeight( relation: Map, key: SourceKey, @@ -179,10 +187,9 @@ function sourceChangesFor( return changes } -function expectExactSourceRelation( +function expectTrackerMatchesSource( sourceRows: ReadonlyMap, sentRows: ReadonlyMap, - relation: ReadonlyMap, ): void { const compareEntries = ( [a]: readonly [SourceKey, SourceRow], @@ -191,14 +198,17 @@ function expectExactSourceRelation( expect([...sentRows.entries()].sort(compareEntries)).toEqual( [...sourceRows.entries()].sort(compareEntries), ) +} + +function expectWeightedRelationMatchesSource( + sourceRows: ReadonlyMap, + relation: ReadonlyMap, +): void { expect( [...relation.entries()].sort(([a], [b]) => a.localeCompare(b)), ).toEqual( [...sourceRows.entries()] - .map( - ([key, row]) => - [`${typeof key}:${String(key)}|${rowIdentity(row)}`, 1] as const, - ) + .map(([key, row]) => [expectedWeightedRowIdentity(key, row), 1] as const) .sort(([a], [b]) => a.localeCompare(b)), ) } @@ -245,7 +255,8 @@ function applyReconciliationStep( } if (model.graphActive) { - expectExactSourceRelation(model.sourceRows, model.sentRows, model.relation) + expectTrackerMatchesSource(model.sourceRows, model.sentRows) + expectWeightedRelationMatchesSource(model.sourceRows, model.relation) } else { expect(model.sentRows.size).toBe(0) expect(model.relation.size).toBe(0) @@ -556,6 +567,26 @@ it(`replaces the retained live-query source row after an ordered truncate`, asyn } }) +it(`keeps revision and value in weighted row identity`, () => { + const key = `row` + const base = { id: 1, revision: 1, value: 1 } + const differentRevision = { id: 1, revision: 2, value: 1 } + const differentValue = { id: 1, revision: 1, value: 2 } + const relation = new Map() + + addWeight(relation, key, base, 1) + addWeight(relation, key, differentRevision, 1) + addWeight(relation, key, differentValue, 1) + + expect(relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(key, base), 1], + [expectedWeightedRowIdentity(key, differentRevision), 1], + [expectedWeightedRowIdentity(key, differentValue), 1], + ]), + ) +}) + it(`preserves external source rows across graph teardown and restart`, () => { const model = createReconciliationModel() const row = { id: 1, revision: 1, value: 1 } @@ -572,7 +603,7 @@ it(`preserves external source rows across graph teardown and restart`, () => { expect(model.sourceRows).toEqual(new Map([[`row`, row]])) expect(model.sentRows).toEqual(new Map([[`row`, row]])) expect(model.relation).toEqual( - new Map([[`string:row|${rowIdentity(row)}`, 1]]), + new Map([[expectedWeightedRowIdentity(`row`, row), 1]]), ) }) @@ -592,7 +623,7 @@ it(`replays external source changes made while the graph is down`, () => { expect(model.sourceRows).toEqual(new Map([[`row`, replacement]])) expect(model.sentRows).toEqual(new Map([[`row`, replacement]])) expect(model.relation).toEqual( - new Map([[`string:row|${rowIdentity(replacement)}`, 1]]), + new Map([[expectedWeightedRowIdentity(`row`, replacement), 1]]), ) }) From ad725550e8e7ed53c70ea54d1af1fad5806d69c6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:17:34 -0600 Subject: [PATCH 282/327] test(db): pin mixed D2 source keys --- ...rce-reconciliation-oracle.property.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index 5f7c99adc..b36d9a209 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -587,6 +587,51 @@ it(`keeps revision and value in weighted row identity`, () => { ) }) +it(`keeps numeric and string source keys distinct across restart`, () => { + const model = createReconciliationModel() + const row = { id: 1, revision: 1, value: 1 } + const keys = [0, `0`] as const + + applyReconciliationStep(model, { + type: `batch`, + operations: keys.map((key) => ({ + type: `upsert` as const, + key, + row, + reportedPreviousValue: row, + })), + }) + expect(model.sourceRows).toEqual( + new Map([ + [0, row], + [`0`, row], + ]), + ) + expect(model.sentRows).toEqual(model.sourceRows) + expect(model.relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(0, row), 1], + [expectedWeightedRowIdentity(`0`, row), 1], + ]), + ) + + applyReconciliationStep(model, { type: `teardown` }) + applyReconciliationStep(model, { type: `restart` }) + expect(model.sourceRows).toEqual( + new Map([ + [0, row], + [`0`, row], + ]), + ) + expect(model.sentRows).toEqual(model.sourceRows) + expect(model.relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(0, row), 1], + [expectedWeightedRowIdentity(`0`, row), 1], + ]), + ) +}) + it(`preserves external source rows across graph teardown and restart`, () => { const model = createReconciliationModel() const row = { id: 1, revision: 1, value: 1 } From 3c7957179bf82ff67977ce562575da80403b7dd3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:24:59 -0600 Subject: [PATCH 283/327] test(db): cross pending publication query shapes --- .../query/includes-publication-oracle.test.ts | 221 ++++++++++++------ 1 file changed, 149 insertions(+), 72 deletions(-) diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index c44c06270..e7e4c00e1 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -13,6 +13,7 @@ import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, withExpectedRejection } from '../utils.js' import { createControlledCollection } from './includes-oracle-helpers.js' +import type { Collection } from '../../src/collection/index.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' import type { SyncConfig } from '../../src/types.js' @@ -43,6 +44,7 @@ type Q2Shape = `passThrough` | `where` | `orderBy` | `select` type Q1Shape = `direct` | `joined` type PendingPublicationOperation = `insert` | `update` | `delete` type PendingPublicationDepth = `direct` | `layered` +type PendingPublicationShape = `passThrough` | `orderBy` | `select` type SourceConfirmationOperation = `insert` | `update` | `delete` type SourceConfirmationInterleaving = | `handlerEcho` @@ -420,11 +422,12 @@ const q1Shapes = [`direct`, `joined`] as const const pendingPublicationOperations = [`insert`, `update`, `delete`] as const const pendingPublicationDepths = [`direct`, `layered`] as const +const pendingPublicationShapes = [`passThrough`, `orderBy`, `select`] as const const optimisticExistingRow: PendingPublicationRow = { id: 1, value: 10 } const sourceExistingRow: PendingPublicationRow = { id: 2, value: 20 } const optimisticInsertedRow: PendingPublicationRow = { id: 3, value: 30 } -const sourceInsertedRow: PendingPublicationRow = { id: 4, value: 40 } +const sourceInsertedRow: PendingPublicationRow = { id: 4, value: 15 } function pendingOperationRow( operation: PendingPublicationOperation, @@ -437,7 +440,7 @@ function pendingOperationRow( } if (operation === `insert`) return { ...sourceInsertedRow } - if (operation === `update`) return { ...sourceExistingRow, value: 21 } + if (operation === `update`) return { ...sourceExistingRow, value: 5 } return { ...sourceExistingRow } } @@ -452,10 +455,63 @@ function applyPendingOperation( function expectedPendingRows( rows: ReadonlyMap, + shape: PendingPublicationShape, ): Array { return [...rows.values()] .map((row) => ({ ...row })) - .sort((left, right) => left.id - right.id) + .sort((left, right) => + shape === `orderBy` + ? left.value - right.value || left.id - right.id + : left.id - right.id, + ) +} + +function createPendingPublicationQuery( + source: Collection, + shape: PendingPublicationShape, +) { + return createLiveQueryCollection({ + id: `pending-publication-${shape}-${nextCollectionId++}`, + query: (query) => { + const rows = query.from({ row: source }) + if (shape === `orderBy`) { + return rows.orderBy(({ row }) => row.value) + } + if (shape === `select`) { + return rows.select(({ row }) => ({ id: row.id, value: row.value })) + } + return rows + }, + getKey: (row) => row.id, + }) +} + +function observePendingPublication( + collection: Collection, + shape: PendingPublicationShape, +) { + const batches: Array< + Array<{ type: `insert` | `update` | `delete`; key: number }> + > = [] + const callbackSnapshots: Array> = [] + const currentRows = () => { + const rows = collection.toArray.map((row) => ({ + id: row.id, + value: row.value, + })) + return shape === `orderBy` + ? rows + : rows.sort((left, right) => left.id - right.id) + } + const subscription = collection.subscribeChanges( + (changes) => { + batches.push(changes.map(({ type, key }) => ({ type, key: Number(key) }))) + callbackSnapshots.push(currentRows()) + }, + { includeInitialState: false }, + ) + + return { batches, callbackSnapshots, currentRows, subscription } } function inversePendingOperation( @@ -470,6 +526,7 @@ async function expectSourcePublicationDuringPendingMutation( optimisticOperation: PendingPublicationOperation, sourceOperation: PendingPublicationOperation, depth: PendingPublicationDepth, + shape: PendingPublicationShape, sameKey = false, ): Promise { const initialRows = [optimisticExistingRow, sourceExistingRow] @@ -477,45 +534,15 @@ async function expectSourcePublicationDuringPendingMutation( `pending-publication-source`, initialRows, ) - const q1 = createLiveQueryCollection({ - id: `pending-publication-q1-${nextCollectionId++}`, - query: (q) => - q.from({ row: source.collection }).select(({ row }) => ({ - id: row.id, - value: row.value, - })), - getKey: (row) => row.id, - }) - const q2 = createLiveQueryCollection({ - id: `pending-publication-q2-${nextCollectionId++}`, - query: (q) => - q.from({ row: q1 }).select(({ row }) => ({ - id: row.id, - value: row.value, - })), - getKey: (row) => row.id, - }) + const q1 = createPendingPublicationQuery(source.collection, shape) + const q2 = createPendingPublicationQuery(q1, shape) const target = depth === `direct` ? q1 : q2 const persistence = createDeferred() - const observedBatches: Array< - Array<{ type: `insert` | `update` | `delete`; key: number }> - > = [] - const callbackSnapshots: Array> = [] - const currentRows = () => - target.toArray - .map((row) => ({ id: row.id, value: row.value })) - .sort((left, right) => left.id - right.id) await target.preload() - const subscription = target.subscribeChanges( - (changes) => { - observedBatches.push( - changes.map(({ type, key }) => ({ type, key: Number(key) })), - ) - callbackSnapshots.push(currentRows()) - }, - { includeInitialState: false }, - ) + const terminal = observePendingPublication(target, shape) + const intermediate = + depth === `layered` ? observePendingPublication(q1, shape) : undefined const optimisticRow = pendingOperationRow(optimisticOperation, `optimistic`) const sourceRow = sameKey @@ -546,11 +573,25 @@ async function expectSourcePublicationDuringPendingMutation( applyPendingOperation(afterOptimistic, optimisticOperation, optimisticRow) try { - expect(observedBatches).toEqual([ + expect(terminal.batches).toEqual([ [{ type: optimisticOperation, key: optimisticRow.id }], ]) - expect(callbackSnapshots).toEqual([expectedPendingRows(afterOptimistic)]) - expect(currentRows()).toEqual(expectedPendingRows(afterOptimistic)) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape), + ]) + expect(terminal.currentRows()).toEqual( + expectedPendingRows(afterOptimistic, shape), + ) + if (intermediate) { + expect(intermediate.batches).toEqual([]) + expect(intermediate.callbackSnapshots).toEqual([]) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows( + new Map(initialRows.map((row) => [row.id, row] as const)), + shape, + ), + ) + } source.write(sourceOperation, sourceRow) const afterSource = new Map( @@ -560,22 +601,38 @@ async function expectSourcePublicationDuringPendingMutation( const whilePending = new Map(afterSource) applyPendingOperation(whilePending, optimisticOperation, optimisticRow) + if (intermediate) { + expect(intermediate.batches).toEqual([ + [{ type: sourceOperation, key: sourceRow.id }], + ]) + expect(intermediate.callbackSnapshots).toEqual([ + expectedPendingRows(afterSource, shape), + ]) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows(afterSource, shape), + ) + } + if (sameKey) { - expect(observedBatches).toEqual([ + expect(terminal.batches).toEqual([ [{ type: optimisticOperation, key: optimisticRow.id }], ]) - expect(callbackSnapshots).toEqual([expectedPendingRows(whilePending)]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(whilePending, shape), + ]) } else { - expect(observedBatches).toEqual([ + expect(terminal.batches).toEqual([ [{ type: optimisticOperation, key: optimisticRow.id }], [{ type: sourceOperation, key: sourceRow.id }], ]) - expect(callbackSnapshots).toEqual([ - expectedPendingRows(afterOptimistic), - expectedPendingRows(whilePending), + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape), + expectedPendingRows(whilePending, shape), ]) } - expect(currentRows()).toEqual(expectedPendingRows(whilePending)) + expect(terminal.currentRows()).toEqual( + expectedPendingRows(whilePending, shape), + ) persistence.resolve() await transaction.isPersisted.promise @@ -586,16 +643,18 @@ async function expectSourcePublicationDuringPendingMutation( optimisticOperation === `delete` ? [] : [[{ type: `update` as const, key: optimisticRow.id }]] - expect(observedBatches).toEqual([ + expect(terminal.batches).toEqual([ [{ type: optimisticOperation, key: optimisticRow.id }], ...confirmationBatches, ]) - expect(callbackSnapshots).toEqual([ - expectedPendingRows(afterSource), - ...confirmationBatches.map(() => expectedPendingRows(afterSource)), + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterSource, shape), + ...confirmationBatches.map(() => + expectedPendingRows(afterSource, shape), + ), ]) } else { - expect(observedBatches).toEqual([ + expect(terminal.batches).toEqual([ [{ type: optimisticOperation, key: optimisticRow.id }], [{ type: sourceOperation, key: sourceRow.id }], [ @@ -605,17 +664,31 @@ async function expectSourcePublicationDuringPendingMutation( }, ], ]) - expect(callbackSnapshots).toEqual([ - expectedPendingRows(afterOptimistic), - expectedPendingRows(whilePending), - expectedPendingRows(afterSource), + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape), + expectedPendingRows(whilePending, shape), + expectedPendingRows(afterSource, shape), + ]) + } + expect(terminal.currentRows()).toEqual( + expectedPendingRows(afterSource, shape), + ) + if (intermediate) { + expect(intermediate.batches).toEqual([ + [{ type: sourceOperation, key: sourceRow.id }], + ]) + expect(intermediate.callbackSnapshots).toEqual([ + expectedPendingRows(afterSource, shape), ]) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows(afterSource, shape), + ) } - expect(currentRows()).toEqual(expectedPendingRows(afterSource)) } finally { persistence.resolve() await transaction.isPersisted.promise.catch(() => undefined) - subscription.unsubscribe() + intermediate?.subscription.unsubscribe() + terminal.subscription.unsubscribe() await q2.cleanup() await q1.cleanup() await source.collection.cleanup() @@ -1053,25 +1126,29 @@ describe(`source publication across pending derived mutations`, () => { } for (const depth of pendingPublicationDepths) { - for (const optimisticOperation of pendingPublicationOperations) { - for (const sourceOperation of pendingPublicationOperations) { - it(`publishes a disjoint source ${sourceOperation} through a ${depth} query while an optimistic ${optimisticOperation} persists`, async () => { + for (const shape of pendingPublicationShapes) { + for (const optimisticOperation of pendingPublicationOperations) { + for (const sourceOperation of pendingPublicationOperations) { + it(`publishes a disjoint source ${sourceOperation} through a ${depth} ${shape} query while an optimistic ${optimisticOperation} persists`, async () => { + await expectSourcePublicationDuringPendingMutation( + optimisticOperation, + sourceOperation, + depth, + shape, + ) + }) + } + + it(`retains a same-key source ${optimisticOperation} through a ${depth} ${shape} query while its optimistic confirmation persists`, async () => { await expectSourcePublicationDuringPendingMutation( optimisticOperation, - sourceOperation, + optimisticOperation, depth, + shape, + true, ) }) } - - it(`retains a same-key source ${optimisticOperation} through a ${depth} query while its optimistic confirmation persists`, async () => { - await expectSourcePublicationDuringPendingMutation( - optimisticOperation, - optimisticOperation, - depth, - true, - ) - }) } } }) From 22f38ce277134f6f3e83aa5a97c1797d8c738b27 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:31:38 -0600 Subject: [PATCH 284/327] test(db): assert pending publication payloads --- .../query/includes-publication-oracle.test.ts | 194 +++++++++++++----- 1 file changed, 137 insertions(+), 57 deletions(-) diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index e7e4c00e1..4dbe1abaf 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -15,7 +15,7 @@ import { flushPromises, withExpectedRejection } from '../utils.js' import { createControlledCollection } from './includes-oracle-helpers.js' import type { Collection } from '../../src/collection/index.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' -import type { SyncConfig } from '../../src/types.js' +import type { ChangeMessage, SyncConfig } from '../../src/types.js' type ParentRow = { id: number @@ -45,6 +45,7 @@ type Q1Shape = `direct` | `joined` type PendingPublicationOperation = `insert` | `update` | `delete` type PendingPublicationDepth = `direct` | `layered` type PendingPublicationShape = `passThrough` | `orderBy` | `select` +type PendingPublicationSettlement = `succeeds` | `rejects` type SourceConfirmationOperation = `insert` | `update` | `delete` type SourceConfirmationInterleaving = | `handlerEcho` @@ -57,6 +58,19 @@ type PendingPublicationRow = { value: number } +type PendingPublicationEvent = + | { + type: `insert` | `delete` + key: number + value: PendingPublicationRow + } + | { + type: `update` + key: number + value: PendingPublicationRow + previousValue: PendingPublicationRow + } + const initialParent: ParentRow = { id: 1, group: 10, value: 0 } const initialChild: ChildRow = { id: 100, parentGroup: 10, value: 1 } const initialChildren: ReadonlyArray = [ @@ -423,6 +437,7 @@ const q1Shapes = [`direct`, `joined`] as const const pendingPublicationOperations = [`insert`, `update`, `delete`] as const const pendingPublicationDepths = [`direct`, `layered`] as const const pendingPublicationShapes = [`passThrough`, `orderBy`, `select`] as const +const pendingPublicationSettlements = [`succeeds`, `rejects`] as const const optimisticExistingRow: PendingPublicationRow = { id: 1, value: 10 } const sourceExistingRow: PendingPublicationRow = { id: 2, value: 20 } @@ -466,6 +481,44 @@ function expectedPendingRows( ) } +function expectedPendingEvent( + type: PendingPublicationOperation, + key: number, + before: ReadonlyMap, + after: ReadonlyMap, +): PendingPublicationEvent { + if (type === `insert`) { + return { type, key, value: { ...after.get(key)! } } + } + if (type === `delete`) { + return { type, key, value: { ...before.get(key)! } } + } + return { + type, + key, + value: { ...after.get(key)! }, + previousValue: { ...before.get(key)! }, + } +} + +function pendingPublicationEvent( + change: ChangeMessage, +): PendingPublicationEvent { + const value = { id: change.value.id, value: change.value.value } + if (change.type !== `update`) { + return { type: change.type, key: Number(change.key), value } + } + return { + type: `update`, + key: Number(change.key), + value, + previousValue: { + id: change.previousValue!.id, + value: change.previousValue!.value, + }, + } +} + function createPendingPublicationQuery( source: Collection, shape: PendingPublicationShape, @@ -490,9 +543,7 @@ function observePendingPublication( collection: Collection, shape: PendingPublicationShape, ) { - const batches: Array< - Array<{ type: `insert` | `update` | `delete`; key: number }> - > = [] + const batches: Array> = [] const callbackSnapshots: Array> = [] const currentRows = () => { const rows = collection.toArray.map((row) => ({ @@ -505,7 +556,7 @@ function observePendingPublication( } const subscription = collection.subscribeChanges( (changes) => { - batches.push(changes.map(({ type, key }) => ({ type, key: Number(key) }))) + batches.push(changes.map(pendingPublicationEvent)) callbackSnapshots.push(currentRows()) }, { includeInitialState: false }, @@ -527,9 +578,13 @@ async function expectSourcePublicationDuringPendingMutation( sourceOperation: PendingPublicationOperation, depth: PendingPublicationDepth, shape: PendingPublicationShape, + settlement: PendingPublicationSettlement, sameKey = false, ): Promise { const initialRows = [optimisticExistingRow, sourceExistingRow] + const initialState = new Map( + initialRows.map((row) => [row.id, { ...row }] as const), + ) const source = createControlledCollection( `pending-publication-source`, initialRows, @@ -538,6 +593,7 @@ async function expectSourcePublicationDuringPendingMutation( const q2 = createPendingPublicationQuery(q1, shape) const target = depth === `direct` ? q1 : q2 const persistence = createDeferred() + const settlementError = new Error(`pending publication rollback`) await target.preload() const terminal = observePendingPublication(target, shape) @@ -567,15 +623,17 @@ async function expectSourcePublicationDuringPendingMutation( }) const transaction = mutate(optimisticOperation) - const afterOptimistic = new Map( - initialRows.map((row) => [row.id, { ...row }] as const), - ) + const afterOptimistic = new Map(initialState) applyPendingOperation(afterOptimistic, optimisticOperation, optimisticRow) + const optimisticEvent = expectedPendingEvent( + optimisticOperation, + optimisticRow.id, + initialState, + afterOptimistic, + ) try { - expect(terminal.batches).toEqual([ - [{ type: optimisticOperation, key: optimisticRow.id }], - ]) + expect(terminal.batches).toEqual([[optimisticEvent]]) expect(terminal.callbackSnapshots).toEqual([ expectedPendingRows(afterOptimistic, shape), ]) @@ -586,10 +644,7 @@ async function expectSourcePublicationDuringPendingMutation( expect(intermediate.batches).toEqual([]) expect(intermediate.callbackSnapshots).toEqual([]) expect(intermediate.currentRows()).toEqual( - expectedPendingRows( - new Map(initialRows.map((row) => [row.id, row] as const)), - shape, - ), + expectedPendingRows(initialState, shape), ) } @@ -600,11 +655,21 @@ async function expectSourcePublicationDuringPendingMutation( applyPendingOperation(afterSource, sourceOperation, sourceRow) const whilePending = new Map(afterSource) applyPendingOperation(whilePending, optimisticOperation, optimisticRow) + const intermediateSourceEvent = expectedPendingEvent( + sourceOperation, + sourceRow.id, + initialState, + afterSource, + ) + const terminalSourceEvent = expectedPendingEvent( + sourceOperation, + sourceRow.id, + afterOptimistic, + whilePending, + ) if (intermediate) { - expect(intermediate.batches).toEqual([ - [{ type: sourceOperation, key: sourceRow.id }], - ]) + expect(intermediate.batches).toEqual([[intermediateSourceEvent]]) expect(intermediate.callbackSnapshots).toEqual([ expectedPendingRows(afterSource, shape), ]) @@ -614,16 +679,14 @@ async function expectSourcePublicationDuringPendingMutation( } if (sameKey) { - expect(terminal.batches).toEqual([ - [{ type: optimisticOperation, key: optimisticRow.id }], - ]) + expect(terminal.batches).toEqual([[optimisticEvent]]) expect(terminal.callbackSnapshots).toEqual([ expectedPendingRows(whilePending, shape), ]) } else { expect(terminal.batches).toEqual([ - [{ type: optimisticOperation, key: optimisticRow.id }], - [{ type: sourceOperation, key: sourceRow.id }], + [optimisticEvent], + [terminalSourceEvent], ]) expect(terminal.callbackSnapshots).toEqual([ expectedPendingRows(afterOptimistic, shape), @@ -634,35 +697,50 @@ async function expectSourcePublicationDuringPendingMutation( expectedPendingRows(whilePending, shape), ) - persistence.resolve() - await transaction.isPersisted.promise + if (settlement === `succeeds`) { + persistence.resolve() + await transaction.isPersisted.promise + } else { + persistence.reject(settlementError) + await expect(transaction.isPersisted.promise).rejects.toBe( + settlementError, + ) + } await flushPromises() if (sameKey) { - const confirmationBatches = + const settlementBatches: Array> = optimisticOperation === `delete` ? [] - : [[{ type: `update` as const, key: optimisticRow.id }]] + : [ + [ + expectedPendingEvent( + `update`, + optimisticRow.id, + whilePending, + afterSource, + ), + ], + ] expect(terminal.batches).toEqual([ - [{ type: optimisticOperation, key: optimisticRow.id }], - ...confirmationBatches, + [optimisticEvent], + ...settlementBatches, ]) expect(terminal.callbackSnapshots).toEqual([ expectedPendingRows(afterSource, shape), - ...confirmationBatches.map(() => - expectedPendingRows(afterSource, shape), - ), + ...settlementBatches.map(() => expectedPendingRows(afterSource, shape)), ]) } else { + const settlementEvent = expectedPendingEvent( + inversePendingOperation(optimisticOperation), + optimisticRow.id, + whilePending, + afterSource, + ) expect(terminal.batches).toEqual([ - [{ type: optimisticOperation, key: optimisticRow.id }], - [{ type: sourceOperation, key: sourceRow.id }], - [ - { - type: inversePendingOperation(optimisticOperation), - key: optimisticRow.id, - }, - ], + [optimisticEvent], + [terminalSourceEvent], + [settlementEvent], ]) expect(terminal.callbackSnapshots).toEqual([ expectedPendingRows(afterOptimistic, shape), @@ -674,9 +752,7 @@ async function expectSourcePublicationDuringPendingMutation( expectedPendingRows(afterSource, shape), ) if (intermediate) { - expect(intermediate.batches).toEqual([ - [{ type: sourceOperation, key: sourceRow.id }], - ]) + expect(intermediate.batches).toEqual([[intermediateSourceEvent]]) expect(intermediate.callbackSnapshots).toEqual([ expectedPendingRows(afterSource, shape), ]) @@ -1127,27 +1203,31 @@ describe(`source publication across pending derived mutations`, () => { for (const depth of pendingPublicationDepths) { for (const shape of pendingPublicationShapes) { - for (const optimisticOperation of pendingPublicationOperations) { - for (const sourceOperation of pendingPublicationOperations) { - it(`publishes a disjoint source ${sourceOperation} through a ${depth} ${shape} query while an optimistic ${optimisticOperation} persists`, async () => { + for (const settlement of pendingPublicationSettlements) { + for (const optimisticOperation of pendingPublicationOperations) { + for (const sourceOperation of pendingPublicationOperations) { + it(`publishes a disjoint source ${sourceOperation} through a ${depth} ${shape} query while an optimistic ${optimisticOperation} ${settlement}`, async () => { + await expectSourcePublicationDuringPendingMutation( + optimisticOperation, + sourceOperation, + depth, + shape, + settlement, + ) + }) + } + + it(`retains a same-key source ${optimisticOperation} through a ${depth} ${shape} query while its optimistic mutation ${settlement}`, async () => { await expectSourcePublicationDuringPendingMutation( optimisticOperation, - sourceOperation, + optimisticOperation, depth, shape, + settlement, + true, ) }) } - - it(`retains a same-key source ${optimisticOperation} through a ${depth} ${shape} query while its optimistic confirmation persists`, async () => { - await expectSourcePublicationDuringPendingMutation( - optimisticOperation, - optimisticOperation, - depth, - shape, - true, - ) - }) } } } From 3bf1f8c71cd5e8e2619c10496db1773450153ea6 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:43:45 -0600 Subject: [PATCH 285/327] fix(db): retain root base beneath optimistic deletes --- .../query/live/collection-config-builder.ts | 7 +- .../query/includes-publication-oracle.test.ts | 313 +++++++++++------- 2 files changed, 207 insertions(+), 113 deletions(-) diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 3f052cfaf..80bd9b155 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -1235,6 +1235,11 @@ export class CollectionConfigBuilder< // Store the key of the result so that we can retrieve it in the // getKey function this.resultKeys.set(value, key) + const resultKey = collection.getKeyFromItem(value) + // Graph deltas update the synced base. A pending optimistic delete can + // hide that base row from the public Collection view, so collection.has() + // cannot distinguish a base update from a delete. + const hasSyncedRow = collection._state.syncedData.has(resultKey) // Store the orderBy index if it exists if (orderByIndex !== undefined) { @@ -1252,7 +1257,7 @@ export class CollectionConfigBuilder< inserts > deletes || // Just update(s) but the item is already in the collection (so // was inserted previously). - (inserts === deletes && collection.has(collection.getKeyFromItem(value))) + (inserts === deletes && hasSyncedRow) ) { write({ value, diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 4dbe1abaf..4fe59fe59 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -71,6 +71,21 @@ type PendingPublicationEvent = previousValue: PendingPublicationRow } +type PendingPublicationSourceChange = { + operation: PendingPublicationOperation + row: PendingPublicationRow +} + +type PendingPublicationScenario = { + optimisticOperation: PendingPublicationOperation + sourceChanges: ReadonlyArray + sameKey: boolean +} + +type OffDiagonalSameKeyHistory = PendingPublicationScenario & { + name: string +} + const initialParent: ParentRow = { id: 1, group: 10, value: 0 } const initialChild: ChildRow = { id: 100, parentGroup: 10, value: 1 } const initialChildren: ReadonlyArray = [ @@ -444,6 +459,39 @@ const sourceExistingRow: PendingPublicationRow = { id: 2, value: 20 } const optimisticInsertedRow: PendingPublicationRow = { id: 3, value: 30 } const sourceInsertedRow: PendingPublicationRow = { id: 4, value: 15 } +const offDiagonalSameKeyHistories = [ + { + name: `source inserts then updates the optimistic insert key`, + optimisticOperation: `insert`, + sourceChanges: [ + { operation: `insert`, row: { id: 3, value: 20 } }, + { operation: `update`, row: { id: 3, value: 15 } }, + ], + sameKey: true, + }, + { + name: `source inserts then deletes the optimistic insert key`, + optimisticOperation: `insert`, + sourceChanges: [ + { operation: `insert`, row: { id: 3, value: 20 } }, + { operation: `delete`, row: { id: 3, value: 20 } }, + ], + sameKey: true, + }, + { + name: `source deletes the optimistic update key`, + optimisticOperation: `update`, + sourceChanges: [{ operation: `delete`, row: { ...optimisticExistingRow } }], + sameKey: true, + }, + { + name: `source updates the optimistic delete key`, + optimisticOperation: `delete`, + sourceChanges: [{ operation: `update`, row: { id: 1, value: 5 } }], + sameKey: true, + }, +] as const satisfies ReadonlyArray + function pendingOperationRow( operation: PendingPublicationOperation, owner: `optimistic` | `source`, @@ -471,14 +519,23 @@ function applyPendingOperation( function expectedPendingRows( rows: ReadonlyMap, shape: PendingPublicationShape, + orderedBase: ReadonlyMap = rows, ): Array { - return [...rows.values()] - .map((row) => ({ ...row })) - .sort((left, right) => - shape === `orderBy` - ? left.value - right.value || left.id - right.id - : left.id - right.id, - ) + if (shape !== `orderBy`) { + return [...rows.values()] + .map((row) => ({ ...row })) + .sort((left, right) => left.id - right.id) + } + + const baseKeys = [...orderedBase.values()] + .sort((left, right) => left.value - right.value || left.id - right.id) + .map((row) => row.id) + const optimisticOnlyKeys = [...rows.keys()] + .filter((key) => !orderedBase.has(key)) + .sort((left, right) => left - right) + return [...baseKeys, ...optimisticOnlyKeys] + .filter((key) => rows.has(key)) + .map((key) => ({ ...rows.get(key)! })) } function expectedPendingEvent( @@ -501,6 +558,38 @@ function expectedPendingEvent( } } +function pendingPublicationRowsEqual( + left: PendingPublicationRow | undefined, + right: PendingPublicationRow | undefined, +): boolean { + return left?.id === right?.id && left?.value === right?.value +} + +function expectedPendingTransition( + key: number, + before: ReadonlyMap, + after: ReadonlyMap, + includeLogicalNoopUpdate = false, +): PendingPublicationEvent | undefined { + const previousValue = before.get(key) + const value = after.get(key) + if (!previousValue && !value) return undefined + if (!previousValue) return { type: `insert`, key, value: { ...value! } } + if (!value) return { type: `delete`, key, value: { ...previousValue } } + if ( + !includeLogicalNoopUpdate && + pendingPublicationRowsEqual(previousValue, value) + ) { + return undefined + } + return { + type: `update`, + key, + value: { ...value }, + previousValue: { ...previousValue }, + } +} + function pendingPublicationEvent( change: ChangeMessage, ): PendingPublicationEvent { @@ -565,22 +654,13 @@ function observePendingPublication( return { batches, callbackSnapshots, currentRows, subscription } } -function inversePendingOperation( - operation: PendingPublicationOperation, -): PendingPublicationOperation { - if (operation === `insert`) return `delete` - if (operation === `delete`) return `insert` - return `update` -} - async function expectSourcePublicationDuringPendingMutation( - optimisticOperation: PendingPublicationOperation, - sourceOperation: PendingPublicationOperation, + scenario: PendingPublicationScenario, depth: PendingPublicationDepth, shape: PendingPublicationShape, settlement: PendingPublicationSettlement, - sameKey = false, ): Promise { + const { optimisticOperation, sourceChanges, sameKey } = scenario const initialRows = [optimisticExistingRow, sourceExistingRow] const initialState = new Map( initialRows.map((row) => [row.id, { ...row }] as const), @@ -601,9 +681,6 @@ async function expectSourcePublicationDuringPendingMutation( depth === `layered` ? observePendingPublication(q1, shape) : undefined const optimisticRow = pendingOperationRow(optimisticOperation, `optimistic`) - const sourceRow = sameKey - ? { ...optimisticRow } - : pendingOperationRow(sourceOperation, `source`) const insertTarget = target.insert.bind(target) as unknown as ( row: PendingPublicationRow, ) => unknown @@ -635,10 +712,10 @@ async function expectSourcePublicationDuringPendingMutation( try { expect(terminal.batches).toEqual([[optimisticEvent]]) expect(terminal.callbackSnapshots).toEqual([ - expectedPendingRows(afterOptimistic, shape), + expectedPendingRows(afterOptimistic, shape, initialState), ]) expect(terminal.currentRows()).toEqual( - expectedPendingRows(afterOptimistic, shape), + expectedPendingRows(afterOptimistic, shape, initialState), ) if (intermediate) { expect(intermediate.batches).toEqual([]) @@ -648,53 +725,60 @@ async function expectSourcePublicationDuringPendingMutation( ) } - source.write(sourceOperation, sourceRow) - const afterSource = new Map( - initialRows.map((row) => [row.id, { ...row }] as const), - ) - applyPendingOperation(afterSource, sourceOperation, sourceRow) - const whilePending = new Map(afterSource) - applyPendingOperation(whilePending, optimisticOperation, optimisticRow) - const intermediateSourceEvent = expectedPendingEvent( - sourceOperation, - sourceRow.id, - initialState, - afterSource, - ) - const terminalSourceEvent = expectedPendingEvent( - sourceOperation, - sourceRow.id, - afterOptimistic, - whilePending, - ) + const afterSource = new Map(initialState) + let whilePending = new Map(afterOptimistic) + const intermediateSourceBatches: Array> = [] + const intermediateSourceSnapshots: Array> = [] + const terminalSourceBatches: Array> = [] + const terminalSourceSnapshots: Array> = [] + + for (const { operation, row } of sourceChanges) { + const beforeSource = new Map(afterSource) + const beforeTerminal = new Map(whilePending) + source.write(operation, row) + applyPendingOperation(afterSource, operation, row) + + intermediateSourceBatches.push([ + expectedPendingEvent(operation, row.id, beforeSource, afterSource), + ]) + intermediateSourceSnapshots.push(expectedPendingRows(afterSource, shape)) + + const nextTerminal = new Map(afterSource) + applyPendingOperation(nextTerminal, optimisticOperation, optimisticRow) + const terminalSourceEvent = expectedPendingTransition( + row.id, + beforeTerminal, + nextTerminal, + ) + if (terminalSourceEvent) { + terminalSourceBatches.push([terminalSourceEvent]) + terminalSourceSnapshots.push( + expectedPendingRows(nextTerminal, shape, afterSource), + ) + } + whilePending = nextTerminal + } if (intermediate) { - expect(intermediate.batches).toEqual([[intermediateSourceEvent]]) - expect(intermediate.callbackSnapshots).toEqual([ - expectedPendingRows(afterSource, shape), - ]) + expect(intermediate.batches).toEqual(intermediateSourceBatches) + expect(intermediate.callbackSnapshots).toEqual( + intermediateSourceSnapshots, + ) expect(intermediate.currentRows()).toEqual( expectedPendingRows(afterSource, shape), ) } - if (sameKey) { - expect(terminal.batches).toEqual([[optimisticEvent]]) - expect(terminal.callbackSnapshots).toEqual([ - expectedPendingRows(whilePending, shape), - ]) - } else { - expect(terminal.batches).toEqual([ - [optimisticEvent], - [terminalSourceEvent], - ]) - expect(terminal.callbackSnapshots).toEqual([ - expectedPendingRows(afterOptimistic, shape), - expectedPendingRows(whilePending, shape), - ]) - } + expect(terminal.batches).toEqual([ + [optimisticEvent], + ...terminalSourceBatches, + ]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape, initialState), + ...terminalSourceSnapshots, + ]) expect(terminal.currentRows()).toEqual( - expectedPendingRows(whilePending, shape), + expectedPendingRows(whilePending, shape, afterSource), ) if (settlement === `succeeds`) { @@ -708,54 +792,33 @@ async function expectSourcePublicationDuringPendingMutation( } await flushPromises() - if (sameKey) { - const settlementBatches: Array> = - optimisticOperation === `delete` - ? [] - : [ - [ - expectedPendingEvent( - `update`, - optimisticRow.id, - whilePending, - afterSource, - ), - ], - ] - expect(terminal.batches).toEqual([ - [optimisticEvent], - ...settlementBatches, - ]) - expect(terminal.callbackSnapshots).toEqual([ - expectedPendingRows(afterSource, shape), - ...settlementBatches.map(() => expectedPendingRows(afterSource, shape)), - ]) - } else { - const settlementEvent = expectedPendingEvent( - inversePendingOperation(optimisticOperation), - optimisticRow.id, - whilePending, - afterSource, - ) - expect(terminal.batches).toEqual([ - [optimisticEvent], - [terminalSourceEvent], - [settlementEvent], - ]) - expect(terminal.callbackSnapshots).toEqual([ - expectedPendingRows(afterOptimistic, shape), - expectedPendingRows(whilePending, shape), - expectedPendingRows(afterSource, shape), - ]) - } + const settlementEvent = expectedPendingTransition( + optimisticRow.id, + whilePending, + afterSource, + sameKey, + ) + const settlementBatches = settlementEvent ? [[settlementEvent]] : [] + expect(terminal.batches).toEqual([ + [optimisticEvent], + ...terminalSourceBatches, + ...settlementBatches, + ]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape, initialState), + ...terminalSourceSnapshots, + ...settlementBatches.map(() => + expectedPendingRows(afterSource, shape, afterSource), + ), + ]) expect(terminal.currentRows()).toEqual( - expectedPendingRows(afterSource, shape), + expectedPendingRows(afterSource, shape, afterSource), ) if (intermediate) { - expect(intermediate.batches).toEqual([[intermediateSourceEvent]]) - expect(intermediate.callbackSnapshots).toEqual([ - expectedPendingRows(afterSource, shape), - ]) + expect(intermediate.batches).toEqual(intermediateSourceBatches) + expect(intermediate.callbackSnapshots).toEqual( + intermediateSourceSnapshots, + ) expect(intermediate.currentRows()).toEqual( expectedPendingRows(afterSource, shape), ) @@ -1208,8 +1271,16 @@ describe(`source publication across pending derived mutations`, () => { for (const sourceOperation of pendingPublicationOperations) { it(`publishes a disjoint source ${sourceOperation} through a ${depth} ${shape} query while an optimistic ${optimisticOperation} ${settlement}`, async () => { await expectSourcePublicationDuringPendingMutation( - optimisticOperation, - sourceOperation, + { + optimisticOperation, + sourceChanges: [ + { + operation: sourceOperation, + row: pendingOperationRow(sourceOperation, `source`), + }, + ], + sameKey: false, + }, depth, shape, settlement, @@ -1219,12 +1290,30 @@ describe(`source publication across pending derived mutations`, () => { it(`retains a same-key source ${optimisticOperation} through a ${depth} ${shape} query while its optimistic mutation ${settlement}`, async () => { await expectSourcePublicationDuringPendingMutation( - optimisticOperation, - optimisticOperation, + { + optimisticOperation, + sourceChanges: [ + { + operation: optimisticOperation, + row: pendingOperationRow(optimisticOperation, `optimistic`), + }, + ], + sameKey: true, + }, + depth, + shape, + settlement, + ) + }) + } + + for (const history of offDiagonalSameKeyHistories) { + it(`retains the synced base when the ${history.name} through a ${depth} ${shape} query and the optimistic mutation ${settlement}`, async () => { + await expectSourcePublicationDuringPendingMutation( + history, depth, shape, settlement, - true, ) }) } From ce1c93937d82a44827c453e3dd3b760be0e3a901 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 19:47:52 -0600 Subject: [PATCH 286/327] test(db): bound immediate graph publication --- .../query/includes-publication-oracle.test.ts | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 4fe59fe59..5f5794d70 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -964,6 +964,105 @@ describe(`layered-query publication oracle`, () => { }) describe(`source publication across pending derived mutations`, () => { + for (const settlement of pendingPublicationSettlements) { + it(`keeps ordinary source sync parked while layered graph publication ${settlement}`, async () => { + let sync!: Parameters< + SyncConfig[`sync`] + >[0] + const source = createCollection({ + id: `ordinary-source-prefix-${nextCollectionId++}`, + getKey: (row) => row.id, + sync: { + sync: (methods) => { + sync = methods + methods.markReady() + }, + }, + }) + await source.preload() + sync.begin() + sync.write({ type: `insert`, value: { ...optimisticExistingRow } }) + sync.write({ type: `insert`, value: { ...sourceExistingRow } }) + const initialReceipt = sync.commit() + if (initialReceipt !== true) await initialReceipt + + const q1 = createPendingPublicationQuery(source, `passThrough`) + const q2 = createPendingPublicationQuery(q1, `select`) + await q2.preload() + const observed = observePendingPublication(q2, `select`) + const persistence = createDeferred() + const settlementError = new Error(`ordinary source prefix rollback`) + const mutate = createOptimisticAction({ + onMutate: () => { + source.update(1, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(q2.get(1)?.value).toBe(11) + observed.batches.length = 0 + observed.callbackSnapshots.length = 0 + + sync.begin() + sync.write({ type: `update`, value: { id: 2, value: 5 } }) + const parkedReceipt = sync.commit() + expect(parkedReceipt).not.toBe(true) + if (parkedReceipt === true) { + throw new Error(`ordinary source sync did not park`) + } + let parkedReceiptSettled = false + void parkedReceipt.then(() => { + parkedReceiptSettled = true + }) + await flushPromises() + + expect(parkedReceiptSettled).toBe(false) + expect(source.get(2)?.value).toBe(20) + expect(q2.get(2)?.value).toBe(20) + expect( + observed.batches.flat().filter((event) => event.key === 2), + ).toEqual([]) + + if (settlement === `succeeds`) { + persistence.resolve() + await transaction.isPersisted.promise + } else { + persistence.reject(settlementError) + await expect(transaction.isPersisted.promise).rejects.toBe( + settlementError, + ) + } + await parkedReceipt + await flushPromises() + + expect(parkedReceiptSettled).toBe(true) + expect(source.get(2)?.value).toBe(5) + expect(q2.get(2)?.value).toBe(5) + expect( + observed.batches.flat().filter((event) => event.key === 2), + ).toEqual([ + { + type: `update`, + key: 2, + value: { id: 2, value: 5 }, + previousValue: { id: 2, value: 20 }, + }, + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + observed.subscription.unsubscribe() + await q2.cleanup() + await q1.cleanup() + await source.cleanup() + } + }) + } + async function expectSourceConfirmationPreservesGraphIntegrity( operation: SourceConfirmationOperation, depth: PendingPublicationDepth, From fc6facee6a22636420702ac04618a029200e130c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:00:56 -0600 Subject: [PATCH 287/327] fix(db): defer rollback-sensitive settlement --- packages/db/src/collection/changes.ts | 18 ++++- packages/db/src/collection/state.ts | 15 ++-- packages/db/src/query/live/ARCHITECTURE.md | 5 +- ...ncludes-collection-oracle.property.test.ts | 81 ++++++++++++++----- 4 files changed, 92 insertions(+), 27 deletions(-) diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 4d37f16df..93be8ef11 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -39,6 +39,7 @@ type PublicationDeferralState< }> stateRevisionDelta: number layoutRevisionDelta: number + afterPublication: Array<() => void> prepared: | { changes: Array> @@ -205,6 +206,7 @@ export class CollectionChangesManager< publications: [], stateRevisionDelta: 0, layoutRevisionDelta: 0, + afterPublication: [], prepared: undefined, published: false, } @@ -222,7 +224,10 @@ export class CollectionChangesManager< if (this.publicationDeferral === publicationDeferral) { this.publicationDeferral = undefined } - if (publicationDeferral.discard) return + if (publicationDeferral.discard) { + publicationDeferral.afterPublication = [] + return + } this.stateRevision += publicationDeferral.stateRevisionDelta this.layoutRevision += publicationDeferral.layoutRevisionDelta @@ -258,11 +263,22 @@ export class CollectionChangesManager< if (publication) { this.publishEvents(publication.changes, publication.layoutChanged) } + runAllCallbacks(publicationDeferral.afterPublication.splice(0)) }, discard: () => prepare(true), } } + /** Run work only after a held coherent publication becomes public. */ + public afterPublication(callback: () => void): void { + const publicationDeferral = this.publicationDeferral + if (publicationDeferral) { + publicationDeferral.afterPublication.push(callback) + return + } + callback() + } + private publishEvents( rawEvents: Array>, layoutChanged: boolean, diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 2682de9e9..523afa8a0 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1712,11 +1712,14 @@ export class CollectionStateManager< this.preSyncVisibleState.clear() this.preSyncVirtualState.clear() - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - if (this.syncSessionGeneration === syncSessionGeneration) { - this.recentlySyncedKeys.clear() - } + // A coherent multi-Collection publication can still roll back this + // commit. Start its cleanup tail only after that publication succeeds. + this.changes.afterPublication(() => { + Promise.resolve().then(() => { + if (this.syncSessionGeneration === syncSessionGeneration) { + this.recentlySyncedKeys.clear() + } + }) }) // Mark that we've received the first commit (for tracking purposes) @@ -1726,7 +1729,7 @@ export class CollectionStateManager< } for (const transaction of committedSyncedTransactions) { - transaction.applied.resolve() + this.changes.afterPublication(() => transaction.applied.resolve()) } return { processed: true, publicationError } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index f85a835e7..92183762d 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1219,6 +1219,9 @@ read another participating Collection without seeing new rows behind an old revision. If a later root or containing-facade application fails before that release, rollback restores the installed state and discards both the held events and their revision advances. Routing and identity remain inside D2. +Applied receipts and asynchronous cleanup tails created by a participating +Collection join that same release. Rollback leaves restored receipts pending +and discards cleanup tails that could otherwise mutate the restored state. The root and facade adapters retain the graph deltas consumed by that failed attempt. A later graph turn retries the whole uncommitted relation even when the source emits only an unrelated root delta; D2 does not replay a delta that @@ -1406,7 +1409,7 @@ create recursive Collection machinery. | Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | | Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | | Exact metadata settlement, cancellation ownership, and rollback recovery | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | -| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | +| Collection facades, event/receipt rollback, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | | Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | | Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | | Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index fcff8aac4..b46be9c74 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -52,9 +52,7 @@ type FacadeCandidateScanScenario = { class ThrowingUpdateIndex extends BasicIndex { updateFailure: { error: unknown } | undefined - buildFailure: - | { error: unknown; stage: `before` | `after` } - | undefined + buildFailure: { error: unknown; stage: `before` | `after` } | undefined buildCalls = 0 override update(key: number, oldItem: unknown, newItem: unknown): void { @@ -1133,8 +1131,28 @@ describe(`Collection-valued includes oracle`, () => { const rootIndex = live.createIndex((row) => row.value, { indexType: ThrowingUpdateIndex, }) as ThrowingUpdateIndex + const pendingApplied = createDeferred() + void pendingApplied.promise.catch(() => undefined) + const pendingFacadeSync = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map([ + [initialChild.id, { type: `set` as const, value: `pending` }], + ]), + collectionMetadataWrites: new Map(), + applied: pendingApplied, + } + facade._state.pendingSyncedTransactions.push(pendingFacadeSync) + facade._state.capturePreSyncVisibleState() + const recentlySyncedBeforeFailure = new Set( + facade._state.recentlySyncedKeys, + ) const rootPublications: Array = [] const childPublications: Array = [] + const childReceiptStates: Array = [] const rootCallbackFacadeSnapshots: Array<{ rows: Array<{ id: number; value: number }> stateRevision: number @@ -1152,7 +1170,10 @@ describe(`Collection-valued includes oracle`, () => { { includeInitialState: false }, ) const childSubscription = facade.subscribeChanges( - (batch) => childPublications.push(...batch), + (batch) => { + childPublications.push(...batch) + childReceiptStates.push(pendingApplied.isPending()) + }, { includeInitialState: false }, ) const childObserver = createLiveQueryObserver(facade) @@ -1164,10 +1185,11 @@ describe(`Collection-valued includes oracle`, () => { const rootLayoutRevisionBeforeFailure = live._layoutRevision const childStateRevisionBeforeFailure = facade._stateRevision const childLayoutRevisionBeforeFailure = facade._layoutRevision - rootIndex.updateFailure = { error: new Error(`root index failed`) } + const rootFailure = new Error(`root index failed`) + rootIndex.updateFailure = { error: rootFailure } try { - expect(() => + const failure = captureFailure(() => nodes.writeBatch([ { type: `update`, @@ -1182,7 +1204,8 @@ describe(`Collection-valued includes oracle`, () => { value: { ...initialSibling, value: 0 }, }, ]), - ).toThrow(`root index failed`) + ) + expect(failure?.error).toBe(rootFailure) expect(live.get(1)!.value).toBe(1) expect([...rootIndex.equalityLookup(1)]).toEqual([1]) expect([...rootIndex.equalityLookup(2)]).toEqual([]) @@ -1192,6 +1215,7 @@ describe(`Collection-valued includes oracle`, () => { ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) + expect(childReceiptStates).toEqual([]) expect(rootCallbackFacadeSnapshots).toEqual([]) expect(live._stateRevision).toBe(rootStateRevisionBeforeFailure) expect(live._layoutRevision).toBe(rootLayoutRevisionBeforeFailure) @@ -1199,12 +1223,31 @@ describe(`Collection-valued includes oracle`, () => { expect(facade._layoutRevision).toBe(childLayoutRevisionBeforeFailure) expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) expect(observerNotifications).toBe(0) + expect(facade._state.pendingSyncedTransactions).toHaveLength(1) + expect(facade._state.pendingSyncedTransactions[0]).toBe( + pendingFacadeSync, + ) + expect( + facade._state.pendingSyncedTransactions[0]!.applied.isPending(), + ).toBe(true) + expect(facade._state.recentlySyncedKeys).toEqual( + recentlySyncedBeforeFailure, + ) + await Promise.resolve() + expect(facade._state.recentlySyncedKeys).toEqual( + recentlySyncedBeforeFailure, + ) rootIndex.updateFailure = undefined // Only the root changes on retry. The child deltas consumed by the // failed graph turn must remain staged until the whole publication // commits; the source will not emit them again. nodes.write(`update`, { ...initialParent, value: 3 }) + expect(pendingApplied.isPending()).toBe(false) + await pendingApplied.promise + expect(facade._state.syncedMetadata.get(initialChild.id)).toBe( + `pending`, + ) expect(live.get(1)!.value).toBe(3) expect([...rootIndex.equalityLookup(1)]).toEqual([]) expect([...rootIndex.equalityLookup(3)]).toEqual([1]) @@ -1214,6 +1257,7 @@ describe(`Collection-valued includes oracle`, () => { ]) expect(rootPublications).toHaveLength(1) expect(childPublications).toHaveLength(2) + expect(childReceiptStates).toEqual([true]) expect(rootCallbackFacadeSnapshots).toEqual([ { rows: [ @@ -1518,19 +1562,18 @@ describe(`Collection-valued includes oracle`, () => { { id: 10, value: 2 }, { id: 11, value: 20 }, ]) - const rootPublicationsAfterRecovery: Array< - Array - > = [ - [], + const rootPublicationsAfterRecovery: Array> = [ - { - type: `update`, - key: 1, - value: { id: 1, value: 5, preservesFacade: true }, - previousValue: { id: 1, value: 1, preservesFacade: true }, - }, - ], - ] + [], + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 5, preservesFacade: true }, + previousValue: { id: 1, value: 1, preservesFacade: true }, + }, + ], + ] const childPublicationsAfterRecovery: Array< Array > = [ From 8c7e453782e4e4c770ae2ea4f11b58483264a81b Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:02:16 -0600 Subject: [PATCH 288/327] chore(db): keep oracle formatting stable --- ...ncludes-collection-oracle.property.test.ts | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index b46be9c74..e90c8d55a 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -52,7 +52,9 @@ type FacadeCandidateScanScenario = { class ThrowingUpdateIndex extends BasicIndex { updateFailure: { error: unknown } | undefined - buildFailure: { error: unknown; stage: `before` | `after` } | undefined + buildFailure: + | { error: unknown; stage: `before` | `after` } + | undefined buildCalls = 0 override update(key: number, oldItem: unknown, newItem: unknown): void { @@ -1562,18 +1564,19 @@ describe(`Collection-valued includes oracle`, () => { { id: 10, value: 2 }, { id: 11, value: 20 }, ]) - const rootPublicationsAfterRecovery: Array> = + const rootPublicationsAfterRecovery: Array< + Array + > = [ + [], [ - [], - [ - { - type: `update`, - key: 1, - value: { id: 1, value: 5, preservesFacade: true }, - previousValue: { id: 1, value: 1, preservesFacade: true }, - }, - ], - ] + { + type: `update`, + key: 1, + value: { id: 1, value: 5, preservesFacade: true }, + previousValue: { id: 1, value: 1, preservesFacade: true }, + }, + ], + ] const childPublicationsAfterRecovery: Array< Array > = [ From 17a79e091e716a03e2754c4672ba6a5cc0b94c61 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:06:54 -0600 Subject: [PATCH 289/327] test(db): restore metadata transition axes --- ...tadata-publication-oracle.property.test.ts | 72 ++++++++++++++----- 1 file changed, 56 insertions(+), 16 deletions(-) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index abea7069c..f21c127fb 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -18,6 +18,8 @@ type SyncActions = Parameters[`sync`]>[0] type MetadataOperation = { type: `set`; value: unknown } | { type: `delete` } +type MetadataEntryState = { present: false } | { present: true; value: unknown } + type MetadataWrite = { key: number } & MetadataOperation type PublicationRound = { @@ -65,6 +67,14 @@ const metadataOperationArbitrary: fc.Arbitrary = fc.oneof( fc.constant({ type: `delete` as const }), ) +const metadataEntryStateArbitrary: fc.Arbitrary = fc.oneof( + fc.constant({ present: false as const }), + metadataValueArbitrary.map((value) => ({ + present: true as const, + value, + })), +) + const metadataWriteArbitrary = fc .tuple(fc.integer({ min: 0, max: 2 }), metadataOperationArbitrary) .map(([key, operation]) => ({ key, ...operation })) @@ -95,18 +105,16 @@ const metadataCancellationArbitrary = fc.record({ }), canceledOperation: metadataOperationArbitrary, retainedOperation: metadataOperationArbitrary, + initialMetadata: fc.array(metadataEntryStateArbitrary, { + minLength: 3, + maxLength: 3, + }), }) -const metadataRollbackCaseArbitrary = fc.oneof( - metadataValueArbitrary.map((value) => ({ - initialMetadata: { present: false as const }, - pendingOperation: { type: `set` as const, value }, - })), - metadataValueArbitrary.map((value) => ({ - initialMetadata: { present: true as const, value }, - pendingOperation: { type: `delete` as const }, - })), -) +const metadataRollbackCaseArbitrary = fc.record({ + initialMetadata: metadataEntryStateArbitrary, + pendingOperation: metadataOperationArbitrary, +}) const metadataRollbackArbitrary = fc .record({ @@ -346,13 +354,13 @@ async function expectMetadataCancellationOwnership( retainedKeys: ReadonlyArray, canceledOperation: MetadataOperation, retainedOperation: MetadataOperation, + initialMetadataState: ReadonlyArray, ): Promise { const harness = await createPublicationHarness() - const initialMetadata = new Map([ - [0, undefined], - [1, false], - [2, null], - ]) + const initialMetadata = new Map() + for (const [key, state] of initialMetadataState.entries()) { + if (state.present) initialMetadata.set(key, state.value) + } const initialSync = harness.getSync() initialSync.begin() for (const [key, value] of initialMetadata) { @@ -592,6 +600,21 @@ it(`releases only canceled metadata keys while another sync remains pending`, as [1, 2], { type: `delete` }, { type: `set`, value: false }, + [ + { present: true, value: undefined }, + { present: true, value: false }, + { present: true, value: null }, + ], + ) +}) + +it(`does not apply canceled metadata to an absent base key`, async () => { + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `set`, value: `canceled` }, + { type: `set`, value: `retained` }, + [{ present: false }, { present: false }, { present: false }], ) }) @@ -605,6 +628,16 @@ it(`restores pending metadata when a derived publication fails`, async () => { }) }) +it(`restores an existing metadata value after a failed replacement`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + initialMetadata: { present: true, value: `before` }, + pendingOperation: { type: `set`, value: `after` }, + }) +}) + fcTest.prop( [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], oraclePropertyOptions(50, `collection-publication.metadata-only`), @@ -618,12 +651,19 @@ fcTest.prop( oraclePropertyOptions(50, `collection-publication.metadata-cancellation`), )( `keeps metadata suppression owned by the remaining pending transactions`, - ({ canceledKeys, retainedKeys, canceledOperation, retainedOperation }) => + ({ + canceledKeys, + retainedKeys, + canceledOperation, + retainedOperation, + initialMetadata, + }) => expectMetadataCancellationOwnership( canceledKeys, retainedKeys, canceledOperation, retainedOperation, + initialMetadata, ), ) fcTest.prop( From 68c65963894f857922e570d7ec66763421c72451 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:09:51 -0600 Subject: [PATCH 290/327] test(db): complete metadata cancellation histories --- ...tadata-publication-oracle.property.test.ts | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index f21c127fb..4ad1d39cb 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -105,6 +105,7 @@ const metadataCancellationArbitrary = fc.record({ }), canceledOperation: metadataOperationArbitrary, retainedOperation: metadataOperationArbitrary, + canceledFirst: fc.boolean(), initialMetadata: fc.array(metadataEntryStateArbitrary, { minLength: 3, maxLength: 3, @@ -354,6 +355,7 @@ async function expectMetadataCancellationOwnership( retainedKeys: ReadonlyArray, canceledOperation: MetadataOperation, retainedOperation: MetadataOperation, + canceledFirst: boolean, initialMetadataState: ReadonlyArray, ): Promise { const harness = await createPublicationHarness() @@ -400,8 +402,14 @@ async function expectMetadataCancellationOwnership( return { receipt, transaction } } - const canceled = stageMetadata(canceledKeys, canceledOperation) - const retained = stageMetadata(retainedKeys, retainedOperation) + const first = canceledFirst + ? stageMetadata(canceledKeys, canceledOperation) + : stageMetadata(retainedKeys, retainedOperation) + const second = canceledFirst + ? stageMetadata(retainedKeys, retainedOperation) + : stageMetadata(canceledKeys, canceledOperation) + const canceled = canceledFirst ? first : second + const retained = canceledFirst ? second : first try { harness.rows._state.capturePreSyncVisibleState() @@ -421,6 +429,9 @@ async function expectMetadataCancellationOwnership( expect(harness.rows._state.pendingSyncedTransactions).toEqual([ retained.transaction, ]) + expect(retained.transaction.rowMetadataWrites).toEqual( + new Map(retainedKeys.map((key) => [key, retainedOperation])), + ) expect(harness.rows._state.recentlySyncedKeys).toEqual(expectedAfter) expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( expectedAfter, @@ -433,9 +444,24 @@ async function expectMetadataCancellationOwnership( await expect(canceled.receipt).rejects.toBeInstanceOf( SyncTransactionAbortedError, ) + + persistence.resolve() + await heldTransaction.isPersisted.promise + await expect(retained.receipt).resolves.toBeUndefined() + const expectedMetadata = new Map(initialMetadata) + for (const key of retainedKeys) { + if (retainedOperation.type === `set`) { + expectedMetadata.set(key, retainedOperation.value) + } else { + expectedMetadata.delete(key) + } + } + expect(harness.rows._state.syncedMetadata).toEqual(expectedMetadata) } finally { - harness.rows._state.cancelPendingSyncedTransaction(retained.transaction) - await retained.receipt.catch(() => undefined) + if (retained.transaction.applied.isPending()) { + harness.rows._state.cancelPendingSyncedTransaction(retained.transaction) + await retained.receipt.catch(() => undefined) + } persistence.resolve() await heldTransaction.isPersisted.promise.catch(() => undefined) harness.unsubscribe() @@ -600,6 +626,7 @@ it(`releases only canceled metadata keys while another sync remains pending`, as [1, 2], { type: `delete` }, { type: `set`, value: false }, + true, [ { present: true, value: undefined }, { present: true, value: false }, @@ -614,10 +641,26 @@ it(`does not apply canceled metadata to an absent base key`, async () => { [1, 2], { type: `set`, value: `canceled` }, { type: `set`, value: `retained` }, + true, [{ present: false }, { present: false }, { present: false }], ) }) +it(`settles an older metadata owner after canceling the newer owner`, async () => { + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `delete` }, + { type: `set`, value: `retained` }, + false, + [ + { present: true, value: undefined }, + { present: false }, + { present: true, value: false }, + ], + ) +}) + it(`restores pending metadata when a derived publication fails`, async () => { await expectMetadataRollbackRecovery({ sourceKey: 0, @@ -656,6 +699,7 @@ fcTest.prop( retainedKeys, canceledOperation, retainedOperation, + canceledFirst, initialMetadata, }) => expectMetadataCancellationOwnership( @@ -663,6 +707,7 @@ fcTest.prop( retainedKeys, canceledOperation, retainedOperation, + canceledFirst, initialMetadata, ), ) From 036b96724deea07374ef8e39d8f672388b2ef91c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:13:26 -0600 Subject: [PATCH 291/327] test(db): prove multi-key metadata rollback --- ...tadata-publication-oracle.property.test.ts | 81 +++++++++++++++---- 1 file changed, 67 insertions(+), 14 deletions(-) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 4ad1d39cb..5f5224d33 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -475,12 +475,18 @@ async function expectMetadataRollbackRecovery({ sourceDelta, initialMetadata, pendingOperation, + additionalMetadata = [], }: { sourceKey: number metadataKey: number sourceDelta: number - initialMetadata: { present: false } | { present: true; value: unknown } + initialMetadata: MetadataEntryState pendingOperation: MetadataOperation + additionalMetadata?: ReadonlyArray<{ + key: number + initialMetadata: MetadataEntryState + pendingOperation: MetadataOperation + }> }): Promise { const harnessId = nextMetadataRollbackHarnessId++ const source = await createPublicationHarness() @@ -496,7 +502,13 @@ async function expectMetadataRollbackRecovery({ }) await derived.preload() - const stageMetadata = (key: number, operation: MetadataOperation) => { + const metadataCases = [ + { key: metadataKey, initialMetadata, pendingOperation }, + ...additionalMetadata, + ] + const stageMetadata = ( + writes: ReadonlyArray<{ key: number; operation: MetadataOperation }>, + ) => { const applied = createDeferred() void applied.promise.catch(() => undefined) const transaction = { @@ -505,7 +517,9 @@ async function expectMetadataRollbackRecovery({ layoutChanged: false, operations: [], deletedKeys: new Set(), - rowMetadataWrites: new Map([[key, operation]]), + rowMetadataWrites: new Map( + writes.map(({ key, operation }) => [key, operation]), + ), collectionMetadataWrites: new Map(), applied, } @@ -513,15 +527,31 @@ async function expectMetadataRollbackRecovery({ return transaction } - if (initialMetadata.present) { - stageMetadata(metadataKey, { - type: `set`, - value: initialMetadata.value, - }) + const initialWrites = metadataCases.flatMap( + ({ key, initialMetadata: state }) => + state.present + ? [ + { + key, + operation: { + type: `set` as const, + value: state.value, + }, + }, + ] + : [], + ) + if (initialWrites.length > 0) { + stageMetadata(initialWrites) derived._state.commitPendingTransactions() } - const pending = stageMetadata(metadataKey, pendingOperation) + const pending = stageMetadata( + metadataCases.map(({ key, pendingOperation: operation }) => ({ + key, + operation, + })), + ) const sourceRowsBefore = [...rows.values()].map((row) => ({ ...row })) const rowsBefore = [...derived.values()].map((row) => ({ ...row })) const originBefore = new Map(derived._state.rowOrigins) @@ -551,7 +581,8 @@ async function expectMetadataRollbackRecovery({ try { const previousSourceRow = rows.get(sourceKey)! - expect(() => { + let thrown: unknown + try { getSync().begin() getSync().write({ type: `update`, @@ -561,7 +592,10 @@ async function expectMetadataRollbackRecovery({ }, }) getSync().commit() - }).toThrow(publicationFailure) + } catch (error) { + thrown = error + } + expect(thrown).toBe(publicationFailure) expect(rows.get(sourceKey)?.position).toBe( previousSourceRow.position + sourceDelta, @@ -575,9 +609,11 @@ async function expectMetadataRollbackRecovery({ ) expect([...derived.values()].map((row) => ({ ...row }))).toEqual(rowsBefore) expect(derived._state.syncedMetadata).toEqual( - initialMetadata.present - ? new Map([[metadataKey, initialMetadata.value]]) - : new Map(), + new Map( + metadataCases.flatMap(({ key, initialMetadata: state }) => + state.present ? [[key, state.value]] : [], + ), + ), ) expect(derived._state.pendingSyncedTransactions).toHaveLength(1) expect(derived._state.pendingSyncedTransactions[0]).toBe(pending) @@ -681,6 +717,23 @@ it(`restores an existing metadata value after a failed replacement`, async () => }) }) +it(`restores every metadata key after one failed publication`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + initialMetadata: { present: true, value: `before` }, + pendingOperation: { type: `set`, value: `after` }, + additionalMetadata: [ + { + key: 2, + initialMetadata: { present: true, value: false }, + pendingOperation: { type: `delete` }, + }, + ], + }) +}) + fcTest.prop( [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], oraclePropertyOptions(50, `collection-publication.metadata-only`), From 30db62b95f69d3da9d8c8164d19d351623a98a6e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:17:28 -0600 Subject: [PATCH 292/327] test(db): isolate metadata publication affectedness --- ...tadata-publication-oracle.property.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 5f5224d33..f30a7a322 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -656,6 +656,37 @@ it(`publishes one event per key when metadata-only sync retires optimistic work` ]) }) +it(`includes metadata-only keys in a publication snapshot`, async () => { + const harness = await createPublicationHarness() + const applied = createDeferred() + void applied.promise.catch(() => undefined) + const transaction = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map([[1, { type: `set` as const, value: false }]]), + collectionMetadataWrites: new Map(), + applied, + } + harness.rows._state.pendingSyncedTransactions.push(transaction) + + try { + const snapshot = harness.rows._state.snapshotPublicationState([]) + expect([...snapshot.keys.keys()]).toEqual([1]) + expect(snapshot.keys.get(1)?.syncedMetadata).toEqual({ + present: false, + value: undefined, + }) + expect(snapshot.pendingSyncedTransactions).toEqual([transaction]) + } finally { + harness.rows._state.cancelPendingSyncedTransaction(transaction) + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +}) + it(`releases only canceled metadata keys while another sync remains pending`, async () => { await expectMetadataCancellationOwnership( [0, 1], From ddcf839747ded246f2c053ceffb5c6709d904cfe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:22:43 -0600 Subject: [PATCH 293/327] test(db): preserve metadata rollback order --- ...tadata-publication-oracle.property.test.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index f30a7a322..009ef110b 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -476,6 +476,7 @@ async function expectMetadataRollbackRecovery({ initialMetadata, pendingOperation, additionalMetadata = [], + separatePendingTransactions = false, }: { sourceKey: number metadataKey: number @@ -487,6 +488,7 @@ async function expectMetadataRollbackRecovery({ initialMetadata: MetadataEntryState pendingOperation: MetadataOperation }> + separatePendingTransactions?: boolean }): Promise { const harnessId = nextMetadataRollbackHarnessId++ const source = await createPublicationHarness() @@ -546,12 +548,12 @@ async function expectMetadataRollbackRecovery({ derived._state.commitPendingTransactions() } - const pending = stageMetadata( - metadataCases.map(({ key, pendingOperation: operation }) => ({ - key, - operation, - })), + const pendingWrites = metadataCases.map( + ({ key, pendingOperation: operation }) => ({ key, operation }), ) + const pendingTransactions = separatePendingTransactions + ? pendingWrites.map((write) => stageMetadata([write])) + : [stageMetadata(pendingWrites)] const sourceRowsBefore = [...rows.values()].map((row) => ({ ...row })) const rowsBefore = [...derived.values()].map((row) => ({ ...row })) const originBefore = new Map(derived._state.rowOrigins) @@ -615,9 +617,13 @@ async function expectMetadataRollbackRecovery({ ), ), ) - expect(derived._state.pendingSyncedTransactions).toHaveLength(1) - expect(derived._state.pendingSyncedTransactions[0]).toBe(pending) - expect(pending.applicationStarted).toBe(false) + expect(derived._state.pendingSyncedTransactions).toEqual( + pendingTransactions, + ) + for (const pending of pendingTransactions) { + expect(pending.applicationStarted).toBe(false) + expect(pending.applied.isPending()).toBe(true) + } expect(derived._state.rowOrigins).toEqual(originBefore) expect(derived._state.hydrationSeedKeys).toEqual(hydrationSeedsBefore) expect(derived._state.hydratedKeys).toEqual(hydratedBefore) @@ -628,7 +634,9 @@ async function expectMetadataRollbackRecovery({ expect(published).toEqual([]) } finally { derived._state.commitPendingTransactions = commitPendingTransactions - derived._state.cancelPendingSyncedTransaction(pending) + for (const pending of pendingTransactions) { + derived._state.cancelPendingSyncedTransaction(pending) + } subscription.unsubscribe() source.unsubscribe() await Promise.all([ @@ -755,6 +763,7 @@ it(`restores every metadata key after one failed publication`, async () => { sourceDelta: 1, initialMetadata: { present: true, value: `before` }, pendingOperation: { type: `set`, value: `after` }, + separatePendingTransactions: true, additionalMetadata: [ { key: 2, From f0144f94a2dcc4604dddecd83868ca37616c4b24 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:28:55 -0600 Subject: [PATCH 294/327] test(db): order restarted sync receipts --- ...on-state-retention-oracle.property.test.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 4b6da233c..8b35e35cb 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -68,7 +68,8 @@ const retentionActionArbitrary: fc.Arbitrary = fc.oneof( }, { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, { - weight: 1, + // Keep each phase at least as likely as the original unsplit restart arm. + weight: 3, arbitrary: fc .tuple( retainedRowArbitrary, @@ -195,7 +196,11 @@ async function runRetentionHistory( let restarted = false let restartedSync: SyncActions | undefined let restartedReceipt: true | Promise | undefined + let restartedReceiptOutcome: Promise | undefined let restartedReceiptSettled = false + const settlementTimeline: Array< + `checkpoint` | `publication` | `receipt` + > = [] const batches: Array<{ changes: Array<{ type: string @@ -224,6 +229,9 @@ async function runRetentionHistory( .map(({ id, value }) => ({ id, value })) .sort((left, right) => left.id - right.id), }) + if (changes.some(({ key }) => key === restartedRow.id)) { + queueMicrotask(() => settlementTimeline.push(`publication`)) + } if (restarted) return restarted = true cleanup = collection.cleanup() @@ -234,10 +242,13 @@ async function runRetentionHistory( if (action.commitPhase === `insideListener`) { restartedReceipt = restartedSync.commit() if (restartedReceipt !== true) { - void restartedReceipt.then(() => { + restartedReceiptOutcome = restartedReceipt.then((value) => { + settlementTimeline.push(`receipt`) restartedReceiptSettled = true + return value }) } + queueMicrotask(() => settlementTimeline.push(`checkpoint`)) } else { collection._state.preSyncVisibleState.set(-1, retainedMarker) collection._state.recentlySyncedKeys.add(restartedRow.id) @@ -257,12 +268,20 @@ async function runRetentionHistory( if (action.commitPhase === `insideListener`) { expect(restartedReceipt).toBeDefined() expect(restartedReceipt).not.toBe(true) + expect(restartedReceipt).toBeInstanceOf(Promise) expect(restartedReceiptSettled).toBe(false) + expect(settlementTimeline).toEqual([]) if (restartedReceipt === undefined || restartedReceipt === true) { throw new Error(`restarted sync receipt was not parked`) } - await restartedReceipt + expect(restartedReceiptOutcome).toBeDefined() + await expect(restartedReceiptOutcome).resolves.toBeUndefined() expect(restartedReceiptSettled).toBe(true) + expect(settlementTimeline).toEqual([ + `checkpoint`, + `publication`, + `receipt`, + ]) } else { expect(collection._state.preSyncVisibleState).toEqual( new Map([[-1, retainedMarker]]), From 0f86d22114b4e52c4b2e51c51c5f5d2c0a380f3f Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:39:39 -0600 Subject: [PATCH 295/327] test(db): cross restart trigger shapes --- ...on-state-retention-oracle.property.test.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 8b35e35cb..de1e63ef2 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -250,6 +250,9 @@ async function runRetentionHistory( } queueMicrotask(() => settlementTimeline.push(`checkpoint`)) } else { + // Synthetic generation canary: seed restarted-session + // publication state so the old publication tail cannot clear it. + // The batch assertions below exercise the public restart path. collection._state.preSyncVisibleState.set(-1, retainedMarker) collection._state.recentlySyncedKeys.add(restartedRow.id) } @@ -371,11 +374,19 @@ it(`retains a missing row introduced by a sync update`, async () => { await runRetentionHistory([{ type: `update`, row: { id: 1, value: 1 } }]) }) -it.each([`insideListener`, `afterOldReturn`] as const)( - `retains a restarted row committed %s`, - async (commitPhase) => { +it.each( + ([`insert`, `update`] as const).flatMap((triggerType) => + ([`insideListener`, `afterOldReturn`] as const).map( + (commitPhase) => [triggerType, commitPhase] as const, + ), + ), +)( + `retains an old-session %s and a restarted row committed %s`, + async (triggerType, commitPhase) => { await runRetentionHistory([ - { type: `insert`, row: { id: 1, value: 1 } }, + ...(triggerType === `update` + ? ([{ type: `insert`, row: { id: 1, value: 1 } }] as const) + : []), { type: `reentrantRestart`, row: { id: 1, value: 1 }, From dbed47e375d552375b448ec3c423e4fa7f22eb97 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:51:01 -0600 Subject: [PATCH 296/327] fix(db): keep repeated rollback inert --- packages/db/src/query/live/ARCHITECTURE.md | 3 +- packages/db/src/transactions.ts | 1 + packages/db/tests/transactions.test.ts | 35 ++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 92183762d..50bbcc0e5 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -969,7 +969,8 @@ priority merely to make a subset load settle. A rollback that wins while an optimistic transaction's `mutationFn` is still in flight is terminal. A later resolve or rejection from that function cannot change its outcome, run rollback again, affect newer transactions, or republish -its overlay. +its overlay. Repeating rollback on that failed transaction is inert and cannot +cascade into transactions created after the first rollback. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index 13c2d2abc..194af4f43 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -535,6 +535,7 @@ class Transaction> { if (this.state === `completed`) { throw new TransactionAlreadyCompletedRollbackError() } + if (this.state === `failed`) return this this.setState(`failed`) diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index 0d519ea01..34d9d69c6 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -338,6 +338,41 @@ describe(`Transactions`, () => { await collection.cleanup() } }) + it(`keeps repeated rollback from affecting newer transactions`, async () => { + type Row = { id: number; owner: string } + const collection = createCollection({ + id: `repeated-rollback-is-terminal`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const first = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + const second = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + + try { + void first.isPersisted.promise.catch(() => undefined) + first.mutate(() => collection.insert({ id: 1, owner: `first` })) + first.rollback() + + second.mutate(() => collection.insert({ id: 1, owner: `second` })) + expect(second.state).toBe(`pending`) + + expect(first.rollback()).toBe(first) + expect(first.state).toBe(`failed`) + expect(second.state).toBe(`pending`) + expect(collection.get(1)).toMatchObject({ id: 1, owner: `second` }) + } finally { + if (second.state === `pending`) { + second.rollback({ isSecondaryRollback: true }) + } + await collection.cleanup() + } + }) it(`should rollback if the mutationFn throws an error`, async () => { const transaction = createTransaction({ mutationFn: async () => { From 3b1bda2c489cd7cfccb5237817d8973a4142fe75 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 20:53:34 -0600 Subject: [PATCH 297/327] test(db): vary late rejection values --- packages/db/tests/transactions.test.ts | 166 +++++++++++++------------ 1 file changed, 88 insertions(+), 78 deletions(-) diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index 34d9d69c6..d69932dc6 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -256,88 +256,98 @@ describe(`Transactions`, () => { await collection.cleanup() } }) - it(`ignores a late persistence rejection after rollback wins`, async () => { - type Row = { id: number; owner: string } - let rejectPersistence!: (reason: unknown) => void - const persistence = new Promise((_resolve, reject) => { - rejectPersistence = reject - }) - const collection = createCollection({ - id: `late-persistence-rejection`, - getKey: (item) => item.id, - sync: { sync: () => {} }, - }) - const batches: Array> = [] - const subscription = collection.subscribeChanges( - (changes) => { - batches.push( - changes.map(({ type, key }) => ({ - type, - key, - })), - ) - }, - { includeInitialState: false }, - ) - const first = createTransaction({ - autoCommit: false, - mutationFn: () => persistence, - }) - const second = createTransaction({ - autoCommit: false, - mutationFn: async () => {}, - }) - - try { - const persisted = first.isPersisted.promise.then( - (value) => ({ status: `fulfilled` as const, value }), - (reason: unknown) => ({ status: `rejected` as const, reason }), - ) - first.mutate(() => collection.insert({ id: 1, owner: `first` })) - const commit = first.commit().then( - (value) => ({ status: `fulfilled` as const, value }), - (reason: unknown) => ({ status: `rejected` as const, reason }), + it.each([ + [`Error`, () => new Error(`late persistence rejection`)], + [`undefined`, () => undefined], + [`false`, () => false], + [`zero`, () => 0], + [`NaN`, () => Number.NaN], + [`string`, () => `late persistence rejection`], + [`object`, () => ({ late: true })], + ] as const)( + `ignores a late %s persistence rejection after rollback wins`, + async (reasonName, createReason) => { + type Row = { id: number; owner: string } + let rejectPersistence!: (reason: unknown) => void + const persistence = new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + const collection = createCollection({ + id: `late-persistence-rejection-${reasonName}`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const batches: Array> = [] + const subscription = collection.subscribeChanges( + (changes) => { + batches.push( + changes.map(({ type, key }) => ({ + type, + key, + })), + ) + }, + { includeInitialState: false }, ) - - first.rollback() - second.mutate(() => collection.insert({ id: 1, owner: `second` })) - const lateError = new Error(`late persistence rejection`) - rejectPersistence(lateError) - - const commitOutcome = await commit - expect(commitOutcome.status).toBe(`fulfilled`) - if (commitOutcome.status === `fulfilled`) { - expect(commitOutcome.value).toBe(first) - } - expect(await persisted).toEqual({ - status: `rejected`, - reason: undefined, + const first = createTransaction({ + autoCommit: false, + mutationFn: () => persistence, }) - expect(first.state).toBe(`failed`) - expect(first.error).toBeUndefined() - expect(second.state).toBe(`pending`) - expect(collection.get(1)).toEqual({ - id: 1, - owner: `second`, - $collectionId: collection.id, - $key: 1, - $origin: `local`, - $synced: false, + const second = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, }) - expect(batches).toEqual([ - [{ type: `insert`, key: 1 }], - [{ type: `delete`, key: 1 }], - [{ type: `insert`, key: 1 }], - ]) - } finally { - rejectPersistence(new Error(`test cleanup`)) - if (second.state === `pending`) { - second.rollback({ isSecondaryRollback: true }) + + try { + const persisted = first.isPersisted.promise.then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + first.mutate(() => collection.insert({ id: 1, owner: `first` })) + const commit = first.commit().then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + first.rollback() + second.mutate(() => collection.insert({ id: 1, owner: `second` })) + rejectPersistence(createReason()) + + const commitOutcome = await commit + expect(commitOutcome.status).toBe(`fulfilled`) + if (commitOutcome.status === `fulfilled`) { + expect(commitOutcome.value).toBe(first) + } + expect(await persisted).toEqual({ + status: `rejected`, + reason: undefined, + }) + expect(first.state).toBe(`failed`) + expect(first.error).toBeUndefined() + expect(second.state).toBe(`pending`) + expect(collection.get(1)).toEqual({ + id: 1, + owner: `second`, + $collectionId: collection.id, + $key: 1, + $origin: `local`, + $synced: false, + }) + expect(batches).toEqual([ + [{ type: `insert`, key: 1 }], + [{ type: `delete`, key: 1 }], + [{ type: `insert`, key: 1 }], + ]) + } finally { + rejectPersistence(new Error(`test cleanup`)) + if (second.state === `pending`) { + second.rollback({ isSecondaryRollback: true }) + } + subscription.unsubscribe() + await collection.cleanup() } - subscription.unsubscribe() - await collection.cleanup() - } - }) + }, + ) it(`keeps repeated rollback from affecting newer transactions`, async () => { type Row = { id: number; owner: string } const collection = createCollection({ From f12dfbfb9528692d4fec6aa9128615b88c286369 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:03:01 -0600 Subject: [PATCH 298/327] test(db): restore virtual state on rollback --- .../query/includes-collection-oracle.property.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index e90c8d55a..8f11a3a21 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1152,6 +1152,10 @@ describe(`Collection-valued includes oracle`, () => { const recentlySyncedBeforeFailure = new Set( facade._state.recentlySyncedKeys, ) + const preSyncVirtualBeforeFailure = new Map( + facade._state.preSyncVirtualState, + ) + expect([...preSyncVirtualBeforeFailure.keys()]).toEqual([initialChild.id]) const rootPublications: Array = [] const childPublications: Array = [] const childReceiptStates: Array = [] @@ -1235,10 +1239,16 @@ describe(`Collection-valued includes oracle`, () => { expect(facade._state.recentlySyncedKeys).toEqual( recentlySyncedBeforeFailure, ) + expect(facade._state.preSyncVirtualState).toEqual( + preSyncVirtualBeforeFailure, + ) await Promise.resolve() expect(facade._state.recentlySyncedKeys).toEqual( recentlySyncedBeforeFailure, ) + expect(facade._state.preSyncVirtualState).toEqual( + preSyncVirtualBeforeFailure, + ) rootIndex.updateFailure = undefined // Only the root changes on retry. The child deltas consumed by the From b9d9f1cd7430f16e48eddd4015490d180dffb19a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:06:08 -0600 Subject: [PATCH 299/327] test(db): preserve canceled virtual snapshots --- ...tadata-publication-oracle.property.test.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts index 009ef110b..0586c142c 100644 --- a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -410,6 +410,18 @@ async function expectMetadataCancellationOwnership( : stageMetadata(canceledKeys, canceledOperation) const canceled = canceledFirst ? first : second const retained = canceledFirst ? second : first + const expectedVirtualSnapshots = (keys: ReadonlyArray) => + new Map( + [...new Set(keys)].map((key) => [ + key, + { + $collectionId: harness.rows.id, + $key: key, + $origin: `remote`, + $synced: true, + }, + ]), + ) try { harness.rows._state.capturePreSyncVisibleState() @@ -418,8 +430,8 @@ async function expectMetadataCancellationOwnership( expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( expectedBefore, ) - expect(new Set(harness.rows._state.preSyncVirtualState.keys())).toEqual( - expectedBefore, + expect(harness.rows._state.preSyncVirtualState).toEqual( + expectedVirtualSnapshots([...canceledKeys, ...retainedKeys]), ) const batchCountBefore = harness.batches.length @@ -436,8 +448,8 @@ async function expectMetadataCancellationOwnership( expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( expectedAfter, ) - expect(new Set(harness.rows._state.preSyncVirtualState.keys())).toEqual( - expectedAfter, + expect(harness.rows._state.preSyncVirtualState).toEqual( + expectedVirtualSnapshots(retainedKeys), ) expect(harness.batches).toHaveLength(batchCountBefore) expect(harness.rows._state.syncedMetadata).toEqual(initialMetadata) @@ -448,6 +460,9 @@ async function expectMetadataCancellationOwnership( persistence.resolve() await heldTransaction.isPersisted.promise await expect(retained.receipt).resolves.toBeUndefined() + expect(harness.rows._state.preSyncVisibleState.size).toBe(0) + expect(harness.rows._state.preSyncVirtualState.size).toBe(0) + expect(harness.rows._state.recentlySyncedKeys.size).toBe(0) const expectedMetadata = new Map(initialMetadata) for (const key of retainedKeys) { if (retainedOperation.type === `set`) { From 85005ab4413780fdc93cd9730c3ad01cb195b2b2 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:14:49 -0600 Subject: [PATCH 300/327] test(db): verify parked receipt outcomes --- .../collection-state-retention-oracle.property.test.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index de1e63ef2..eca8c100c 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -723,6 +723,7 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme if (syncReceipt === undefined || syncReceipt === true) { throw new Error(`restarted sync receipt was not parked`) } + expect(syncReceipt).toBeInstanceOf(Promise) expect(syncReceiptOutcome).toBeDefined() expect(rollbackMutation).toBeDefined() await Promise.resolve() @@ -736,13 +737,17 @@ it(`publishes a virtual-state update when a restarted optimistic row is confirme expect(syncReceiptSettled).toBe(true) expect(settlementTimeline).toEqual([`publication`, `receipt`]) expect(publications).toEqual(expectedPublications) - expect([...collection.state.keys()]).toEqual([2]) + expect([...collection.state.values()].map(snapshotRow)).toEqual([ + remoteRow(2), + ]) releaseMutation() await mutationCommit expect(readMutationState?.()).toBe(`failed`) expect(publications).toEqual(expectedPublications) - expect([...collection.state.keys()]).toEqual([2]) + expect([...collection.state.values()].map(snapshotRow)).toEqual([ + remoteRow(2), + ]) } finally { releaseMutation() await mutationCommit From 2b196be4827c304ef3510ac9627fdd5828eb42fe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:28:39 -0600 Subject: [PATCH 301/327] test(db): observe falsy publication delivery --- packages/db/tests/query/scheduler.test.ts | 74 +++++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 1b709ff90..bf68144b1 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -465,6 +465,8 @@ describe(`live query scheduler`, () => { { name: `null`, failure: null }, { name: `false`, failure: false }, { name: `zero`, failure: 0 }, + { name: `negative zero`, failure: -0 }, + { name: `bigint zero`, failure: 0n }, { name: `empty string`, failure: `` }, { name: `NaN`, failure: Number.NaN }, ])( @@ -473,7 +475,21 @@ describe(`live query scheduler`, () => { let begin!: () => void let write!: (message: { type: `insert`; value: User }) => void let commit!: () => void - const laterListener = vi.fn() + type UserObservation = { + changes: Array<{ + type: string + key: string | number + value: User + previousValue: User | undefined + }> + rows: Array + } + const sourceObservations: Array = [] + const dependentObservations: Array = [] + const snapshotUser = ({ id, name: userName }: User): User => ({ + id, + name: userName, + }) const source = createCollection({ id: `falsy-row-listener-${name.replaceAll(` `, `-`)}`, getKey: (user) => user.id, @@ -495,9 +511,23 @@ describe(`live query scheduler`, () => { }, { includeInitialState: false }, ) - const laterSubscription = source.subscribeChanges(laterListener, { - includeInitialState: false, - }) + const laterSubscription = source.subscribeChanges( + (changes) => { + sourceObservations.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotUser(value), + previousValue: + previousValue === undefined + ? undefined + : snapshotUser(previousValue), + })), + rows: [...source.state.values()].map(snapshotUser), + }) + }, + { includeInitialState: false }, + ) const live = createLiveQueryCollection({ id: `falsy-row-listener-dependent-${name.replaceAll(` `, `-`)}`, startSync: true, @@ -506,9 +536,29 @@ describe(`live query scheduler`, () => { .from({ user: source }) .select(({ user }) => ({ id: user.id, name: user.name })), }) + let dependentSubscription: + | ReturnType + | undefined try { await live.preload() + dependentSubscription = live.subscribeChanges( + (changes) => { + dependentObservations.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotUser(value), + previousValue: + previousValue === undefined + ? undefined + : snapshotUser(previousValue), + })), + rows: [...live.state.values()].map(snapshotUser), + }) + }, + { includeInitialState: false }, + ) begin() write({ type: `insert`, value: { id: 1, name: `Ada` } }) let didThrow = false @@ -522,11 +572,23 @@ describe(`live query scheduler`, () => { expect(didThrow).toBe(true) expect(Object.is(thrown, failure)).toBe(true) - expect(laterListener).toHaveBeenCalledOnce() - expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + const expectedObservation: UserObservation = { + changes: [ + { + type: `insert`, + key: 1, + value: { id: 1, name: `Ada` }, + previousValue: undefined, + }, + ], + rows: [{ id: 1, name: `Ada` }], + } + expect(sourceObservations).toEqual([expectedObservation]) + expect(dependentObservations).toEqual([expectedObservation]) } finally { throwingSubscription.unsubscribe() laterSubscription.unsubscribe() + dependentSubscription?.unsubscribe() await live.cleanup() await source.cleanup() } From c05d22b8573c7b7b9c521535bc5dc24a9410325a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:41:19 -0600 Subject: [PATCH 302/327] test(db): preserve publication result state --- packages/db/tests/query/scheduler.test.ts | 57 ++++++++++++++++------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index bf68144b1..fc07a3159 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -479,16 +479,27 @@ describe(`live query scheduler`, () => { changes: Array<{ type: string key: string | number - value: User - previousValue: User | undefined + value: UserWithVirtual + previousValue: UserWithVirtual | undefined }> - rows: Array + rows: Array } const sourceObservations: Array = [] const dependentObservations: Array = [] - const snapshotUser = ({ id, name: userName }: User): User => ({ + const snapshotUser = ({ id, name: userName, + $collectionId, + $key, + $origin, + $synced, + }: UserWithVirtual): UserWithVirtual => ({ + id, + name: userName, + $collectionId, + $key, + $origin, + $synced, }) const source = createCollection({ id: `falsy-row-listener-${name.replaceAll(` `, `-`)}`, @@ -572,19 +583,33 @@ describe(`live query scheduler`, () => { expect(didThrow).toBe(true) expect(Object.is(thrown, failure)).toBe(true) - const expectedObservation: UserObservation = { - changes: [ - { - type: `insert`, - key: 1, - value: { id: 1, name: `Ada` }, - previousValue: undefined, - }, - ], - rows: [{ id: 1, name: `Ada` }], + const expectedObservation = (collectionId: string): UserObservation => { + const row: UserWithVirtual = { + id: 1, + name: `Ada`, + $collectionId: collectionId, + $key: 1, + $origin: `remote`, + $synced: true, + } + return { + changes: [ + { + type: `insert`, + key: 1, + value: row, + previousValue: undefined, + }, + ], + rows: [row], + } } - expect(sourceObservations).toEqual([expectedObservation]) - expect(dependentObservations).toEqual([expectedObservation]) + const expectedDependent = expectedObservation(live.id) + expect(sourceObservations).toEqual([expectedObservation(source.id)]) + expect(dependentObservations).toEqual([expectedDependent]) + expect([...live.state.values()].map(snapshotUser)).toEqual( + expectedDependent.rows, + ) } finally { throwingSubscription.unsubscribe() laterSubscription.unsubscribe() From 36caec93f35a038ba44e28e81c26036ac1e4f6f3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:48:59 -0600 Subject: [PATCH 303/327] test(db): vary filtered listener failures --- packages/db/tests/query/scheduler.test.ts | 169 ++++++++++++---------- 1 file changed, 94 insertions(+), 75 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index fc07a3159..e3b2da895 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -26,6 +26,17 @@ interface User { name: string } +const falsyListenerFailureCases = [ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `negative zero`, failure: -0 }, + { name: `bigint zero`, failure: 0n }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, +] + type UserWithVirtual = OutputWithVirtual interface Task { @@ -460,16 +471,7 @@ describe(`live query scheduler`, () => { } }) - it.each([ - { name: `undefined`, failure: undefined }, - { name: `null`, failure: null }, - { name: `false`, failure: false }, - { name: `zero`, failure: 0 }, - { name: `negative zero`, failure: -0 }, - { name: `bigint zero`, failure: 0n }, - { name: `empty string`, failure: `` }, - { name: `NaN`, failure: Number.NaN }, - ])( + it.each(falsyListenerFailureCases)( `preserves an exact $name row-listener failure after later delivery`, async ({ name, failure }) => { let begin!: () => void @@ -620,75 +622,92 @@ describe(`live query scheduler`, () => { }, ) - it(`preserves a filtered row-listener failure after later delivery`, async () => { - let begin!: () => void - let write!: (message: { type: `insert`; value: User }) => void - let commit!: () => void - const failure = new Error(`filtered source listener failed`) - const filteredCalls = vi.fn() - const laterListener = vi.fn() - const source = createCollection({ - id: `filtered-throwing-listener-source`, - getKey: (user) => user.id, - startSync: true, - sync: { - sync: (actions) => { - begin = actions.begin - write = actions.write - commit = () => { - actions.commit() - } - actions.markReady() + it.each([ + { + name: `Error`, + failure: new Error(`filtered source listener failed`), + }, + ...falsyListenerFailureCases, + ])( + `preserves an exact $name filtered row-listener failure`, + async ({ name, failure }) => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const filteredCalls = vi.fn() + const laterListener = vi.fn() + const source = createCollection({ + id: `filtered-throwing-listener-source-${name.replaceAll(` `, `-`)}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, }, - }, - }) - const throwingSubscription = source.subscribeChanges( - (changes) => { - filteredCalls(changes) - throw failure - }, - { + }) + const throwingSubscription = source.subscribeChanges( + (changes) => { + filteredCalls(changes) + throw failure + }, + { + includeInitialState: false, + where: (user) => eq(user.name, `Ada`), + }, + ) + const laterSubscription = source.subscribeChanges(laterListener, { includeInitialState: false, - where: (user) => eq(user.name, `Ada`), - }, - ) - const laterSubscription = source.subscribeChanges(laterListener, { - includeInitialState: false, - }) - const live = createLiveQueryCollection({ - id: `filtered-throwing-listener-dependent`, - startSync: true, - query: (q) => - q - .from({ user: source }) - .select(({ user }) => ({ id: user.id, name: user.name })), - }) + }) + const live = createLiveQueryCollection({ + id: `filtered-throwing-listener-dependent-${name.replaceAll(` `, `-`)}`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) - try { - await live.preload() - begin() - write({ type: `insert`, value: { id: 1, name: `Ada` } }) - expect(() => commit()).toThrow(failure) - expect(filteredCalls).toHaveBeenCalledOnce() - expect(filteredCalls.mock.calls[0]?.[0]).toEqual([ - expect.objectContaining({ type: `insert`, key: 1 }), - ]) - expect(laterListener).toHaveBeenCalledOnce() - expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + let didThrow = false + let thrown: unknown + try { + commit() + } catch (error) { + didThrow = true + thrown = error + } + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + expect(filteredCalls).toHaveBeenCalledOnce() + expect(filteredCalls.mock.calls[0]?.[0]).toEqual([ + expect.objectContaining({ type: `insert`, key: 1 }), + ]) + expect(laterListener).toHaveBeenCalledOnce() + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) - begin() - write({ type: `insert`, value: { id: 2, name: `Grace` } }) - expect(() => commit()).not.toThrow() - expect(filteredCalls).toHaveBeenCalledOnce() - expect(laterListener).toHaveBeenCalledTimes(2) - expect(live.get(2)).toEqual(expect.objectContaining({ name: `Grace` })) - } finally { - throwingSubscription.unsubscribe() - laterSubscription.unsubscribe() - await live.cleanup() - await source.cleanup() - } - }) + begin() + write({ type: `insert`, value: { id: 2, name: `Grace` } }) + expect(() => commit()).not.toThrow() + expect(filteredCalls).toHaveBeenCalledOnce() + expect(laterListener).toHaveBeenCalledTimes(2) + expect(live.get(2)).toEqual(expect.objectContaining({ name: `Grace` })) + } finally { + throwingSubscription.unsubscribe() + laterSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }, + ) it(`keeps a nested ready failure when a later outer listener throws`, async () => { let markInnerReady!: () => void From 2b99270ba2376c771a1d6a6d43d115e445cdbc22 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 21:56:28 -0600 Subject: [PATCH 304/327] test(db): preserve falsy failures through cleanup --- packages/db/tests/query/scheduler.test.ts | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index e3b2da895..f759bb1fb 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -214,10 +214,16 @@ describe(`Collection publication scheduler context`, () => { removeAdded?.() }) - it.each([`publication`, `graph`] as const)( - `does not replace a $source failure with a clear-listener failure`, - (source) => { - const primaryFailure = new Error(`${source} failed`) + it.each([ + { source: `publication`, failureKind: `Error` }, + { source: `publication`, failureKind: `undefined` }, + { source: `graph`, failureKind: `Error` }, + { source: `graph`, failureKind: `undefined` }, + ] as const)( + `does not replace a $failureKind $source failure with a clear-listener failure`, + ({ source, failureKind }) => { + const primaryFailure = + failureKind === `Error` ? new Error(`${source} failed`) : undefined const clearFailure = new Error(`clear listener failed`) const laterClear = vi.fn() const removeThrowingClear = transactionScopedScheduler.onClear(() => { @@ -226,6 +232,7 @@ describe(`Collection publication scheduler context`, () => { const removeLaterClear = transactionScopedScheduler.onClear(laterClear) try { + let didThrow = false let thrown: unknown try { withPublicationContext(() => { @@ -240,10 +247,12 @@ describe(`Collection publication scheduler context`, () => { }) }) } catch (error) { + didThrow = true thrown = error } - expect(thrown).toBe(primaryFailure) + expect(didThrow).toBe(true) + expect(Object.is(thrown, primaryFailure)).toBe(true) expect(laterClear).toHaveBeenCalledOnce() } finally { removeThrowingClear() From d1a30b66e8e74e404187f32deda64ae3b71436f4 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 22:02:12 -0600 Subject: [PATCH 305/327] test(db): keep first-ready callbacks one-shot --- packages/db/tests/collection-lifecycle.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index bd9cba15b..08878a2ba 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -549,6 +549,7 @@ describe(`Collection Lifecycle Management`, () => { } finally { subscription.unsubscribe() await collection.cleanup() + expect(calls).toEqual([`first`, `nested`, `later`, `after`]) } }) From 73ee2454c0e3a4cb531477c9cd0b405e038e4763 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 22:13:51 -0600 Subject: [PATCH 306/327] test(db): observe mark-ready transitions --- .../db/tests/collection-lifecycle.test.ts | 77 +++++++++++++++---- 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 08878a2ba..90a688459 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -558,48 +558,38 @@ describe(`Collection Lifecycle Management`, () => { from: `ready`, expectedStatus: `ready`, expectedFirstReadyCalls: 1, - expectedDependentReadyEvents: 1, invalid: false, }, { from: `error`, expectedStatus: `ready`, expectedFirstReadyCalls: 1, - expectedDependentReadyEvents: 2, invalid: false, }, { from: `idle`, expectedStatus: `idle`, expectedFirstReadyCalls: 0, - expectedDependentReadyEvents: 0, invalid: true, }, { from: `cleaned-up`, expectedStatus: `cleaned-up`, expectedFirstReadyCalls: 0, - expectedDependentReadyEvents: 0, invalid: true, }, ] as const)( `defines the $from -> ready transition`, - async ({ - from, - expectedStatus, - expectedFirstReadyCalls, - expectedDependentReadyEvents, - invalid, - }) => { + async ({ from, expectedStatus, expectedFirstReadyCalls, invalid }) => { const syncFailure = new Error(`sync failed before recovery`) let firstReadyCalls = 0 + let recoveryFirstReadyCalls = 0 const collection = createCollection<{ id: string; name: string }>({ id: `mark-ready-from-${from}`, getKey: (item) => item.id, startSync: false, sync: { sync: () => {} }, }) - const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) collection.onFirstReady(() => { firstReadyCalls++ }) @@ -616,6 +606,47 @@ describe(`Collection Lifecycle Management`, () => { } expect(collection.status).toBe(from) + if (from === `error`) { + collection.onFirstReady(() => { + recoveryFirstReadyCalls++ + }) + expect(recoveryFirstReadyCalls).toBe(1) + } + + const transitionTrace: Array< + | { + kind: `status` + previousStatus: string + status: string + syncError: unknown + } + | { + kind: `dependent-ready` + status: string + syncError: unknown + } + > = [] + collection.on(`status:change`, ({ previousStatus, status }) => { + transitionTrace.push({ + kind: `status`, + previousStatus, + status, + syncError: collection._lifecycle.getSyncError(), + }) + }) + const originalEmitEmptyReadyEvent = + collection._changes.emitEmptyReadyEvent.bind(collection._changes) + vi.spyOn(collection._changes, `emitEmptyReadyEvent`).mockImplementation( + () => { + transitionTrace.push({ + kind: `dependent-ready`, + status: collection.status, + syncError: collection._lifecycle.getSyncError(), + }) + originalEmitEmptyReadyEvent() + }, + ) + let didThrow = false let thrown: unknown try { @@ -628,10 +659,30 @@ describe(`Collection Lifecycle Management`, () => { expect(didThrow).toBe(invalid) if (invalid) { expect(thrown).toBeInstanceOf(InvalidCollectionStatusTransitionError) + expect((thrown as Error).message).toBe( + `Invalid collection status transition from "${from}" to "ready" for collection "mark-ready-from-${from}"`, + ) } expect(collection.status).toBe(expectedStatus) expect(firstReadyCalls).toBe(expectedFirstReadyCalls) - expect(readyEvent).toHaveBeenCalledTimes(expectedDependentReadyEvents) + expect(recoveryFirstReadyCalls).toBe(from === `error` ? 1 : 0) + expect(transitionTrace).toEqual( + from === `error` + ? [ + { + kind: `status`, + previousStatus: `error`, + status: `ready`, + syncError: undefined, + }, + { + kind: `dependent-ready`, + status: `ready`, + syncError: undefined, + }, + ] + : [], + ) expect(collection._lifecycle.getSyncError()).toBeUndefined() await collection.cleanup() From a0189e6423367fc81f94250b436ba1a8caadf3e0 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 22:25:19 -0600 Subject: [PATCH 307/327] fix(db): fence superseded ready transitions --- packages/db/src/collection/lifecycle.ts | 15 ++-- packages/db/src/query/live/ARCHITECTURE.md | 11 +-- .../db/tests/collection-lifecycle.test.ts | 71 +++++++++++++++++++ 3 files changed, 89 insertions(+), 8 deletions(-) diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index be51f1bf4..a4ca225c7 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -38,6 +38,7 @@ export class CollectionLifecycleManager< public onFirstReadyCallbacks: Array<() => void> = [] private idleCallbackId: number | null = null private syncError: unknown + private statusRevision = 0 /** * Creates a new CollectionLifecycleManager instance @@ -105,6 +106,7 @@ export class CollectionLifecycleManager< ) } this.validateStatusTransition(this.status, newStatus) + this.statusRevision++ const previousStatus = this.status this.status = newStatus @@ -148,12 +150,17 @@ export class CollectionLifecycleManager< // A successful initial sync or recovery establishes a ready snapshot. if (this.status === `loading` || this.status === `error`) { this.syncError = undefined + const readyRevision = this.statusRevision + 1 this.setStatus(`ready`, true) - // A status listener can synchronously supersede this transition, for - // example by cleaning up the Collection. Do not publish ready effects - // for a snapshot that is no longer ready. - if ((this.status as CollectionStatus) !== `ready`) return undefined + // A status listener can synchronously supersede this transition, even + // when it restarts the Collection back to ready before returning. + if ( + (this.status as CollectionStatus) !== `ready` || + this.statusRevision !== readyRevision + ) { + return undefined + } const readyEffects: Array<() => void> = [] diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 50bbcc0e5..a1bea44f8 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1145,10 +1145,13 @@ sync error and emits a dependent-ready event, but does not start a second first-ready cycle. `idle` and `cleaned-up` cannot transition directly to `ready`; sync must establish `loading` first. -The `status:ready` event precedes first-ready effects. If one of its listeners -synchronously moves the Collection away from `ready`, that newer lifecycle -transition supersedes the current one. Core does not resume first-ready effects -or emit a dependent-ready event for the superseded snapshot. +The `status:ready` event precedes the ready effects captured by that transition. +If one of its listeners synchronously performs another lifecycle transition, +that newer transition supersedes the current one, even if a restart returns the +Collection to `ready` before the listener returns. Core does not resume the +captured effects or emit a dependent-ready event for the superseded snapshot. +Cleanup may separately drain pending first-ready callbacks, including preload +waiters, so they settle; that cleanup-owned drain is not a ready publication. A ready-effect failure does not undo effects already attempted in that cycle. After cleanup, the next sync is a new first-ready cycle with a fresh preload diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 90a688459..c5fff9744 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -719,6 +719,77 @@ describe(`Collection Lifecycle Management`, () => { removeLater() }) + it(`does not resume ready effects after a status listener enters error`, async () => { + const failure = new Error(`ready listener failed the sync`) + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-error-test`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const firstReady = vi.fn() + collection.onFirstReady(firstReady) + collection.on(`status:ready`, () => { + collection._lifecycle.markError(failure) + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(failure) + expect(collection._lifecycle.hasBeenReady).toBe(false) + expect(firstReady).not.toHaveBeenCalled() + expect(readyEvent).not.toHaveBeenCalled() + await collection.cleanup() + }) + + it(`does not resume an outer ready transition after a synchronous restart`, async () => { + let syncStarts = 0 + let restartedPreload: Promise | undefined + let restartOnce = true + let lateSubscription: { unsubscribe: () => void } | undefined + const lateReadyBatches: Array> = [] + const firstReadyStatuses: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-aba-test`, + getKey: (item) => item.id, + startSync: false, + sync: { + sync: ({ markReady }) => { + syncStarts++ + markReady() + }, + }, + }) + const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + collection.onFirstReady(() => { + firstReadyStatuses.push(collection.status) + }) + collection.on(`status:ready`, () => { + if (!restartOnce) return + restartOnce = false + void collection.cleanup() + restartedPreload = collection.preload() + lateSubscription = collection.subscribeChanges((batch) => { + lateReadyBatches.push(batch) + }) + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + await restartedPreload + + expect(syncStarts).toBe(1) + expect(collection.status).toBe(`ready`) + expect(firstReadyStatuses).toEqual([`ready`]) + expect(lateReadyBatches).toEqual([]) + expect(readyEvent).toHaveBeenCalledOnce() + lateSubscription!.unsubscribe() + await collection.cleanup() + }) + it(`starts a fresh first-ready cycle after cleanup of a failed ready effect`, async () => { const readyCallbacks: Array<() => void> = [] const firstFailure = new Error(`first ready cycle failed exactly`) From 0ff39e99df97c9f9b03e27af5923f0c2885f3d2a Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 22:33:37 -0600 Subject: [PATCH 308/327] test(db): fence ready restart generations --- .../db/tests/collection-lifecycle.test.ts | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index c5fff9744..2cf7b9434 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -813,6 +813,13 @@ describe(`Collection Lifecycle Management`, () => { trace.push(`first later:${collection.status}`) }) const firstPreload = collection.preload() + let firstPreloadSettled = false + void firstPreload.then(() => { + firstPreloadSettled = true + }) + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(firstPreloadSettled).toBe(false) let thrown: unknown try { @@ -835,10 +842,26 @@ describe(`Collection Lifecycle Management`, () => { expect(trace).toEqual([`first failure:ready`, `first later:ready`]) const secondPreload = collection.preload() + let secondPreloadSettled = false + void secondPreload.then(() => { + secondPreloadSettled = true + }) expect(secondPreload).not.toBe(firstPreload) expect(readyCallbacks).toHaveLength(2) + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(secondPreloadSettled).toBe(false) + + readyCallbacks[0]!() + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(secondPreloadSettled).toBe(false) + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + expect(readyEvent).toHaveBeenCalledOnce() + readyCallbacks[1]!() await expect(secondPreload).resolves.toBeUndefined() + expect(secondPreloadSettled).toBe(true) expect(trace).toEqual([ `first failure:ready`, From 5beb0b864c759dc4b09dfee463215ec66d39569c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 22:53:28 -0600 Subject: [PATCH 309/327] test(db): isolate retained state model inputs --- ...on-state-retention-oracle.property.test.ts | 71 ++++++++++++------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index eca8c100c..92d6fd6f9 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -36,6 +36,10 @@ const retainedRowArbitrary = fc.record({ value: fc.integer({ min: -2, max: 2 }), }) +function snapshotRetainedRow(row: RetainedRow): RetainedRow { + return { id: row.id, value: row.value } +} + const retentionActionArbitrary: fc.Arbitrary = fc.oneof( { weight: 4, @@ -114,18 +118,23 @@ function applyAction( case `insert`: { const previous = model.get(action.row.id) if (previous !== undefined && previous.value !== action.row.value) { - expect(() => sync.write({ type: `insert`, value: action.row })).toThrow( - DuplicateKeySyncError, - ) + expect(() => + sync.write({ + type: `insert`, + value: snapshotRetainedRow(action.row), + }), + ).toThrow(DuplicateKeySyncError) break } - sync.write({ type: `insert`, value: action.row }) - model.set(action.row.id, action.row) + const expectedRow = snapshotRetainedRow(action.row) + sync.write({ type: `insert`, value: snapshotRetainedRow(action.row) }) + model.set(expectedRow.id, expectedRow) break } case `update`: { - sync.write({ type: action.type, value: action.row }) - model.set(action.row.id, action.row) + const expectedRow = snapshotRetainedRow(action.row) + sync.write({ type: action.type, value: snapshotRetainedRow(action.row) }) + model.set(expectedRow.id, expectedRow) break } case `delete`: @@ -136,8 +145,9 @@ function applyAction( sync.truncate() model.clear() for (const row of action.rows) { - sync.write({ type: `insert`, value: row }) - model.set(row.id, row) + const expectedRow = snapshotRetainedRow(row) + sync.write({ type: `insert`, value: snapshotRetainedRow(row) }) + model.set(expectedRow.id, expectedRow) } break case `restart`: @@ -187,11 +197,14 @@ async function runRetentionHistory( id: action.row.id, value: (model.get(action.row.id)?.value ?? action.row.value) + 1, } + const expectedTriggerRow = snapshotRetainedRow(triggerRow) const restartedRow = { id: (action.row.id + 1) % 4, value: action.row.value + 1, } + const expectedRestartedRow = snapshotRetainedRow(restartedRow) const retainedMarker = { id: -1, value: action.row.value } + const expectedRetainedMarker = snapshotRetainedRow(retainedMarker) let cleanup: Promise | undefined let restarted = false let restartedSync: SyncActions | undefined @@ -229,7 +242,7 @@ async function runRetentionHistory( .map(({ id, value }) => ({ id, value })) .sort((left, right) => left.id - right.id), }) - if (changes.some(({ key }) => key === restartedRow.id)) { + if (changes.some(({ key }) => key === expectedRestartedRow.id)) { queueMicrotask(() => settlementTimeline.push(`publication`)) } if (restarted) return @@ -238,7 +251,10 @@ async function runRetentionHistory( collection.startSyncImmediate() restartedSync = harness.sync restartedSync.begin() - restartedSync.write({ type: `insert`, value: restartedRow }) + restartedSync.write({ + type: `insert`, + value: snapshotRetainedRow(restartedRow), + }) if (action.commitPhase === `insideListener`) { restartedReceipt = restartedSync.commit() if (restartedReceipt !== true) { @@ -254,14 +270,17 @@ async function runRetentionHistory( // publication state so the old publication tail cannot clear it. // The batch assertions below exercise the public restart path. collection._state.preSyncVisibleState.set(-1, retainedMarker) - collection._state.recentlySyncedKeys.add(restartedRow.id) + collection._state.recentlySyncedKeys.add(expectedRestartedRow.id) } }, { includeInitialState: false }, ) oldSync.begin() - oldSync.write({ type: `update`, value: triggerRow }) + oldSync.write({ + type: `update`, + value: snapshotRetainedRow(triggerRow), + }) expect(oldSync.commit()).toBe(true) expect(restarted).toBe(true) expect(restartedSync).toBeDefined() @@ -287,19 +306,19 @@ async function runRetentionHistory( ]) } else { expect(collection._state.preSyncVisibleState).toEqual( - new Map([[-1, retainedMarker]]), + new Map([[-1, expectedRetainedMarker]]), ) expect(collection._state.recentlySyncedKeys).toEqual( - new Set([restartedRow.id]), + new Set([expectedRestartedRow.id]), ) expect(collection._state.hasReceivedFirstCommit).toBe(false) await Promise.resolve() expect(collection._state.preSyncVisibleState).toEqual( - new Map([[-1, retainedMarker]]), + new Map([[-1, expectedRetainedMarker]]), ) expect(collection._state.recentlySyncedKeys).toEqual( - new Set([restartedRow.id]), + new Set([expectedRestartedRow.id]), ) expect(collection._state.hasReceivedFirstCommit).toBe(false) @@ -307,21 +326,21 @@ async function runRetentionHistory( expect(collection._state.preSyncVisibleState.size).toBe(0) expect(collection._state.hasReceivedFirstCommit).toBe(true) expect(collection._state.recentlySyncedKeys).toEqual( - new Set([restartedRow.id]), + new Set([expectedRestartedRow.id]), ) await Promise.resolve() expect(collection._state.recentlySyncedKeys.size).toBe(0) } const triggerRows = new Map(model) - triggerRows.set(triggerRow.id, triggerRow) + triggerRows.set(expectedTriggerRow.id, expectedTriggerRow) expect(batches).toEqual([ { changes: [ { type: triggerType, - key: triggerRow.id, - row: triggerRow, - previousRow: model.get(triggerRow.id), + key: expectedTriggerRow.id, + row: expectedTriggerRow, + previousRow: model.get(expectedTriggerRow.id), }, ], rows: [...triggerRows.values()].sort( @@ -336,19 +355,19 @@ async function runRetentionHistory( changes: [ { type: `insert`, - key: restartedRow.id, - row: restartedRow, + key: expectedRestartedRow.id, + row: expectedRestartedRow, previousRow: undefined, }, ], - rows: [restartedRow], + rows: [expectedRestartedRow], }, ]) subscription.unsubscribe() await cleanup model.clear() - model.set(restartedRow.id, restartedRow) + model.set(expectedRestartedRow.id, expectedRestartedRow) } else { applyAction(action, model, harness.sync) } From d2c0974c942ed9714be5d053e4b2c00f217c63ee Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 23:25:09 -0600 Subject: [PATCH 310/327] test(db): reject stale row origins --- .../tests/collection-state-retention-oracle.property.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts index 92d6fd6f9..0bc0e6a4a 100644 --- a/packages/db/tests/collection-state-retention-oracle.property.test.ts +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -170,6 +170,11 @@ function expectRetainedState( expect([...collection._state.syncedKeys].sort((a, b) => a - b)).toEqual( expectedRows.map(([key]) => key), ) + expect( + [...collection._state.rowOrigins.keys()] + .filter((key) => !model.has(key)) + .sort((a, b) => a - b), + ).toEqual([]) expect( [...collection.state.entries()] .map(([key, row]) => [key, { id: row.id, value: row.value }] as const) From c54588bfd5987b8da01a0536ff8605da733ffc22 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 23:31:25 -0600 Subject: [PATCH 311/327] test(db): retire hydration ownership --- packages/db/tests/db-client.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/db/tests/db-client.test.ts b/packages/db/tests/db-client.test.ts index fe8a0db0e..6fdf08c34 100644 --- a/packages/db/tests/db-client.test.ts +++ b/packages/db/tests/db-client.test.ts @@ -1207,6 +1207,21 @@ describe(`DbClient`, () => { expect(() => adapterWrite({ id: `1`, name: `adapter` })).not.toThrow() expect(collection.get(`1`)?.name).toBe(`adapter`) + expect(collection._state.hydrationSeedKeys.has(`1`)).toBe(false) + expect(collection._state.hydratedKeys.has(`1`)).toBe(false) + + client.hydrate({ + collections: [ + { + collectionId: `ready-hydration-seed`, + rows: [{ key: `1`, value: { id: `1`, name: `late hydration` } }], + }, + ], + }) + + expect(collection.get(`1`)?.name).toBe(`adapter`) + expect(collection._state.hydrationSeedKeys.has(`1`)).toBe(false) + expect(collection._state.hydratedKeys.has(`1`)).toBe(false) }) it(`does not let a late stream chunk overwrite adapter rows or metadata`, async () => { From eb8fba475450afcaab3acaf08a11664db63cfb52 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Tue, 1 Sep 2026 23:39:24 -0600 Subject: [PATCH 312/327] test(db): retire confirmed optimistic ownership --- packages/db/tests/collection-subscribe-changes.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/db/tests/collection-subscribe-changes.test.ts b/packages/db/tests/collection-subscribe-changes.test.ts index 08dce9199..2a7eb9e28 100644 --- a/packages/db/tests/collection-subscribe-changes.test.ts +++ b/packages/db/tests/collection-subscribe-changes.test.ts @@ -2340,6 +2340,8 @@ describe(`Virtual properties`, () => { ) expect(optimisticInsert).toBeDefined() expect(optimisticInsert!.value.$synced).toBe(false) + expect(collection._state.pendingLocalOrigins.has(`row-1`)).toBe(true) + expect(collection._state.pendingOptimisticUpserts.has(`row-1`)).toBe(true) changes.length = 0 @@ -2361,6 +2363,8 @@ describe(`Virtual properties`, () => { expect(confirmedUpdate).toBeDefined() expect(confirmedUpdate!.value.$synced).toBe(true) expect(confirmedUpdate!.previousValue?.$synced).toBe(false) + expect(collection._state.pendingLocalOrigins.size).toBe(0) + expect(collection._state.pendingOptimisticUpserts.size).toBe(0) subscription.unsubscribe() }) From 4aede1258ac203ec66cd1c713368aedc240639b9 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 00:14:00 -0600 Subject: [PATCH 313/327] fix(query-db): preserve active row ownership --- packages/query-db-collection/src/query.ts | 28 +-- .../tests/ownership-lifecycle.oracle.test.ts | 207 +++++++----------- .../query-db-collection/tests/query.test.ts | 25 +-- 3 files changed, 106 insertions(+), 154 deletions(-) diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 11d498b3e..cd6b52537 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -1575,14 +1575,20 @@ export function queryCollectionOptions( newItemsMap.forEach((newItem, key) => { const owners = getPersistedOwners(key) - if (!owners.has(hashedQueryKey)) { + const addsOwner = !owners.has(hashedQueryKey) + const insertsRow = !currentSyncedItems.has(key) + if (addsOwner) { owners.add(hashedQueryKey) - setPersistedOwners(key, owners) } addRowOwner(key, hashedQueryKey) - if (!currentSyncedItems.has(key)) { + if (insertsRow) { write({ type: `insert`, value: newItem }) } + if (addsOwner || insertsRow) { + // An insert clears stale metadata for its key. Stage ownership + // afterward so rows and ownership commit as one state change. + setPersistedOwners(key, owners) + } }) const applied = commit(signal) @@ -1946,6 +1952,12 @@ export function queryCollectionOptions( unsubscribePendingReadyListeners(hashedQueryKey) } + // Refcounts are explicit ownership tokens. A cache event can remove the + // observer while an active acquisition still owns this query. + if (refcount > 0) { + return + } + const hasListeners = observer?.hasListeners() ?? false if (hasListeners) { @@ -1955,16 +1967,6 @@ export function queryCollectionOptions( return } - // No listeners means the query is truly idle. - // Even if refcount > 0, we treat hasListeners as authoritative to prevent leaks. - // This can happen if subscriptions are GC'd without calling unloadSubset. - if (refcount > 0) { - console.warn( - `[cleanupQueryIfIdle] Invariant violation: refcount=${refcount} but no listeners. Cleaning up to prevent leak.`, - { hashedQueryKey }, - ) - } - if ( effectivePersistedGcTime !== undefined && metadata && diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 4dddbf356..4540ed369 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1,7 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { QueryClient } from '@tanstack/query-core' import { createCollection, eq } from '@tanstack/db' -import { expectAssertionFailure } from '../../db/tests/expected-failure.js' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' import type { Collection, SyncMetadataApi } from '@tanstack/db' @@ -140,82 +139,6 @@ function assertCheckpoint( } } -function asRecords({ - actual, - expected, -}: { - actual: unknown - expected: unknown -}): - | { - observed: Record - wanted: Record - } - | undefined { - if ( - !actual || - typeof actual !== `object` || - !expected || - typeof expected !== `object` - ) { - return undefined - } - - return { - observed: actual as Record, - wanted: expected as Record, - } -} - -function classifyInsertedOwnerMetadataLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - Array.isArray(observed.persistedOwners) && - observed.persistedOwners.length === 0 && - Array.isArray(observed.metadataSetKeys) && - observed.metadataSetKeys.length === 1 && - observed.metadataSetKeys[0] === shared.id && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 1 && - typeof wanted.persistedOwners[0] === `string` && - Array.isArray(wanted.metadataSetKeys) && - wanted.metadataSetKeys.length === 1 && - wanted.metadataSetKeys[0] === shared.id - ) -} - -function sameArray(actual: unknown, expected: unknown): boolean { - return ( - Array.isArray(actual) && - Array.isArray(expected) && - actual.length === expected.length && - actual.every((value, index) => value === expected[index]) - ) -} - -function classifyPersistedBaselineLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - sameArray(observed.liveOwners, wanted.liveOwners) && - sameArray(observed.persistedOwners, wanted.insertedOwners) && - Array.isArray(observed.insertedOwners) && - observed.insertedOwners.length === 0 && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 2 && - sameArray(observed.metadataSetKeys, wanted.metadataSetKeys) - ) -} - function recordMetadataWrites( metadata: SyncMetadataApi, recorder: MetadataRecorder, @@ -602,10 +525,58 @@ describe(`query collection ownership lifecycle oracle`, () => { } }) - it(`#1656 keeps the first persisted owner when a second query inserts another row`, async () => { + it(`keeps an active on-demand owner when its cache entry is removed`, async () => { + const id = `ownership-active-cache-removal` + const { collection, maps, queryClient } = createOwnershipFixture({ + id, + results: [[shared]], + }) + const subset = { where: eq(`category`, `detail`) } + + await collection._sync.loadSubset(subset) + const queryHash = onlyOwner(maps, shared.id) + const subscription = collection.subscribeChanges(() => {}) + subscription.unsubscribe() + assertCheckpoint(0, collection.subscriberCount, 0) + + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + try { + queryClient.removeQueries({ queryKey: [id] }) + + assertCheckpoint( + 1, + { + rows: collectionRows(collection), + owners: ownersOf(maps, shared.id), + ownedRows: rowsOwnedBy(maps, queryHash), + }, + { + rows: [shared.id], + owners: [queryHash], + ownedRows: [shared.id], + }, + ) + expect(warning).not.toHaveBeenCalled() + } finally { + warning.mockRestore() + } + + collection._sync.unloadSubset(subset) + assertCheckpoint( + 2, + { + rows: collectionRows(collection), + ownershipRows: maps.rowToQueries.size, + ownershipQueries: maps.queryToRows.size, + }, + { rows: [], ownershipRows: 0, ownershipQueries: 0 }, + ) + }) + + it(`keeps every persisted owner when overlapping queries insert rows`, async () => { const metadataRecorder: MetadataRecorder = { rowWrites: [] } const { collection, maps } = createOwnershipFixture({ - id: `ownership-persisted-baseline-1656`, + id: `ownership-persisted-baseline`, results: [[shared], [shared, listOnly]], metadataRecorder, }) @@ -614,59 +585,41 @@ describe(`query collection ownership lifecycle oracle`, () => { await collection._sync.loadSubset(detailSubset) const detailHash = onlyOwner(maps, shared.id) - // The production metadata API records the owner write, but the insert's - // commit currently loses it. Accept only that exact #1656 boundary. - const assertInsertedOwnerPersists = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 0, - { - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, - ) - }), - { checkpoint: 0, classify: classifyInsertedOwnerMetadataLoss }, + assertCheckpoint( + 0, + { + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, ) - await assertInsertedOwnerPersists() await collection._sync.loadSubset(listSubset) const listHash = otherOwner(maps, shared.id, detailHash) - // A second insert loses its own owner and rebuilds the persisted baseline - // with only the later query, while the in-memory ownership remains sound. - const assertPersistedBaselineSurvives = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 1, - { - liveOwners: ownersOf(maps, shared.id), - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - insertedOwners: persistedOwners( - collection._state.syncedMetadata, - listOnly.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { - liveOwners: sorted([detailHash, listHash]), - persistedOwners: sorted([detailHash, listHash]), - insertedOwners: [listHash], - metadataSetKeys: [listOnly.id, shared.id], - }, - ) - }), - { checkpoint: 1, classify: classifyPersistedBaselineLoss }, + assertCheckpoint( + 1, + { + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + insertedOwners: persistedOwners( + collection._state.syncedMetadata, + listOnly.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { + liveOwners: sorted([detailHash, listHash]), + persistedOwners: sorted([detailHash, listHash]), + insertedOwners: [listHash], + metadataSetKeys: [listOnly.id, shared.id], + }, ) - await assertPersistedBaselineSurvives() collection._sync.unloadSubset(listSubset) assertCheckpoint( diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index b6f826681..9f4c38f70 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -7401,11 +7401,7 @@ describe(`QueryCollection`, () => { } }) - it(`should reset refcount after query GC and reload (stale refcount bug)`, async () => { - // This test catches Bug 2: stale refcounts after GC/remove - // When TanStack Query GCs a query, the refcount should be cleaned up - // Otherwise, reloading the same subset will start with a stale count - + it(`should reload a released subset without retaining a stale refcount`, async () => { const baseQueryKey = [`stale-refcount-test`] const items: Array = [ { id: `1`, name: `Item 1`, category: `A` }, @@ -7443,13 +7439,17 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Force GC by calling removeQueries (simulates gcTime expiry) + // Release the first acquisition before its cache entry is removed. + // Cache events do not revoke active collection ownership. + await query1.cleanup() + await vi.waitFor(() => { + expect(collection.size).toBe(0) + }) + + // Force GC by calling removeQueries (simulates gcTime expiry). queryClient.removeQueries({ queryKey: baseQueryKey }) await flushPromises() - // BUG: queryRefCounts still has stale count, wasn't cleaned up by cleanupQuery - // When we load again, the refcount will be wrong (starts at 1 instead of 0, or accumulates) - // Reload the same query const query2 = createLiveQueryCollection({ query: (q) => @@ -7466,14 +7466,11 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Cleanup - this should properly decrement from 1 to 0 and clean up + // Cleanup should decrement the new acquisition from one to zero. await query2.cleanup() await vi.waitFor(() => { - expect(collection.size).toBe(0) // Should be cleaned up + expect(collection.size).toBe(0) }) - - // BUG SYMPTOM: If refcount was stale (e.g. was 2, decremented to 1), - // the observer won't be destroyed and data won't be cleaned up }) it(`should handle mount/unmount/remount without breaking cache (destroyed observer bug)`, async () => { From e1147c693b8ed1e2439c13ef0a6f56efac76bdce Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 00:35:11 -0600 Subject: [PATCH 314/327] test(query-db): separate persisted owner insertion --- .../tests/ownership-lifecycle.oracle.test.ts | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 4540ed369..90f9d003a 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { QueryClient } from '@tanstack/query-core' +import { QueryClient, hashKey } from '@tanstack/query-core' import { createCollection, eq } from '@tanstack/db' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' @@ -30,6 +30,7 @@ type OwnershipFixtureOptions = { results: Array> syncMode?: `eager` | `on-demand` metadataRecorder?: MetadataRecorder + setupMetadata?: (metadata: SyncMetadataApi) => void } type OwnershipFixture = { @@ -169,6 +170,7 @@ function createOwnershipFixture({ results, syncMode = `on-demand`, metadataRecorder, + setupMetadata, }: OwnershipFixtureOptions): OwnershipFixture { const queryClient = createQueryClient() const queryFn = vi.fn<() => Promise>>() @@ -186,7 +188,7 @@ function createOwnershipFixture({ const maps = inspectOwnershipMaps(baseOptions) const originalSync = baseOptions.sync const collection = createCollection( - metadataRecorder + metadataRecorder || setupMetadata ? { ...baseOptions, sync: { @@ -194,12 +196,16 @@ function createOwnershipFixture({ if (!params.metadata) { throw new Error(`Sync metadata API is unavailable`) } + if (setupMetadata) { + params.begin() + setupMetadata(params.metadata) + params.commit() + } return originalSync.sync({ ...params, - metadata: recordMetadataWrites( - params.metadata, - metadataRecorder, - ), + metadata: metadataRecorder + ? recordMetadataWrites(params.metadata, metadataRecorder) + : params.metadata, }) }, }, @@ -639,4 +645,37 @@ describe(`query collection ownership lifecycle oracle`, () => { }, ) }) + + it(`restages an existing persisted owner when its absent row is inserted`, async () => { + const id = `ownership-existing-metadata-before-insert` + const queryHash = hashKey([id]) + const { collection, maps } = createOwnershipFixture({ + id, + syncMode: `eager`, + results: [[shared]], + setupMetadata: (metadata) => { + metadata.row.set(shared.id, { + queryCollection: { owners: { [queryHash]: true } }, + }) + }, + }) + + await collection.stateWhenReady() + assertCheckpoint( + 0, + { + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + }, + { + rows: [shared.id], + liveOwners: [queryHash], + persistedOwners: [queryHash], + }, + ) + }) }) From aae958e797b730a150418ef909cb13229096e91e Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 00:49:18 -0600 Subject: [PATCH 315/327] test(query-db): prove persisted owner precondition --- .../tests/ownership-lifecycle.oracle.test.ts | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 90f9d003a..a45341524 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' import { QueryClient, hashKey } from '@tanstack/query-core' import { createCollection, eq } from '@tanstack/db' +import { createDeferred } from '../../db/src/deferred.js' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' import type { Collection, SyncMetadataApi } from '@tanstack/db' @@ -27,7 +28,7 @@ type MetadataRecorder = { type OwnershipFixtureOptions = { id: string - results: Array> + results: Array | Promise>> syncMode?: `eager` | `on-demand` metadataRecorder?: MetadataRecorder setupMetadata?: (metadata: SyncMetadataApi) => void @@ -174,7 +175,9 @@ function createOwnershipFixture({ }: OwnershipFixtureOptions): OwnershipFixture { const queryClient = createQueryClient() const queryFn = vi.fn<() => Promise>>() - results.forEach((result) => queryFn.mockResolvedValueOnce(result)) + results.forEach((result) => + queryFn.mockImplementationOnce(() => Promise.resolve(result)), + ) queryFn.mockRejectedValue(new Error(`Unexpected ownership-oracle refetch`)) const baseOptions = queryCollectionOptions({ id, @@ -187,6 +190,7 @@ function createOwnershipFixture({ }) const maps = inspectOwnershipMaps(baseOptions) const originalSync = baseOptions.sync + let pendingSetupMetadata = setupMetadata const collection = createCollection( metadataRecorder || setupMetadata ? { @@ -196,10 +200,11 @@ function createOwnershipFixture({ if (!params.metadata) { throw new Error(`Sync metadata API is unavailable`) } - if (setupMetadata) { + if (pendingSetupMetadata) { params.begin() - setupMetadata(params.metadata) + pendingSetupMetadata(params.metadata) params.commit() + pendingSetupMetadata = undefined } return originalSync.sync({ ...params, @@ -649,10 +654,11 @@ describe(`query collection ownership lifecycle oracle`, () => { it(`restages an existing persisted owner when its absent row is inserted`, async () => { const id = `ownership-existing-metadata-before-insert` const queryHash = hashKey([id]) - const { collection, maps } = createOwnershipFixture({ + const result = createDeferred>() + const { collection, maps, queryFn } = createOwnershipFixture({ id, syncMode: `eager`, - results: [[shared]], + results: [result.promise], setupMetadata: (metadata) => { metadata.row.set(shared.id, { queryCollection: { owners: { [queryHash]: true } }, @@ -660,7 +666,7 @@ describe(`query collection ownership lifecycle oracle`, () => { }, }) - await collection.stateWhenReady() + expect(queryFn).toHaveBeenCalledTimes(1) assertCheckpoint( 0, { @@ -671,6 +677,25 @@ describe(`query collection ownership lifecycle oracle`, () => { shared.id, ), }, + { + rows: [], + liveOwners: [], + persistedOwners: [queryHash], + }, + ) + + result.resolve([shared]) + await collection.stateWhenReady() + assertCheckpoint( + 1, + { + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + }, { rows: [shared.id], liveOwners: [queryHash], From 854c86ac90b1930e50149d7aad377e997fdc23ca Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 00:56:17 -0600 Subject: [PATCH 316/327] test(query-db): prove metadata setup is one-shot --- .../tests/ownership-lifecycle.oracle.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index a45341524..c4ac3b5a1 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -655,11 +655,13 @@ describe(`query collection ownership lifecycle oracle`, () => { const id = `ownership-existing-metadata-before-insert` const queryHash = hashKey([id]) const result = createDeferred>() + let setupCalls = 0 const { collection, maps, queryFn } = createOwnershipFixture({ id, syncMode: `eager`, - results: [result.promise], + results: [result.promise, [shared]], setupMetadata: (metadata) => { + setupCalls += 1 metadata.row.set(shared.id, { queryCollection: { owners: { [queryHash]: true } }, }) @@ -702,5 +704,9 @@ describe(`query collection ownership lifecycle oracle`, () => { persistedOwners: [queryHash], }, ) + + await collection.cleanup() + await collection.preload() + assertCheckpoint(2, setupCalls, 1) }) }) From c311063fa36a1cf46ed0a44dd182709dd4ad58ac Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 01:06:30 -0600 Subject: [PATCH 317/327] test(query-db): verify metadata setup restart --- .../tests/ownership-lifecycle.oracle.test.ts | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index c4ac3b5a1..d684f6158 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -659,7 +659,7 @@ describe(`query collection ownership lifecycle oracle`, () => { const { collection, maps, queryFn } = createOwnershipFixture({ id, syncMode: `eager`, - results: [result.promise, [shared]], + results: [result.promise, [{ ...shared, name: `Restarted` }]], setupMetadata: (metadata) => { setupCalls += 1 metadata.row.set(shared.id, { @@ -706,7 +706,33 @@ describe(`query collection ownership lifecycle oracle`, () => { ) await collection.cleanup() + assertCheckpoint(2, collection.status, `cleaned-up`) await collection.preload() - assertCheckpoint(2, setupCalls, 1) + await vi.waitFor(() => { + expect(queryFn).toHaveBeenCalledTimes(2) + expect(collection.get(shared.id)?.name).toBe(`Restarted`) + }) + assertCheckpoint( + 3, + { + status: collection.status, + fetches: queryFn.mock.calls.length, + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + setupCalls, + }, + { + status: `ready`, + fetches: 2, + rows: [shared.id], + liveOwners: [queryHash], + persistedOwners: [queryHash], + setupCalls: 1, + }, + ) }) }) From e381471fa638e58765f968e6fcf6544217f0e731 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 01:34:13 -0600 Subject: [PATCH 318/327] fix(db): preserve predicate subtraction semantics --- packages/db/package.json | 2 +- packages/db/src/query/predicate-utils.ts | 74 ++- packages/db/tests/load-subset-outcome.test.ts | 6 +- packages/db/tests/oracle-config.ts | 3 + ...dicate-subtraction-oracle.property.test.ts | 593 ++++++++++++++++++ .../db/tests/query/predicate-utils.test.ts | 115 +++- packages/db/tests/query/subset-dedupe.test.ts | 104 ++- packages/db/tests/utils.test.ts | 7 + 8 files changed, 822 insertions(+), 82 deletions(-) create mode 100644 packages/db/tests/query/predicate-subtraction-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index d504fd2a6..a4bf3d9e1 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts tests/query/predicate-subtraction-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index bff11e345..921e01766 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -380,15 +380,10 @@ export function minusWherePredicates( ) } - // If from is undefined then we are asking for all data - // so we need to load all data minus what we already loaded - // i.e. we need to load NOT(subtractPredicate) if (fromPredicate === undefined) { - return { - type: `func`, - name: `not`, - args: [subtractPredicate], - } as BasicExpression + // NOT(subtractPredicate) would also filter UNKNOWN rows under three-valued + // logic, even though those rows were not loaded. Fall back to the full request. + return null } // Check if fromPredicate is entirely contained in subtractPredicate @@ -404,21 +399,25 @@ export function minusWherePredicates( ) if (commonConditions.length > 0) { // Extract predicates without common conditions - const fromWithoutCommon = removeConditions(fromPredicate, commonConditions) - const subtractWithoutCommon = removeConditions( + const fromRemoval = removeConditions(fromPredicate, commonConditions) + const subtractRemoval = removeConditions( subtractPredicate, commonConditions, ) - // Recursively compute difference on simplified predicates - const simplifiedDifference = minusWherePredicates( - fromWithoutCommon, - subtractWithoutCommon, - ) + // Recurse only when both operands lost at least one flattened AND term. + // This strict decrease is the termination measure for common-condition + // simplification. + if (fromRemoval.removed && subtractRemoval.removed) { + const simplifiedDifference = minusWherePredicates( + fromRemoval.predicate, + subtractRemoval.predicate, + ) - if (simplifiedDifference !== null) { - // Combine the simplified difference with common conditions - return combineConditions([...commonConditions, simplifiedDifference]) + if (simplifiedDifference !== null) { + // Combine the simplified difference with common conditions + return combineConditions([...commonConditions, simplifiedDifference]) + } } } @@ -1037,21 +1036,40 @@ function extractAllConditions( /** * Remove specified conditions from a predicate. - * Returns the predicate with the specified conditions removed, or undefined if all conditions are removed. + * Reports whether removal made progress and returns the remaining predicate, + * or undefined when all conditions were removed. */ function removeConditions( predicate: BasicExpression, conditionsToRemove: Array>, -): BasicExpression | undefined { - const remaining = extractAllConditions(predicate).filter( - (candidate) => - !conditionsToRemove.some((condition) => - areExpressionsEqual(candidate, condition), - ), - ) +): { + predicate: BasicExpression | undefined + removed: boolean +} { + const conditions = extractAllConditions(predicate) + const remainingConditions = [...conditions] + + // Consume one occurrence per common term so duplicate predicates remain. + for (const conditionToRemove of conditionsToRemove) { + const matchingIndex = remainingConditions.findIndex((condition) => + areExpressionsEqual(condition, conditionToRemove), + ) + if (matchingIndex !== -1) { + remainingConditions.splice(matchingIndex, 1) + } + } - if (remaining.length === 0) return undefined - return combineConditions(remaining) + if (remainingConditions.length === conditions.length) { + return { predicate, removed: false } + } + + return { + predicate: + remainingConditions.length === 0 + ? undefined + : combineConditions(remainingConditions), + removed: true, + } } /** diff --git a/packages/db/tests/load-subset-outcome.test.ts b/packages/db/tests/load-subset-outcome.test.ts index 6d869c912..65da4d0e1 100644 --- a/packages/db/tests/load-subset-outcome.test.ts +++ b/packages/db/tests/load-subset-outcome.test.ts @@ -1309,7 +1309,7 @@ describe(`loadSubset outcomes`, () => { }, ) - it(`scopes source extent to a narrowed physical acquisition`, async () => { + it(`preserves source extent for a conservative full acquisition`, async () => { const adapterCalls: Array = [] const deduplicated = new DeduplicatedLoadSubset({ loadSubset: (options) => { @@ -1348,8 +1348,8 @@ describe(`loadSubset outcomes`, () => { const outcome = collection._sync.loadSubset({}) expect(adapterCalls).toHaveLength(2) - expect(adapterCalls[1]?.where).toBeDefined() - await expect(outcome).resolves.toMatchObject({ extent: `unknown` }) + expect(adapterCalls[1]).toEqual({}) + await expect(outcome).resolves.toMatchObject({ extent: `exhausted` }) } finally { await collection.cleanup() } diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index c9c70ca3c..04546a3d6 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -66,6 +66,9 @@ const staticOracleProperties = [ `pagination.pending-history`, `pagination.pending-mutation`, `pagination.window-transition`, + `predicate-subtraction.duplicate-terms`, + `predicate-subtraction.finite-world`, + `predicate-subtraction.unbounded`, `subscription-replay.completion`, `subscription-replay.optimistic`, `subscription-replay.ownership`, diff --git a/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts b/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts new file mode 100644 index 000000000..3e4de6796 --- /dev/null +++ b/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts @@ -0,0 +1,593 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { minusWherePredicates } from '../../src/query/predicate-utils' +import { Func, PropRef, Value } from '../../src/query/ir' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config' +import type { BasicExpression } from '../../src/query/ir' + +type Field = `score` | `rank` + +type PredicateSpec = + | { kind: `eq`; field: Field; value: number | null } + | { + kind: `range` + field: Field + operator: `gt` | `gte` | `lt` | `lte` + value: number + } + | { kind: `in`; field: Field; values: Array } + | { kind: `not`; predicate: AtomicPredicateSpec } + | { + kind: `or` + left: AtomicPredicateSpec + right: AtomicPredicateSpec + } + +type AtomicPredicateSpec = Exclude + +type Association = `flat` | `left` | `right` +type ScenarioFamily = + | `general residuals` + | `ordered range overlap` + | `set overlap` + +interface DifferenceScenario { + family: ScenarioFamily + shared: Array + fromResidual: AtomicPredicateSpec + subtractResidual: AtomicPredicateSpec + fromAssociation: Association + subtractAssociation: Association + reverseFrom: boolean + reverseSubtract: boolean + duplicateFrom: boolean + duplicateSubtract: boolean +} + +type DifferenceOutcome = + | `successful narrowing` + | `unchanged fallback` + | `conservative bailout` + +type DifferenceObservation = `${ScenarioFamily} / ${DifferenceOutcome}` + +const finiteWorldProperty = `predicate-subtraction.finite-world` +const unboundedProperty = `predicate-subtraction.unbounded` +const duplicateProperty = `predicate-subtraction.duplicate-terms` + +const scalarArbitrary = fc.oneof( + fc.integer({ min: -2, max: 2 }), + fc.constant(null), +) +const fieldArbitrary = fc.constantFrom(`score`, `rank`) + +const atomicPredicateArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + kind: fc.constant(`eq` as const), + field: fieldArbitrary, + value: scalarArbitrary, + }), + fc.record({ + kind: fc.constant(`range` as const), + field: fieldArbitrary, + operator: fc.constantFrom<`gt` | `gte` | `lt` | `lte`>( + `gt`, + `gte`, + `lt`, + `lte`, + ), + value: fc.integer({ min: -2, max: 2 }), + }), + fc.record({ + kind: fc.constant(`in` as const), + field: fieldArbitrary, + values: fc.uniqueArray(scalarArbitrary, { minLength: 1, maxLength: 4 }), + }), +) + +const predicateArbitrary: fc.Arbitrary = fc.oneof( + atomicPredicateArbitrary, + atomicPredicateArbitrary.map((predicate) => ({ + kind: `not` as const, + predicate, + })), + fc + .tuple(atomicPredicateArbitrary, atomicPredicateArbitrary) + .map(([left, right]) => ({ kind: `or` as const, left, right })), +) + +const residualPairArbitrary = fc.oneof( + fc + .tuple(atomicPredicateArbitrary, atomicPredicateArbitrary) + .map(([fromResidual, subtractResidual]) => ({ + family: `general residuals` as const, + fromResidual, + subtractResidual, + })), + fc + .tuple(fieldArbitrary, fc.integer({ min: -2, max: 1 })) + .map(([field, boundary]) => ({ + family: `ordered range overlap` as const, + fromResidual: { + kind: `range` as const, + field, + operator: `gt` as const, + value: boundary, + }, + subtractResidual: { + kind: `range` as const, + field, + operator: `gt` as const, + value: boundary + 1, + }, + })), + fc + .tuple( + fieldArbitrary, + fc.uniqueArray(fc.integer({ min: -2, max: 2 }), { + minLength: 2, + maxLength: 4, + }), + ) + .map(([field, values]) => ({ + family: `set overlap` as const, + fromResidual: { kind: `in` as const, field, values }, + subtractResidual: { + kind: `in` as const, + field, + values: values.slice(1), + }, + })), +) + +const scenarioShapeArbitrary = fc.record({ + shared: fc.array(predicateArbitrary, { minLength: 1, maxLength: 3 }), + fromAssociation: fc.constantFrom(`flat`, `left`, `right`), + subtractAssociation: fc.constantFrom(`flat`, `left`, `right`), + reverseFrom: fc.boolean(), + reverseSubtract: fc.boolean(), + duplicateFrom: fc.boolean(), + duplicateSubtract: fc.boolean(), +}) + +const scenarioArbitrary: fc.Arbitrary = fc + .tuple(scenarioShapeArbitrary, residualPairArbitrary) + .map(([shape, residuals]) => ({ ...shape, ...residuals })) + +const refs: Record = { + score: new PropRef([`score`]), + rank: new PropRef([`rank`]), +} + +function value(input: unknown): Value { + return new Value(input) +} + +function call( + name: string, + ...args: Array +): BasicExpression { + return new Func(name, args) as BasicExpression +} + +function buildAtomic(spec: AtomicPredicateSpec): BasicExpression { + const ref = refs[spec.field] + if (spec.kind === `in`) { + return call(`in`, ref, value(spec.values)) + } + if (spec.kind === `range`) { + return call(spec.operator, ref, value(spec.value)) + } + return call(`eq`, ref, value(spec.value)) +} + +function buildPredicate(spec: PredicateSpec): BasicExpression { + if (spec.kind === `not`) { + return call(`not`, buildAtomic(spec.predicate)) + } + if (spec.kind === `or`) { + return call(`or`, buildAtomic(spec.left), buildAtomic(spec.right)) + } + return buildAtomic(spec) +} + +function predicateFields(spec: PredicateSpec): Array { + if (spec.kind === `not`) return [spec.predicate.field] + if (spec.kind === `or`) return [spec.left.field, spec.right.field] + return [spec.field] +} + +function scenarioFields(scenario: DifferenceScenario): Array { + return [ + ...scenario.shared.flatMap(predicateFields), + scenario.fromResidual.field, + scenario.subtractResidual.field, + ] +} + +function associateAnd( + terms: Array>, + association: Association, +): BasicExpression { + if (terms.length === 1) return terms[0]! + if (association === `flat`) return call(`and`, ...terms) + + if (association === `left`) { + return terms + .slice(1) + .reduce((left, right) => call(`and`, left, right), terms[0]!) + } + + return terms + .slice(0, -1) + .reduceRight((right, left) => call(`and`, left, right), terms.at(-1)!) +} + +function buildOperand( + sharedSpecs: Array, + residualSpec: AtomicPredicateSpec, + association: Association, + reverse: boolean, + duplicate: boolean, +): BasicExpression { + const shared = sharedSpecs.map(buildPredicate) + const residual = buildAtomic(residualSpec) + const terms = reverse ? [residual, ...shared] : [...shared, residual] + if (duplicate) terms.splice(1, 0, terms[0]!) + return associateAnd(terms, association) +} + +const finiteValues = [-3, -2, -1, 0, 1, 2, 3, null] +const finiteRows = finiteValues.flatMap((score) => + finiteValues.map((rank) => ({ score, rank })), +) + +function evaluatePredicate( + expression: BasicExpression, + row: Record, +): unknown { + if (expression.type === `val`) return expression.value + if (expression.type === `ref`) { + const [field, ...remainingPath] = expression.path + if (remainingPath.length > 0 || (field !== `score` && field !== `rank`)) { + throw new Error(`Unsupported reference path ${expression.path.join(`.`)}`) + } + return row[field] + } + + const args = expression.args.map((argument) => + evaluatePredicate(argument, row), + ) + const isUnknown = (candidate: unknown) => + candidate === null || candidate === undefined + switch (expression.name) { + case `and`: + return args.includes(false) ? false : args.some(isUnknown) ? null : true + case `or`: + return args.includes(true) ? true : args.some(isUnknown) ? null : false + case `not`: + return isUnknown(args[0]) ? null : !args[0] + case `eq`: + return isUnknown(args[0]) || isUnknown(args[1]) + ? null + : args[0] === args[1] + case `gt`: + case `gte`: + case `lt`: + case `lte`: { + if (isUnknown(args[0]) || isUnknown(args[1])) return null + const left = args[0] as number + const right = args[1] as number + if (expression.name === `gt`) return left > right + if (expression.name === `gte`) return left >= right + if (expression.name === `lt`) return left < right + return left <= right + } + case `in`: + if (isUnknown(args[0])) return null + return Array.isArray(args[1]) && args[1].includes(args[0]) + default: + throw new Error(`Unsupported predicate ${expression.name}`) + } +} + +function assertSemanticDifference( + scenario: DifferenceScenario, + override?: { result: BasicExpression | null }, +): void { + const difference = evaluateDifference(scenario) + const { requested, loaded } = difference + const result = override === undefined ? difference.result : override.result + + assertExpressionDifference(requested, loaded, result) +} + +function assertExpressionDifference( + requested: BasicExpression, + loaded: BasicExpression, + result: BasicExpression | null, +): void { + if (result === null) return + + for (const row of finiteRows) { + const expected = + evaluatePredicate(requested, row) === true && + evaluatePredicate(loaded, row) !== true + expect(evaluatePredicate(result, row) === true).toBe(expected) + } +} + +function assertUnboundedDifference(spec: PredicateSpec): void { + const loaded = buildPredicate(spec) + const result = minusWherePredicates(undefined, loaded) + assertExpressionDifference( + value(true) as BasicExpression, + loaded, + result, + ) +} + +function assertDuplicateTermDifference(field: Field, boundary: number): void { + const shared = buildAtomic({ + kind: `range`, + field, + operator: `gt`, + value: boundary, + }) + const nullableChoice = call( + `or`, + buildAtomic({ kind: `eq`, field, value: null }), + buildAtomic({ kind: `eq`, field, value: boundary + 1 }), + ) + const membership = buildAtomic({ + kind: `in`, + field, + values: [boundary + 1, boundary], + }) + const requested = call( + `and`, + shared, + nullableChoice, + membership, + buildAtomic({ + kind: `range`, + field, + operator: `gt`, + value: boundary - 1, + }), + ) + const loaded = call(`and`, shared, nullableChoice, membership, shared) + const result = minusWherePredicates(requested, loaded) + + assertExpressionDifference(requested, loaded, result) +} + +function evaluateDifference(scenario: DifferenceScenario): { + requested: BasicExpression + loaded: BasicExpression + result: BasicExpression | null +} { + const requested = buildOperand( + scenario.shared, + scenario.fromResidual, + scenario.fromAssociation, + scenario.reverseFrom, + scenario.duplicateFrom, + ) + const loaded = buildOperand( + scenario.shared, + scenario.subtractResidual, + scenario.subtractAssociation, + scenario.reverseSubtract, + scenario.duplicateSubtract, + ) + + return { + requested, + loaded, + result: minusWherePredicates(requested, loaded), + } +} + +function classifyDifferenceOutcome( + scenario: DifferenceScenario, +): DifferenceOutcome { + const { requested, result } = evaluateDifference(scenario) + if (result === null) return `conservative bailout` + + for (const row of finiteRows) { + if ( + (evaluatePredicate(requested, row) === true) !== + (evaluatePredicate(result, row) === true) + ) { + return `successful narrowing` + } + } + + return `unchanged fallback` +} + +function expectEveryDifferenceOutcome(parameters: { + numRuns: number + seed: number +}): void { + const counts = new Map() + + for (const scenario of fc.sample(scenarioArbitrary, parameters)) { + const observation: DifferenceObservation = `${scenario.family} / ${classifyDifferenceOutcome(scenario)}` + counts.set(observation, (counts.get(observation) ?? 0) + 1) + } + + const requiredObservations: Array = [ + `general residuals / unchanged fallback`, + `general residuals / conservative bailout`, + `ordered range overlap / successful narrowing`, + `set overlap / successful narrowing`, + ] + const diagnostics = `seed=${parameters.seed} counts=${JSON.stringify(Object.fromEntries(counts))}` + + for (const observation of requiredObservations) { + expect(counts.get(observation) ?? 0, diagnostics).toBeGreaterThanOrEqual(10) + } +} + +function calibrationScenario( + fromResidual: AtomicPredicateSpec, + subtractResidual: AtomicPredicateSpec, +): DifferenceScenario { + return { + family: `general residuals`, + shared: [], + fromResidual, + subtractResidual, + fromAssociation: `flat`, + subtractAssociation: `flat`, + reverseFrom: false, + reverseSubtract: false, + duplicateFrom: false, + duplicateSubtract: false, + } +} + +const outcomeCalibrations: Record = { + 'successful narrowing': calibrationScenario( + { kind: `range`, field: `score`, operator: `gt`, value: -1 }, + { kind: `range`, field: `score`, operator: `gt`, value: 0 }, + ), + 'unchanged fallback': calibrationScenario( + { kind: `eq`, field: `score`, value: 0 }, + { kind: `eq`, field: `score`, value: 1 }, + ), + 'conservative bailout': calibrationScenario( + { kind: `eq`, field: `score`, value: 0 }, + { kind: `eq`, field: `rank`, value: 0 }, + ), +} + +if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + fc.statistics( + scenarioArbitrary, + (scenario) => `${scenario.family} / ${classifyDifferenceOutcome(scenario)}`, + oraclePropertyOptions(1_000, finiteWorldProperty), + ) +} + +describe(`predicate subtraction oracle`, () => { + it(`resolves each generated reference path independently`, () => { + const row = { score: 1, rank: 2 } + + expect(evaluatePredicate(refs.score, row)).toBe(1) + expect(evaluatePredicate(refs.rank, row)).toBe(2) + }) + + it(`evaluates the Cartesian product of reference values`, () => { + const encodedRows = new Set( + finiteRows.map(({ score, rank }) => `${String(score)}:${String(rank)}`), + ) + + expect(finiteRows).toHaveLength(finiteValues.length ** 2) + expect(encodedRows).toHaveLength(finiteValues.length ** 2) + expect(finiteRows).toContainEqual({ score: -3, rank: null }) + expect(finiteRows).toContainEqual({ score: null, rank: -3 }) + }) + + it(`covers both reference paths in the fixed replay corpus`, () => { + const fields = new Set( + fc + .sample(scenarioArbitrary, { + numRuns: oracleRuns(250), + seed: 1777, + }) + .flatMap(scenarioFields), + ) + + expect(fields).toEqual(new Set([`score`, `rank`])) + }) + + it(`calibrates every subtraction outcome label`, () => { + for (const [expected, scenario] of Object.entries( + outcomeCalibrations, + ) as Array<[DifferenceOutcome, DifferenceScenario]>) { + expect(classifyDifferenceOutcome(scenario)).toBe(expected) + assertSemanticDifference(scenario) + } + }) + + it(`rejects a subtraction result with the wrong finite-world meaning`, () => { + const scenario = outcomeCalibrations[`successful narrowing`] + const { requested } = evaluateDifference(scenario) + + expect(() => + assertSemanticDifference(scenario, { result: requested }), + ).toThrow() + }) + + it(`calibrates runtime IN null semantics under NOT and OR`, () => { + const membership = buildAtomic({ + kind: `in`, + field: `score`, + values: [null, 1], + }) + const negated = call(`not`, membership) + const disjunction = call( + `or`, + negated, + buildAtomic({ kind: `eq`, field: `rank`, value: 2 }), + ) + + expect(evaluatePredicate(membership, { score: null, rank: 0 })).toBeNull() + expect(evaluatePredicate(membership, { score: 0, rank: 0 })).toBe(false) + expect(evaluatePredicate(negated, { score: 0, rank: 0 })).toBe(true) + expect(evaluatePredicate(disjunction, { score: null, rank: 0 })).toBeNull() + expect(evaluatePredicate(disjunction, { score: null, rank: 2 })).toBe(true) + }) + + fcTest.prop([scenarioArbitrary], { numRuns: oracleRuns(250), seed: 1777 })( + `preserves finite-world subtraction for a fixed replay corpus`, + assertSemanticDifference, + ) + + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(250, finiteWorldProperty), + )( + `preserves finite-world subtraction for a random or replayed seed`, + assertSemanticDifference, + ) + + fcTest.prop([predicateArbitrary], { numRuns: oracleRuns(100), seed: 1778 })( + `preserves unbounded subtraction across UNKNOWN rows for a fixed replay corpus`, + assertUnboundedDifference, + ) + + fcTest.prop( + [predicateArbitrary], + oraclePropertyOptions(100, unboundedProperty), + )( + `preserves unbounded subtraction across UNKNOWN rows for a random or replayed seed`, + assertUnboundedDifference, + ) + + fcTest.prop([fieldArbitrary, fc.integer({ min: -2, max: 2 })], { + numRuns: oracleRuns(100), + seed: 1779, + })( + `preserves duplicate common terms for a fixed replay corpus`, + assertDuplicateTermDifference, + ) + + fcTest.prop( + [fieldArbitrary, fc.integer({ min: -2, max: 2 })], + oraclePropertyOptions(100, duplicateProperty), + )( + `preserves duplicate common terms for a random or replayed seed`, + assertDuplicateTermDifference, + ) + + it(`covers every difference outcome in the fixed replay corpus`, () => { + expectEveryDifferenceOutcome({ + numRuns: oracleRuns(1_000), + seed: 1777, + }) + }) +}) diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index cdd1c785a..a18b26673 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -10,6 +10,7 @@ import { unionWherePredicates, } from '../../src/query/predicate-utils' import { Func, PropRef, Value } from '../../src/query/ir' +import { evaluateReferenceExpression } from '../reference-expression' import type { BasicExpression, OrderBy, @@ -58,6 +59,10 @@ function or(...args: Array): Func { return func(`or`, ...args) } +function not(arg: BasicExpression): Func { + return func(`not`, arg) +} + function inOp(left: BasicExpression, values: Array): Func { return func(`in`, left, val(values)) } @@ -1194,11 +1199,22 @@ describe(`minusWherePredicates`, () => { const subtract = gt(ref(`age`), val(10)) const result = minusWherePredicates(undefined, subtract) - expect(result).toEqual({ - type: `func`, - name: `not`, - args: [subtract], - }) + expect(result).toBeNull() + }) + + it(`falls back before negating an IN predicate`, () => { + const subtract = inOp(ref(`status`), [`active`, null]) + + expect(minusWherePredicates(undefined, subtract)).toBeNull() + }) + + it(`falls back before negating an OR predicate`, () => { + const subtract = or( + eq(ref(`status`), val(`active`)), + eq(ref(`status`), val(null)), + ) + + expect(minusWherePredicates(undefined, subtract)).toBeNull() }) it(`should return empty set when from is subset of subtract`, () => { @@ -1431,22 +1447,85 @@ describe(`minusWherePredicates`, () => { }) describe(`common conditions`, () => { - it(`removes a reordered IN condition from a nested conjunction`, () => { - const requested = inOp(ref(`score`), [2, -2, 0, -3, -1, 3]) - const alreadyLoaded = and( - inOp(ref(`score`), [0]), - and( - lt(ref(`score`), val(1)), - inOp(ref(`score`), [2, -3, -2, 3, 0, -1]), - ), + it(`falls back before negating a nullable residual field`, () => { + const shared = lt(ref(`rank`), val(1)) + const requested = and(shared, shared) + const loaded = and(shared, shared, eq(ref(`score`), val(0))) + + expect(minusWherePredicates(requested, loaded)).toBeNull() + }) + + it(`removes only one matching occurrence for each common condition`, () => { + const score = ref(`score`) + const requested = and( + gt(score, val(0)), + or(eq(score, val(null)), eq(score, val(1))), + inOp(score, [1, 0]), + gt(score, val(-1)), + ) + const loaded = and( + gt(score, val(0)), + or(eq(score, val(null)), eq(score, val(1))), + inOp(score, [1, 0]), + gt(score, val(0)), ) - expect(minusWherePredicates(requested, alreadyLoaded)).toEqual( - and( - requested, - func(`not`, and(inOp(ref(`score`), [0]), lt(ref(`score`), val(1)))), - ), + const result = minusWherePredicates(requested, loaded) + + expect(result).not.toBeNull() + for (const value of [-1, 0, 1, null]) { + const row = { score: value } + const expected = + evaluateReferenceExpression(requested, row) === true && + evaluateReferenceExpression(loaded, row) !== true + expect(evaluateReferenceExpression(result!, row)).toBe(expected) + } + }) + + it(`falls back when nested subtraction would negate an unknown value`, () => { + const score = ref(`score`) + const requested = eq(score, val(0)) + const loaded = and( + not(eq(score, val(-1))), + and(eq(score, val(0)), lt(score, val(1))), ) + + expect(minusWherePredicates(requested, loaded)).toBeNull() + }) + + it(`falls back across nested equality, range, and NOT terms`, () => { + const score = ref(`score`) + const ranges = [gt, gte, lt, lte] + + for (const requestedValue of [-1, 0, 1]) { + const requested = eq(score, val(requestedValue)) + for (const excludedValue of [-1, 0, 1]) { + const negatedEquality = not(eq(score, val(excludedValue))) + for (const range of ranges) { + for (const boundary of [-1, 0, 1]) { + const rangePredicate = range(score, val(boundary)) + const loadedPredicates = [ + and( + negatedEquality, + and(eq(score, val(requestedValue)), rangePredicate), + ), + and( + and(negatedEquality, eq(score, val(requestedValue))), + rangePredicate, + ), + and( + rangePredicate, + and(negatedEquality, eq(score, val(requestedValue))), + ), + ] + + for (const loaded of loadedPredicates) { + expect(minusWherePredicates(requested, loaded)).toBeNull() + } + } + } + } + } }) it(`should handle common conditions: (age > 10 AND status = 'active') - (age > 20 AND status = 'active') = (age > 10 AND age <= 20 AND status = 'active')`, () => { diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index b0c79ef2c..3658042aa 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -45,10 +45,6 @@ function lte(left: BasicExpression, right: BasicExpression): Func { return new Func(`lte`, [left, right]) } -function not(expression: BasicExpression): Func { - return new Func(`not`, [expression]) -} - describe(`createDeduplicatedLoadSubset`, () => { it(`does not let mutation rewrite settled large-binary coverage`, () => { const loadSubset = vi.fn(() => true as const) @@ -1348,6 +1344,37 @@ describe(`createDeduplicatedLoadSubset`, () => { }) }) + it(`tracks the original demand while a narrowed transport is in flight`, async () => { + let resolveNarrowed: (() => void) | undefined + const calls: Array = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: (options) => { + calls.push(cloneOptions(options)) + if (calls.length === 1) return Promise.resolve() + return new Promise((resolve) => { + resolveNarrowed = resolve + }) + }, + }) + + await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) + + const wider = { where: gt(ref(`age`), val(10)) } + const first = deduplicated.loadSubset(wider) + const second = deduplicated.loadSubset(wider) + + expect(calls).toHaveLength(2) + expect(calls[1]?.where).toEqual( + and(gt(ref(`age`), val(10)), lte(ref(`age`), val(20))), + ) + expect(first).toBeInstanceOf(Promise) + expect(second).toBeInstanceOf(Promise) + + resolveNarrowed?.() + await Promise.all([first, second]) + expect(deduplicated.loadSubset(wider)).toBe(true) + }) + it(`should request only the difference for set predicates`, async () => { let callCount = 0 const calls: Array = [] @@ -1503,11 +1530,11 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(1) // Second call: no where clause (all data) - // Should request all data except what we already loaded - // i.e. should request NOT (age > 20) + // The missing difference is not safe to express under three-valued + // logic, so the adapter receives the full all-data request. await deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ where: not(gt(ref(`age`), val(20))) }) + expect(calls[1]).toEqual({}) // After loading all data, subsequent calls should be deduplicated const result = await deduplicated.loadSubset({ @@ -1517,6 +1544,37 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(2) }) + it(`retries a full-request fallback after transport failure`, async () => { + let rejectAllData: ((error: Error) => void) | undefined + const calls: Array = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: (options) => { + calls.push(cloneOptions(options)) + if (calls.length === 1 || calls.length === 3) { + return Promise.resolve() + } + return new Promise((_resolve, reject) => { + rejectAllData = reject + }) + }, + }) + + await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) + + const failed = deduplicated.loadSubset({}) + const rejected = expect(failed).rejects.toThrow(`all-data failed`) + rejectAllData?.(new Error(`all-data failed`)) + await rejected + + expect(calls).toHaveLength(2) + expect(calls[1]).toEqual({}) + + await deduplicated.loadSubset({}) + expect(calls).toHaveLength(3) + expect(calls[2]).toEqual({}) + expect(deduplicated.loadSubset({})).toBe(true) + }) + describe(`hasLoadedAllData after loading filtered + unfiltered data`, () => { it(`should set hasLoadedAllData after a filtered load followed by an unfiltered load`, async () => { let callCount = 0 @@ -1538,9 +1596,7 @@ describe(`createDeduplicatedLoadSubset`, () => { await deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ - where: not(inOp(ref(`task_id`), [`id1`, `id2`, `id3`])), - }) + expect(calls[1]).toEqual({}) const result = await deduplicated.loadSubset({}) expect(result).toBe(true) @@ -1632,9 +1688,7 @@ describe(`createDeduplicatedLoadSubset`, () => { await deduplicated.loadSubset({}) - expect(calls[2]).toEqual({ - where: not(inOp(ref(`task_id`), [`uuid-1`, `uuid-2`])), - }) + expect(calls[2]).toEqual({}) expect((deduplicated as any).hasLoadedAllData).toBe(true) expect((deduplicated as any).unlimitedWhere).toBeUndefined() @@ -1695,11 +1749,8 @@ describe(`createDeduplicatedLoadSubset`, () => { const secondAllDataLoad = deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ - where: not(eq(ref(`task_id`), val(`uuid-1`))), - }) - expect(firstAllDataLoad).toBeInstanceOf(Promise) - expect(secondAllDataLoad).toBeInstanceOf(Promise) + expect(calls[1]).toEqual({}) + expect(secondAllDataLoad).toBe(firstAllDataLoad) resolveAllDataLoad?.() await firstAllDataLoad @@ -1732,23 +1783,12 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(10) // Now load all data (no WHERE clause) - // This should send NOT(IN(...)) to the backend but track as "all data loaded" + // The adapter receives the full request because NOT(IN(...)) would drop + // rows whose task_id is null under three-valued logic. await deduplicated.loadSubset({}) expect(callCount).toBe(11) - // The load request should be NOT(IN(task_id, [all accumulated uuids])) - const loadWhere = calls[10]!.where as any - expect(loadWhere.name).toBe(`not`) - expect(loadWhere.args[0].name).toBe(`in`) - expect(loadWhere.args[0].args[0].path).toEqual([`task_id`]) - const loadedUuids = ( - loadWhere.args[0].args[1].value as Array - ).sort() - const expectedUuids = Array.from( - { length: 10 }, - (_, i) => `uuid-${i}`, - ).sort() - expect(loadedUuids).toEqual(expectedUuids) + expect(calls[10]).toEqual({}) // Critical: after loading all data, subsequent requests should be deduplicated const result1 = await deduplicated.loadSubset({ diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index ae46c102d..170853787 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest' import { Temporal } from 'temporal-polyfill' +import packageJson from '../package.json' import { deepEquals } from '../src/utils' import { isPromiseLike } from '../src/utils/type-guards' import { @@ -9,6 +10,12 @@ import { } from './oracle-config' describe(`oracle run configuration`, () => { + it(`runs the predicate subtraction oracle in the oracle campaign`, () => { + expect(packageJson.scripts[`test:oracles`]).toContain( + `tests/query/predicate-subtraction-oracle.property.test.ts`, + ) + }) + it(`reads the multiplier and replay coordinates from an explicit environment`, () => { expect( readOracleRunConfig({ From 79bf85f1b1af8c79fde302ca04b91e01510c6cd3 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 02:40:03 -0600 Subject: [PATCH 319/327] test(db): clear settled dedupe evidence on reset --- packages/db/tests/query/subset-dedupe.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 0b25f089d..033a52f3d 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -1034,6 +1034,25 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(removeSpy).toHaveBeenCalledOnce() }) + it(`releases settled exact acquisition evidence when reset`, () => { + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: () => true, + }) + const retainedAcquisitions = () => + ( + deduplicated as unknown as { + exactAcquisitions: ReadonlyArray + } + ).exactAcquisitions.length + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(retainedAcquisitions()).toBe(1) + + deduplicated.reset() + + expect(retainedAcquisitions()).toBe(0) + }) + it(`starts new work immediately after reset and protects it from old completion`, async () => { const releases: Array<() => void> = [] const loadSubset = vi.fn( From 2fa5e390e2b8e181a92b33a29aa850bd113e211c Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 03:23:15 -0600 Subject: [PATCH 320/327] test(powersync): prove failed demand retirement --- .../powersync-db-collection/src/internal.ts | 5 +++++ .../powersync-db-collection/src/powersync.ts | 4 ++++ .../tests/on-demand-sync.test.ts | 18 +++++++++++++++--- 3 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 packages/powersync-db-collection/src/internal.ts diff --git a/packages/powersync-db-collection/src/internal.ts b/packages/powersync-db-collection/src/internal.ts new file mode 100644 index 000000000..c7122835f --- /dev/null +++ b/packages/powersync-db-collection/src/internal.ts @@ -0,0 +1,5 @@ +export const POWERSYNC_TEST_HOOKS = Symbol(`powerSyncTestHooks`) + +export type PowerSyncTestHooks = { + getDemandCount: () => number +} diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index e8a5336cc..736caaecf 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -5,6 +5,7 @@ import { PendingOperationStore } from './PendingOperationStore' import { PowerSyncTransactor } from './PowerSyncTransactor' import { DEFAULT_BATCH_SIZE } from './definitions' import { asPowerSyncRecord, mapOperation } from './helpers' +import { POWERSYNC_TEST_HOOKS } from './internal' import { convertTableToSchema } from './schema' import { serializeForSQLite } from './serialization' import type { @@ -883,6 +884,9 @@ function createPowerSyncCollectionConfig< markReady() return { + [POWERSYNC_TEST_HOOKS]: { + getDemandCount: () => demands.size, + }, cleanup: () => { stopped = true lifecycleGeneration++ diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index b2ae51c19..d2b575810 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -17,10 +17,12 @@ import { import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' +import { POWERSYNC_TEST_HOOKS } from '../src/internal' import { projectRetainedRowKeys, projectTransportLoads, } from '../../db/tests/load-subset-full-flow-model' +import type {PowerSyncTestHooks} from '../src/internal'; import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' import type { Scheduler } from 'fast-check' @@ -3378,6 +3380,8 @@ describe(`On-Demand Sync Mode`, () => { const onLoadSubset = vi .fn() .mockRejectedValueOnce(hookFailure) + .mockRejectedValueOnce(hookFailure) + .mockRejectedValueOnce(hookFailure) .mockResolvedValueOnce(undefined) const createDiffTrigger = vi .spyOn(db.triggers, `createDiffTrigger`) @@ -3400,11 +3404,19 @@ describe(`On-Demand Sync Mode`, () => { if (!sync || typeof sync === `function` || !sync.loadSubset) { throw new Error(`Expected on-demand sync controls`) } + const { getDemandCount } = ( + sync as typeof sync & { + [POWERSYNC_TEST_HOOKS]: PowerSyncTestHooks + } + )[POWERSYNC_TEST_HOOKS] try { - await expect( - sync.loadSubset({ where: eq(`category`, `electronics`) }), - ).rejects.toBe(hookFailure) + for (const category of [`electronics`, `clothing`, `outdoors`]) { + await expect( + sync.loadSubset({ where: eq(`category`, category) }), + ).rejects.toBe(hookFailure) + expect(getDemandCount()).toBe(0) + } await sync.loadSubset({ where: eq(`category`, `clothing`) }) const when = createDiffTrigger.mock.calls.at(-1)?.[0].when From 28f4399575d99bf115cd93ca581d22984b74e959 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 03:31:20 -0600 Subject: [PATCH 321/327] style(powersync): format test hook imports --- packages/powersync-db-collection/tests/on-demand-sync.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index d2b575810..9021f5f10 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -17,12 +17,12 @@ import { import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' -import { POWERSYNC_TEST_HOOKS } from '../src/internal' +import { POWERSYNC_TEST_HOOKS } from '../src/internal' import { projectRetainedRowKeys, projectTransportLoads, } from '../../db/tests/load-subset-full-flow-model' -import type {PowerSyncTestHooks} from '../src/internal'; +import type { PowerSyncTestHooks } from '../src/internal' import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' import type { Scheduler } from 'fast-check' From 0c1af71f4ac7e06c004eac7e04f0061e042d8012 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 03:40:58 -0600 Subject: [PATCH 322/327] test(db): restore full-suite type safety --- .../db/tests/collection-lifecycle.test.ts | 51 +++++++++++++------ ...rce-reconciliation-oracle.property.test.ts | 2 +- .../query/includes-publication-oracle.test.ts | 41 +++++++++++---- packages/db/tests/transactions.test.ts | 14 ++--- 4 files changed, 74 insertions(+), 34 deletions(-) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index 2cf7b9434..1c84158cf 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -12,6 +12,16 @@ import { const originalSetTimeout = global.setTimeout const originalClearTimeout = global.clearTimeout +function getChangesManager(collection: object): { + emitEmptyReadyEvent: () => void +} { + return ( + collection as unknown as { + _changes: { emitEmptyReadyEvent: () => void } + } + )._changes +} + describe(`Collection Lifecycle Management`, () => { let mockSetTimeout: ReturnType let mockClearTimeout: ReturnType @@ -634,18 +644,17 @@ describe(`Collection Lifecycle Management`, () => { syncError: collection._lifecycle.getSyncError(), }) }) + const changes = getChangesManager(collection) const originalEmitEmptyReadyEvent = - collection._changes.emitEmptyReadyEvent.bind(collection._changes) - vi.spyOn(collection._changes, `emitEmptyReadyEvent`).mockImplementation( - () => { - transitionTrace.push({ - kind: `dependent-ready`, - status: collection.status, - syncError: collection._lifecycle.getSyncError(), - }) - originalEmitEmptyReadyEvent() - }, - ) + changes.emitEmptyReadyEvent.bind(changes) + vi.spyOn(changes, `emitEmptyReadyEvent`).mockImplementation(() => { + transitionTrace.push({ + kind: `dependent-ready`, + status: collection.status, + syncError: collection._lifecycle.getSyncError(), + }) + originalEmitEmptyReadyEvent() + }) let didThrow = false let thrown: unknown @@ -696,7 +705,10 @@ describe(`Collection Lifecycle Management`, () => { startSync: false, sync: { sync: () => {} }, }) - const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) const firstReadyStatuses: Array = [] collection.onFirstReady(() => { firstReadyStatuses.push(collection.status) @@ -727,7 +739,10 @@ describe(`Collection Lifecycle Management`, () => { startSync: false, sync: { sync: () => {} }, }) - const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) const firstReady = vi.fn() collection.onFirstReady(firstReady) collection.on(`status:ready`, () => { @@ -763,7 +778,10 @@ describe(`Collection Lifecycle Management`, () => { }, }, }) - const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) collection.onFirstReady(() => { firstReadyStatuses.push(collection.status) }) @@ -804,7 +822,10 @@ describe(`Collection Lifecycle Management`, () => { }, }, }) - const readyEvent = vi.spyOn(collection._changes, `emitEmptyReadyEvent`) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) collection.onFirstReady(() => { trace.push(`first failure:${collection.status}`) throw firstFailure diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts index b36d9a209..a9d5d9347 100644 --- a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -525,7 +525,7 @@ it(`replaces the retained live-query source row after an ordered truncate`, asyn .limit(1), startSync: true, }) - const batches: Array>> = [] + const batches: Array>> = [] try { await live.preload() diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index 5f5794d70..d95fdd1c9 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -13,9 +13,10 @@ import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, withExpectedRejection } from '../utils.js' import { createControlledCollection } from './includes-oracle-helpers.js' +import type { StandardSchemaV1 } from '@standard-schema/spec' import type { Collection } from '../../src/collection/index.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' -import type { ChangeMessage, SyncConfig } from '../../src/types.js' +import type { ChangeMessage, SyncConfig, UtilsRecord } from '../../src/types.js' type ParentRow = { id: number @@ -590,9 +591,10 @@ function expectedPendingTransition( } } -function pendingPublicationEvent( - change: ChangeMessage, -): PendingPublicationEvent { +function pendingPublicationEvent< + TRow extends PendingPublicationRow, + TKey extends string | number, +>(change: ChangeMessage): PendingPublicationEvent { const value = { id: change.value.id, value: change.value.value } if (change.type !== `update`) { return { type: change.type, key: Number(change.key), value } @@ -608,14 +610,25 @@ function pendingPublicationEvent( } } -function createPendingPublicationQuery( - source: Collection, +function createPendingPublicationQuery< + TRow extends PendingPublicationRow, + TKey extends string | number, + TUtils extends UtilsRecord, + TSchema extends StandardSchemaV1, + TInput extends object, +>( + source: Collection, shape: PendingPublicationShape, ) { return createLiveQueryCollection({ id: `pending-publication-${shape}-${nextCollectionId++}`, query: (query) => { - const rows = query.from({ row: source }) + const rows = query.from({ + row: source as unknown as Collection< + PendingPublicationRow, + string | number + >, + }) if (shape === `orderBy`) { return rows.orderBy(({ row }) => row.value) } @@ -628,8 +641,14 @@ function createPendingPublicationQuery( }) } -function observePendingPublication( - collection: Collection, +function observePendingPublication< + TRow extends PendingPublicationRow, + TKey extends string | number, + TUtils extends UtilsRecord, + TSchema extends StandardSchemaV1, + TInput extends object, +>( + collection: Collection, shape: PendingPublicationShape, ) { const batches: Array> = [] @@ -645,7 +664,7 @@ function observePendingPublication( } const subscription = collection.subscribeChanges( (changes) => { - batches.push(changes.map(pendingPublicationEvent)) + batches.push(changes.map((change) => pendingPublicationEvent(change))) callbackSnapshots.push(currentRows()) }, { includeInitialState: false }, @@ -992,7 +1011,7 @@ describe(`source publication across pending derived mutations`, () => { const observed = observePendingPublication(q2, `select`) const persistence = createDeferred() const settlementError = new Error(`ordinary source prefix rollback`) - const mutate = createOptimisticAction({ + const mutate = createOptimisticAction({ onMutate: () => { source.update(1, (draft) => { draft.value = 11 diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index d69932dc6..e6179c99e 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -257,13 +257,13 @@ describe(`Transactions`, () => { } }) it.each([ - [`Error`, () => new Error(`late persistence rejection`)], - [`undefined`, () => undefined], - [`false`, () => false], - [`zero`, () => 0], - [`NaN`, () => Number.NaN], - [`string`, () => `late persistence rejection`], - [`object`, () => ({ late: true })], + [`Error`, (): unknown => new Error(`late persistence rejection`)], + [`undefined`, (): unknown => undefined], + [`false`, (): unknown => false], + [`zero`, (): unknown => 0], + [`NaN`, (): unknown => Number.NaN], + [`string`, (): unknown => `late persistence rejection`], + [`object`, (): unknown => ({ late: true })], ] as const)( `ignores a late %s persistence rejection after rollback wins`, async (reasonName, createReason) => { From 303a19ca253b7e84d2ea22c84da4502657d5babe Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 04:43:00 -0600 Subject: [PATCH 323/327] style(db): format integrated oracle changes --- packages/db/src/collection/index.ts | 5 +--- packages/db/src/indexes/base-index.ts | 4 +-- .../tests/collection-sync-reentrancy.test.ts | 4 +-- packages/db/tests/effect.test.ts | 4 ++- ...ncludes-collection-oracle.property.test.ts | 27 +++++++++---------- .../query/includes-temporal-oracle.test.ts | 18 +++++-------- 6 files changed, 25 insertions(+), 37 deletions(-) diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 319f7daea..447ed9261 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -458,10 +458,7 @@ export class CollectionImpl< /** Capture mutable state before a coherent graph publication is installed. */ public _snapshotPublicationState( keys: Iterable, - ): CollectionPublicationStateSnapshot< - TOutput, - TKey - > { + ): CollectionPublicationStateSnapshot { return this._state.snapshotPublicationState(keys) } diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 9c6507030..ecbd86852 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -197,8 +197,8 @@ export abstract class BaseIndex< /** * Checks if the compare options match the index's compare options. * Reversing an index also reverses null placement, so opposite directions - * are compatible only when their requested null placement is opposite too. - */ + * are compatible only when their requested null placement is opposite too. + */ matchesCompareOptions(compareOptions: CompareOptions): boolean { const indexCompareOptions = this.compareOptions const indexUsesLocale = usesLocaleCollation(indexCompareOptions) diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index 16ca7afd7..946a45637 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -222,9 +222,7 @@ describe(`sync publication reentrancy`, () => { it.each([`open`, `prepared`, `published`] as const)( `starts a second publication cycle with the first cycle %s`, async (firstCycleState) => { - const harness = createSyncHarness( - `publication-cycle-${firstCycleState}`, - ) + const harness = createSyncHarness(`publication-cycle-${firstCycleState}`) const { collection } = harness const callbacks: Array<{ changes: Array diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index fb5eb3ae9..8c9c64446 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -2160,7 +2160,9 @@ describe(`createEffect`, () => { ) let loadCount = 0 let unloadCount = 0 - const consoleError = vi.spyOn(console, `error`).mockImplementation(() => {}) + const consoleError = vi + .spyOn(console, `error`) + .mockImplementation(() => {}) const issues = createCollection({ id: `effect-obsolete-release-issues`, getKey: (issue) => issue.id, diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 8f11a3a21..6b3180321 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -52,9 +52,7 @@ type FacadeCandidateScanScenario = { class ThrowingUpdateIndex extends BasicIndex { updateFailure: { error: unknown } | undefined - buildFailure: - | { error: unknown; stage: `before` | `after` } - | undefined + buildFailure: { error: unknown; stage: `before` | `after` } | undefined buildCalls = 0 override update(key: number, oldItem: unknown, newItem: unknown): void { @@ -1574,19 +1572,18 @@ describe(`Collection-valued includes oracle`, () => { { id: 10, value: 2 }, { id: 11, value: 20 }, ]) - const rootPublicationsAfterRecovery: Array< - Array - > = [ - [], + const rootPublicationsAfterRecovery: Array> = [ - { - type: `update`, - key: 1, - value: { id: 1, value: 5, preservesFacade: true }, - previousValue: { id: 1, value: 1, preservesFacade: true }, - }, - ], - ] + [], + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 5, preservesFacade: true }, + previousValue: { id: 1, value: 1, preservesFacade: true }, + }, + ], + ] const childPublicationsAfterRecovery: Array< Array > = [ diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index a4892697c..f7612bf98 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -877,20 +877,16 @@ async function expectDemandReactivationRetriesAfterReleaseFailure( } try { - expect(controller.setDemand(subscription, plan, new Set(keys))).toMatchObject( - { changed: true, empty: false }, - ) + expect( + controller.setDemand(subscription, plan, new Set(keys)), + ).toMatchObject({ changed: true, empty: false }) expect(loadCount).toBe(1) const retired = controller.setDemand(subscription, plan, new Set()) expect(retired).toMatchObject({ changed: true, empty: true }) expect(retired.releaseFailure?.error).toBe(releaseError) - const reactivated = controller.setDemand( - subscription, - plan, - new Set(keys), - ) + const reactivated = controller.setDemand(subscription, plan, new Set(keys)) expect(reactivated).toMatchObject({ changed: true, empty: false }) expect(loadCount).toBe(2) } finally { @@ -1334,10 +1330,8 @@ describe(`includes temporal oracle`, () => { expectFailedDemandRetriesSameCoverage, ) - it( - `reactivated demand retries after its prior release fails`, - () => expectDemandReactivationRetriesAfterReleaseFailure([1]), - ) + it(`reactivated demand retries after its prior release fails`, () => + expectDemandReactivationRetriesAfterReleaseFailure([1])) fcTest.prop( [ From 0b3aa0f4f151ef4b59ae7f3be1dfdf42e2d60141 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 05:10:43 -0600 Subject: [PATCH 324/327] docs: add integrated loadSubset changeset --- .changeset/harden-load-subset-lifecycle.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/harden-load-subset-lifecycle.md diff --git a/.changeset/harden-load-subset-lifecycle.md b/.changeset/harden-load-subset-lifecycle.md new file mode 100644 index 000000000..c417a68e9 --- /dev/null +++ b/.changeset/harden-load-subset-lifecycle.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Harden on-demand query refinement across predicate subtraction, replay and publication rollback, readiness restarts, and resource cleanup. From 79781c8f9648bf5612e4f4eac0158ed091b3bb87 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 05:43:40 -0600 Subject: [PATCH 325/327] fix(db): preserve outcome-free paging compatibility --- packages/db/src/collection/subscription.ts | 20 +++- packages/db/src/query/effect.ts | 9 +- packages/db/src/query/live/ARCHITECTURE.md | 14 ++- .../src/query/live/collection-subscriber.ts | 5 +- packages/db/src/query/live/window-state.ts | 34 ++++++- ...d-subset-full-flow-oracle.property.test.ts | 93 ++++++++++++++++++- packages/db/tests/query/window-state.test.ts | 9 ++ 7 files changed, 168 insertions(+), 16 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 2ce24ce60..69ff37dd7 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1305,6 +1305,18 @@ export class CollectionSubscription ) } + get hasOrderedResultForActiveWindow(): boolean { + if (!this.hasActiveOrderedDemand() || !this.orderedWindow) return false + return this.retainedOrderedPublication + ? this.orderedWindow.coversActiveWindow + : this.orderedWindow.satisfiesActiveWindow + } + + settleOrderedResultAfterNoProgress(): boolean { + if (this.retainedOrderedPublication || !this.orderedWindow) return false + return this.orderedWindow.settleLocalRequestAfterNoProgress() + } + get orderedBoundaryRow(): object | undefined { if (!this.hasActiveOrderedDemand()) return undefined const boundary = this.retainedOrderedPublication @@ -2492,7 +2504,7 @@ export class CollectionSubscription if (changes.length > 0) this.callback(changes) - if (!retainedPublication && this.orderedWindow.coversActiveWindow) { + if (!retainedPublication && this.orderedWindow.satisfiesActiveWindow) { // No adapter request was made. Use an impossible zero-window demand so // direct tracking can finish without claiming another demand's outcome. onLoadSubsetResult?.(true, { @@ -2645,7 +2657,11 @@ export class CollectionSubscription const rowKeys = outcome?.appliedRowKeys const exhausted = outcome?.extent === `exhausted` - if (outcome !== undefined && rowKeys === undefined && !exhausted) { + if ( + outcome?.extent === `unknown` && + rowKeys === undefined && + !exhausted + ) { window.recordLocalRequestSatisfaction(ordered.requestedPrefix) } else if (!ordered.hadBoundary && !ordered.requiresUnboundedRefinement) { window.recordInitialCoverage(rowKeys, exhausted) diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 20cc28f9a..e87465fa2 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -1013,7 +1013,7 @@ class EffectPipelineRunner { const missingResultRows = orderByInfo.dataNeeded() if ( (!orderByInfo.refillFromResultDeficit || missingResultRows === 0) && - subscription.hasOrderedCoverageForActiveWindow + subscription.hasOrderedResultForActiveWindow ) { continue } @@ -1028,7 +1028,7 @@ class EffectPipelineRunner { subscription.orderedRetainedWindowSize + missingResultRows, ) } - if (subscription.hasOrderedCoverageForActiveWindow) { + if (subscription.hasOrderedResultForActiveWindow) { continue } @@ -1089,7 +1089,10 @@ class EffectPipelineRunner { subscription.orderedRetainedWindowSize, subscription.orderedBoundaryKey, ) - if (!cursor) return // Duplicate request — skip + if (!cursor) { + subscription.settleOrderedResultAfterNoProgress() + return + } this.lastLoadRequestKey.set(sourceId, cursor.loadRequestKey) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a1bea44f8..e4ce9dfb3 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -772,11 +772,14 @@ continuation after all replacement acquisitions have settled. An outcome-free completion (`true` or `Promise`) supplies no reusable row provenance, source extent, or CoverageFact. Its exact request has still settled, -so the owning subscription may admit only the current local prefix. A short -page remains uncovered and triggers another pass. The admitted local boundary -may distinguish those immediate passes, but it is scheduling state, not a -transport cursor. If the window later grows, core refreshes the required prefix -from the start instead of continuing from those rows as a cursor boundary. +so the owning subscription may admit and publish only the current local prefix, +even when that prefix is short. Core keeps loading while the local boundary +advances, then treats a repeated request with no new progress as caller-locally +satisfied. This stops work only for that exact active window. It is not +coverage: it cannot establish source extent, satisfy a replacement epoch, or +become a transport cursor. If the window later grows, core refreshes the +required prefix from the start. An explicit continuing outcome is not +outcome-free and cannot use this fallback. A bare child query is a Collection-valued include. It exposes one stable public Collection facade per active bucket in that edge: @@ -1422,6 +1425,7 @@ create recursive Collection machinery. | Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | | Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | | End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | +| Framework paging, final partial pages, and peek-ahead compatibility | `packages/db/tests/conformance/infinite-suite.ts` | | Shared subset acquisition, readiness, receipt, and replay interpreter | `packages/db/tests/query/load-subset-refinement-model.property.test.ts` | | Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.test.ts` | | Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index d3a363f3d..205636dfa 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -514,7 +514,7 @@ export class CollectionSubscriber< const missingResultRows = refillFromResultDeficit ? dataNeeded() : 0 if ( missingResultRows === 0 && - subscription.hasOrderedCoverageForActiveWindow + subscription.hasOrderedResultForActiveWindow ) { return true } @@ -542,7 +542,7 @@ export class CollectionSubscriber< subscription.orderedRetainedWindowSize + missingResultRows, ) } - if (subscription.hasOrderedCoverageForActiveWindow) { + if (subscription.hasOrderedResultForActiveWindow) { return true } @@ -657,6 +657,7 @@ export class CollectionSubscriber< subscription.orderedBoundaryKey, ) if (!cursor) { + if (subscription.settleOrderedResultAfterNoProgress()) return if (this.lastNoProgressRequestKey !== this.lastLoadRequestKey) { this.lastNoProgressRequestKey = this.lastLoadRequestKey this.collectionConfigBuilder.recordSubsetError( diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index d773cce36..d6905fb27 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -45,6 +45,8 @@ export class WindowState< private hasFullCoverage = false private needsFullRefinement = false private needsPrefixRefresh = false + private locallySettledSize = 0 + private hasOutcomeFreeSettlement = false private hasInitialCoverage = false private hasUnsettledInitialMutation = false private revision = 0 @@ -101,6 +103,11 @@ export class WindowState< return this.hasFullCoverage || this.coveredSize >= this.activeSize } + /** Whether this caller may stop loading its exact active window. */ + get satisfiesActiveWindow(): boolean { + return this.coversActiveWindow || this.locallySettledSize >= this.activeSize + } + get coveredPrefixSize(): number { return this.coveredSize } @@ -133,6 +140,8 @@ export class WindowState< this.hasFullCoverage = false this.needsFullRefinement = false this.needsPrefixRefresh = false + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = false this.hasUnsettledInitialMutation = false this.candidateKeys.clear() @@ -144,6 +153,8 @@ export class WindowState< rowKeys: ReadonlyArray | undefined, exhausted: boolean, ): void { + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = true if (exhausted) { this.hasUnsettledInitialMutation = false @@ -176,6 +187,8 @@ export class WindowState< requestedPrefix: number, requestRevision: number, ): void { + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = true if (exhausted) { this.establishFullCoverage() @@ -222,6 +235,7 @@ export class WindowState< * proof. */ recordLocalRequestSatisfaction(requestedPrefix: number): void { + this.hasOutcomeFreeSettlement = true this.candidateKeys.clear() this.provenanceKeys.clear() this.admittedKeys.clear() @@ -229,20 +243,30 @@ export class WindowState< this.admittedKeys.add(change.key) } // Outcome-free completions (`true` and Promise) do not prove - // exhaustion. Only count rows that are now present, so a short synchronous - // page can request another pass until the active prefix is actually filled. + // exhaustion. Keep their exact settled request separate from the applied + // row count so this caller can publish without creating reusable evidence. this.coveredSize = Math.min(requestedPrefix, this.admittedKeys.size) + if (this.admittedKeys.size >= requestedPrefix) { + this.locallySettledSize = requestedPrefix + } this.needsFullRefinement = false this.needsPrefixRefresh = true } + /** Stop a legacy outcome-free request only after its boundary stops moving. */ + settleLocalRequestAfterNoProgress(): boolean { + if (!this.hasOutcomeFreeSettlement) return false + this.locallySettledSize = this.activeSize + return true + } + admitChanges(changes: ReadonlyArray>): void { if (this.hasFullCoverage) return // Initial applied rows remain candidates until their boundary equivalence // class is refined. Live source changes during that request still belong // to the same ordered prefix and must survive its later settlement. - if (this.admittedKeys.size === 0) { + if (this.admittedKeys.size === 0 && this.locallySettledSize === 0) { if (this.hasInitialCoverage) { if (this.updateKnownPrefix(this.candidateKeys, changes)) { this.revision++ @@ -281,6 +305,8 @@ export class WindowState< this.provenanceKeys.clear() this.needsFullRefinement = false this.needsPrefixRefresh = true + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false } } @@ -397,6 +423,8 @@ export class WindowState< this.hasFullCoverage = true this.needsFullRefinement = false this.needsPrefixRefresh = false + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.coveredSize = Number.POSITIVE_INFINITY this.candidateKeys.clear() this.provenanceKeys.clear() diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index 1de937821..afe6d6db6 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -39,7 +39,11 @@ import { readOracleRunConfig, } from '../oracle-config.js' import type { InitialQueryBuilder } from '../../src/query/builder/index.js' -import type { LoadSubsetOptions, WritableDeep } from '../../src/types.js' +import type { + LoadSubsetOptions, + LoadSubsetResult, + WritableDeep, +} from '../../src/types.js' import type { LoadSubsetFullFlowEvent, OrderedSourceStep, @@ -4392,6 +4396,26 @@ it.each([`sync`, `async`] as const)( expect(demands).toHaveLength(2) expect(demands[1]).toMatchObject({ limit: 2, offset: 0 }) expect(demands[1]?.cursor).toBeUndefined() + + await live.utils.setWindow({ offset: 0, limit: 4 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(4) + expect(demands[2]).toMatchObject({ limit: 4, offset: 0 }) + expect(demands[2]?.cursor).toBeUndefined() + expect(demands[3]).toMatchObject({ limit: 4, offset: 0 }) + expect(demands[3]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + await live.utils.setWindow({ offset: 0, limit: 5 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(5) + expect(demands[4]).toMatchObject({ limit: 5, offset: 0 }) + expect(demands[4]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) } finally { await live.cleanup() await source.cleanup() @@ -4399,6 +4423,73 @@ it.each([`sync`, `async`] as const)( }, ) +it(`does not treat explicit continuation as outcome-free satisfaction`, async () => { + type Row = { id: number; rank: number } + const pending: Array>> = [] + const calls: Array = [] + const source = createCollection({ + id: `full-flow-explicit-continuation-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options) => { + calls.push(options) + if (calls.length === 1) { + begin() + write({ type: `insert`, value: { id: 1, rank: 1 } }) + commit() + } + const deferred = createDeferred() + pending.push(deferred) + return deferred.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `full-flow-explicit-continuation-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(2), + startSync: true, + }) + const preload = live.preload() + + try { + expect(pending).toHaveLength(1) + pending[0]!.resolve({ hasMore: true }) + await flushPromises() + + expect(pending).toHaveLength(2) + expect(calls[1]?.limit).toBeUndefined() + const [subscription] = Object.values( + live.utils[LIVE_QUERY_INTERNAL].getBuilder().subscriptions, + ) + expect(subscription?.hasOrderedResultForActiveWindow).toBe(false) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [] }) + await preload + expect(live.toArray.map(({ id }) => id)).toEqual([1]) + } finally { + for (const request of pending) { + request.resolve({ hasMore: false, appliedRowKeys: [] }) + } + await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await source.cleanup() + } +}) + it.each([ { name: `continues past an excluded source row`, diff --git a/packages/db/tests/query/window-state.test.ts b/packages/db/tests/query/window-state.test.ts index 2828c5baf..c0ef4e1a8 100644 --- a/packages/db/tests/query/window-state.test.ts +++ b/packages/db/tests/query/window-state.test.ts @@ -235,9 +235,18 @@ describe(`WindowState`, () => { expect(window.localPrefixSize).toBe(Math.min(requestedPrefix, 3)) expect(window.coversActiveWindow).toBe(requestedPrefix <= 3) + expect(window.satisfiesActiveWindow).toBe(requestedPrefix <= 3) expect(window.requestBoundary()).toBeUndefined() expect(window.progressBoundary()?.key).toBe(Math.min(requestedPrefix, 3)) expect(window.requiresPrefixRefresh).toBe(true) + + if (requestedPrefix > 3) { + expect(window.settleLocalRequestAfterNoProgress()).toBe(true) + expect(window.satisfiesActiveWindow).toBe(true) + } + + window.ensureSize(requestedPrefix + 1) + expect(window.satisfiesActiveWindow).toBe(false) }, ) }) From 3be29b3812014cab1a538530620b832a9d56c2d7 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 06:20:10 -0600 Subject: [PATCH 326/327] fix(db): keep outcome-free paging local --- packages/db/src/collection/subscription.ts | 4 + packages/db/src/query/effect.ts | 14 +- .../src/query/live/collection-subscriber.ts | 1 + packages/db/src/query/live/utils.ts | 2 + packages/db/src/query/live/window-state.ts | 24 ++- ...ubscription-replay-oracle.property.test.ts | 41 ++-- ...d-subset-full-flow-oracle.property.test.ts | 188 +++++++++++++++++- packages/db/tests/query/window-state.test.ts | 111 ++++++++++- .../tests/on-demand-sync.test.ts | 1 + 9 files changed, 358 insertions(+), 28 deletions(-) diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 69ff37dd7..bc0ce64c0 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -1294,6 +1294,10 @@ export class CollectionSubscription return this.orderedWindow?.retainedPrefixSize ?? 0 } + get orderedCoverageRevision(): number { + return this.orderedWindow?.coverageRevision ?? 0 + } + get requiresOrderedPrefixRefresh(): boolean { return this.orderedWindow?.requiresPrefixRefresh ?? false } diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index e87465fa2..fb1225cf5 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -971,8 +971,7 @@ class EffectPipelineRunner { limit: offset + limit, orderBy: normalizedOrderBy, trackLoadSubsetPromise: false, - onLoadSubsetResult: (result) => - this.trackOrderedLoad(result, orderByInfo.sourceId), + onLoadSubsetResult: (result) => this.trackOrderedLoad(result), }) } else { // Without an index there is no sound cursor continuation. Load the full @@ -1037,15 +1036,9 @@ class EffectPipelineRunner { } } - private trackOrderedLoad( - result: LoadSubsetRequestResult, - sourceId: string, - ): void { + private trackOrderedLoad(result: LoadSubsetRequestResult): void { const continueAfterFulfillment = () => { if (this.disposed) return - if (this.subscriptions[sourceId]?.requiresOrderedPrefixRefresh) { - this.lastLoadRequestKey.delete(sourceId) - } this.loadMoreIfNeeded() } if (!(result instanceof Promise)) { @@ -1088,6 +1081,7 @@ class EffectPipelineRunner { n, subscription.orderedRetainedWindowSize, subscription.orderedBoundaryKey, + subscription.orderedCoverageRevision, ) if (!cursor) { subscription.settleOrderedResultAfterNoProgress() @@ -1104,7 +1098,7 @@ class EffectPipelineRunner { minValues: cursor.minValues, trackLoadSubsetPromise: false, onLoadSubsetResult: (loadResult: LoadSubsetRequestResult) => - this.trackOrderedLoad(loadResult, sourceId), + this.trackOrderedLoad(loadResult), }) } catch (error) { if ( diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 205636dfa..68052e5bd 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -655,6 +655,7 @@ export class CollectionSubscriber< n, subscription.orderedRetainedWindowSize, subscription.orderedBoundaryKey, + subscription.orderedCoverageRevision, ) if (!cursor) { if (subscription.settleOrderedResultAfterNoProgress()) return diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index 5b27e8873..4ba81dd62 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -284,6 +284,7 @@ export function computeOrderedLoadCursor( limit: number, demandedPrefix = limit, boundaryKey?: string | number, + progressRevision = 0, ): | { minValues: Array | undefined @@ -315,6 +316,7 @@ export function computeOrderedLoadCursor( boundaryKey: boundaryKey ?? null, offset, demandedPrefix, + progressRevision, }) if (lastLoadRequestKey === loadRequestKey) { return undefined diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index d6905fb27..71fb61723 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -78,6 +78,25 @@ export class WindowState< } ensureSize(size: number): void { + if ( + size > this.activeSize && + this.hasOutcomeFreeSettlement && + this.locallySettledSize === this.activeSize + ) { + // The same numerical window can recur after a shrink. Give that growth + // a new request generation so an old no-progress key cannot suppress + // the required refresh from the start. + this.revision++ + } + if ( + size < this.activeSize && + this.hasOutcomeFreeSettlement && + this.localPrefixSize >= size + ) { + // A shrink can reuse the already-published local prefix. Keep the + // settlement exact to the smaller window so any later growth refreshes. + this.locallySettledSize = size + } this.activeSize = size this.retainedSize = Math.max(this.retainedSize, size) } @@ -105,7 +124,9 @@ export class WindowState< /** Whether this caller may stop loading its exact active window. */ get satisfiesActiveWindow(): boolean { - return this.coversActiveWindow || this.locallySettledSize >= this.activeSize + return ( + this.coversActiveWindow || this.locallySettledSize === this.activeSize + ) } get coveredPrefixSize(): number { @@ -245,7 +266,6 @@ export class WindowState< // Outcome-free completions (`true` and Promise) do not prove // exhaustion. Keep their exact settled request separate from the applied // row count so this caller can publish without creating reusable evidence. - this.coveredSize = Math.min(requestedPrefix, this.admittedKeys.size) if (this.admittedKeys.size >= requestedPrefix) { this.locallySettledSize = requestedPrefix } diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index e59a00399..48f9b0336 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -3774,7 +3774,7 @@ describe(`CollectionSubscription replay oracle`, () => { } await flushPromises() - if (replacementResult === `return` || replacementResult === `resolve`) { + if (replacementResult === `resolve`) { expect(errorObservations).toEqual([]) expect([...visible.keys()]).toEqual([`y`]) expect(subscription.orderedBoundaryKey).toBe(`y`) @@ -3785,6 +3785,12 @@ describe(`CollectionSubscription replay oracle`, () => { await flushPromises() expect([...visible.keys()]).toEqual([`z`]) expect(subscription.orderedBoundaryKey).toBe(`z`) + } else if (replacementResult === `return`) { + // A synchronous outcome-free result can settle this acquisition, + // but cannot prove that the replay is a complete replacement. + expect(errorObservations).toEqual([]) + expect([...visible.keys()]).toEqual([]) + expect(subscription.orderedBoundaryKey).toBe(`y`) } else { expect(errorObservations).toEqual([[`x`]]) expect([...visible.keys()]).toEqual([`x`]) @@ -4920,7 +4926,12 @@ describe(`CollectionSubscription replay oracle`, () => { value: { id: `b`, rank: 2, version: 2 }, }) commit(options.signal) - return settlement === `sync` ? true : Promise.resolve() + return settlement === `sync` + ? true + : Promise.resolve({ + hasMore: false, + appliedRowKeys: [`b`] as const, + }) }, unloadSubset: (options) => { unloads.push(options) @@ -5001,8 +5012,7 @@ describe(`CollectionSubscription replay oracle`, () => { await flushPromises() const publishesReplacement = - callback === `none` || - (callback === `cleanup-succeed` && settlement === `sync`) + settlement === `async` && callback === `none` expect(subscription.status).toBe(`ready`) expect(escapedCallbackError).toBeUndefined() expect( @@ -8963,7 +8973,8 @@ describe(`CollectionSubscription replay oracle`, () => { direction === `asc` ? ([`three`, `four`] as const) : ([`four`, `three`] as const) - const succeeds = delivery === `return` || delivery === `resolve` + const sourceSucceeded = delivery === `return` || delivery === `resolve` + const publishesReplacement = delivery === `resolve` const expectedIds = identity === `changed` ? replacementIds : initialIds try { @@ -9030,14 +9041,12 @@ describe(`CollectionSubscription replay oracle`, () => { } await flushPromises() expect(collection.toArray.map(({ id }) => id).sort()).toEqual( - succeeds ? [...expectedIds].sort() : [], + sourceSucceeded ? [...expectedIds].sort() : [], ) expect(publicationSnapshots).toEqual( delivery === `resolve` ? [[initialIds[0]], [...expectedIds].sort()] - : delivery === `return` && identity === `changed` - ? [[initialIds[0]], [expectedIds[0]]] - : [[initialIds[0]]], + : [[initialIds[0]]], ) const loadCountBeforeWiden = loadOptions.length @@ -9046,13 +9055,17 @@ describe(`CollectionSubscription replay oracle`, () => { limit: 1, minValues: [direction === `asc` ? 2 : 1], }) - if (succeeds) { + if (publishesReplacement) { expect(loadOptions).toHaveLength(loadCountBeforeWiden) } else { - expect(loadOptions[loadCountBeforeWiden]).toMatchObject({ - offset: 1, - cursor: { lastKey: initialIds[0] }, - }) + expect(loadOptions[loadCountBeforeWiden]).toMatchObject( + delivery === `return` + ? { offset: 1, cursor: undefined } + : { + offset: 1, + cursor: { lastKey: initialIds[0] }, + }, + ) } } finally { subscription.unsubscribe() diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index afe6d6db6..5a79036b1 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -4416,6 +4416,20 @@ it.each([`sync`, `async`] as const)( expect(demands[4]).toMatchObject({ limit: 5, offset: 0 }) expect(demands[4]?.cursor).toBeUndefined() expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + await live.utils.setWindow({ offset: 0, limit: 2 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(demands).toHaveLength(5) + + await live.utils.setWindow({ offset: 0, limit: 5 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(6) + expect(demands[5]).toMatchObject({ limit: 5, offset: 0 }) + expect(demands[5]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) } finally { await live.cleanup() await source.cleanup() @@ -4490,6 +4504,99 @@ it(`does not treat explicit continuation as outcome-free satisfaction`, async () } }) +it(`keeps the prior ordered publication until truncate replay gains authoritative coverage`, async () => { + type Row = { id: number; rank: number } + const oldRows: ReadonlyArray = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ] + const replacementRows: ReadonlyArray = [ + { id: 3, rank: 3 }, + { id: 4, rank: 4 }, + ] + const authoritative = createDeferred() + let calls = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `full-flow-outcome-free-truncate-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: () => { + calls++ + const rows = calls === 1 ? oldRows : replacementRows + if (calls <= 2) { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + commit() + } + if (calls === 1) { + return Promise.resolve({ + hasMore: false, + appliedRowKeys: oldRows.map(({ id }) => id), + }) + } + if (calls === 2) return Promise.resolve() + if (calls === 3) return authoritative.promise + throw new Error(`Unexpected fourth replay request`) + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `full-flow-outcome-free-truncate-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(2), + startSync: true, + }) + const preload = live.preload() + + try { + await preload + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + + begin() + truncate() + const replacement = commit() + await flushPromises() + + expect(calls).toBe(3) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + authoritative.resolve({ + hasMore: false, + appliedRowKeys: replacementRows.map(({ id }) => id), + }) + if (replacement !== true) await replacement + await flushPromises() + + expect(live.toArray.map(({ id }) => id)).toEqual([3, 4]) + } finally { + authoritative.resolve({ hasMore: false, appliedRowKeys: [] }) + await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await source.cleanup() + } +}) + it.each([ { name: `continues past an excluded source row`, @@ -5079,7 +5186,7 @@ it(`retries an evidence-free ordered Effect after truncate`, async () => { } }) -it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { +it(`rechecks an ordered Effect until truncate replay proves replacement coverage`, async () => { type Row = { id: number; rank: number } type Result = { hasMore: boolean @@ -5123,6 +5230,10 @@ it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { begin() write({ type: `insert`, value: finalRow }) commit() + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [finalRow.id], + }) } return true }, @@ -5169,6 +5280,81 @@ it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { } }) +it(`settles an outcome-free ordered Effect when its boundary stops advancing`, async () => { + type Row = { id: number; rank: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ] + const visible = new Map() + let calls = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `full-flow-effect-outcome-free-no-progress`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + calls++ + if (calls <= rows.length) { + begin() + write({ type: `insert`, value: rows[calls - 1]! }) + commit() + } + + // Bound the old loop. A correct implementation stops when the + // fourth request completes without moving the local boundary. + if (calls === 5) { + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [], + }) + } + return Promise.resolve() + }, + unloadSubset: () => {}, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(4), + onBatch: (events) => { + for (const event of events) { + if (event.type === `exit`) visible.delete(event.key) + else visible.set(event.key, event.value) + } + }, + }) + + try { + await flushPromises() + + expect([...visible.keys()]).toEqual([1, 2, 3]) + expect(calls).toBe(4) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + it(`replaces an ordered Effect only after a rejected continuation disposes it`, async () => { type Row = { id: number; rank: number } const firstRow: Row = { id: 1, rank: 1 } diff --git a/packages/db/tests/query/window-state.test.ts b/packages/db/tests/query/window-state.test.ts index c0ef4e1a8..6a5b0b6f4 100644 --- a/packages/db/tests/query/window-state.test.ts +++ b/packages/db/tests/query/window-state.test.ts @@ -234,7 +234,7 @@ describe(`WindowState`, () => { window.recordLocalRequestSatisfaction(requestedPrefix) expect(window.localPrefixSize).toBe(Math.min(requestedPrefix, 3)) - expect(window.coversActiveWindow).toBe(requestedPrefix <= 3) + expect(window.coversActiveWindow).toBe(false) expect(window.satisfiesActiveWindow).toBe(requestedPrefix <= 3) expect(window.requestBoundary()).toBeUndefined() expect(window.progressBoundary()?.key).toBe(Math.min(requestedPrefix, 3)) @@ -249,4 +249,113 @@ describe(`WindowState`, () => { expect(window.satisfiesActiveWindow).toBe(false) }, ) + + it(`refreshes an outcome-free window after shrinking and regrowing`, () => { + const window = new WindowState( + mockCollection([ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ]), + [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + undefined, + 4, + ) + + window.recordLocalRequestSatisfaction(4) + expect(window.settleLocalRequestAfterNoProgress()).toBe(true) + expect(window.satisfiesActiveWindow).toBe(true) + expect(window.coversRetainedWindow).toBe(false) + + window.ensureSize(2) + expect(window.satisfiesActiveWindow).toBe(true) + + window.ensureSize(3) + expect(window.satisfiesActiveWindow).toBe(false) + expect(window.requestBoundary()).toBeUndefined() + expect(window.coverageRevision).toBe(1) + + window.recordLocalRequestSatisfaction(3) + expect(window.satisfiesActiveWindow).toBe(true) + + window.ensureSize(2) + window.ensureSize(3) + expect(window.satisfiesActiveWindow).toBe(false) + expect(window.coverageRevision).toBe(2) + }) + + it.each([ + { + transition: `coverage reset`, + apply: (window: WindowState) => window.resetCoverage(), + expectedCoverage: false, + expectedSatisfaction: false, + }, + { + transition: `continuing authoritative result`, + apply: (window: WindowState) => + window.recordContinuationCoverage( + [], + false, + 4, + window.coverageRevision, + ), + expectedCoverage: false, + expectedSatisfaction: false, + }, + { + transition: `exhausted authoritative result`, + apply: (window: WindowState) => + window.recordContinuationCoverage([], true, 4, window.coverageRevision), + expectedCoverage: true, + expectedSatisfaction: true, + }, + { + transition: `prefix-invalidating live change`, + apply: (window: WindowState) => + window.admitChanges([ + { + type: `delete`, + key: 1, + value: { id: 1, rank: 1 }, + }, + ]), + expectedCoverage: false, + expectedSatisfaction: false, + }, + ])( + `clears local outcome-free satisfaction after $transition`, + ({ apply, expectedCoverage, expectedSatisfaction }) => { + const window = new WindowState( + mockCollection([ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ]), + [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + undefined, + 4, + ) + + window.recordLocalRequestSatisfaction(4) + window.settleLocalRequestAfterNoProgress() + expect(window.satisfiesActiveWindow).toBe(true) + expect(window.coversActiveWindow).toBe(false) + + apply(window) + + expect(window.coversActiveWindow).toBe(expectedCoverage) + expect(window.satisfiesActiveWindow).toBe(expectedSatisfaction) + }, + ) }) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 9021f5f10..84ee7cad2 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -3147,6 +3147,7 @@ describe(`On-Demand Sync Mode`, () => { async (scheduler) => { await expectScheduledLifecycleMatches(scheduler, secondOutcome) }, + 15_000, ) } From 2143f4814f635618c7f7133f13d930db2ba5ce37 Mon Sep 17 00:00:00 2001 From: Kyle Mathews Date: Wed, 2 Sep 2026 06:36:18 -0600 Subject: [PATCH 327/327] test(db): prove outcome-free settlement retirement --- packages/db/tests/query/window-state.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db/tests/query/window-state.test.ts b/packages/db/tests/query/window-state.test.ts index 6a5b0b6f4..98afb999e 100644 --- a/packages/db/tests/query/window-state.test.ts +++ b/packages/db/tests/query/window-state.test.ts @@ -356,6 +356,7 @@ describe(`WindowState`, () => { expect(window.coversActiveWindow).toBe(expectedCoverage) expect(window.satisfiesActiveWindow).toBe(expectedSatisfaction) + expect(window.settleLocalRequestAfterNoProgress()).toBe(false) }, ) })