diff --git a/.changeset/cancel-electric-refresh-wait.md b/.changeset/cancel-electric-refresh-wait.md new file mode 100644 index 0000000000..311ce222a5 --- /dev/null +++ b/.changeset/cancel-electric-refresh-wait.md @@ -0,0 +1,5 @@ +--- +'@tanstack/electric-db-collection': patch +--- + +Cancel an on-demand refresh wait when its request or collection is cleaned up, preventing snapshots from starting after teardown. diff --git a/.changeset/fix-powersync-tracking-startup.md b/.changeset/fix-powersync-tracking-startup.md new file mode 100644 index 0000000000..eeb52f8a5d --- /dev/null +++ b/.changeset/fix-powersync-tracking-startup.md @@ -0,0 +1,5 @@ +--- +'@tanstack/powersync-db-collection': patch +--- + +Serialize PowerSync tracking startup so changes and cleanup cannot race an unpublished trigger, and cancel subset or load-hook work released while startup is suspended. diff --git a/.changeset/harden-load-subset-lifecycle.md b/.changeset/harden-load-subset-lifecycle.md new file mode 100644 index 0000000000..c417a68e9e --- /dev/null +++ b/.changeset/harden-load-subset-lifecycle.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Harden on-demand query refinement across predicate subtraction, replay and publication rollback, readiness restarts, and resource cleanup. diff --git a/.changeset/lazy-runtime-reference-identities.md b/.changeset/lazy-runtime-reference-identities.md deleted file mode 100644 index 807b0f5ce7..0000000000 --- a/.changeset/lazy-runtime-reference-identities.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@tanstack/db': patch ---- - -Lazily initialize runtime reference identities to avoid generating random values during Cloudflare Worker module evaluation. diff --git a/docs/guides/error-handling.md b/docs/guides/error-handling.md index bff185c1a9..8de4468d6c 100644 --- a/docs/guides/error-handling.md +++ b/docs/guides/error-handling.md @@ -141,17 +141,23 @@ console.log(subscription.lastError) ``` For ordered live queries, `utils.setWindow()` rejects with the same error. The -last failure is also available as `utils.lastSubsetError`, while the last -successful snapshot remains readable: +last failure is also available as `utils.lastSubsetError`, while +`utils.hasSubsetError` distinguishes a thrown `undefined` from no observed +failure. The last successful snapshot remains readable: ```ts try { await liveTodos.utils.setWindow({ offset: 0, limit: 100 }) } catch (error) { - console.error(liveTodos.utils.lastSubsetError) + if (liveTodos.utils.hasSubsetError) { + console.error(liveTodos.utils.lastSubsetError) + } } ``` +Both diagnostic values reset together when the live query starts a new sync +session. + Effects report subset failures through `onSourceError` and dispose because their incremental result can no longer be kept complete. diff --git a/docs/reference/classes/BTreeIndex.md b/docs/reference/classes/BTreeIndex.md index e62c8fd562..1f82ddaef2 100644 --- a/docs/reference/classes/BTreeIndex.md +++ b/docs/reference/classes/BTreeIndex.md @@ -68,7 +68,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:56](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) #### Inherited from @@ -82,7 +82,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) #### Inherited from @@ -96,7 +96,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -113,7 +113,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) #### Inherited from @@ -127,7 +127,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) #### Inherited from @@ -141,7 +141,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) #### Inherited from @@ -155,7 +155,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) #### Inherited from @@ -183,7 +183,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:39](https://github.com/TanSt protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) #### Inherited from @@ -281,7 +281,7 @@ Defined in: [packages/db/src/indexes/btree-index.ts:452](https://github.com/TanS get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -427,7 +427,7 @@ Performs an equality lookup protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -451,7 +451,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -549,7 +549,7 @@ Performs a lookup operation matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -576,7 +576,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -602,7 +602,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -709,7 +709,7 @@ Removes a value from the index supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -891,7 +891,7 @@ The last n items protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -952,7 +952,7 @@ Updates a value in the index protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/classes/BaseIndex.md b/docs/reference/classes/BaseIndex.md index d0ae8debaf..5a282c2224 100644 --- a/docs/reference/classes/BaseIndex.md +++ b/docs/reference/classes/BaseIndex.md @@ -5,7 +5,7 @@ title: BaseIndex # Abstract Class: BaseIndex\ -Defined in: [packages/db/src/indexes/base-index.ts:91](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L91) +Defined in: [packages/db/src/indexes/base-index.ts:113](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L113) Base abstract class that all index types extend @@ -36,7 +36,7 @@ new BaseIndex( options?): BaseIndex; ``` -Defined in: [packages/db/src/indexes/base-index.ts:110](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L110) +Defined in: [packages/db/src/indexes/base-index.ts:132](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L132) #### Parameters @@ -68,7 +68,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:110](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) *** @@ -78,7 +78,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) *** @@ -88,7 +88,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -101,7 +101,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) *** @@ -111,7 +111,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) *** @@ -121,7 +121,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) *** @@ -131,7 +131,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) *** @@ -141,7 +141,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanSta abstract readonly supportedOperations: Set<"eq" | "gt" | "gte" | "lt" | "lte" | "in" | "like" | "ilike">; ``` -Defined in: [packages/db/src/indexes/base-index.ts:97](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L97) +Defined in: [packages/db/src/indexes/base-index.ts:119](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L119) *** @@ -151,7 +151,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:97](https://github.com/TanSta protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) ## Accessors @@ -163,7 +163,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanSt get abstract indexedKeysSet(): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:155](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L155) +Defined in: [packages/db/src/indexes/base-index.ts:177](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L177) ##### Returns @@ -183,7 +183,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:155](https://github.com/TanSt get abstract keyCount(): number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L148) +Defined in: [packages/db/src/indexes/base-index.ts:170](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L170) ##### Returns @@ -203,7 +203,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanSt get abstract orderedEntriesArray(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:153](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L153) +Defined in: [packages/db/src/indexes/base-index.ts:175](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L175) ##### Returns @@ -223,7 +223,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:153](https://github.com/TanSt get abstract orderedEntriesArrayReversed(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:154](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L154) +Defined in: [packages/db/src/indexes/base-index.ts:176](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L176) ##### Returns @@ -243,7 +243,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:154](https://github.com/TanSt get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -269,7 +269,7 @@ a full scan when this is `false`. get abstract valueMapData(): Map>; ``` -Defined in: [packages/db/src/indexes/base-index.ts:156](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L156) +Defined in: [packages/db/src/indexes/base-index.ts:178](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L178) ##### Returns @@ -287,7 +287,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:156](https://github.com/TanSt abstract add(key, item): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) +Defined in: [packages/db/src/indexes/base-index.ts:146](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L146) #### Parameters @@ -315,7 +315,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanSt abstract build(entries): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:127](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L127) +Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L149) #### Parameters @@ -339,7 +339,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:127](https://github.com/TanSt abstract clear(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:128](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L128) +Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L150) #### Returns @@ -357,7 +357,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:128](https://github.com/TanSt abstract equalityLookup(value): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L149) +Defined in: [packages/db/src/indexes/base-index.ts:171](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L171) #### Parameters @@ -381,7 +381,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:149](https://github.com/TanSt protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -401,7 +401,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -419,7 +419,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanSt abstract inArrayLookup(values): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L150) +Defined in: [packages/db/src/indexes/base-index.ts:172](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L172) #### Parameters @@ -443,7 +443,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:150](https://github.com/TanSt abstract protected initialize(options?): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:212](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L212) +Defined in: [packages/db/src/indexes/base-index.ts:244](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L244) #### Parameters @@ -463,7 +463,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:212](https://github.com/TanSt abstract lookup(operation, value): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:129](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L129) +Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L151) #### Parameters @@ -491,7 +491,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:129](https://github.com/TanSt matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -518,7 +518,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -544,7 +544,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -568,7 +568,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanSt abstract rangeQuery(options): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L151) +Defined in: [packages/db/src/indexes/base-index.ts:173](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L173) #### Parameters @@ -592,7 +592,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:151](https://github.com/TanSt abstract rangeQueryReversed(options): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L152) +Defined in: [packages/db/src/indexes/base-index.ts:174](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L174) #### Parameters @@ -616,7 +616,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanSt abstract remove(key, item): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:125](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L125) +Defined in: [packages/db/src/indexes/base-index.ts:147](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L147) #### Parameters @@ -644,7 +644,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:125](https://github.com/TanSt supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -671,7 +671,7 @@ abstract take( filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) +Defined in: [packages/db/src/indexes/base-index.ts:152](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L152) #### Parameters @@ -703,7 +703,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanSt abstract takeFromStart(n, filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:135](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L135) +Defined in: [packages/db/src/indexes/base-index.ts:157](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L157) #### Parameters @@ -734,7 +734,7 @@ abstract takeReversed( filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:139](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L139) +Defined in: [packages/db/src/indexes/base-index.ts:161](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L161) #### Parameters @@ -766,7 +766,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:139](https://github.com/TanSt abstract takeReversedFromEnd(n, filterFn?): TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:144](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L144) +Defined in: [packages/db/src/indexes/base-index.ts:166](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L166) #### Parameters @@ -794,7 +794,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:144](https://github.com/TanSt protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -817,7 +817,7 @@ abstract update( newItem): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:126](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L126) +Defined in: [packages/db/src/indexes/base-index.ts:148](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L148) #### Parameters @@ -849,7 +849,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:126](https://github.com/TanSt protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/classes/BasicIndex.md b/docs/reference/classes/BasicIndex.md index 4ca10dfd5b..5100dd6537 100644 --- a/docs/reference/classes/BasicIndex.md +++ b/docs/reference/classes/BasicIndex.md @@ -74,7 +74,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:64](https://github.com/TanSt protected compareOptions: CompareOptions; ``` -Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L102) +Defined in: [packages/db/src/indexes/base-index.ts:124](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L124) #### Inherited from @@ -88,7 +88,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:102](https://github.com/TanSt readonly expression: BasicExpression; ``` -Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L96) +Defined in: [packages/db/src/indexes/base-index.ts:118](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L118) #### Inherited from @@ -102,7 +102,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:96](https://github.com/TanSta protected hasCustomComparator: boolean = false; ``` -Defined in: [packages/db/src/indexes/base-index.ts:108](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L108) +Defined in: [packages/db/src/indexes/base-index.ts:130](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L130) Set by subclasses when constructed with a user-supplied comparator, whose ordering may not match the WHERE evaluator's relational operators. @@ -119,7 +119,7 @@ ordering may not match the WHERE evaluator's relational operators. readonly id: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L94) +Defined in: [packages/db/src/indexes/base-index.ts:116](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L116) #### Inherited from @@ -133,7 +133,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:94](https://github.com/TanSta protected lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) +Defined in: [packages/db/src/indexes/base-index.ts:123](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L123) #### Inherited from @@ -147,7 +147,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanSt protected lookupCount: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L99) +Defined in: [packages/db/src/indexes/base-index.ts:121](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L121) #### Inherited from @@ -161,7 +161,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:99](https://github.com/TanSta readonly optional name: string; ``` -Defined in: [packages/db/src/indexes/base-index.ts:95](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L95) +Defined in: [packages/db/src/indexes/base-index.ts:117](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L117) #### Inherited from @@ -189,7 +189,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:46](https://github.com/TanSt protected totalLookupTime: number = 0; ``` -Defined in: [packages/db/src/indexes/base-index.ts:100](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L100) +Defined in: [packages/db/src/indexes/base-index.ts:122](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L122) #### Inherited from @@ -287,7 +287,7 @@ Defined in: [packages/db/src/indexes/basic-index.ts:530](https://github.com/TanS get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:163](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L163) +Defined in: [packages/db/src/indexes/base-index.ts:185](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L185) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -433,7 +433,7 @@ Performs an equality lookup - O(1) protected evaluateIndexExpression(item): any; ``` -Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L214) +Defined in: [packages/db/src/indexes/base-index.ts:246](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L246) #### Parameters @@ -457,7 +457,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:214](https://github.com/TanSt getStats(): IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:202](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L202) +Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) #### Returns @@ -555,7 +555,7 @@ Performs a lookup operation matchesCompareOptions(compareOptions): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:179](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L179) +Defined in: [packages/db/src/indexes/base-index.ts:201](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L201) Checks if the compare options match the index's compare options. The direction is ignored because the index can be reversed if the direction is different. @@ -582,7 +582,7 @@ The direction is ignored because the index can be reversed if the direction is d matchesDirection(direction): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:198](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L198) +Defined in: [packages/db/src/indexes/base-index.ts:230](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L230) Checks if the index matches the provided direction. @@ -608,7 +608,7 @@ Checks if the index matches the provided direction. matchesField(fieldPath): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:167](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L167) +Defined in: [packages/db/src/indexes/base-index.ts:189](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L189) #### Parameters @@ -714,7 +714,7 @@ Removes a value from the index supports(operation): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:159](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L159) +Defined in: [packages/db/src/indexes/base-index.ts:181](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L181) #### Parameters @@ -872,7 +872,7 @@ Returns the first n items in reverse sorted order (from the end) protected trackLookup(startTime): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:220](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L220) +Defined in: [packages/db/src/indexes/base-index.ts:252](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L252) #### Parameters @@ -933,7 +933,7 @@ Updates a value in the index protected updateTimestamp(): void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:226](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L226) +Defined in: [packages/db/src/indexes/base-index.ts:258](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L258) #### Returns diff --git a/docs/reference/interfaces/IndexInterface.md b/docs/reference/interfaces/IndexInterface.md index 48c8c6bdf5..415f739966 100644 --- a/docs/reference/interfaces/IndexInterface.md +++ b/docs/reference/interfaces/IndexInterface.md @@ -5,7 +5,7 @@ title: IndexInterface # Interface: IndexInterface\ -Defined in: [packages/db/src/indexes/base-index.ts:29](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L29) +Defined in: [packages/db/src/indexes/base-index.ts:51](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L51) ## Type Parameters @@ -21,7 +21,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:29](https://github.com/TanSta add: (key, item) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:32](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L32) +Defined in: [packages/db/src/indexes/base-index.ts:54](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L54) #### Parameters @@ -45,7 +45,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:32](https://github.com/TanSta build: (entries) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:36](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L36) +Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L58) #### Parameters @@ -65,7 +65,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:36](https://github.com/TanSta clear: () => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:37](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L37) +Defined in: [packages/db/src/indexes/base-index.ts:59](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L59) #### Returns @@ -79,7 +79,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:37](https://github.com/TanSta equalityLookup: (value) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:41](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L41) +Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L63) #### Parameters @@ -99,7 +99,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:41](https://github.com/TanSta getStats: () => IndexStats; ``` -Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L85) +Defined in: [packages/db/src/indexes/base-index.ts:107](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L107) #### Returns @@ -113,7 +113,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanSta inArrayLookup: (values) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:42](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L42) +Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L64) #### Parameters @@ -133,7 +133,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:42](https://github.com/TanSta lookup: (operation, value) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:39](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L39) +Defined in: [packages/db/src/indexes/base-index.ts:61](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L61) #### Parameters @@ -157,7 +157,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:39](https://github.com/TanSta matchesCompareOptions: (compareOptions) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:82](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L82) +Defined in: [packages/db/src/indexes/base-index.ts:104](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L104) #### Parameters @@ -177,7 +177,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:82](https://github.com/TanSta matchesDirection: (direction) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:83](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L83) +Defined in: [packages/db/src/indexes/base-index.ts:105](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L105) #### Parameters @@ -197,7 +197,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:83](https://github.com/TanSta matchesField: (fieldPath) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:81](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L81) +Defined in: [packages/db/src/indexes/base-index.ts:103](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L103) #### Parameters @@ -217,7 +217,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:81](https://github.com/TanSta rangeQuery: (options) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L44) +Defined in: [packages/db/src/indexes/base-index.ts:66](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L66) #### Parameters @@ -237,7 +237,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanSta rangeQueryReversed: (options) => Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L45) +Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L67) #### Parameters @@ -257,7 +257,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanSta remove: (key, item) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:33](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L33) +Defined in: [packages/db/src/indexes/base-index.ts:55](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L55) #### Parameters @@ -281,7 +281,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:33](https://github.com/TanSta supports: (operation) => boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:70](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L70) +Defined in: [packages/db/src/indexes/base-index.ts:92](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L92) #### Parameters @@ -301,7 +301,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:70](https://github.com/TanSta take: (n, from, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L47) +Defined in: [packages/db/src/indexes/base-index.ts:69](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L69) #### Parameters @@ -329,7 +329,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanSta takeFromStart: (n, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:52](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L52) +Defined in: [packages/db/src/indexes/base-index.ts:74](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L74) #### Parameters @@ -353,7 +353,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:52](https://github.com/TanSta takeReversed: (n, from, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:53](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L53) +Defined in: [packages/db/src/indexes/base-index.ts:75](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L75) #### Parameters @@ -381,7 +381,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:53](https://github.com/TanSta takeReversedFromEnd: (n, filterFn?) => TKey[]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L58) +Defined in: [packages/db/src/indexes/base-index.ts:80](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L80) #### Parameters @@ -405,7 +405,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:58](https://github.com/TanSta update: (key, oldItem, newItem) => void; ``` -Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) +Defined in: [packages/db/src/indexes/base-index.ts:56](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L56) #### Parameters @@ -435,7 +435,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanSta get indexedKeysSet(): Set; ``` -Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L67) +Defined in: [packages/db/src/indexes/base-index.ts:89](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L89) ##### Returns @@ -451,7 +451,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:67](https://github.com/TanSta get keyCount(): number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L63) +Defined in: [packages/db/src/indexes/base-index.ts:85](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L85) ##### Returns @@ -467,7 +467,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:63](https://github.com/TanSta get orderedEntriesArray(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L64) +Defined in: [packages/db/src/indexes/base-index.ts:86](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L86) ##### Returns @@ -483,7 +483,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:64](https://github.com/TanSta get orderedEntriesArrayReversed(): [any, Set][]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:65](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L65) +Defined in: [packages/db/src/indexes/base-index.ts:87](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L87) ##### Returns @@ -499,7 +499,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:65](https://github.com/TanSta get supportsRangeOptimization(): boolean; ``` -Defined in: [packages/db/src/indexes/base-index.ts:79](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L79) +Defined in: [packages/db/src/indexes/base-index.ts:101](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L101) Whether range lookups (gt/gte/lt/lte) on this index can be trusted to return every matching key. Range traversal relies on the index ordering, so @@ -521,7 +521,7 @@ a full scan when this is `false`. get valueMapData(): Map>; ``` -Defined in: [packages/db/src/indexes/base-index.ts:68](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L68) +Defined in: [packages/db/src/indexes/base-index.ts:90](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L90) ##### Returns diff --git a/docs/reference/interfaces/IndexStats.md b/docs/reference/interfaces/IndexStats.md index 7852397f00..2fec3b7709 100644 --- a/docs/reference/interfaces/IndexStats.md +++ b/docs/reference/interfaces/IndexStats.md @@ -5,7 +5,7 @@ title: IndexStats # Interface: IndexStats -Defined in: [packages/db/src/indexes/base-index.ts:22](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L22) +Defined in: [packages/db/src/indexes/base-index.ts:44](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L44) Statistics about index usage and performance @@ -17,7 +17,7 @@ Statistics about index usage and performance readonly averageLookupTime: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:25](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L25) +Defined in: [packages/db/src/indexes/base-index.ts:47](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L47) *** @@ -27,7 +27,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:25](https://github.com/TanSta readonly entryCount: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:23](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L23) +Defined in: [packages/db/src/indexes/base-index.ts:45](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L45) *** @@ -37,7 +37,7 @@ Defined in: [packages/db/src/indexes/base-index.ts:23](https://github.com/TanSta readonly lastUpdated: Date; ``` -Defined in: [packages/db/src/indexes/base-index.ts:26](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L26) +Defined in: [packages/db/src/indexes/base-index.ts:48](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L48) *** @@ -47,4 +47,4 @@ Defined in: [packages/db/src/indexes/base-index.ts:26](https://github.com/TanSta readonly lookupCount: number; ``` -Defined in: [packages/db/src/indexes/base-index.ts:24](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L24) +Defined in: [packages/db/src/indexes/base-index.ts:46](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L46) diff --git a/docs/reference/type-aliases/IndexConstructor.md b/docs/reference/type-aliases/IndexConstructor.md index e3ec6a2936..4e0b4bbda9 100644 --- a/docs/reference/type-aliases/IndexConstructor.md +++ b/docs/reference/type-aliases/IndexConstructor.md @@ -9,7 +9,7 @@ title: IndexConstructor type IndexConstructor = (id, expression, name?, options?) => BaseIndex; ``` -Defined in: [packages/db/src/indexes/base-index.ts:234](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L234) +Defined in: [packages/db/src/indexes/base-index.ts:266](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L266) Type for index constructor diff --git a/docs/reference/type-aliases/IndexOperation-1.md b/docs/reference/type-aliases/IndexOperation-1.md index 28b8a867c5..025823309a 100644 --- a/docs/reference/type-aliases/IndexOperation-1.md +++ b/docs/reference/type-aliases/IndexOperation-1.md @@ -9,6 +9,6 @@ title: IndexOperation type IndexOperation = typeof comparisonFunctions[number]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:12](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L12) +Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) Type for index operation values diff --git a/docs/reference/type-aliases/IndexOperation.md b/docs/reference/type-aliases/IndexOperation.md index 6816e9788f..42c757f781 100644 --- a/docs/reference/type-aliases/IndexOperation.md +++ b/docs/reference/type-aliases/IndexOperation.md @@ -9,6 +9,6 @@ title: IndexOperation type IndexOperation = readonly ["eq", "gt", "gte", "lt", "lte", "in", "like", "ilike"]; ``` -Defined in: [packages/db/src/indexes/base-index.ts:12](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L12) +Defined in: [packages/db/src/indexes/base-index.ts:34](https://github.com/TanStack/db/blob/main/packages/db/src/indexes/base-index.ts#L34) Operations that indexes can support, imported from available comparison functions diff --git a/docs/reference/type-aliases/LiveQueryCollectionUtils.md b/docs/reference/type-aliases/LiveQueryCollectionUtils.md index 620333d20e..acdaf4473a 100644 --- a/docs/reference/type-aliases/LiveQueryCollectionUtils.md +++ b/docs/reference/type-aliases/LiveQueryCollectionUtils.md @@ -52,6 +52,15 @@ Gets the current window (offset and limit) for an ordered query. The current window settings, or `undefined` if the query is not windowed +### hasSubsetError + +```ts +readonly hasSubsetError: boolean; +``` + +Whether this live query has observed a subset-load failure in its current sync +session. + ### lastSubsetError ```ts diff --git a/examples/angular/todos/package.json b/examples/angular/todos/package.json index 26bd22f872..f875a94367 100644 --- a/examples/angular/todos/package.json +++ b/examples/angular/todos/package.json @@ -28,8 +28,8 @@ "@angular/forms": "^20.3.16", "@angular/platform-browser": "^20.3.16", "@angular/router": "^20.3.16", - "@tanstack/angular-db": "^0.1.86", - "@tanstack/db": "^0.8.5", + "@tanstack/angular-db": "^0.1.88", + "@tanstack/db": "^0.8.7", "rxjs": "^7.8.2", "tslib": "^2.8.1", "zone.js": "~0.15.0" diff --git a/examples/electron/offline-first/package.json b/examples/electron/offline-first/package.json index 80e1cbdc17..c40b53e277 100644 --- a/examples/electron/offline-first/package.json +++ b/examples/electron/offline-first/package.json @@ -13,11 +13,11 @@ "postinstall": "prebuild-install --runtime electron --target 40.2.1 --arch arm64 || echo 'prebuild-install failed, try: npx @electron/rebuild'" }, "dependencies": { - "@tanstack/electron-db-sqlite-persistence": "^0.1.30", - "@tanstack/node-db-sqlite-persistence": "^0.2.18", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/electron-db-sqlite-persistence": "^0.1.32", + "@tanstack/node-db-sqlite-persistence": "^0.2.20", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-query": "^5.90.20", "better-sqlite3": "^12.6.2", "react": "^19.2.4", diff --git a/examples/react-native/offline-transactions/package.json b/examples/react-native/offline-transactions/package.json index 59ea587b6f..c1d9b21cd5 100644 --- a/examples/react-native/offline-transactions/package.json +++ b/examples/react-native/offline-transactions/package.json @@ -15,11 +15,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.18", + "@tanstack/db": "^0.8.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.20", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react-native/shopping-list/package.json b/examples/react-native/shopping-list/package.json index 1cce0f1a6d..8a580bbae5 100644 --- a/examples/react-native/shopping-list/package.json +++ b/examples/react-native/shopping-list/package.json @@ -18,11 +18,11 @@ "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "2.1.2", "@react-native-community/netinfo": "11.4.1", - "@tanstack/db": "^0.8.5", - "@tanstack/electric-db-collection": "^0.4.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/react-db": "^0.3.5", - "@tanstack/react-native-db-sqlite-persistence": "^0.2.18", + "@tanstack/db": "^0.8.7", + "@tanstack/electric-db-collection": "^0.4.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/react-db": "^0.3.7", + "@tanstack/react-native-db-sqlite-persistence": "^0.2.20", "@tanstack/react-query": "^5.90.20", "expo": "~53.0.26", "expo-constants": "~17.1.0", diff --git a/examples/react/next-ssr-e2e/package.json b/examples/react/next-ssr-e2e/package.json index 3b584c5fdc..680de9dabf 100644 --- a/examples/react/next-ssr-e2e/package.json +++ b/examples/react/next-ssr-e2e/package.json @@ -9,8 +9,8 @@ "test:e2e": "pnpm --filter @tanstack/db build && pnpm --filter @tanstack/react-db build && playwright test" }, "dependencies": { - "@tanstack/db": "^0.8.5", - "@tanstack/react-db": "^0.3.5", + "@tanstack/db": "^0.8.7", + "@tanstack/react-db": "^0.3.7", "next": "^16.3.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/offline-transactions/package.json b/examples/react/offline-transactions/package.json index 6bb399af9f..790e21f41d 100644 --- a/examples/react/offline-transactions/package.json +++ b/examples/react/offline-transactions/package.json @@ -8,11 +8,11 @@ "build": "vite build && tsc --noEmit" }, "dependencies": { - "@tanstack/browser-db-sqlite-persistence": "^0.2.18", - "@tanstack/db": "^0.8.5", - "@tanstack/offline-transactions": "^1.0.51", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/browser-db-sqlite-persistence": "^0.2.20", + "@tanstack/db": "^0.8.7", + "@tanstack/offline-transactions": "^1.0.53", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-query": "^5.90.20", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", diff --git a/examples/react/paced-mutations-demo/package.json b/examples/react/paced-mutations-demo/package.json index 318832aec4..aaed91570c 100644 --- a/examples/react/paced-mutations-demo/package.json +++ b/examples/react/paced-mutations-demo/package.json @@ -9,8 +9,8 @@ "preview": "vite preview" }, "dependencies": { - "@tanstack/db": "^0.8.5", - "@tanstack/react-db": "^0.3.5", + "@tanstack/db": "^0.8.7", + "@tanstack/react-db": "^0.3.7", "mitt": "^3.0.1", "react": "^19.2.4", "react-dom": "^19.2.4" diff --git a/examples/react/projects/package.json b/examples/react/projects/package.json index 1454e644d5..3251b24643 100644 --- a/examples/react/projects/package.json +++ b/examples/react/projects/package.json @@ -17,8 +17,8 @@ "dependencies": { "@tailwindcss/vite": "^4.1.18", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-devtools": "^1.159.5", "@tanstack/react-router-with-query": "^1.130.17", diff --git a/examples/react/start-ssr-e2e/package.json b/examples/react/start-ssr-e2e/package.json index 3361ed0123..b521504a65 100644 --- a/examples/react/start-ssr-e2e/package.json +++ b/examples/react/start-ssr-e2e/package.json @@ -10,7 +10,7 @@ "test:e2e:hosted": "playwright test" }, "dependencies": { - "@tanstack/react-db": "^0.3.5", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-router-with-db": "^0.1.0", "@tanstack/react-start": "^1.159.5", diff --git a/examples/react/todo/package.json b/examples/react/todo/package.json index 4df1dea31d..6264ccbf6c 100644 --- a/examples/react/todo/package.json +++ b/examples/react/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.1.27", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.5", + "@tanstack/electric-db-collection": "^0.4.7", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/react-db": "^0.3.5", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/react-db": "^0.3.7", "@tanstack/react-router": "^1.159.5", "@tanstack/react-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.104", + "@tanstack/trailbase-db-collection": "^0.1.106", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/examples/solid/todo/package.json b/examples/solid/todo/package.json index 2d6ae6a1ad..562c72f4e4 100644 --- a/examples/solid/todo/package.json +++ b/examples/solid/todo/package.json @@ -3,13 +3,13 @@ "private": true, "version": "0.0.36", "dependencies": { - "@tanstack/electric-db-collection": "^0.4.5", + "@tanstack/electric-db-collection": "^0.4.7", "@tanstack/query-core": "^5.90.20", - "@tanstack/query-db-collection": "^1.2.10", - "@tanstack/solid-db": "^0.2.40", + "@tanstack/query-db-collection": "^1.2.12", + "@tanstack/solid-db": "^0.2.42", "@tanstack/solid-router": "^1.159.5", "@tanstack/solid-start": "^1.159.5", - "@tanstack/trailbase-db-collection": "^0.1.104", + "@tanstack/trailbase-db-collection": "^0.1.106", "cors": "^2.8.6", "drizzle-orm": "^0.45.1", "drizzle-zod": "^0.8.3", diff --git a/packages/angular-db/CHANGELOG.md b/packages/angular-db/CHANGELOG.md index 51d6114d38..4573819d56 100644 --- a/packages/angular-db/CHANGELOG.md +++ b/packages/angular-db/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/angular-db +## 0.1.88 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.1.87 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.86 ### Patch Changes diff --git a/packages/angular-db/package.json b/packages/angular-db/package.json index 96cabe10b6..dbfce25ea1 100644 --- a/packages/angular-db/package.json +++ b/packages/angular-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/angular-db", - "version": "0.1.86", + "version": "0.1.88", "description": "Angular integration for @tanstack/db", "author": "Ethan McDaniel", "license": "MIT", diff --git a/packages/browser-db-sqlite-persistence/CHANGELOG.md b/packages/browser-db-sqlite-persistence/CHANGELOG.md index 56adbe19e5..56534f46ee 100644 --- a/packages/browser-db-sqlite-persistence/CHANGELOG.md +++ b/packages/browser-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/browser-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/browser-db-sqlite-persistence/package.json b/packages/browser-db-sqlite-persistence/package.json index 68ed0ff201..a5b695feea 100644 --- a/packages/browser-db-sqlite-persistence/package.json +++ b/packages/browser-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/browser-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Browser wa-sqlite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md index ea93ca36b2..256149571e 100644 --- a/packages/capacitor-db-sqlite-persistence/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/capacitor-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md index 58a7cb8096..f063fefbbf 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,21 @@ # @tanstack/capacitor-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/capacitor-db-sqlite-persistence@0.2.20 + +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/capacitor-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json index f9098decc6..cab7d1ec61 100644 --- a/packages/capacitor-db-sqlite-persistence/e2e/app/package.json +++ b/packages/capacitor-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.32", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/capacitor-db-sqlite-persistence/package.json b/packages/capacitor-db-sqlite-persistence/package.json index 07156ec2d9..9aecec8263 100644 --- a/packages/capacitor-db-sqlite-persistence/package.json +++ b/packages/capacitor-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/capacitor-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Capacitor SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md index 029aec5d22..595601afdd 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/cloudflare-durable-objects-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json index 77c98ed41b..70de8645fa 100644 --- a/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json +++ b/packages/cloudflare-durable-objects-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/cloudflare-durable-objects-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Cloudflare Durable Object SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db-sqlite-persistence-core/CHANGELOG.md b/packages/db-sqlite-persistence-core/CHANGELOG.md index 06933cdf19..400abead4d 100644 --- a/packages/db-sqlite-persistence-core/CHANGELOG.md +++ b/packages/db-sqlite-persistence-core/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/db-sqlite-persistence-core +## 0.2.20 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.2.18 ### Patch Changes diff --git a/packages/db-sqlite-persistence-core/package.json b/packages/db-sqlite-persistence-core/package.json index 5dcc96de29..e4979c8946 100644 --- a/packages/db-sqlite-persistence-core/package.json +++ b/packages/db-sqlite-persistence-core/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db-sqlite-persistence-core", - "version": "0.2.18", + "version": "0.2.20", "description": "SQLite persisted collection core for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/db/CHANGELOG.md b/packages/db/CHANGELOG.md index 6f0ecfe61b..2b2a07a8e0 100644 --- a/packages/db/CHANGELOG.md +++ b/packages/db/CHANGELOG.md @@ -1,5 +1,17 @@ # @tanstack/db +## 0.8.7 + +### Patch Changes + +- Match index collation options by their effective values so indexes remain reusable when optional locale fields are omitted, set to `undefined`, or use equivalent locale identifiers. ([#1788](https://github.com/TanStack/db/pull/1788)) + +## 0.8.6 + +### Patch Changes + +- Lazily initialize runtime reference identities to avoid generating random values during Cloudflare Worker module evaluation. ([#1782](https://github.com/TanStack/db/pull/1782)) + ## 0.8.5 ### Patch Changes diff --git a/packages/db/package.json b/packages/db/package.json index c9db40aff0..a4bf3d9e1e 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/db", - "version": "0.8.5", + "version": "0.8.7", "description": "A reactive client store for building super fast apps on sync", "author": "Kyle Mathews", "license": "MIT", @@ -21,7 +21,7 @@ "dev": "vite build --watch", "lint": "eslint . --fix", "test": "vitest --run", - "test:oracles": "vitest --run tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts" + "test:oracles": "vitest --run tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-sync-reentrancy.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/query/coverage-registry-oracle.property.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-projection-oracle.property.test.ts tests/query/load-subset-lifecycle-oracle.property.test.ts tests/query/load-subset-full-flow-oracle.property.test.ts tests/query/load-subset-refinement-model.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/pagination-oracle.property.test.ts tests/query/predicate-subtraction-oracle.property.test.ts" }, "type": "module", "main": "dist/cjs/index.cjs", diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index d78f2f45a0..9d23ebaf6c 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -1,3 +1,4 @@ +import { compareKeys } from '@tanstack/db-ivm' import { createSingleRowRefProxy, toExpression, @@ -21,10 +22,26 @@ import type { SubscribeChangesOptions, } from '../types' import type { CollectionImpl } from './index.js' +import type { IndexInterface } from '../indexes/base-index.js' import type { SingleRowRefProxy } from '../query/builder/ref-proxy' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { WithVirtualProps } from '../virtual-props.js' +type OrderedBucketIndex = { + orderedBuckets: () => IterableIterator]> +} + +function getOrderedBuckets( + index: IndexInterface, +): IterableIterator]> | undefined { + if (index instanceof ReverseIndex && !index.supportsOrderedBucketIteration) { + return + } + return ( + index as IndexInterface & Partial> + ).orderedBuckets?.() +} + /** * Returns the current state of the collection as an array of changes * @param collection - The collection to get changes from @@ -86,6 +103,12 @@ export function currentStateAsChanges< throw new Error(`limit cannot be used without orderBy`) } + // An empty ordered window has no source work. Return before compiling its + // predicate or finding, creating, and traversing an order index. + if (options.limit === 0) { + return [] + } + // First check if orderBy is present (optionally with limit) if (options.orderBy) { // Create where filter function if present @@ -346,7 +369,7 @@ function getOrderedKeys( // Find the index const index = findIndexForField(collection, fieldPath, compareOpts) - if (index && index.supports(`gt`)) { + if (index && index.supports(`gt`) && index.supportsRangeOptimization) { // Use index optimization const filterFn = (key: TKey): boolean => { const value = collection.get(key) @@ -356,33 +379,47 @@ function getOrderedKeys( return whereFilter?.(value) ?? true } - // Take the keys that match the filter and limit - // if no limit is provided `index.keyCount` is used, - // i.e. we will take all keys that match the filter - if (!(index instanceof ReverseIndex)) { - return index.takeFromStart(limit ?? index.keyCount, filterFn) + const orderedBuckets = getOrderedBuckets(index) + + // Public custom indexes predate lazy bucket iteration. Preserve their + // semantics with the full TotalOrder refinement instead of assuming + // their materialized entries expose complete comparator tie classes. + if (!orderedBuckets) { + const totalOrder = new TotalOrder(orderBy, collection) + const indexedEntries = index + .takeFromStart(index.keyCount, filterFn) + .flatMap((key) => { + const value = collection.get(key) + return value === undefined ? [] : [{ key, value }] + }) + indexedEntries.sort((left, right) => + totalOrder.compareEntries( + [left.key, left.value], + [right.key, right.value], + ), + ) + return indexedEntries + .slice(0, limit ?? indexedEntries.length) + .map(({ key }) => key) } - // Reversing a value index also reverses keys inside an equal-value - // bucket, but query TotalOrder keeps its public-key tie-break ascending. - // Refine all matching indexed rows locally so a limit cannot cut the - // wrong side of a tied boundary. - const totalOrder = new TotalOrder(orderBy, collection) - const indexedEntries = index - .takeFromStart(index.keyCount, filterFn) - .flatMap((key) => { - const value = collection.get(key) - return value === undefined ? [] : [{ key, value }] - }) - indexedEntries.sort((left, right) => - totalOrder.compareEntries( - [left.key, left.value], - [right.key, right.value], - ), - ) - return indexedEntries - .slice(0, limit ?? indexedEntries.length) - .map(({ key }) => key) + // Value order comes from the matching index or its reverse view. The + // public-key suffix remains ascending in both directions. Stop after + // the first complete bucket that proves the requested prefix because + // filtering can otherwise select the wrong key from a boundary tie. + const keys: Array = [] + for (const [, bucket] of orderedBuckets) { + const matchingKeys = [...bucket].sort(compareKeys).filter(filterFn) + const remaining = + limit === undefined ? undefined : limit - keys.length + keys.push( + ...(remaining === undefined + ? matchingKeys + : matchingKeys.slice(0, remaining)), + ) + if (limit !== undefined && keys.length === limit) break + } + return keys } } } diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index 00523a2f48..93be8ef110 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -1,5 +1,9 @@ import { NegativeActiveSubscribersError } from '../errors' -import { withPublicationContext } from '../scheduler.js' +import { + deferPublicationFailure, + withPublicationContext, +} from '../scheduler.js' +import { runAllCallbacks } from '../utils/callbacks.js' import { createSingleRowRefProxy, toExpression, @@ -15,10 +19,36 @@ import type { CollectionStateManager } from './state.js' import type { WithVirtualProps } from '../virtual-props.js' export type PublicationDeferral = { + /** Irrevocably advance held revision clocks without invoking callbacks. */ + prepare: () => void + /** Prepare if needed, then release the held callbacks. */ publish: () => void + /** Discard held clocks and callbacks before preparation. */ discard: () => void } +type PublicationDeferralState< + TOutput extends object, + TKey extends string | number, +> = { + depth: number + discard: boolean + publications: Array<{ + changes: Array> + layoutChanged: boolean + }> + stateRevisionDelta: number + layoutRevisionDelta: number + afterPublication: Array<() => void> + prepared: + | { + changes: Array> + layoutChanged: boolean + } + | undefined + published: boolean +} + export class CollectionChangesManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -35,12 +65,12 @@ export class CollectionChangesManager< public changeSubscriptions = new Set() public batchedEvents: Array> = [] public shouldBatchEvents = false - private publicationDeferralDepth = 0 - private discardDeferredPublications = false - private deferredPublications: Array<{ - changes: Array> - layoutChanged: boolean - }> = [] + private publicationDeferral: + | PublicationDeferralState + | undefined + private preparedPublicationDeferral: + | PublicationDeferralState + | undefined private layoutChangeListeners = new Set<() => void>() /** @@ -83,8 +113,17 @@ export class CollectionChangesManager< */ public emitEmptyReadyEvent(): void { withPublicationContext(() => { - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents([]) + try { + runAllCallbacks( + [...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents([]), + ), + ) + } catch (error) { + // The ready snapshot is already public. Keep the callback failure on + // the shared publication so nested graph work can drain before the + // outer boundary rethrows it. + deferPublicationFailure(error) } }) } @@ -107,10 +146,16 @@ export class CollectionChangesManager< forceEmit = false, layoutChanged = false, ): void { - // The visible state was already committed by the caller, so the revision - // advances even when the events below end up batched for later emission. - if (changes.length > 0) this.stateRevision++ - if (layoutChanged) this.layoutRevision++ + // A coherent multi-Collection publication may still roll back. Hold its + // revision clocks with its events so discard leaves no public trace. + const publicationDeferral = this.publicationDeferral + if (publicationDeferral) { + if (changes.length > 0) publicationDeferral.stateRevisionDelta++ + if (layoutChanged) publicationDeferral.layoutRevisionDelta++ + } else { + if (changes.length > 0) this.stateRevision++ + if (layoutChanged) this.layoutRevision++ + } // Skip batching for user actions (forceEmit=true) to keep UI responsive if (this.shouldBatchEvents && !forceEmit) { @@ -133,8 +178,11 @@ export class CollectionChangesManager< this.shouldBatchEvents = false } - if (this.publicationDeferralDepth > 0) { - this.deferredPublications.push({ changes: rawEvents, layoutChanged }) + if (publicationDeferral) { + publicationDeferral.publications.push({ + changes: rawEvents, + layoutChanged, + }) return } @@ -147,34 +195,88 @@ export class CollectionChangesManager< * normal transaction boundaries. */ public deferPublication(): PublicationDeferral { - this.publicationDeferralDepth++ - let closed = false - - const close = (discard: boolean) => { - if (closed) return - closed = true - if (this.publicationDeferralDepth === 0) return - this.discardDeferredPublications ||= discard - - this.publicationDeferralDepth-- - if (this.publicationDeferralDepth > 0) return - - const publications = this.deferredPublications - this.deferredPublications = [] - if (this.discardDeferredPublications) { - this.discardDeferredPublications = false + if (this.preparedPublicationDeferral) { + throw new Error( + `Cannot start a publication cycle while another is prepared`, + ) + } + const publicationDeferral = this.publicationDeferral ?? { + depth: 0, + discard: false, + publications: [], + stateRevisionDelta: 0, + layoutRevisionDelta: 0, + afterPublication: [], + prepared: undefined, + published: false, + } + this.publicationDeferral = publicationDeferral + publicationDeferral.depth++ + let handleState: `open` | `prepared` | `discarded` = `open` + + const prepare = (discard: boolean) => { + if (handleState !== `open`) return + handleState = discard ? `discarded` : `prepared` + publicationDeferral.discard ||= discard + publicationDeferral.depth-- + if (publicationDeferral.depth > 0) return + + if (this.publicationDeferral === publicationDeferral) { + this.publicationDeferral = undefined + } + if (publicationDeferral.discard) { + publicationDeferral.afterPublication = [] return } - this.publishEvents( - publications.flatMap(({ changes }) => changes), - publications.some(({ layoutChanged }) => layoutChanged), - ) + + this.stateRevision += publicationDeferral.stateRevisionDelta + this.layoutRevision += publicationDeferral.layoutRevisionDelta + publicationDeferral.prepared = { + changes: publicationDeferral.publications.flatMap( + ({ changes }) => changes, + ), + layoutChanged: publicationDeferral.publications.some( + ({ layoutChanged }) => layoutChanged, + ), + } + this.preparedPublicationDeferral = publicationDeferral } return { - publish: () => close(false), - discard: () => close(true), + prepare: () => prepare(false), + publish: () => { + if (handleState === `discarded`) return + prepare(false) + if ( + publicationDeferral.depth > 0 || + publicationDeferral.discard || + publicationDeferral.published + ) { + return + } + publicationDeferral.published = true + if (this.preparedPublicationDeferral === publicationDeferral) { + this.preparedPublicationDeferral = undefined + } + const publication = publicationDeferral.prepared + publicationDeferral.prepared = undefined + if (publication) { + this.publishEvents(publication.changes, publication.layoutChanged) + } + runAllCallbacks(publicationDeferral.afterPublication.splice(0)) + }, + discard: () => prepare(true), + } + } + + /** Run work only after a held coherent publication becomes public. */ + public afterPublication(callback: () => void): void { + const publicationDeferral = this.publicationDeferral + if (publicationDeferral) { + publicationDeferral.afterPublication.push(callback) + return } + callback() } private publishEvents( @@ -194,15 +296,24 @@ export class CollectionChangesManager< // Every subscriber sees one committed source batch before dependent query // graphs run. This keeps repeated aliases and sibling subqueries coherent. withPublicationContext(() => { - // Notify both internal layout consumers and the public subscription API. - // Public subscribers historically receive an empty batch for order-only - // moves because there is no row-value ChangeMessage to publish. - if (rawEvents.length === 0) { - for (const listener of this.layoutChangeListeners) listener() - } - - for (const subscription of this.changeSubscriptions) { - subscription.emitEvents(enrichedEvents) + const callbacks = [ + // Notify both internal layout consumers and the public subscription API. + // Public subscribers historically receive an empty batch for order-only + // moves because there is no row-value ChangeMessage to publish. + ...(rawEvents.length === 0 + ? [...this.layoutChangeListeners].map((listener) => () => listener()) + : []), + ...[...this.changeSubscriptions].map( + (subscription) => () => subscription.emitEvents(enrichedEvents), + ), + ] + try { + runAllCallbacks(callbacks) + } catch (error) { + // The committed batch is already public. Keep the first callback + // failure on the publication while later subscribers and graph work + // finish observing the same snapshot. + deferPublicationFailure(error) } }) } @@ -347,7 +458,17 @@ export class CollectionChangesManager< public cleanup(): void { this.batchedEvents = [] this.shouldBatchEvents = false - this.deferredPublications = [] - this.publicationDeferralDepth = 0 + if (this.publicationDeferral) { + this.publicationDeferral.discard = true + this.publicationDeferral.publications = [] + this.publicationDeferral.prepared = undefined + } + this.publicationDeferral = undefined + if (this.preparedPublicationDeferral) { + this.preparedPublicationDeferral.discard = true + this.preparedPublicationDeferral.publications = [] + this.preparedPublicationDeferral.prepared = undefined + } + this.preparedPublicationDeferral = undefined } } diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 7b61a13503..447ed92611 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -14,6 +14,7 @@ import { CollectionIndexesManager } from './indexes' import { CollectionMutationsManager } from './mutations' import { CollectionEventsManager } from './events.js' import type { PublicationDeferral } from './changes' +import type { CollectionPublicationStateSnapshot } from './state' import type { CollectionSubscription } from './subscription' import type { AllCollectionEvents, @@ -454,6 +455,21 @@ export class CollectionImpl< return this._changes.deferPublication() } + /** Capture mutable state before a coherent graph publication is installed. */ + public _snapshotPublicationState( + keys: Iterable, + ): CollectionPublicationStateSnapshot { + return this._state.snapshotPublicationState(keys) + } + + /** Restore a failed coherent graph publication and rebuild its indexes. */ + public _restorePublicationState( + snapshot: CollectionPublicationStateSnapshot, + ): void { + this._state.restorePublicationState(snapshot) + this._indexes.rebuildIndexes() + } + /** * Register a callback to be executed when the collection first becomes ready * Useful for preloading collections diff --git a/packages/db/src/collection/indexes.ts b/packages/db/src/collection/indexes.ts index 84e45d6fcd..e6bdc1e2e8 100644 --- a/packages/db/src/collection/indexes.ts +++ b/packages/db/src/collection/indexes.ts @@ -369,6 +369,13 @@ export class CollectionIndexesManager< } } + /** Rebuild every retained index from the Collection's current state. */ + public rebuildIndexes(): void { + for (const index of this.indexes.values()) { + index.build(this.state.entries()) + } + } + /** * Clean up indexes */ diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 661e8410d0..a4ca225c7d 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -7,6 +7,7 @@ import { safeCancelIdleCallback, safeRequestIdleCallback, } from '../utils/browser-polyfills' +import { runAllCallbacks } from '../utils/callbacks' import { CleanupQueue } from './cleanup-queue' import type { IdleCallbackDeadline } from '../utils/browser-polyfills' import type { StandardSchemaV1 } from '@standard-schema/spec' @@ -37,6 +38,7 @@ export class CollectionLifecycleManager< public onFirstReadyCallbacks: Array<() => void> = [] private idleCallbackId: number | null = null private syncError: unknown + private statusRevision = 0 /** * Creates a new CollectionLifecycleManager instance @@ -104,6 +106,7 @@ export class CollectionLifecycleManager< ) } this.validateStatusTransition(this.status, newStatus) + this.statusRevision++ const previousStatus = this.status this.status = newStatus @@ -133,12 +136,34 @@ export class CollectionLifecycleManager< * @private - Should only be called by sync implementations */ public markReady(): void { + const failure = this.applyReadyTransition() + if (failure) throw failure.error + } + + /** @internal Capture ready-effect failures while the sync entry completes. */ + public markReadyDuringSyncStart(): { error: unknown } | undefined { + return this.applyReadyTransition() + } + + private applyReadyTransition(): { error: unknown } | undefined { this.validateStatusTransition(this.status, `ready`) // A successful initial sync or recovery establishes a ready snapshot. if (this.status === `loading` || this.status === `error`) { this.syncError = undefined + const readyRevision = this.statusRevision + 1 this.setStatus(`ready`, true) + // A status listener can synchronously supersede this transition, even + // when it restarts the Collection back to ready before returning. + if ( + (this.status as CollectionStatus) !== `ready` || + this.statusRevision !== readyRevision + ) { + return undefined + } + + const readyEffects: Array<() => void> = [] + // Call any registered first ready callbacks (only on first time becoming ready) if (!this.hasBeenReady) { this.hasBeenReady = true @@ -148,16 +173,19 @@ export class CollectionLifecycleManager< this.hasReceivedFirstCommit = true } - const callbacks = [...this.onFirstReadyCallbacks] + readyEffects.push(...this.onFirstReadyCallbacks) this.onFirstReadyCallbacks = [] - callbacks.forEach((callback) => callback()) } // Notify dependents when markReady is called, after status is set // This ensures live queries get notified when their dependencies become ready - if (this.changes.changeSubscriptions.size > 0) { - this.changes.emitEmptyReadyEvent() + readyEffects.push(() => this.changes.emitEmptyReadyEvent()) + try { + runAllCallbacks(readyEffects) + } catch (error) { + return { error } } } + return undefined } /** Mark an asynchronous sync failure after sync has started. */ diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 9c91789780..967898d84d 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -37,6 +37,19 @@ import type { TransactionScope } from '../transactions' import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionStateManager } from './state' +function copyNonEnumerableSymbols( + target: T, + source: object, +): T { + for (const symbol of Object.getOwnPropertySymbols(source)) { + const descriptor = Object.getOwnPropertyDescriptor(source, symbol) + if (descriptor && !descriptor.enumerable) { + Object.defineProperty(target, symbol, descriptor) + } + } + return target +} + export class CollectionMutationsManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -369,10 +382,9 @@ export class CollectionMutationsManager< ) // Construct the full modified item by applying the validated update payload to the original item - const modifiedItem = Object.assign( - {}, + const modifiedItem = copyNonEnumerableSymbols( + Object.assign({}, originalItem, validatedUpdatePayload), originalItem, - validatedUpdatePayload, ) // Check if the ID of the item is being changed diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 7f422cfc85..523afa8a0b 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -28,6 +28,33 @@ import type { CollectionIndexesManager } from './indexes' import type { CollectionEventsManager } from './events' import type { Deferred } from '../deferred' +function replaceMap( + target: { clear: () => void; set: (key: K, value: V) => unknown }, + entries: Iterable, +): void { + target.clear() + for (const [key, value] of entries) target.set(key, value) +} + +function replaceSet(target: Set, values: Iterable): void { + target.clear() + for (const value of values) target.add(value) +} + +function restoreMapEntry( + target: { set: (key: K, value: V) => unknown; delete: (key: K) => unknown }, + key: K, + entry: { present: boolean; value: V | undefined }, +): void { + if (entry.present) target.set(key, entry.value as V) + else target.delete(key) +} + +function restoreSetEntry(target: Set, value: T, present: boolean): void { + if (present) target.add(value) + else target.delete(value) +} + interface PendingSyncedTransaction< T extends object = Record, TKey extends string | number = string | number, @@ -57,6 +84,40 @@ interface PendingSyncedTransaction< immediate?: boolean } +export type CollectionPublicationStateSnapshot< + TOutput extends object, + TKey extends string | number, +> = { + pendingSyncedTransactions: Array> + applicationStarted: Map, boolean> + keys: Map< + TKey, + { + syncedData: { present: boolean; value: TOutput | undefined } + syncedMetadata: { present: boolean; value: unknown } + rowOrigin: { present: boolean; value: VirtualOrigin | undefined } + hydrationSeed: boolean + hydrated: boolean + synced: boolean + } + > + syncedCollectionMetadata: Array<[string, unknown]> + optimisticUpserts: Map + optimisticDeletes: Set + pendingOptimisticUpserts: Map + pendingOptimisticDeletes: Set + pendingOptimisticDirectUpserts: Set + pendingOptimisticDirectDeletes: Set + pendingLocalChanges: Set + pendingLocalOrigins: Set + size: number + preSyncVisibleState: Map + preSyncVirtualState: Map> + recentlySyncedKeys: Set + hasReceivedFirstCommit: boolean + isCommittingSyncTransactions: boolean +} + type PendingMetadataWrite = { type: `set`; value: unknown } | { type: `delete` } type InternalChangeMessage< @@ -137,10 +198,12 @@ export class CollectionStateManager< // State used for computing the change events public syncedKeys = new Set() public preSyncVisibleState = new Map() + public preSyncVirtualState = new Map>() public recentlySyncedKeys = new Set() public hasReceivedFirstCommit = false public isCommittingSyncTransactions = false private isDrainingSyncTransactions = false + private syncSessionGeneration = 0 public isLocalOnly = false /** @@ -171,6 +234,126 @@ export class CollectionStateManager< this._events = deps.events } + /** Collect every row key whose visible state a sync transaction can change. */ + private collectAffectedKeys( + transactions: Iterable>, + ): Set { + const keys = new Set() + for (const transaction of transactions) { + for (const operation of transaction.operations) { + keys.add(operation.key as TKey) + } + for (const key of transaction.rowMetadataWrites.keys()) { + keys.add(key) + } + } + return keys + } + + public snapshotPublicationState( + keys: Iterable, + ): CollectionPublicationStateSnapshot { + const affectedKeys = new Set(keys) + for (const key of this.collectAffectedKeys( + this.pendingSyncedTransactions, + )) { + affectedKeys.add(key) + } + + return { + pendingSyncedTransactions: [...this.pendingSyncedTransactions], + applicationStarted: new Map( + this.pendingSyncedTransactions.map((transaction) => [ + transaction, + transaction.applicationStarted, + ]), + ), + keys: new Map( + [...affectedKeys].map((key) => [ + key, + { + syncedData: { + present: this.syncedData.has(key), + value: this.syncedData.get(key), + }, + syncedMetadata: { + present: this.syncedMetadata.has(key), + value: this.syncedMetadata.get(key), + }, + rowOrigin: { + present: this.rowOrigins.has(key), + value: this.rowOrigins.get(key), + }, + hydrationSeed: this.hydrationSeedKeys.has(key), + hydrated: this.hydratedKeys.has(key), + synced: this.syncedKeys.has(key), + }, + ]), + ), + syncedCollectionMetadata: [...this.syncedCollectionMetadata.entries()], + optimisticUpserts: new Map(this.optimisticUpserts), + optimisticDeletes: new Set(this.optimisticDeletes), + pendingOptimisticUpserts: new Map(this.pendingOptimisticUpserts), + pendingOptimisticDeletes: new Set(this.pendingOptimisticDeletes), + pendingOptimisticDirectUpserts: new Set( + this.pendingOptimisticDirectUpserts, + ), + pendingOptimisticDirectDeletes: new Set( + this.pendingOptimisticDirectDeletes, + ), + pendingLocalChanges: new Set(this.pendingLocalChanges), + pendingLocalOrigins: new Set(this.pendingLocalOrigins), + size: this.size, + preSyncVisibleState: new Map(this.preSyncVisibleState), + preSyncVirtualState: new Map(this.preSyncVirtualState), + recentlySyncedKeys: new Set(this.recentlySyncedKeys), + hasReceivedFirstCommit: this.hasReceivedFirstCommit, + isCommittingSyncTransactions: this.isCommittingSyncTransactions, + } + } + + public restorePublicationState( + snapshot: CollectionPublicationStateSnapshot, + ): void { + this.pendingSyncedTransactions = [...snapshot.pendingSyncedTransactions] + for (const [ + transaction, + applicationStarted, + ] of snapshot.applicationStarted) { + transaction.applicationStarted = applicationStarted + } + for (const [key, state] of snapshot.keys) { + restoreMapEntry(this.syncedData, key, state.syncedData) + restoreMapEntry(this.syncedMetadata, key, state.syncedMetadata) + restoreMapEntry(this.rowOrigins, key, state.rowOrigin) + restoreSetEntry(this.hydrationSeedKeys, key, state.hydrationSeed) + restoreSetEntry(this.hydratedKeys, key, state.hydrated) + restoreSetEntry(this.syncedKeys, key, state.synced) + } + replaceMap(this.syncedCollectionMetadata, snapshot.syncedCollectionMetadata) + replaceMap(this.optimisticUpserts, snapshot.optimisticUpserts) + replaceSet(this.optimisticDeletes, snapshot.optimisticDeletes) + replaceMap(this.pendingOptimisticUpserts, snapshot.pendingOptimisticUpserts) + replaceSet(this.pendingOptimisticDeletes, snapshot.pendingOptimisticDeletes) + replaceSet( + this.pendingOptimisticDirectUpserts, + snapshot.pendingOptimisticDirectUpserts, + ) + replaceSet( + this.pendingOptimisticDirectDeletes, + snapshot.pendingOptimisticDirectDeletes, + ) + replaceSet(this.pendingLocalChanges, snapshot.pendingLocalChanges) + replaceSet(this.pendingLocalOrigins, snapshot.pendingLocalOrigins) + this.size = snapshot.size + replaceMap(this.preSyncVisibleState, snapshot.preSyncVisibleState) + replaceMap(this.preSyncVirtualState, snapshot.preSyncVirtualState) + replaceSet(this.recentlySyncedKeys, snapshot.recentlySyncedKeys) + this.hasReceivedFirstCommit = snapshot.hasReceivedFirstCommit + this.isCommittingSyncTransactions = snapshot.isCommittingSyncTransactions + this.virtualPropsCache = new WeakMap() + } + /** * Checks whether this row currently has no pending local optimistic writes. * @@ -527,6 +710,21 @@ export class CollectionStateManager< if (!this.isThisCollection(mutation.collection)) { continue } + + // Direct mutation handlers may publish their authoritative echo + // before the transaction completes. That sync commit consumes the + // pending-origin marker. Do not recreate optimistic state after the + // same mutation has already been confirmed. + const wasConfirmedDuringDirectMutation = + isDirectTransaction && !this.pendingLocalOrigins.has(mutation.key) + if (wasConfirmedDuringDirectMutation) { + this.pendingOptimisticUpserts.delete(mutation.key) + this.pendingOptimisticDeletes.delete(mutation.key) + this.pendingOptimisticDirectUpserts.delete(mutation.key) + this.pendingOptimisticDirectDeletes.delete(mutation.key) + continue + } + this.pendingLocalOrigins.add(mutation.key) if (!mutation.optimistic) { continue @@ -862,6 +1060,8 @@ export class CollectionStateManager< processed: boolean publicationError?: { error: unknown } } { + const syncSessionGeneration = this.syncSessionGeneration + // Check if there are any persisting transaction let hasPersistingTransaction = false for (const transaction of this.transactions.values()) { @@ -923,6 +1123,10 @@ export class CollectionStateManager< // non-immediate transactions would be applied later and could overwrite newer state. // Processing all committed transactions together preserves causal ordering. if (!hasPersistingTransaction || hasTruncateSync || hasImmediateSync) { + // Every committed transaction below belongs to one applied causal prefix. + // Compare its final layout with the one public state immediately before + // application, not with snapshots taken when older work first queued. + const visibleKeysBeforeCommit = layoutChanged ? [...this.keys()] : [] // This queue remains authoritative while user callbacks run. Transactions // opened by a callback must not be overwritten by this batch's snapshot. this.pendingSyncedTransactions = uncommittedSyncedTransactions @@ -945,17 +1149,7 @@ export class CollectionStateManager< let truncatePendingLocalChanges: Set | undefined let truncatePendingLocalOrigins: Set | undefined - // First collect all keys that will be affected by sync operations - const changedKeys = new Set() - for (const transaction of committedSyncedTransactions) { - for (const operation of transaction.operations) { - const key = operation.key as TKey - changedKeys.add(key) - } - for (const [key] of transaction.rowMetadataWrites) { - changedKeys.add(key) - } - } + const changedKeys = this.collectAffectedKeys(committedSyncedTransactions) type AppliedRequestProvenance = { version: { @@ -1084,6 +1278,12 @@ export class CollectionStateManager< this.hydrationSeedKeys.clear() this.hydratedKeys.clear() this.clearOriginTrackingState() + for (const key of truncatePendingLocalChanges) { + this.pendingLocalChanges.add(key) + } + for (const key of truncatePendingLocalOrigins) { + this.pendingLocalOrigins.add(key) + } // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations // are compared against the post-truncate state (undefined) rather than pre-truncate state @@ -1118,13 +1318,17 @@ export class CollectionStateManager< case `insert`: this.syncedData.set(key, operation.value) this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + // Ordinary same-key sync confirms pending local work. A full + // replacement has no causal link to that mutation and must + // preserve its optimistic overlay until a later echo arrives. + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break case `update`: { if (rowUpdateMode === `partial`) { @@ -1138,26 +1342,29 @@ export class CollectionStateManager< this.syncedData.set(key, operation.value) } this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break } case `delete`: this.syncedData.delete(key) + this.syncedKeys.delete(key) this.syncedMetadata.delete(key) - // Clean up origin and pending tracking for deleted rows this.rowOrigins.delete(key) - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) + if (!transaction.truncate) { + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + } break } recordRequestProvenance(key, transaction.requestSignal) @@ -1273,6 +1480,23 @@ export class CollectionStateManager< } } + // An ordinary sync for another key must not create a hidden interval + // where a completed direct mutation disappears. Same-key confirmation + // removed these pending entries above; everything left is still an + // authoritative optimistic overlay for this publication turn. + for (const [key, value] of this.pendingOptimisticUpserts) { + if (this.pendingOptimisticDirectUpserts.has(key)) { + this.optimisticUpserts.set(key, value) + this.optimisticDeletes.delete(key) + } + } + for (const key of this.pendingOptimisticDeletes) { + if (this.pendingOptimisticDirectDeletes.has(key)) { + this.optimisticUpserts.delete(key) + this.optimisticDeletes.add(key) + } + } + // Always overlay any still-active optimistic transactions so mutations that started // after the truncate snapshot are preserved. for (const transaction of this.transactions.values()) { @@ -1302,45 +1526,54 @@ export class CollectionStateManager< } } - // A completed optimistic insert may have used a temporary client key while - // the sync confirmation used a different server-generated key. Once a - // sync commit has been applied, stop retaining completed optimistic keys - // that were not confirmed by this commit so the temporary row is removed. - for (const key of this.pendingOptimisticDirectUpserts) { - if (!changedKeys.has(key)) { - changedKeys.add(key) - if (!currentVisibleState.has(key)) { - const previousValue = previousOptimisticUpserts.get(key) - if (previousValue !== undefined) { - currentVisibleState.set(key, previousValue) + if ( + committedSyncedTransactions.some((transaction) => !transaction.truncate) + ) { + // A completed optimistic insert may have used a temporary client key while + // the sync confirmation used a different server-generated key. Once an + // ordinary sync commit has been applied, stop retaining completed + // optimistic keys that were not confirmed by this commit so the temporary + // row is removed. Truncate replacement is not confirmation. + for (const key of this.pendingOptimisticDirectUpserts) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + if (!currentVisibleState.has(key)) { + const previousValue = previousOptimisticUpserts.get(key) + if (previousValue !== undefined) { + currentVisibleState.set(key, previousValue) + } } + this.pendingOptimisticUpserts.delete(key) + this.pendingLocalOrigins.delete(key) + this.optimisticUpserts.delete(key) } - this.pendingOptimisticUpserts.delete(key) - this.pendingLocalOrigins.delete(key) + requestProvenanceByKey.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) } - requestProvenanceByKey.delete(key) - } - for (const key of this.pendingOptimisticDirectDeletes) { - if (!changedKeys.has(key)) { - changedKeys.add(key) + for (const key of this.pendingOptimisticDirectDeletes) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + } + this.pendingOptimisticDeletes.delete(key) + this.pendingLocalOrigins.delete(key) + this.optimisticDeletes.delete(key) + requestProvenanceByKey.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) } - this.pendingOptimisticDeletes.delete(key) - this.pendingLocalOrigins.delete(key) - requestProvenanceByKey.delete(key) } - this.pendingOptimisticDirectUpserts.clear() - this.pendingOptimisticDirectDeletes.clear() // Now check what actually changed in the final visible state for (const key of changedKeys) { const previousVisibleValue = currentVisibleState.get(key) const newVisibleValue = this.get(key) // This returns the new derived state - const previousVirtualProps = this.getVirtualPropsSnapshotForState(key, { - rowOrigins: previousRowOrigins, - optimisticUpserts: previousOptimisticUpserts, - optimisticDeletes: previousOptimisticDeletes, - completedOptimisticKeys: completedOptimisticOps, - }) + const previousVirtualProps = + this.preSyncVirtualState.get(key) ?? + this.getVirtualPropsSnapshotForState(key, { + rowOrigins: previousRowOrigins, + optimisticUpserts: previousOptimisticUpserts, + optimisticDeletes: previousOptimisticDeletes, + completedOptimisticKeys: completedOptimisticOps, + }) const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) const virtualChanged = previousVirtualProps.$synced !== nextVirtualProps.$synced || @@ -1458,30 +1691,45 @@ export class CollectionStateManager< } // End batching and emit all events (combines any batched events with sync events) + const visibleKeysAfterCommit = layoutChanged ? [...this.keys()] : [] + const visibleLayoutChanged = + layoutChanged && + (visibleKeysBeforeCommit.length !== visibleKeysAfterCommit.length || + visibleKeysBeforeCommit.some( + (key, index) => key !== visibleKeysAfterCommit[index], + )) let publicationError: { error: unknown } | undefined try { - this.changes.emitEvents(events, true, layoutChanged) + this.changes.emitEvents(events, true, visibleLayoutChanged) } catch (error) { // The state is already committed. Finish this batch and drain any work // queued by earlier listeners before surfacing their publication error. publicationError = { error } } - // Clear the pre-sync state since sync operations are complete - this.preSyncVisibleState.clear() - - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - this.recentlySyncedKeys.clear() - }) + if (this.syncSessionGeneration === syncSessionGeneration) { + // Clear the pre-sync state since sync operations are complete + this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() + + // A coherent multi-Collection publication can still roll back this + // commit. Start its cleanup tail only after that publication succeeds. + this.changes.afterPublication(() => { + Promise.resolve().then(() => { + if (this.syncSessionGeneration === syncSessionGeneration) { + this.recentlySyncedKeys.clear() + } + }) + }) - // Mark that we've received the first commit (for tracking purposes) - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true + // Mark that we've received the first commit (for tracking purposes) + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } } for (const transaction of committedSyncedTransactions) { - transaction.applied.resolve() + this.changes.afterPublication(() => transaction.applied.resolve()) } return { processed: true, publicationError } @@ -1536,22 +1784,20 @@ export class CollectionStateManager< this.pendingSyncedTransactions.splice(index, 1) transaction.applied.reject(new SyncTransactionAbortedError()) - const remainingPendingKeys = new Set() - for (const pending of this.pendingSyncedTransactions) { - for (const operation of pending.operations) { - remainingPendingKeys.add(operation.key as TKey) - } - } - for (const operation of transaction.operations) { - const key = operation.key as TKey + const remainingPendingKeys = this.collectAffectedKeys( + this.pendingSyncedTransactions, + ) + for (const key of this.collectAffectedKeys([transaction])) { if (!remainingPendingKeys.has(key)) { this.recentlySyncedKeys.delete(key) this.preSyncVisibleState.delete(key) + this.preSyncVirtualState.delete(key) } } if (this.pendingSyncedTransactions.length === 0) { this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() this.recentlySyncedKeys.clear() this.changes.emitEvents([], true) } else { @@ -1591,13 +1837,7 @@ export class CollectionStateManager< public capturePreSyncVisibleState(): void { if (this.pendingSyncedTransactions.length === 0) return - // Get all keys that will be affected by sync operations - const syncedKeys = new Set() - for (const transaction of this.pendingSyncedTransactions) { - for (const operation of transaction.operations) { - syncedKeys.add(operation.key as TKey) - } - } + const syncedKeys = this.collectAffectedKeys(this.pendingSyncedTransactions) // Mark keys as about to be synced to suppress intermediate events from recomputeOptimisticState for (const key of syncedKeys) { @@ -1612,6 +1852,10 @@ export class CollectionStateManager< const currentValue = this.get(key) if (currentValue !== undefined) { this.preSyncVisibleState.set(key, currentValue) + this.preSyncVirtualState.set( + key, + this.getVirtualPropsSnapshotForState(key), + ) } } } @@ -1637,6 +1881,7 @@ export class CollectionStateManager< * This can be called manually or automatically by garbage collection */ public cleanup(): void { + this.syncSessionGeneration++ for (const transaction of this.pendingSyncedTransactions) { transaction.applied.reject(new SyncTransactionAbortedError()) } @@ -1656,6 +1901,9 @@ export class CollectionStateManager< this.size = 0 this.pendingSyncedTransactions = [] this.syncedKeys.clear() + this.preSyncVisibleState.clear() + this.preSyncVirtualState.clear() + this.recentlySyncedKeys.clear() this.hasReceivedFirstCommit = false } } diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 3678ab11bf..bc0ce64c0e 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -6,6 +6,7 @@ import { getSyncRequestProvenance, isLoadSubsetRequestSignalFor, } from '../load-subset-request-provenance.js' +import { cloneLoadSubsetOptions } from '../query/load-subset-options.js' import { buildCursor, buildCursorEquality, @@ -116,6 +117,7 @@ type ReplayHandoffResult = type SubsetDemand = SubsetAcquisition & { requestOptions: LoadSubsetOptions + matchesWhere: (row: object) => boolean onLoadSubsetResult?: ( result: LoadSubsetRequestResult, demand: LoadSubsetOptions, @@ -227,6 +229,8 @@ function createSubsetCleanupError(errors: ReadonlyArray): unknown { return new SubsetCleanupAggregateError(errors) } +const matchesEveryRow = () => true + export class CollectionSubscription extends EventEmitter implements Subscription @@ -262,9 +266,14 @@ export class CollectionSubscription private stalePublication: PublicationState | undefined private filteredCallback: (changes: Array>) => boolean + // Execution uses the frozen predicate; release keeps the caller's handle. + private readonly whereExpression: BasicExpression | undefined + private readonly releaseWhereExpression: BasicExpression | undefined private orderByIndex: IndexInterface | undefined private orderedWindow: WindowState | undefined + // The first ordered request fixes this subscription's total order. + private orderedRequestOptions: LoadSubsetOptions | undefined // Status tracking private _status: SubscriptionStatus = `ready` @@ -315,13 +324,7 @@ export class CollectionSubscription private activeAdditionalFilters(): Array<(row: object) => boolean> { return this.subsetDemands .filter((demand) => demand.active && demand.ordered === undefined) - .map((demand) => - demand.requestOptions.where - ? createFilterFunctionFromExpression( - demand.requestOptions.where, - ) - : () => true, - ) + .map((demand) => demand.matchesWhere) } private diffPublishedRows( @@ -412,9 +415,13 @@ export class CollectionSubscription constructor( private collection: CollectionImpl, private callback: (changes: Array>) => void, - private options: CollectionSubscriptionOptions, + options: CollectionSubscriptionOptions, ) { super() + this.releaseWhereExpression = options.whereExpression + this.whereExpression = cloneLoadSubsetOptions({ + where: options.whereExpression, + }).where if (options.onUnsubscribe) { this.on(`unsubscribed`, options.onUnsubscribe) } @@ -423,8 +430,8 @@ export class CollectionSubscription } // Auto-index for where expressions if enabled - if (options.whereExpression) { - ensureIndexForExpression(options.whereExpression, this.collection) + if (this.whereExpression) { + ensureIndexForExpression(this.whereExpression, this.collection) } const callbackWithSentKeysTracking = ( @@ -439,8 +446,11 @@ export class CollectionSubscription this.callback = callbackWithSentKeysTracking // Create a filtered callback if where clause is provided - this.filteredCallback = options.whereExpression - ? createFilteredCallback(this.callback, options) + this.filteredCallback = this.whereExpression + ? createFilteredCallback(this.callback, { + ...options, + whereExpression: this.whereExpression, + }) : (changes) => { this.callback(changes) return true @@ -1177,11 +1187,7 @@ export class CollectionSubscription const merged = [...session.buffer.flat(), ...retainedDeletes] const activeDemandFilters = this.subsetDemands .filter((demand) => demand.active) - .map((demand) => - demand.requestOptions.where - ? createFilterFunctionFromExpression(demand.requestOptions.where) - : undefined, - ) + .map((demand) => demand.matchesWhere) // The raw replay buffer can contain rows retained for another demand or // outside the ordered prefix. Publish the settled ordered reconciliation // as the replacement's one atomic batch. @@ -1190,8 +1196,7 @@ export class CollectionSubscription : this.createPublicationDiff( session.publicationState.publishedRows, merged, - (value) => - activeDemandFilters.some((filter) => filter?.(value) ?? true), + (value) => activeDemandFilters.some((filter) => filter(value)), ) if (replacement.length > 0) this.filteredCallback(replacement) // Buffering records every source key before active-demand filtering. Reset @@ -1244,8 +1249,11 @@ export class CollectionSubscription return this.truncateReplaySession !== undefined } - setOrderByIndex(index: IndexInterface) { + private expandOrderedSourceTies = false + + setOrderByIndex(index: IndexInterface, expandSourceOrderTies = false) { this.orderByIndex = index + this.expandOrderedSourceTies = expandSourceOrderTies } /** @@ -1286,6 +1294,10 @@ export class CollectionSubscription return this.orderedWindow?.retainedPrefixSize ?? 0 } + get orderedCoverageRevision(): number { + return this.orderedWindow?.coverageRevision ?? 0 + } + get requiresOrderedPrefixRefresh(): boolean { return this.orderedWindow?.requiresPrefixRefresh ?? false } @@ -1297,6 +1309,18 @@ export class CollectionSubscription ) } + get hasOrderedResultForActiveWindow(): boolean { + if (!this.hasActiveOrderedDemand() || !this.orderedWindow) return false + return this.retainedOrderedPublication + ? this.orderedWindow.coversActiveWindow + : this.orderedWindow.satisfiesActiveWindow + } + + settleOrderedResultAfterNoProgress(): boolean { + if (this.retainedOrderedPublication || !this.orderedWindow) return false + return this.orderedWindow.settleLocalRequestAfterNoProgress() + } + get orderedBoundaryRow(): object | undefined { if (!this.hasActiveOrderedDemand()) return undefined const boundary = this.retainedOrderedPublication @@ -1346,6 +1370,45 @@ export class CollectionSubscription return changes } + /** Apply logical demand release to the public baseline of a private replay. */ + private reconcileBufferedOrderedPublicationOnRelease(): Array< + ChangeMessage + > { + const publication = this.truncateReplaySession?.publicationState + const ordered = publication?.ordered + const window = this.orderedWindow + if (!publication || !ordered || !window) return [] + + const orderedRows = [...ordered.candidateRows] + .sort((left, right) => window.totalOrder.compareEntries(left, right)) + .slice(0, ordered.prefixSize) + const desired = new Map(orderedRows) + const additionalFilters = this.activeAdditionalFilters() + for (const [key, row] of publication.publishedRows) { + if (additionalFilters.some((filter) => filter(row))) { + desired.set(key, row) + } + } + + const lastOrderedRow = orderedRows.at(-1) + const nextOrdered: OrderedPublicationState = { + prefixSize: orderedRows.length, + boundary: + lastOrderedRow === undefined + ? undefined + : window.totalOrder.boundary(lastOrderedRow[1], lastOrderedRow[0]), + candidateRows: ordered.candidateRows, + } + publication.publishedRows = new Map(desired) + publication.sentKeys = new Set(desired.keys()) + publication.ordered = nextOrdered + this.orderedPublication = { + ...nextOrdered, + candidateRows: new Map(nextOrdered.candidateRows), + } + return this.diffPublishedRows(desired) + } + /** * Evolve a failed replay's last good ordered publication without admitting * rows installed by the rejected replacement. Later source deltas form a @@ -1367,8 +1430,8 @@ export class CollectionSubscription const window = this.orderedWindow if (!stalePublication || !ordered || !window) return [] - const orderedFilter = this.options.whereExpression - ? createFilterFunctionFromExpression(this.options.whereExpression) + const orderedFilter = this.whereExpression + ? createFilterFunctionFromExpression(this.whereExpression) : undefined const additionalFilters = this.activeAdditionalFilters() const isOrderedRow = (row: object) => orderedFilter?.(row) ?? true @@ -1695,7 +1758,7 @@ export class CollectionSubscription return { options: { - ...request.options, + ...cloneLoadSubsetOptions(request.options), signal: abortController.signal, }, ordered: request.ordered, @@ -1906,15 +1969,20 @@ export class CollectionSubscription private startSubsetDemand( requestOptions: LoadSubsetOptions, ordered?: SubsetDemand[`ordered`], + releaseWhere = requestOptions.where, ): { demand: SubsetDemand acquisition: SubsetAcquisition & { abortController: AbortController } result: LoadSubsetRequestResult replayContext: TruncateReplayContext | undefined } { + const stableRequestOptions = cloneLoadSubsetOptions(requestOptions) const demand: SubsetDemand = { - requestOptions, - options: requestOptions, + requestOptions: stableRequestOptions, + options: stableRequestOptions, + matchesWhere: stableRequestOptions.where + ? createFilterFunctionFromExpression(stableRequestOptions.where) + : matchesEveryRow, ...(ordered === undefined ? {} : { ordered }), pendingReplayAcquisitions: new Set(), active: true, @@ -1922,6 +1990,9 @@ export class CollectionSubscription releaseFailed: false, releaseSettled: false, } + if (releaseWhere) { + this.requestedSubsetWhere.set(stableRequestOptions, releaseWhere) + } const acquisition = this.createSubsetAcquisition(demand) demand.options = acquisition.options demand.ordered = acquisition.ordered @@ -2153,7 +2224,7 @@ export class CollectionSubscription } const stateOpts: RequestSnapshotOptions = { - where: this.options.whereExpression, + where: this.whereExpression, optimizedOnly: opts?.optimizedOnly ?? false, } @@ -2185,15 +2256,11 @@ export class CollectionSubscription limit: opts?.limit, } - // Reentrant adapter code must be able to release a request by the exact - // caller predicate even when the subscription predicate was combined into - // the transport predicate. - if (opts?.where) this.requestedSubsetWhere.set(loadOptions, opts.where) const { demand, result: syncResult, replayContext: startedReplayContext, - } = this.startSubsetDemand(loadOptions) + } = this.startSubsetDemand(loadOptions, undefined, opts?.where) const replayTracksCallback = this.retainReplayResultCallback(startedReplayContext) // Replay settlement owns the acquisition even if the result callback @@ -2336,10 +2403,12 @@ export class CollectionSubscription releaseFailure = { error } } finally { this.collectReleasedDemand(demand) - if (this.orderedWindow && !this.isBufferingForTruncate) { - const changes = this.stalePublication?.ordered - ? this.reconcileStaleOrderedPublication([]) - : this.reconcileOrderedWindow() + if (this.orderedWindow) { + const changes = this.isBufferingForTruncate + ? this.reconcileBufferedOrderedPublicationOnRelease() + : this.stalePublication?.ordered + ? this.reconcileStaleOrderedPublication([]) + : this.reconcileOrderedWindow() if (changes.length > 0) this.callback(changes) } } @@ -2365,11 +2434,32 @@ export class CollectionSubscription ) } + this.orderedRequestOptions ??= cloneLoadSubsetOptions({ + where: this.whereExpression, + orderBy, + }) + const orderedRequest = this.orderedRequestOptions + orderBy = orderedRequest.orderBy! + const where = orderedRequest.where + + // Preserve the order for a later positive window without compiling its + // predicate or constructing a coordinator that cannot admit any rows. + if (limit === 0) { + onLoadSubsetResult?.(true, { + where, + orderBy, + limit: 0, + subscription: this, + }) + return + } + this.orderedWindow ??= new WindowState( this.collection, orderBy, - this.options.whereExpression, + where, limit, + this.expandOrderedSourceTies, ) if (this.stalePublication && !this.stalePublication.ordered) { @@ -2384,7 +2474,6 @@ export class CollectionSubscription } } - const where = this.options.whereExpression const retainedPublication = this.retainedOrderedPublication const activeReplacement = this.truncateReplaySession !== undefined const replayOwnsContinuation = @@ -2419,19 +2508,7 @@ export class CollectionSubscription if (changes.length > 0) this.callback(changes) - // A zero window establishes no remote demand, but it must still create the - // ordered coordinator so a later setWindow can load from the same order. - if (limit === 0) { - onLoadSubsetResult?.(true, { - where, - orderBy, - limit: 0, - subscription: this, - }) - return - } - - if (!retainedPublication && this.orderedWindow.coversActiveWindow) { + if (!retainedPublication && this.orderedWindow.satisfiesActiveWindow) { // No adapter request was made. Use an impossible zero-window demand so // direct tracking can finish without claiming another demand's outcome. onLoadSubsetResult?.(true, { @@ -2498,12 +2575,16 @@ export class CollectionSubscription acquisition, result: syncResult, replayContext: startedReplayContext, - } = this.startSubsetDemand(loadOptions, { - requestedPrefix, - hadBoundary: boundary !== undefined || refreshPrefix, - requiresUnboundedRefinement, - revision: this.orderedWindow.coverageRevision, - }) + } = this.startSubsetDemand( + loadOptions, + { + requestedPrefix, + hadBoundary: boundary !== undefined || refreshPrefix, + requiresUnboundedRefinement, + revision: this.orderedWindow.coverageRevision, + }, + this.releaseWhereExpression, + ) // A synchronous continuation can complete ordered coverage. Retain its // callback before applying that evidence so callback failure can still @@ -2580,7 +2661,11 @@ export class CollectionSubscription const rowKeys = outcome?.appliedRowKeys const exhausted = outcome?.extent === `exhausted` - if (outcome !== undefined && rowKeys === undefined && !exhausted) { + if ( + outcome?.extent === `unknown` && + rowKeys === undefined && + !exhausted + ) { window.recordLocalRequestSatisfaction(ordered.requestedPrefix) } else if (!ordered.hadBoundary && !ordered.requiresUnboundedRefinement) { window.recordInitialCoverage(rowKeys, exhausted) diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index b147662054..9031c194ba 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -5,6 +5,7 @@ import { NoPendingSyncTransactionCommitError, NoPendingSyncTransactionWriteError, SyncCleanupError, + SyncTransactionAbortedError, SyncTransactionAlreadyCommittedError, SyncTransactionAlreadyCommittedWriteError, } from '../errors' @@ -222,6 +223,8 @@ export class CollectionSyncManager< const syncEpoch = ++this.syncEpoch const isCurrentSync = () => syncEpoch === this.syncEpoch this.lifecycle.setStatus(`loading`) + let syncEntryActive = true + let readyEffectFailure: { error: unknown } | undefined try { const syncRes = normalizeSyncFnResult( @@ -383,7 +386,12 @@ export class CollectionSyncManager< return receipt }, markReady: () => { - if (isCurrentSync()) this.lifecycle.markReady() + if (!isCurrentSync()) return + if (syncEntryActive) { + readyEffectFailure ??= this.lifecycle.markReadyDuringSyncStart() + } else { + this.lifecycle.markReady() + } }, markError: (error?: unknown) => { if (isCurrentSync()) this.lifecycle.markError(error) @@ -427,6 +435,7 @@ export class CollectionSyncManager< metadata: this.createSyncMetadataApi(isCurrentSync), }), ) + syncEntryActive = false // Store cleanup function if provided this.syncCleanupFn = syncRes?.cleanup ?? null @@ -445,9 +454,11 @@ export class CollectionSyncManager< ) } } catch (error) { + syncEntryActive = false this.lifecycle.markError(error) throw error } + if (readyEffectFailure) throw readyEffectFailure.error } public deferStart(): boolean { @@ -718,10 +729,14 @@ export class CollectionSyncManager< } let settled = false - let startingSync = false + const syncStartState = { active: false, ready: false } let unsubscribeError = () => {} let unsubscribeReady = () => {} const resolveReady = () => { + if (syncStartState.active) { + syncStartState.ready = true + return + } if (settled) return settled = true unsubscribeError() @@ -739,7 +754,7 @@ export class CollectionSyncManager< // Register callback BEFORE starting sync to avoid race condition unsubscribeReady = this.lifecycle.onFirstReady(resolveReady) unsubscribeError = this.collection.on(`status:error`, () => { - if (startingSync) { + if (syncStartState.active) { return } rejectError(this.getPreloadError()) @@ -750,17 +765,24 @@ export class CollectionSyncManager< this.lifecycle.status === `idle` || this.lifecycle.status === `cleaned-up` ) { - startingSync = true + syncStartState.active = true + let startFailure: { error: unknown } | undefined try { this.startSync() } catch (error) { - rejectError(error) - return + startFailure = { error } } finally { - startingSync = false + syncStartState.active = false } if (this.collection.status === `error`) { rejectError(this.getPreloadError()) + } else if (syncStartState.ready) { + // A first-ready listener can throw after readiness is established. + // That failure still escapes direct startSync(), but preload follows + // the final collection state after synchronous adapter entry. + resolveReady() + } else if (startFailure) { + rejectError(startFailure.error) } } }) @@ -800,6 +822,8 @@ export class CollectionSyncManager< cancel: () => void getOutcomes: () => ReadonlyArray } { + // A failed nested operation restores this owner before rollback work. + const previousOperation = this.activeLoadSubsetOperation const operation: LoadSubsetOperation = { pending: new Set(), outcomes: new Map(), @@ -818,7 +842,9 @@ export class CollectionSyncManager< operation.completed = true this.loadSubsetOperations.delete(operation) if (this.activeLoadSubsetOperation === operation) { - this.activeLoadSubsetOperation = undefined + this.activeLoadSubsetOperation = previousOperation?.completed + ? undefined + : previousOperation } }, getOutcomes: () => @@ -986,7 +1012,7 @@ export class CollectionSyncManager< */ public loadSubset(options: LoadSubsetOptions): LoadSubsetRequestResult { if (options.signal?.aborted) { - return true + return Promise.reject(new SyncTransactionAbortedError()) } // Bypass loadSubset when syncMode is 'eager' diff --git a/packages/db/src/indexes/auto-index.ts b/packages/db/src/indexes/auto-index.ts index 350b469a43..3c9f5c9f0c 100644 --- a/packages/db/src/indexes/auto-index.ts +++ b/packages/db/src/indexes/auto-index.ts @@ -71,7 +71,10 @@ export function ensureIndexForField< }, { name: `auto:${fieldPath.join(`.`)}`, - options: compareFn ? { compareFn, compareOptions: compareOpts } : {}, + options: { + compareOptions: compareOpts, + ...(compareFn && { compareFn }), + }, }, ) } catch (error) { diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 26cb09887b..ecbd86852d 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -6,6 +6,28 @@ import type { RangeQueryOptions } from './btree-index.js' import type { CompareOptions } from '../query/builder/types.js' import type { BasicExpression, OrderByDirection } from '../query/ir.js' +function normalizeLocaleOptions(options: object | undefined): object { + return Object.fromEntries( + Object.entries(options ?? {}).filter(([, value]) => value !== undefined), + ) +} + +function canonicalizeLocale(locale: string | undefined): string | undefined { + return locale === undefined ? undefined : Intl.getCanonicalLocales(locale)[0] +} + +type LocaleCompareOptions = CompareOptions & { + stringSort?: `locale` + locale?: string + localeOptions?: object +} + +function usesLocaleCollation( + options: CompareOptions, +): options is LocaleCompareOptions { + return (options.stringSort ?? DEFAULT_COMPARE_OPTIONS.stringSort) === `locale` +} + /** * Operations that indexes can support, imported from available comparison functions */ @@ -70,11 +92,11 @@ export interface IndexInterface< supports: (operation: IndexOperation) => boolean /** - * Whether range lookups (gt/gte/lt/lte) on this index can be trusted to - * return every matching key. Range traversal relies on the index ordering, so - * it is unsafe when the index uses a custom comparator, whose order may not - * match the WHERE evaluator's relational operators. Callers must fall back to - * a full scan when this is `false`. + * Whether range lookups (gt/gte/lt/lte) and ordered traversal on this index + * can be trusted to match query comparison semantics. Both rely on the index + * ordering, so they are unsafe when the index uses a custom comparator whose + * order may not match the WHERE or ORDER BY evaluator. Callers must fall back + * to a full scan when this is `false`. */ get supportsRangeOptimization(): boolean @@ -174,21 +196,39 @@ export abstract class BaseIndex< /** * Checks if the compare options match the index's compare options. - * The direction is ignored because the index can be reversed if the direction is different. + * Reversing an index also reverses null placement, so opposite directions + * are compatible only when their requested null placement is opposite too. */ matchesCompareOptions(compareOptions: CompareOptions): boolean { - const thisCompareOptionsWithoutDirection = { - ...this.compareOptions, - direction: undefined, + const indexCompareOptions = this.compareOptions + const indexUsesLocale = usesLocaleCollation(indexCompareOptions) + const requestedUsesLocale = usesLocaleCollation(compareOptions) + const reversesDirection = + indexCompareOptions.direction !== compareOptions.direction + const effectiveIndexNulls = reversesDirection + ? indexCompareOptions.nulls === `first` + ? `last` + : `first` + : indexCompareOptions.nulls + + if ( + effectiveIndexNulls !== compareOptions.nulls || + indexUsesLocale !== requestedUsesLocale + ) { + return false } - const compareOptionsWithoutDirection = { - ...compareOptions, - direction: undefined, + + if (!indexUsesLocale || !requestedUsesLocale) { + return true } - return deepEquals( - thisCompareOptionsWithoutDirection, - compareOptionsWithoutDirection, + return ( + canonicalizeLocale(indexCompareOptions.locale) === + canonicalizeLocale(compareOptions.locale) && + deepEquals( + normalizeLocaleOptions(indexCompareOptions.localeOptions), + normalizeLocaleOptions(compareOptions.localeOptions), + ) ) } diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index 8eac6f926a..66a8a3a00b 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -1,12 +1,10 @@ import { areSameValueZeroEqual, defaultComparator, + makeComparator, normalizeValue, } from '../utils/comparison.js' -import { - deleteInSortedArray, - findInsertPositionInArray, -} from '../utils/array-utils.js' +import { findInsertPositionInArray } from '../utils/array-utils.js' import { BaseIndex } from './base-index.js' import type { CompareOptions } from '../query/builder/types.js' import type { BasicExpression } from '../query/ir.js' @@ -68,11 +66,11 @@ export class BasicIndex< options?: any, ) { super(id, expression, name, options) - this.compareFn = options?.compareFn ?? defaultComparator - this.hasCustomComparator = options?.compareFn != null if (options?.compareOptions) { this.compareOptions = options!.compareOptions } + this.compareFn = options?.compareFn ?? makeComparator(this.compareOptions) + this.hasCustomComparator = options?.compareFn != null } protected initialize(_options?: BasicIndexOptions): void {} @@ -151,7 +149,23 @@ export class BasicIndex< if (keySet.size === 0) { // No more keys for this value, remove from map and sorted array this.valueMap.delete(normalizedValue) - deleteInSortedArray(this.sortedValues, normalizedValue, this.compareFn) + const firstEqual = findInsertPositionInArray( + this.sortedValues, + normalizedValue, + this.compareFn, + ) + for ( + let index = firstEqual; + index < this.sortedValues.length; + index++ + ) { + const candidate = this.sortedValues[index] + if (this.compareFn(candidate, normalizedValue) !== 0) break + if (areSameValueZeroEqual(candidate, normalizedValue)) { + this.sortedValues.splice(index, 1) + break + } + } } } } @@ -216,8 +230,13 @@ export class BasicIndex< } } - // Build sorted array from unique values - this.sortedValues = Array.from(this.valueMap.keys()).sort(this.compareFn) + // Array.sort always moves bare undefined elements to the end without + // consulting the comparator. Wrap values while sorting so null placement + // and comparator-equivalent null/undefined tie classes stay authoritative. + this.sortedValues = Array.from(this.valueMap.keys()) + .map((value) => ({ value })) + .sort((left, right) => this.compareFn(left.value, right.value)) + .map(({ value }) => value) this.updateTimestamp() } @@ -529,13 +548,54 @@ export class BasicIndex< get orderedEntriesArrayReversed(): Array<[any, Set]> { const result: Array<[any, Set]> = [] - for (let i = this.sortedValues.length - 1; i >= 0; i--) { - const value = this.sortedValues[i] + for (let index = this.sortedValues.length - 1; index >= 0; index--) { + const value = this.sortedValues[index] result.push([value, this.valueMap.get(value) ?? new Set()]) } return result } + *orderedBuckets(): IterableIterator]> { + yield* this.groupOrderedBuckets(this.sortedValues) + } + + *orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + const reversedValues = function* (values: ReadonlyArray) { + for (let index = values.length - 1; index >= 0; index--) { + yield values[index] + } + } + yield* this.groupOrderedBuckets(reversedValues(this.sortedValues)) + } + + private *groupOrderedBuckets( + values: Iterable, + ): IterableIterator]> { + let hasGroup = false + let groupValue: unknown + let groupKeys = new Set() + + for (const value of values) { + if (!hasGroup) { + groupValue = value + hasGroup = true + } else if (this.compareFn(groupValue, value) !== 0) { + yield [groupValue, groupKeys] + groupKeys = new Set() + groupValue = value + } + for (const key of this.valueMap.get(value) ?? []) { + groupKeys.add(key) + } + } + + if (hasGroup) { + yield [groupValue, groupKeys] + } + } + get valueMapData(): Map> { return this.valueMap } diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 6379b91b52..6d87ed6980 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -2,8 +2,8 @@ import { compareKeys } from '@tanstack/db-ivm' import { BTree } from '../utils/btree.js' import { areSameValueZeroEqual, - defaultComparator, denormalizeUndefined, + makeComparator, normalizeForBTree, } from '../utils/comparison.js' import { BaseIndex } from './base-index.js' @@ -46,12 +46,12 @@ export class BTreeIndex< ]) // Internal data structures - private to hide implementation details - // The `orderedEntries` B+ tree is used for efficient range queries - // The `valueMap` is used for O(1) lookups of PKs by indexed value - private orderedEntries: BTree // we don't associate values with the keys of the B+ tree (the keys are indexed values) - private valueMap = new Map>() // instead we store a mapping of indexed values to a set of PKs + // The `orderedEntries` B+ tree groups every key whose indexed values compare + // equal. `valueMap` keeps exact values separate for equality lookups. + private orderedEntries: BTree> + private valueMap = new Map>() private indexedKeys = new Set() - private compareFn: (a: any, b: any) => number = defaultComparator + private compareFn!: (a: any, b: any) => number constructor( id: number, @@ -61,8 +61,12 @@ export class BTreeIndex< ) { super(id, expression, name, options) - // Get the base compare function - const baseCompareFn = options?.compareFn ?? defaultComparator + if (options?.compareOptions) { + this.compareOptions = options!.compareOptions + } + + const baseCompareFn = + options?.compareFn ?? makeComparator(this.compareOptions) this.hasCustomComparator = options?.compareFn != null // Wrap it to denormalize sentinels before comparison @@ -71,9 +75,6 @@ export class BTreeIndex< this.compareFn = (a: any, b: any) => baseCompareFn(denormalizeUndefined(a), denormalizeUndefined(b)) - if (options?.compareOptions) { - this.compareOptions = options!.compareOptions - } this.orderedEntries = new BTree(this.compareFn) } @@ -104,13 +105,16 @@ export class BTreeIndex< private addToBucket(key: TKey, normalizedValue: unknown): void { const keySet = this.valueMap.get(normalizedValue) if (keySet) { - // Add to existing set keySet.add(key) } else { - // Create new set for this value - const newKeySet = new Set([key]) - this.valueMap.set(normalizedValue, newKeySet) - this.orderedEntries.set(normalizedValue, undefined) + this.valueMap.set(normalizedValue, new Set([key])) + } + + const orderedKeySet = this.orderedEntries.get(normalizedValue) + if (orderedKeySet) { + orderedKeySet.add(key) + } else { + this.orderedEntries.set(normalizedValue, new Set([key])) } } @@ -140,16 +144,16 @@ export class BTreeIndex< private removeFromBucket(key: TKey, normalizedValue: unknown): void { const keySet = this.valueMap.get(normalizedValue) - if (keySet) { - keySet.delete(key) + if (!keySet?.delete(key)) return - // If set is now empty, remove the entry entirely - if (keySet.size === 0) { - this.valueMap.delete(normalizedValue) + if (keySet.size === 0) { + this.valueMap.delete(normalizedValue) + } - // Remove from ordered entries - this.orderedEntries.delete(normalizedValue) - } + const orderedKeySet = this.orderedEntries.get(normalizedValue) + orderedKeySet?.delete(key) + if (orderedKeySet?.size === 0) { + this.orderedEntries.delete(normalizedValue) } } @@ -276,7 +280,7 @@ export class BTreeIndex< fromKey, toKey, toInclusive, - (indexedValue, _) => { + (indexedValue, keys) => { // Only exclude the boundary when an exclusive lower bound was // actually provided. Without a `from` bound, `fromKey` defaults to // the minimum key and must not be dropped. Compare against the @@ -292,10 +296,7 @@ export class BTreeIndex< return } - const keys = this.valueMap.get(indexedValue) - if (keys) { - keys.forEach((key) => result.add(key)) - } + keys.forEach((key) => result.add(key)) }, ) @@ -329,22 +330,20 @@ export class BTreeIndex< */ private takeInternal( n: number, - nextPair: (k?: any) => [any, any] | undefined, + nextPair: (k?: any) => [any, Set] | undefined, from: any, filterFn?: (key: TKey) => boolean, reversed: boolean = false, ): Array { const keysInResult: Set = new Set() const result: Array = [] - let pair: [any, any] | undefined + let pair: [any, Set] | undefined let key = from // Use as-is - it's already normalized by the caller while ((pair = nextPair(key)) !== undefined && result.length < n) { key = pair[0] - const keys = this.valueMap.get(key) as - | Set> - | undefined - if (keys && keys.size > 0) { + const keys = pair[1] + if (keys.size > 0) { // Sort keys for deterministic order, reverse if needed const sorted = Array.from(keys).sort(compareKeys) if (reversed) sorted.reverse() @@ -441,21 +440,39 @@ export class BTreeIndex< } get orderedEntriesArray(): Array<[any, Set]> { - return this.orderedEntries - .keysArray() - .map((key) => [ - denormalizeUndefined(key), - this.valueMap.get(key) ?? new Set(), - ]) + return Array.from(this.orderedBuckets(), ([value, keys]) => [ + value, + keys as Set, + ]) } get orderedEntriesArrayReversed(): Array<[any, Set]> { - return this.takeReversedFromEnd(this.orderedEntries.size).map((key) => [ - denormalizeUndefined(key), - this.valueMap.get(key) ?? new Set(), + return Array.from(this.orderedBucketsReversed(), ([value, keys]) => [ + value, + keys as Set, ]) } + *orderedBuckets(): IterableIterator]> { + let pair = this.orderedEntries.nextHigherPair(undefined) + while (pair !== undefined) { + const value = pair[0] + yield [denormalizeUndefined(value), pair[1]] + pair = this.orderedEntries.nextHigherPair(value) + } + } + + *orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + let pair = this.orderedEntries.nextLowerPair(undefined) + while (pair !== undefined) { + const value = pair[0] + yield [denormalizeUndefined(value), pair[1]] + pair = this.orderedEntries.nextLowerPair(value) + } + } + get valueMapData(): Map> { // Return a new Map with denormalized keys const result = new Map>() diff --git a/packages/db/src/indexes/reverse-index.ts b/packages/db/src/indexes/reverse-index.ts index 6ca61636e1..e5dc0c9b0b 100644 --- a/packages/db/src/indexes/reverse-index.ts +++ b/packages/db/src/indexes/reverse-index.ts @@ -3,6 +3,13 @@ import type { OrderByDirection } from '../query/ir' import type { IndexInterface, IndexOperation, IndexStats } from './base-index' import type { RangeQueryOptions } from './btree-index' +interface OrderedBucketIndex { + orderedBuckets: () => IterableIterator]> + orderedBucketsReversed: () => IterableIterator< + readonly [unknown, ReadonlySet] + > +} + export class ReverseIndex< TKey extends string | number, > implements IndexInterface { @@ -67,6 +74,35 @@ export class ReverseIndex< return this.originalIndex.orderedEntriesArray } + get supportsOrderedBucketIteration(): boolean { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + typeof orderedIndex.orderedBuckets === `function` && + typeof orderedIndex.orderedBucketsReversed === `function` + ) + } + + orderedBuckets(): IterableIterator]> { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + orderedIndex.orderedBucketsReversed?.() ?? + this.originalIndex.orderedEntriesArrayReversed[Symbol.iterator]() + ) + } + + orderedBucketsReversed(): IterableIterator< + readonly [unknown, ReadonlySet] + > { + const orderedIndex = this.originalIndex as IndexInterface & + Partial> + return ( + orderedIndex.orderedBuckets?.() ?? + this.originalIndex.orderedEntriesArray[Symbol.iterator]() + ) + } + // All operations below delegate to the original index supports(operation: IndexOperation): boolean { diff --git a/packages/db/src/query/compiler/order-by.ts b/packages/db/src/query/compiler/order-by.ts index 0166cf44f9..2f98d0b0b3 100644 --- a/packages/db/src/query/compiler/order-by.ts +++ b/packages/db/src/query/compiler/order-by.ts @@ -3,7 +3,12 @@ import { orderByWithFractionalIndex, } from '@tanstack/db-ivm' import { defaultComparator, makeComparator } from '../../utils/comparison.js' -import { PropRef, collectCollectionSources, followRef } from '../ir.js' +import { + PropRef, + collectCollectionSources, + followRef, + isResidualWhere, +} from '../ir.js' import { ensureIndexForField } from '../../indexes/auto-index.js' import { findIndexForField } from '../../utils/index-optimization.js' import { resolveCompareOptions, resolveOrderBy } from '../total-order.js' @@ -37,6 +42,10 @@ export type OrderByOptimizationInfo = { /** Index on the first orderBy column - used for lazy loading */ index?: IndexInterface dataNeeded?: () => number + /** D2 must see the complete source-order tie when later order terms are local. */ + expandSourceOrderTies: boolean + /** Upstream relational operators can discard source rows before top-K. */ + refillFromResultDeficit: boolean } /** @@ -276,6 +285,20 @@ export function processOrderBy( valueExtractorForRawRow: rawRowValueExtractor, index, orderBy: sourceOrderBy, + expandSourceOrderTies: sourceTerms.length < orderByClause.length, + refillFromResultDeficit: + rawQuery.from.type !== `collectionRef` || + rawQuery.from.sourceId !== orderBySourceId || + (rawQuery.join?.some( + ({ type }) => type === `inner` || type === `right`, + ) ?? + false) || + (rawQuery.where?.some(isResidualWhere) ?? false) || + (rawQuery.fnWhere?.length ?? 0) > 0 || + rawQuery.groupBy !== undefined || + rawQuery.having !== undefined || + rawQuery.fnHaving !== undefined || + rawQuery.distinct === true, } // Ordered loading is owned by one lexical source. A collection can occur diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 2390e96ac1..fb1225cf59 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -17,7 +17,7 @@ import { computeSubscriptionOrderByHints, extractCollectionSources, extractCollectionsFromQuery, - filterDuplicateInserts, + reconcileChangesForD2, sendChangesToInput, splitUpdates, trackBiggestSentValue, @@ -264,13 +264,13 @@ export function createEffect< // Abort signal for in-flight handlers abortController.abort() - disposalPromise = (async () => { + const attempt = (async () => { // Tear down the pipeline (unsubscribe from sources, etc.) - let cleanupError: unknown + let cleanupFailure: { error: unknown } | undefined try { runner.dispose() } catch (error) { - cleanupError = error + cleanupFailure = { error } } // Wait for any in-flight async handlers to settle @@ -278,9 +278,13 @@ export function createEffect< await Promise.allSettled([...inFlightHandlers]) } - if (cleanupError !== undefined) throw cleanupError + if (cleanupFailure) throw cleanupFailure.error })() - return disposalPromise + disposalPromise = attempt + void attempt.catch(() => { + if (disposalPromise === attempt) disposalPromise = undefined + }) + return attempt } // Create and start the pipeline @@ -396,10 +400,10 @@ class EffectPipelineRunner { // Subscription management private readonly unsubscribeCallbacks = new Set<() => void>() - // Duplicate insert prevention per lexical source - private readonly sentToD2KeysBySource = new Map< + // Exact D2 contributions per lexical source + private readonly sentToD2RowsBySource = new Map< string, - Set + Map> >() // Output accumulator @@ -521,8 +525,7 @@ class EffectPipelineRunner { const { sourceId, alias, collection } = source const collectionId = collection.id - // Initialise per-source duplicate tracking - this.sentToD2KeysBySource.set(sourceId, new Set()) + this.sentToD2RowsBySource.set(sourceId, new Map()) // Discover dependencies: if source collection is itself a live query // collection, its builder must run first during transaction flushes. @@ -631,7 +634,6 @@ class EffectPipelineRunner { const truncateUnsubscribe = collection.on(`truncate`, () => { this.lastLoadRequestKey.delete(sourceId) this.biggestSentValue.delete(sourceId) - this.sentToD2KeysBySource.get(sourceId)?.clear() this.pendingOrderedLoadPromise = undefined }) this.unsubscribeCallbacks.add(truncateUnsubscribe) @@ -753,6 +755,10 @@ class EffectPipelineRunner { if (this.starting) throw error return } + if (update.releaseFailure) { + this.onSourceError(normaliseError(update.releaseFailure.error)) + return + } if (update.ready instanceof Promise) { // Each segment reports its own failure through the subscription. Consume // the aggregate rejection so Promise.all does not create a second, @@ -831,11 +837,10 @@ class EffectPipelineRunner { const input = this.inputs[sourceId] if (!input) return 0 - // Filter duplicates per lexical source - const sentKeys = this.sentToD2KeysBySource.get(sourceId)! - const filtered = filterDuplicateInserts(changes, sentKeys) + const sentRows = this.sentToD2RowsBySource.get(sourceId)! + const reconciled = reconcileChangesForD2(changes, sentRows) - return sendChangesToInput(input, filtered) + return sendChangesToInput(input, reconciled) } /** @@ -947,8 +952,8 @@ class EffectPipelineRunner { /** * Request the initial ordered snapshot for an alias. - * Uses requestLimitedSnapshot (index-based cursor) or requestSnapshot - * (full load with limit) depending on whether an index is available. + * Uses requestLimitedSnapshot (index-based cursor) or an unbounded + * requestSnapshot depending on whether an index is available. */ private requestInitialOrderedSnapshot( alias: string, @@ -958,19 +963,21 @@ class EffectPipelineRunner { const { orderBy, offset, limit, index } = orderByInfo const normalizedOrderBy = normalizeOrderByPaths(orderBy, alias) + if (limit === 0) return + if (index) { - subscription.setOrderByIndex(index) + subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ limit: offset + limit, orderBy: normalizedOrderBy, trackLoadSubsetPromise: false, - onLoadSubsetResult: (result) => - this.trackOrderedLoad(result, orderByInfo.sourceId), + onLoadSubsetResult: (result) => this.trackOrderedLoad(result), }) } else { + // Without an index there is no sound cursor continuation. Load the full + // ordered source so later relational operators cannot underfill top-K. subscription.requestSnapshot({ orderBy: normalizedOrderBy, - limit: offset + limit, trackLoadSubsetPromise: false, }) } @@ -996,13 +1003,17 @@ class EffectPipelineRunner { this.optimizableOrderByCollections, )) { if (!orderByInfo.dataNeeded || !orderByInfo.index) continue - + if (orderByInfo.limit === 0) continue const subscription = this.subscriptions[orderByInfo.sourceId] if (!subscription) continue subscription.ensureOrderedWindowSize( orderByInfo.offset + orderByInfo.limit, ) - if (subscription.hasOrderedCoverageForActiveWindow) { + const missingResultRows = orderByInfo.dataNeeded() + if ( + (!orderByInfo.refillFromResultDeficit || missingResultRows === 0) && + subscription.hasOrderedResultForActiveWindow + ) { continue } @@ -1011,23 +1022,23 @@ class EffectPipelineRunner { continue } - const n = Math.max( - orderByInfo.dataNeeded(), - subscription.orderedRowsNeeded, - ) + if (orderByInfo.refillFromResultDeficit && missingResultRows > 0) { + subscription.ensureOrderedWindowSize( + subscription.orderedRetainedWindowSize + missingResultRows, + ) + } + if (subscription.hasOrderedResultForActiveWindow) { + continue + } + + const n = Math.max(missingResultRows, subscription.orderedRowsNeeded) this.loadNextItems(orderByInfo, Math.max(1, n)) } } - private trackOrderedLoad( - result: LoadSubsetRequestResult, - sourceId: string, - ): void { + private trackOrderedLoad(result: LoadSubsetRequestResult): void { const continueAfterFulfillment = () => { if (this.disposed) return - if (this.subscriptions[sourceId]?.requiresOrderedPrefixRefresh) { - this.lastLoadRequestKey.delete(sourceId) - } this.loadMoreIfNeeded() } if (!(result instanceof Promise)) { @@ -1070,8 +1081,12 @@ class EffectPipelineRunner { n, subscription.orderedRetainedWindowSize, subscription.orderedBoundaryKey, + subscription.orderedCoverageRevision, ) - if (!cursor) return // Duplicate request — skip + if (!cursor) { + subscription.settleOrderedResultAfterNoProgress() + return + } this.lastLoadRequestKey.set(sourceId, cursor.loadRequestKey) @@ -1083,7 +1098,7 @@ class EffectPipelineRunner { minValues: cursor.minValues, trackLoadSubsetPromise: false, onLoadSubsetResult: (loadResult: LoadSubsetRequestResult) => - this.trackOrderedLoad(loadResult, sourceId), + this.trackOrderedLoad(loadResult), }) } catch (error) { if ( @@ -1110,11 +1125,11 @@ class EffectPipelineRunner { changes: Array>, comparator: (a: any, b: any) => number, ): void { - const sentKeys = this.sentToD2KeysBySource.get(sourceId) ?? new Set() + const sentRows = this.sentToD2RowsBySource.get(sourceId) ?? new Map() const result = trackBiggestSentValue( changes, this.biggestSentValue.get(sourceId), - sentKeys, + sentRows, comparator, ) this.biggestSentValue.set(sourceId, result.biggest) @@ -1125,21 +1140,21 @@ class EffectPipelineRunner { /** Tear down subscriptions and clear state */ dispose(): void { - if (this.disposed) return + if (this.disposed && this.unsubscribeCallbacks.size === 0) return this.disposed = true this.subscribedToAllCollections = false // Immediately unsubscribe from every source, even if one release fails. - let firstCleanupError: unknown + let firstCleanupFailure: { error: unknown } | undefined for (const unsubscribe of this.unsubscribeCallbacks) { try { unsubscribe() + this.unsubscribeCallbacks.delete(unsubscribe) } catch (error) { - firstCleanupError ??= error + firstCleanupFailure ??= { error } } } - this.unsubscribeCallbacks.clear() - this.sentToD2KeysBySource.clear() + this.sentToD2RowsBySource.clear() this.pendingChanges.clear() this.lazySources.clear() this.demand.clear() @@ -1167,7 +1182,7 @@ class EffectPipelineRunner { this.finalCleanup() } - if (firstCleanupError !== undefined) throw firstCleanupError + if (firstCleanupFailure) throw firstCleanupFailure.error } /** Clear graph references — called after graph run completes or immediately from dispose */ diff --git a/packages/db/src/query/expression-value-context.ts b/packages/db/src/query/expression-value-context.ts new file mode 100644 index 0000000000..29389268ef --- /dev/null +++ b/packages/db/src/query/expression-value-context.ts @@ -0,0 +1,253 @@ +export type ExpressionValueContext = + | `exact-output` + | `equality-operand` + | `membership-candidates` + | `ordering-operand` + | `structural-operand` + +/** Describe how each function argument contributes to its observable result. */ +export function getExpressionArgumentValueContext( + name: string, + index: number, + argumentCount: number, + resultContext: ExpressionValueContext, +): ExpressionValueContext { + if (name === `eq`) return `equality-operand` + if (name === `in`) { + return index === 0 ? `equality-operand` : `membership-candidates` + } + if (isOrderingFunction(name)) return `ordering-operand` + + if ( + name === `concat` || + name === `length` || + name === `add` || + name === `subtract` || + name === `multiply` || + name === `divide` || + name === `date` || + name === `datetime` || + name === `strftime` + ) { + return `structural-operand` + } + + if (name === `coalesce` || name === `upper` || name === `lower`) { + return resultContext + } + + if (name === `caseWhen`) { + const isDefault = argumentCount % 2 === 1 && index === argumentCount - 1 + return isDefault || index % 2 === 1 ? resultContext : `exact-output` + } + + return `exact-output` +} + +/** Reject values whose observable scalar behavior cannot be cloned exactly. */ +export function assertSnapshotCapableStructuralValue( + value: unknown, + path = `value`, +): void { + visitStructuralValue(value, path, new WeakSet(), new WeakSet()) +} + +/** + * Read an IN candidate array without invoking caller-defined iteration or + * accessors. The plain result gives later identity and adapter paths one stable + * observation of the request. + */ +export function snapshotMembershipCandidateValues( + value: unknown, + path = `value`, +): Array | undefined { + if (!Array.isArray(value)) return undefined + if (Object.getPrototypeOf(value) !== Array.prototype) { + throwUnsupportedMembership(path, `array subclasses are unsupported`) + } + + const lengthDescriptor = Object.getOwnPropertyDescriptor(value, `length`) + const length = lengthDescriptor?.value + if (typeof length !== `number` || !Number.isInteger(length) || length < 0) { + throwUnsupportedMembership(path, `invalid array length`) + } + + const snapshot = new Array(length) + for (let index = 0; index < length; index++) snapshot[index] = undefined + + for (const key of Reflect.ownKeys(value)) { + if (key === `length`) continue + if (typeof key !== `string` || !isArrayIndex(key)) { + throwUnsupportedMembership(path, `custom properties are unsupported`) + } + + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!descriptor.enumerable || !(`value` in descriptor)) { + throwUnsupportedMembership( + `${path}.${key}`, + `non-enumerable indexed properties and accessors are unsupported`, + ) + } + snapshot[Number(key)] = descriptor.value + } + + return snapshot +} + +function visitStructuralValue( + value: unknown, + path: string, + active: WeakSet, + complete: WeakSet, +): void { + if (typeof value === `function`) { + throwUnsupported(path, `functions may expose mutable coercion hooks`) + } + if (typeof value !== `object` || value === null) return + if (complete.has(value)) return + if (active.has(value)) throwUnsupported(path, `cyclic values are unsupported`) + active.add(value) + + if (value instanceof Date) { + assertPrototype(value, Date.prototype, path) + assertNoOwnProperties(value, path) + } else if (value instanceof ArrayBuffer) { + assertPrototype(value, ArrayBuffer.prototype, path) + assertNoOwnProperties(value, path) + } else if (ArrayBuffer.isView(value)) { + assertSupportedArrayBufferViewPrototype(value, path) + assertOnlyIndexedProperties(value, path, false) + } else if (Array.isArray(value)) { + assertPrototype(value, Array.prototype, path) + assertOnlyIndexedProperties(value, path, true) + value.forEach((entry, index) => + visitStructuralValue(entry, `${path}[${index}]`, active, complete), + ) + } else if (value instanceof Map) { + assertPrototype(value, Map.prototype, path) + assertNoOwnProperties(value, path) + let index = 0 + for (const [key, entryValue] of value) { + visitStructuralValue(key, `${path}.key[${index}]`, active, complete) + visitStructuralValue( + entryValue, + `${path}.value[${index}]`, + active, + complete, + ) + index++ + } + } else if (value instanceof Set) { + assertPrototype(value, Set.prototype, path) + assertNoOwnProperties(value, path) + let index = 0 + for (const entry of value) { + visitStructuralValue(entry, `${path}[${index}]`, active, complete) + index++ + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throwUnsupported(path, `opaque object prototypes are unsupported`) + } + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== `string`) { + throwUnsupported(path, `symbol properties are unsupported`) + } + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!descriptor.enumerable || !(`value` in descriptor)) { + throwUnsupported( + `${path}.${key}`, + `non-enumerable properties and accessors are unsupported`, + ) + } + visitStructuralValue(descriptor.value, `${path}.${key}`, active, complete) + } + } + + active.delete(value) + complete.add(value) +} + +function assertPrototype(value: object, expected: object, path: string): void { + if (Object.getPrototypeOf(value) !== expected) { + throwUnsupported(path, `built-in subclasses are unsupported`) + } +} + +function assertSupportedArrayBufferViewPrototype( + value: ArrayBufferView, + path: string, +): void { + const prototype = Object.getPrototypeOf(value) + const supported = [ + DataView.prototype, + Int8Array.prototype, + Uint8Array.prototype, + Uint8ClampedArray.prototype, + Int16Array.prototype, + Uint16Array.prototype, + Int32Array.prototype, + Uint32Array.prototype, + Float32Array.prototype, + Float64Array.prototype, + BigInt64Array.prototype, + BigUint64Array.prototype, + ...(typeof Buffer === `undefined` ? [] : [Buffer.prototype]), + ] + if (!supported.includes(prototype)) { + throwUnsupported(path, `built-in subclasses are unsupported`) + } +} + +function assertNoOwnProperties(value: object, path: string): void { + if (Reflect.ownKeys(value).length > 0) { + throwUnsupported(path, `custom properties are unsupported`) + } +} + +function assertOnlyIndexedProperties( + value: object, + path: string, + allowLength: boolean, +): void { + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== `string`) { + throwUnsupported(path, `custom properties are unsupported`) + } + + const isLength = allowLength && key === `length` + if (!isLength && !isArrayIndex(key)) { + throwUnsupported(path, `custom properties are unsupported`) + } + + const descriptor = Object.getOwnPropertyDescriptor(value, key)! + if (!(`value` in descriptor) || (!isLength && !descriptor.enumerable)) { + throwUnsupported( + `${path}.${key}`, + `non-enumerable indexed properties and accessors are unsupported`, + ) + } + } +} + +function isArrayIndex(key: string): boolean { + const index = Number(key) + return Number.isInteger(index) && index >= 0 && String(index) === key +} + +function throwUnsupported(path: string, reason: string): never { + throw new TypeError( + `Cannot snapshot structural expression value at ${path}: ${reason}`, + ) +} + +function throwUnsupportedMembership(path: string, reason: string): never { + throw new TypeError( + `Cannot snapshot membership candidates at ${path}: ${reason}`, + ) +} + +function isOrderingFunction(name: string): boolean { + return name === `gt` || name === `gte` || name === `lt` || name === `lte` +} diff --git a/packages/db/src/query/ir-stable-identity.ts b/packages/db/src/query/ir-stable-identity.ts index 425e35d5ab..008beff0c0 100644 --- a/packages/db/src/query/ir-stable-identity.ts +++ b/packages/db/src/query/ir-stable-identity.ts @@ -1,7 +1,19 @@ -import { normalizeValue } from '../utils/comparison.js' +import { + isUint8ArrayCandidate, + normalizeValue, + snapshotTemporalEqualityValue, + snapshotUint8ArrayBytes, +} from '../utils/comparison.js' +import { isTemporal } from '../utils.js' import { isRefProxy, toExpression } from './builder/ref-proxy.js' import { getQueryIR } from './builder/get-query-ir.js' +import { + assertSnapshotCapableStructuralValue, + getExpressionArgumentValueContext, + snapshotMembershipCandidateValues, +} from './expression-value-context.js' import { getRuntimeReferenceIdentity } from './runtime-reference-identity.js' +import type { ExpressionValueContext } from './expression-value-context.js' import type { Aggregate, BasicExpression, @@ -26,11 +38,6 @@ type StableIdentityValue = | Array | { [key: string]: StableIdentityValue } -type ValueIdentityContext = - | `exact-output` - | `equality-operand` - | `ordering-operand` - type OpaqueValueIdentity = `reject` | `runtime-reference` type AliasScope = { @@ -568,7 +575,7 @@ function canonicalizeOrderBy( orderBy: OrderByClause, path: string, seen: WeakSet, - valueContext: ValueIdentityContext = `exact-output`, + valueContext: ExpressionValueContext = `exact-output`, scope?: AliasScope, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { @@ -597,7 +604,7 @@ function canonicalizeExpression( | ConditionalSelect, path: string, seen: WeakSet, - valueContext: ValueIdentityContext = `exact-output`, + valueContext: ExpressionValueContext = `exact-output`, scope?: AliasScope, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { @@ -637,71 +644,49 @@ function canonicalizeExpression( scope, opaqueValueIdentity, ) - : valueContext === `ordering-operand` - ? canonicalizeOrderingRuntimeValue( + : valueContext === `membership-candidates` + ? canonicalizeMembershipCandidates( expression.value, `${path}.value`, seen, + scope, opaqueValueIdentity, ) - : canonicalizeExactOutputRuntimeValue( - expression.value, - `${path}.value`, - seen, - opaqueValueIdentity, - ), + : valueContext === `ordering-operand` + ? canonicalizeOrderingRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ) + : valueContext === `structural-operand` + ? canonicalizeStructuralRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ) + : canonicalizeExactOutputRuntimeValue( + expression.value, + `${path}.value`, + seen, + opaqueValueIdentity, + ), } } if (expression.type === `func`) { - if ( - expression.name === `in` && - expression.args.length === 2 && - expression.args[1]?.type === `val` && - Array.isArray(expression.args[1].value) - ) { - const candidates = expression.args[1].value.map((value, index) => - canonicalizeEqualityRuntimeValue( - value, - `${path}.args[1].value[${index}]`, - seen, - scope, - opaqueValueIdentity, - ), - ) - return canonicalizeFunction(expression.name, [ - canonicalizeExpression( - expression.args[0]!, - `${path}.args[0]`, - seen, - `equality-operand`, - scope, - opaqueValueIdentity, - ), - { - type: `val`, - // IN tests membership. Candidate order and duplicates do not change - // its result, but each candidate keeps its own equality semantics. - value: [`set`, sortUniqueStableIdentityValues(candidates)], - }, - ]) - } - - const operandContext: ValueIdentityContext = - expression.name === `eq` - ? `equality-operand` - : expression.name === `gt` || - expression.name === `gte` || - expression.name === `lt` || - expression.name === `lte` - ? `ordering-operand` - : `exact-output` const args = expression.args.map((arg, index) => canonicalizeExpression( arg, `${path}.args[${index}]`, seen, - operandContext, + getExpressionArgumentValueContext( + expression.name, + index, + expression.args.length, + valueContext, + ), scope, opaqueValueIdentity, ), @@ -1061,11 +1046,13 @@ function canonicalizeEqualityRuntimeValue( // Equality compares Uint8Array and Buffer values by content, independent of // their concrete constructor and size. - const isUint8Array = - (typeof Buffer !== `undefined` && value instanceof Buffer) || - value instanceof Uint8Array - if (isUint8Array) { - return [`binary`, `Uint8Array`, Array.from(value as Uint8Array)] + if (isUint8ArrayCandidate(value)) { + return [`binary`, `Uint8Array`, Array.from(snapshotUint8ArrayBytes(value))] + } + + if (isTemporal(value)) { + const snapshot = snapshotTemporalEqualityValue(value) + return canonicalizeRuntimeValue(normalizeValue(snapshot), path, seen) } const normalized = normalizeValue(value) @@ -1080,12 +1067,121 @@ function canonicalizeEqualityRuntimeValue( return canonicalizeRuntimeValue(value, path, seen) } +function canonicalizeMembershipCandidates( + value: unknown, + path: string, + seen: WeakSet, + scope?: AliasScope, + opaqueValueIdentity: OpaqueValueIdentity = `reject`, +): StableIdentityValue { + if (!Array.isArray(value)) { + return canonicalizeExactOutputRuntimeValue( + value, + path, + seen, + opaqueValueIdentity, + ) + } + + const candidateValues = snapshotMembershipCandidateValues(value, path)! + const candidates = candidateValues.map((candidate, index) => + canonicalizeEqualityRuntimeValue( + candidate, + `${path}[${index}]`, + seen, + scope, + opaqueValueIdentity, + ), + ) + // IN tests membership. Candidate order and duplicates do not change its + // result, but each candidate keeps its own equality semantics. + return [`set`, sortUniqueStableIdentityValues(candidates)] +} + +function canonicalizeStructuralRuntimeValue( + value: unknown, + path: string, + seen: WeakSet, + _opaqueValueIdentity: OpaqueValueIdentity = `reject`, +): StableIdentityValue { + assertSnapshotCapableStructuralValue(value, path) + return canonicalizeSnapshotStructuralValue(value, path, seen) +} + +function canonicalizeSnapshotStructuralValue( + value: unknown, + path: string, + seen: WeakSet, +): StableIdentityValue { + if (typeof value === `symbol`) { + return getRuntimeReferenceIdentity(value) + } + + if (value instanceof Date) { + return Number.isNaN(value.getTime()) + ? [`Date`, `Invalid`] + : canonicalizeRuntimeValue(value, path, seen) + } + + if (Array.isArray(value)) { + return withCircularGuard(value, path, seen, () => [ + `snapshotArray`, + value.length, + Object.keys(value).map((key) => [ + key, + canonicalizeSnapshotStructuralValue( + value[Number(key)], + `${path}[${key}]`, + seen, + ), + ]), + ]) + } + + if (value instanceof Map) { + return withCircularGuard(value, path, seen, () => [ + `snapshotMap`, + Array.from(value.entries(), ([key, entryValue], index) => [ + canonicalizeSnapshotStructuralValue(key, `${path}.key[${index}]`, seen), + canonicalizeSnapshotStructuralValue( + entryValue, + `${path}.value[${index}]`, + seen, + ), + ]), + ]) + } + + if (value instanceof Set) { + return withCircularGuard(value, path, seen, () => [ + `snapshotSet`, + Array.from(value, (entry, index) => + canonicalizeSnapshotStructuralValue(entry, `${path}[${index}]`, seen), + ), + ]) + } + + if (isPlainObject(value)) { + return withCircularGuard(value, path, seen, () => [ + `snapshotObject`, + Object.getPrototypeOf(value) === null ? `null` : `plain`, + Object.keys(value).map((key) => [ + key, + canonicalizeSnapshotStructuralValue(value[key], `${path}.${key}`, seen), + ]), + ]) + } + + return canonicalizeRuntimeValue(value, path, seen) +} + function canonicalizeOrderingRuntimeValue( value: unknown, path: string, seen: WeakSet, opaqueValueIdentity: OpaqueValueIdentity = `reject`, ): StableIdentityValue { + assertSnapshotCapableStructuralValue(value, path) if ( opaqueValueIdentity === `runtime-reference` && (typeof value === `function` || typeof value === `symbol`) @@ -1101,7 +1197,7 @@ function canonicalizeOrderingRuntimeValue( } const normalized = normalizeValue(value) - if (normalized !== value && !(value instanceof Uint8Array)) { + if (normalized !== value && !isUint8ArrayCandidate(value)) { return canonicalizeRuntimeValue(normalized, path, seen) } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 401bfe341d..e4ce9dfb35 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -217,6 +217,21 @@ inside D2 do not need their own lifecycle objects or generations. D2 multisets are the source of truth. A row with positive weight contributes; a row with negative weight retracts the same contribution. +Each lexical source boundary retains the exact row last contributed for every +source key. A replayed insert for an existing key adds no second contribution. +An update or delete retracts the retained row, rather than trusting event +metadata that may describe a newer value, then replaces or removes that entry. +An update for an unknown key becomes an insert, while a delete for an unknown +key contributes nothing. Truncate keeps this boundary state until its later +source batch retracts or replaces the retained rows. A replacement's exact +retraction and new contribution enter the same graph turn, so the public +boundary observes one update rather than an intermediate removal. Graph +teardown clears the tracker and graph together. Thus every source key has +multiplicity zero or one and every negative weight cancels the exact positive +row that entered D2. The source Collection owns its rows independently: graph +teardown does not erase them, source changes continue while the graph is down, +and a new graph replays the source's then-current rows into a fresh tracker. + Internal contribution identity is independent of the user-visible Collection key. When several internal rows collapse to one public key, a keyed D2 reduction retains all contributors and derives at most one canonical row: @@ -265,6 +280,14 @@ row. A negative aggregate is an invariant violation. This is a specialized use of the existing D2 keyed reduction. It is not a separate contribution-ledger subsystem. +Collection-valued facades keep the canonical public key and order token as +non-enumerable row metadata. Optimistic Collection updates preserve that +metadata when they clone a row. It is adapter state, not part of the selected +query value: projections need not expose a key field, and equality or change +payloads must not acquire one. An order update that reuses the canonical row +object clones that row before replacing its order metadata, so the ordered map +can remove the old position before it installs the new one. + ## Routes and buckets are relations For each materialization edge, the compiler produces these keyed relations: @@ -388,6 +411,44 @@ prefix is only a candidate prefix. Core expands the complete source-order boundary class, then applies the public-key tie-break locally. Locale and reference orders that the predicate IR cannot express fetch the full filtered region and refine it locally. +That fallback issues no structural cursor: a lexical predicate is not a locale +boundary. Once the unbounded result proves the needed prefix, widening within +that result performs no more transport work. Bounded locale continuation would +require a future adapter capability with an opaque cursor that preserves the +provider's exact collation and snapshot. Until that contract exists, an +unbounded fetch is the only sound continuation. + +The same rule applies when no range index can support ordered continuation. +Core issues one unbounded ordered acquisition, then lets D2 apply joins, +predicates, and top-K to the full readable source. It must not issue a limited +page and then disable continuation: later relational operators may reject that +page and leave the result window short. + +For an indexed ordered source above a join or later predicate, the visible +window is the direct relational result: source order, then downstream +operators, then offset and top-K. Core may prove that result with the shortest +ordered source prefix, or with other authoritative active demands that +establish all contributors which can precede the boundary. A forward scan +advances its cursor across source rows that the later relation rejects. A +reverse join demand can instead make a later matching row readable without +claiming reusable ordered-prefix coverage for skipped rows. A second source may +settle after a continuation is already in flight, so safe extra primary rows +may become readable. None of these paths may change the direct result or let an +unsettled source region leave a provable window under-filled. A short result is +valid only when authoritative completeness across every involved source region +proves that no remaining row can contribute before the window boundary. + +Test adapters must obey the same boundary contract as production adapters. A +mock that reports exhaustion must have made every matching source row readable +before its result settles. A mock that reports more data must honor later +offset, cursor, and boundary-class refinement requests. Every applied row key +must name a row established by that acquisition. Tests that withhold rows while +claiming exhaustion, or ignore a refinement request, do not model a valid +adapter and cannot establish a runtime defect. +An acquisition may establish a row that is already readable by applying the +same authoritative value again under its own request signal and awaiting that +receipt. Merely observing a row installed by another demand does not transfer +ownership or make it an applied row of the new acquisition. Every continuation boundary comes from rows established by the same ordered demand. Rows retained for another query, join, or window cannot move it. During @@ -651,6 +712,46 @@ start another request only when that prefix grows or that boundary moves. If a continuing page establishes neither fact, core leaves the window uncovered, does not repeat the same request, and records a nonfatal no-progress diagnostic in `lastSubsetError`. +Adapter entry is itself pending work. A request-scoped commit can publish rows +before an async `loadSubset` call returns its Promise, so graph callbacks during +that entry cannot start another ordered request. Once entry returns, the normal +in-flight Promise guard owns the request until settlement. + +An ordered window with an active limit of zero creates no ordered transport +demand. The subscription freezes the order request so a later window change +can load from the same order, but it does not construct `WindowState` until the +window first becomes positive. Neither the initial offset nor a result deficit +may turn the empty window into a positive request. The zero-width path returns +before predicate or order compilation, source enumeration, sorting, or index +creation. The dedupe helper applies the same law when adapters call it +directly: a zero-width request establishes no coverage and owns no physical +acquisition. This also holds when no usable order index exists: core defers the +full-snapshot fallback until the window first becomes positive. One successful +or pending fallback covers that subscription session. A synchronous throw or +rejected fallback clears only that subscription's guard, so the same live query +can retry. Cleanup creates a new subscription and a late settlement from the +old one cannot clear the new guard. Truncate replay belongs to the +subscription's retained demand; the live coordinator must not add a second +fallback while that replay is in flight. A replay that starts after an earlier +rejection reclaims the same subscription +guard. Success keeps it claimed, so replacement publication cannot schedule a +duplicate full-source fallback; rejection releases it for a later retry. +Cleanup also aborts and settles the subscription-visible acquisition before a +raw adapter promise can affect a replacement session. The live coordinator's +subscription-identity check is a second fence, not a substitute for that lower +abort boundary. Session tests must prove the public loading, readiness, error, +and row history rather than depend on reaching either private fence alone. + +The graph loader is part of the same quiescence pass as source processing. If a +window change reaches the pass with no graph work, core calls the loader first. +It then drains every graph step created by a synchronous adapter commit before +publishing. Async settlement schedules another pass under the same rule. A +successful retry therefore cannot commit source rows while leaving the live +result stale until an unrelated later window change. When one pass has several +load callbacks, it attempts all of them and then rethrows the first failure +unchanged, including falsy values such as `undefined`, `false`, `0`, or `NaN`. +This rule applies both to lexical source loaders nested in one graph callback +and to graph callbacks coalesced by the scheduler. Live Collections and Effects keep separate consumer-local continuation state, but obey the same identity and reset law. A settled request remains the @@ -671,11 +772,14 @@ continuation after all replacement acquisitions have settled. An outcome-free completion (`true` or `Promise`) supplies no reusable row provenance, source extent, or CoverageFact. Its exact request has still settled, -so the owning subscription may admit only the current local prefix. A short -page remains uncovered and triggers another pass. The admitted local boundary -may distinguish those immediate passes, but it is scheduling state, not a -transport cursor. If the window later grows, core refreshes the required prefix -from the start instead of continuing from those rows as a cursor boundary. +so the owning subscription may admit and publish only the current local prefix, +even when that prefix is short. Core keeps loading while the local boundary +advances, then treats a repeated request with no new progress as caller-locally +satisfied. This stops work only for that exact active window. It is not +coverage: it cannot establish source extent, satisfy a replacement epoch, or +become a transport cursor. If the window later grows, core refreshes the +required prefix from the start. An explicit continuing outcome is not +outcome-free and cannot use this fallback. A bare child query is a Collection-valued include. It exposes one stable public Collection facade per active bucket in that edge: @@ -731,23 +835,76 @@ type DemandSet = readonly [ ] ``` +### Demand facts + +The demand plane keeps these facts separate. One fact may justify creating the +next, but none is an alias for another. + +| Fact | Meaning | What it does not prove | +| -------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Demand snapshot | Immutable semantic work requested by one caller | That any source work started or any row arrived | +| Logical lease | One active owner of that demand | That it owns a distinct physical request | +| Physical acquisition | One exact adapter attempt, signal, options snapshot, and settlement | That its requested region was applied or is reusable | +| Applied outcome | Extent and row keys established by that acquisition after its writes became visible | Coverage for a different demand or generation | +| Coverage fact | Caller-relative proof that an applied outcome satisfies a demand | Row lifetime or consumer publication | +| Row ownership | Acquisition support for applied row keys | Ordered-prefix membership or public visibility | +| Publication snapshot | The last complete reader-visible rows and ordered boundary | Current private progress, request extent, or source ownership | + +Consumer-local scheduling, loading state, and error state are observations over +these facts. They are not extra coverage or ownership facts. + One request may cover many buckets, and the adapter may coalesce or reuse requests according to the compiled demand plan. A coalesced request has one shared abort lease. If one owner releases its lease, the source request remains active while another owner still needs its coverage. The source signal aborts only after every attached owner has released it. -A Collection subscription installs each logical subset owner before it calls -the source adapter. Reentrant release during `loadSubset` must therefore see and -release that exact acquisition. It also registers the caller's original -predicate before adapter entry, because the transport predicate may combine it -with the subscription predicate. After adapter return, both ordered and -unordered requests recheck logical ownership before they report results, track -loading state, establish coverage, or scan local state; a demand released -during adapter code cannot publish a later snapshot. A synchronous `loadSubset` -throw that did not follow a failed release rolls the tentative owner back before -it emits the error and without calling `unloadSubset`; a failed release keeps -the owner so a later cleanup can retry the same acquisition identity. +A Collection subscription snapshots its predicate when it is constructed. Its +first ordered request also snapshots the total order. Later window requests may +change the requested size, but local reconciliation, boundaries, transport, +replay, and evidence all keep the same predicate and order for that +subscription. + +Each logical subset demand then gets a private snapshot before adapter entry. +Each adapter acquisition gets a separate clone derived from it, so neither +caller nor adapter mutation can rewrite the subscription machine, logical +demand, or another acquisition. Values observed by scalar functions use a +closed snapshot-capable grammar. Its identity preserves every observable part +of the clone, including prototype kind, property order, sparse-array holes, +invalid Dates, and symbol identity. Unsupported coercion hooks, opaque +structural objects, built-in subclasses, accessors, and cycles fail before the +demand is retained. Opaque values used by reference-sensitive equality retain +their identity. The caller's original predicate is retained only as a release +handle, because the transport predicate may combine it with the subscription +predicate. + +The subscription installs the logical owner before adapter entry. Reentrant +release during `loadSubset` must therefore see and release that exact +acquisition. After adapter return, both ordered and unordered requests recheck +logical ownership before they report results, track loading state, establish +coverage, or scan local state; a demand released during adapter code cannot +publish a later snapshot. A synchronous `loadSubset` throw that did not follow +a failed release rolls the tentative owner back before it emits the error and +without calling `unloadSubset`; a failed release keeps the owner so a later +cleanup can retry the same acquisition identity. Reconciliation reuses the +logical owner's evaluator across source changes and truncate acquisition +replacement. A released owner cannot supply a predicate, and a later logical +demand compiles its own evaluator even when it reuses the same expression +object. + +Logical demand state advances even when physical release fails. The failed +acquisition remains retryable cleanup debt in the Collection subscription, but +an aborted segment cannot remain the current demand or suppress a later +incarnation. The demand controller therefore returns the release failure with +the completed logical transition. A live query records that failure and +retires an empty demand without entering a fatal query state; an Effect reports +the same failure through its source-error policy and disposes. Reactivating the +route starts a fresh acquisition. Live-query diagnostics track failure presence +separately from its value so a thrown `undefined` remains observable, and reset +both observations when a new sync session starts. Effect disposal retains +failed unsubscribe callbacks and lets a later `dispose()` retry them instead of +caching a terminal rejected cleanup attempt. + Result callbacks are also arbitrary reentrancy boundaries. After invoking one, the request checks the same exact owner again before it tracks status, applies coverage, or scans local rows. A callback may release or unsubscribe; obsolete @@ -771,14 +928,16 @@ options. Its semantic contract is: -> Every active, satisfiable bucket must be covered by a settled current demand -> request before initial preload completes. +> Every active, satisfiable bucket must have its current demand load settle +> before initial preload completes. -A request may remain in flight after some covered buckets become inactive. +A request may remain in flight after some buckets it targeted become inactive. Those buckets no longer participate in readiness and cannot receive rows through routes that no longer exist. Sharing source work never merges the route rows themselves. +### Adapter obligations + The source contract stays abstract: a demand request eventually establishes one coherent baseline and identifies when that baseline is complete. Each request receives an `AbortSignal`. Cancellation is cooperative at this source @@ -789,6 +948,17 @@ adapter from writing after it ignores that signal. Buffering, snapshot tokens, shape offsets, Collection transactions, and local indexes are source-specific ways to satisfy that contract; they are not materializer state. +A conforming adapter must: + +- treat the received options snapshot and signal as one exact acquisition; +- honor cancellation immediately before publishing request-scoped rows; +- await or return every applied receipt which establishes its result; +- report only row keys established by that acquisition and report source extent + only when it knows it authoritatively; +- make every supplied release callback idempotent and non-throwing, and return + the paired `unloadSubset` callback when it keeps dedupe state across + lifetimes. + Every sync `commit()` returns an applied receipt: `true` when that transaction's writes and events are already visible, or a promise when the transaction is parked in the causal queue. The promise resolves only after the @@ -799,11 +969,43 @@ events are emitted, so an abort raised by a publication observer is already late. A successful `loadSubset` implementation must await or return every receipt for the transactions that establish its result. A source must not add priority merely to make a subset load settle. +A rollback that wins while an optimistic transaction's `mutationFn` is still +in flight is terminal. A later resolve or rejection from that function cannot +change its outcome, run rollback again, affect newer transactions, or republish +its overlay. Repeating rollback on that failed transaction is inert and cannot +cascade into transactions created after the first rollback. Existing immediate bootstrap and persistence-hydration paths, plus truncate, retain their queue-bypass contract; if one applies a parked subset transaction as part of that prefix, the subset receipt settles only after the writes are -visible. Rejected, canceled, and obsolete acquisitions establish no coverage. -Sources must honor cancellation before publishing request-scoped rows. +visible. A quiescent live-query graph output uses the same queue bypass for the +root Collection and regular Collection-valued child-facade changes. A facade +retirement is committed earlier in the same FIFO causal prefix; the immediate +root or containing-facade transaction that removed its final route drains that +retirement too. A direct source change therefore updates the whole derived +publication while an optimistic mutation on either Collection persists. The +normal optimistic overlay still wins for conflicting keys, and the graph +output remains one coherent publication. The overlay replaces the row value, +not the graph-owned key order. A source order move publishes a layout change +only when the complete visible public-key sequence changes after applying the +optimistic overlay. This includes a move beneath an optimistic update. It does +not include a move whose only crossed peers are optimistically deleted. An +optimistic delete hides the key and its order moves. If the synced base is +deleted beneath an optimistic update, the still-visible row moves to the +optimistic-only suffix; that publishes only when the suffix transition changes +the visible sequence. Re-establishing the base applies the inverse rule. A +legal absent-to-present source transition must keep the optimistic value +visible while restoring the graph-owned position in the same publication. +The graph reports a possible layout change before its sync commit. Collection +state captures the visible key sequence immediately before the whole committed +causal prefix applies, then compares it with the final sequence after the sync +writes and active optimistic overlay have both been applied. Queued +transaction-local snapshots are not public boundaries: an overlay may change +before a later immediate transaction drains them. The layout revision advances +only when the exact before/after sequences differ. This final check is shared +by root Collections and child facades; an adapter-local order token is evidence +to check layout, not proof that public layout changed. +Rejected, canceled, and obsolete acquisitions establish no coverage. Sources +must honor cancellation before publishing request-scoped rows. After those writes are applied, `loadSubset` may resolve with `{ hasMore: boolean | undefined, appliedRowKeys?: readonly Key[] }`. Core @@ -874,7 +1076,13 @@ Collection sync boundary; shared rows remain until their final owner retires. An adapter that uses `DeduplicatedLoadSubset` across live-query lifetimes must also return the helper's paired `unloadSubset` callback. That callback invalidates remembered request coverage when core may delete its establishing -rows. A dedupe hit cannot outlive the evidence it claims to reuse. +rows. Core pairs each accepted load with one release of the same options object. +The helper keeps those logical owner reservations across resets so a late +release cannot retire newer-generation work or work still shared by another +owner. Adapter entry is a reentrancy boundary: capture the request generation +before calling adapter code, and do not publish coverage or in-flight work if a +reentrant reset has retired that generation. A dedupe hit cannot outlive the +evidence it claims to reuse. An eager Query DB collection owns its base query for the Collection lifetime. If TanStack Query removes that cache entry while the Collection has no public @@ -889,17 +1097,87 @@ waiting on the preload. Use an adapter's documented mutation acknowledgement helper instead; it must confirm the optimistic write without starting new collection demand. +### Conservative fallbacks + +When evidence is missing, core chooses less reuse or more source work instead +of guessing: + +- an omitted outcome or unknown extent proves no reusable coverage; +- requested limits and current Collection rows never stand in for applied row + evidence; +- an order that lacks an expressible total boundary, exact collation, or usable + range index loads the full filtered source region and lets D2 refine it; +- an unsupported demand value fails before retention instead of receiving a + lossy snapshot or identity; +- a throwing release keeps its lease, acquisition, coverage, and row ownership + as retryable cleanup debt; +- automatic continuation stops when it makes no semantic progress and resumes + only after demand or authoritative evidence changes. + +These fallbacks may cost work or delay reclamation. They must not change the +query result, invent coverage, or expose a private replacement publication. + This project uses a single graph-run order rather than multi-dimensional timely-dataflow frontiers. Do not introduce a general timestamp or frontier framework unless a source contract proves that the generation and up-to-date protocol cannot express its ordering. -**Initial readiness:** preload is complete when every demand currently -reachable from the initial query graph is covered by a settled request. Demand -that is no longer reachable does not block completion. An empty outer relation -has no child demand, but its root demand must still settle. Later readiness -transitions follow the existing Collection contract until an executable test -defines another public behavior. +**Initial readiness:** preload is complete when the current load for every +demand reachable from the initial query graph has settled. An outcome-free load +may settle readiness without proving reusable coverage. Demand that is no +longer reachable does not block completion. An empty outer relation has no +child demand, but its root demand must still settle. Later readiness transitions +follow the existing Collection contract until an executable test defines +another public behavior. + +The first-ready transition is an attempt-all fan-out. One callback failure +cannot suppress later first-ready callbacks, preload settlement, or the empty +ready event that wakes dependent Collections. Core completes every effect, then +rethrows the first failure unchanged, including a falsy value. Status is ready +before these effects run; first-ready callbacks keep registration order, and +the callback set is frozen before delivery. Removing a copied callback during +delivery cannot skip it. Since readiness is already public, a callback added +during delivery runs immediately at its registration point. The dependent-ready +event runs after the frozen first-ready batch. That event snapshots the dependents +present at delivery and attempts every one even if an earlier listener fails. +Removing or adding a dependent during delivery does not change that frozen +batch; an added dependent starts with the next publication. + +`markReady()` from `ready` is a no-op. Recovery from `error` clears the current +sync error and emits a dependent-ready event, but does not start a second +first-ready cycle. `idle` and `cleaned-up` cannot transition directly to +`ready`; sync must establish `loading` first. + +The `status:ready` event precedes the ready effects captured by that transition. +If one of its listeners synchronously performs another lifecycle transition, +that newer transition supersedes the current one, even if a restart returns the +Collection to `ready` before the listener returns. Core does not resume the +captured effects or emit a dependent-ready event for the superseded snapshot. +Cleanup may separately drain pending first-ready callbacks, including preload +waiters, so they settle; that cleanup-owned drain is not a ready publication. + +A ready-effect failure does not undo effects already attempted in that cycle. +After cleanup, the next sync is a new first-ready cycle with a fresh preload +promise. It runs only callbacks registered for that new cycle; completed +callbacks from the prior cycle are not replayed. + +Because the ready snapshot is already public, a listener failure also cannot +discard graph work queued by an earlier listener. Core flushes that work before +it rethrows the first listener failure. If readiness is nested inside an +existing publication, core retains the exact listener failure on that shared +context and the outer boundary rethrows it only after the queued graph work +finishes or the first graph failure stops that turn. When both the listener and +queued graph work fail, the first ready listener failure remains the reported +error and the scheduler clears the turn's remaining work. + +When `markReady()` runs during the synchronous adapter-entry call, core retains +any ready-effect failure until the adapter finishes its own setup. It then +propagates the exact failure without reclassifying it as a sync failure or +moving the Collection to `error`. A later asynchronous `markReady()` call keeps +the ordinary synchronous throw boundary. A preload already pending across this +entry waits for the adapter's final synchronous outcome: a ready-effect failure +alone leaves it resolved, while a later adapter failure rejects it and leaves +the Collection in `error`. Pending demand does not hide the parent row. An active empty bucket gives it the current canonical bucket value, and available partial source rows produce @@ -920,10 +1198,12 @@ For each scheduled graph turn: 1. enqueue all currently committed input deltas into their D2 inputs; 2. run D2 until it has no pending synchronous work; 3. consolidate the already canonical final-output deltas; -4. install child-facade state through normal Collection transactions while - deferring their subscriber delivery; -5. apply direct root insert, update, and delete writes through one normal - Collection transaction; +4. install regular child-facade changes through queue-bypassing Collection + transactions while deferring their subscriber delivery; put route + retirement before the queue-bypassing ancestor that drains its FIFO causal + prefix; +5. apply direct root insert, update, and delete writes through one + queue-bypassing Collection transaction; 6. release the deferred child-facade events after every synchronous read can see the complete root and facade state; 7. allow dependent live-query graphs to run through the existing @@ -938,8 +1218,100 @@ in-place include repair, and forced secondary events are forbidden. Installed state, synchronous reads, change-event payloads, and downstream queries must all observe the same fully materialized commit. The facade adapter -may defer event delivery across its Collection transactions, but it must not -defer state or index installation. Routing and identity remain inside D2. +may defer public revision clocks and event delivery across its Collection +transactions, but it must not defer state or index installation. A successful +coherent publication uses a two-phase release: first advance the clocks of the +root and every changed facade, then deliver any callback. This lets a callback +read another participating Collection without seeing new rows behind an old +revision. If a later root or containing-facade application fails before that +release, rollback restores the installed state and discards both the held +events and their revision advances. Routing and identity remain inside D2. +Applied receipts and asynchronous cleanup tails created by a participating +Collection join that same release. Rollback leaves restored receipts pending +and discards cleanup tails that could otherwise mutate the restored state. +The root and facade adapters retain the graph deltas consumed by that failed +attempt. A later graph turn retries the whole uncommitted relation even when +the source emits only an unrelated root delta; D2 does not replay a delta that +an adapter has already consumed. +Facade rollback restores the Collection's internal publication snapshot. It +must not use a public sync transaction or emit change, layout, readiness, or +truncate lifecycle events for state that never committed. +It also restores the mutable key-to-order map used to classify later layout +changes. Per-object public-key and order WeakMaps are monotonic metadata written +before installation; failed installs do not clear them, so they are retained +rather than copied into rollback state. +Fresh-facade readiness joins the prepared publication release only after every +facade and root install has succeeded, and it precedes root callbacks. A +recovery failure attempts every remaining restore and publication discard, +preserves the original graph-install error, and marks the affected root or +facade as errored so a later successful publication can recover it and restore +readiness. +An index-rebuild failure remains explicit recovery debt. The next graph turn +must attempt every root and facade restore as one recovery preflight, then +finish them all before applying retained deltas or marking any Collection +ready; an ordinary row update cannot repair an unknown partial index rebuild. +Error status is published only after every root and facade recovery attempt and +after each held publication has closed, so a synchronous status observer cannot +see avoidable stale sibling state from a skipped restore. +Once release begins, one subscriber callback failure cannot suppress another +prepared root or facade publication. Release attempts every participant, then +rethrows the first callback failure unchanged, including `null` or `undefined`. +If a callback cleans up another participant after preparation but before its +release, cleanup cancels that participant's held delivery. No callback may run +later against its cleaned-up state. +Nested deferral handles may join one open Collection publication cycle. Once +that cycle is prepared, no independent cycle may begin until it is published, +or canceled by cleanup. An open cycle may instead be discarded. The Collection +rejects prepared-cycle overlap before the newer cycle can install state; +otherwise an older event could be delivered against a newer visible snapshot. + +Every graph-turn origin that can publish rows owns a scheduler publication +context through the complete coherent release. This includes direct window +changes as well as source transactions. Work created by a publication callback +joins that context and runs only after the current root and facade callbacks +finish; it cannot start a second graph turn inside the first one or disappear +through the graph's reentrancy guard. + +Each ordinary Collection publication freezes its layout listeners and public +subscribers, then attempts every callback in registration order. Adding or +removing a listener during delivery does not change that batch. The first exact +callback failure stays on the shared publication context, including when it +came from a nested readiness transition. Later callback failures cannot replace +it. Core runs the dependent graph turn queued by the batch before rethrowing the +retained failure. A graph failure stops that turn and clears its remaining +work. Scheduler context cleanup attempts every clear listener, but a cleanup +failure cannot replace the publication or graph failure that caused the clear. +Scheduler dependencies are not a complete proof that two jobs can commit or +roll back independently. + +Each ordinary Collection batch is a keyed diff and names an affected row key at +most once. Row operations and row-metadata writes use one shared affected-key +derivation for pre-sync capture, commit, cancellation, and rollback snapshots. +A pre-sync capture retains both the exact visible row and its virtual row +properties. Confirmation must therefore publish a same-value update when only +`$origin` or `$synced` changes; its `previousValue` describes the state readers +actually saw before confirmation. +A metadata-only transaction can retire optimistic state and change virtual row +properties, so omitting its key from any of those phases can publish the same +transition twice, leave stale suppression state after cancellation, or retain +applied metadata after a failed coherent publication rolls back. +Affectedness depends on key presence, not metadata value truthiness. In +particular, setting a key to `undefined` is distinct from deleting that key. + +Window metadata follows the same causal order as the published rows. If a +publication callback starts a newer window operation, that newer generation +owns the final public window and the older caller cannot overwrite it when it +resumes. Restoring a rejected window is itself a graph-turn origin: its +callbacks remain inside one publication context, and restoration cannot roll +back a newer nested window generation. A rejected nested operation restores +its immediate parent's effective window, not an older public snapshot; rows +and window metadata therefore describe the same surviving generation. It also +restores the parent's imperative load-operation ownership before rollback +publication. Loads started by rollback or by the parent callback after it +catches the nested error must delay and contribute outcomes to the parent. +If teardown clears the runtime while an accepted window call is unwinding, +that generation remains the desired window for the next sync session. A call +that fails synchronously instead restores its previous effective window. ## External boundaries @@ -984,8 +1356,8 @@ create recursive Collection machinery. materialized output relation of its children. 9. **Publication:** reads, events, and downstream queries observe the same complete graph result. -10. **Initial demand:** preload completes when every initially reachable demand - is covered; obsolete demand does not block it. +10. **Initial demand:** preload completes when the current load for every + initially reachable demand settles; obsolete demand does not block it. 11. **Ownership:** a query-db row exists exactly while an explicit owner remains. 12. **Work:** irrelevant rows do not cause unrelated scans or activate unrelated @@ -1014,6 +1386,17 @@ create recursive Collection machinery. - **Hydration:** establishing an initial snapshot before forwarding later changes. - **Generation:** a token that rejects obsolete asynchronous work. +- **Demand snapshot:** an immutable description of work requested by one + logical caller. +- **Logical lease:** one active owner of a demand. +- **Physical acquisition:** one exact adapter attempt and its settlement. +- **Applied outcome:** the source extent and row keys established by one + acquisition after its writes become visible. +- **Coverage fact:** caller-relative proof that applied evidence satisfies a + demand. +- **Row ownership:** the acquisition support that keeps applied row keys alive. +- **Publication snapshot:** the last complete reader-visible rows and ordered + boundary. - **Source extent:** an authoritative source fact that more rows continue past an exact demand, that the source is exhausted there, or that neither is known. - **Collection facade:** a stable public Collection view shared by the parents @@ -1023,35 +1406,249 @@ create recursive Collection machinery. ## Executable contracts -| Contract | Test suite | -| --------------------------------------------------------------------------- | -------------------------------------------------------------------------- | -| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | -| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | -| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | -| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | -| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | -| Collection facades, event coherence, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | -| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | -| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | -| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | -| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | -| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | -| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | -| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | -| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | - -Each oracle identifies the first divergent checkpoint and compares either the -whole result or one exact structural difference. Correlated-materialization -scenarios use direct assertions. A boundary suite may retain an exact -expected-failure guard for a planner or ownership defect that this graph does -not own. +| Contract | Test suite | +| ---------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| State equivalence, route lifecycle, transition history, and batch partition | `packages/db/tests/query/includes-oracle.property.test.ts` | +| Joined multiplicity, alias identity, and null-key normalization | `packages/db/tests/query/includes-query-shape-oracle.test.ts` | +| Exact D2 source retractions, replay suppression, and boundary lifecycle | `packages/db/tests/d2-source-reconciliation-oracle.property.test.ts` | +| Sync-session retention, cleanup, reentrant restart, and applied receipts | `packages/db/tests/collection-state-retention-oracle.property.test.ts` | +| Demand, cancellation, and progressive timing | `packages/db/tests/query/includes-temporal-oracle.test.ts` | +| Optimistic confirmation, rollback, and later reactivity | `packages/db/tests/query/includes-optimistic-oracle.property.test.ts` | +| Coherent layered publication | `packages/db/tests/query/includes-publication-oracle.test.ts` | +| Exact metadata settlement, cancellation ownership, and rollback recovery | `packages/db/tests/collection-metadata-publication-oracle.property.test.ts` | +| Collection facades, event/receipt rollback, and route activation | `packages/db/tests/query/includes-collection-oracle.property.test.ts` | +| Correlated physical work | `packages/db/tests/query/includes-work-counter-oracle.test.ts` | +| Route-context discovery and transport across recursive and join boundaries | `packages/db/tests/query/includes-context-transport-oracle.test.ts` | +| Ordered source coverage, total boundaries, and window transitions | `packages/db/tests/query/pagination-oracle.property.test.ts` | +| Truncate replacement, retained publication, and boundary provenance | `packages/db/tests/collection-subscription-replay-oracle.property.test.ts` | +| Coverage leases, acquisitions, fact compaction, and row provenance | `packages/db/tests/query/coverage-registry-oracle.property.test.ts` | +| Applied coverage publication through the Collection sync boundary | `packages/db/tests/load-subset-outcome.test.ts` | +| Scheduled acquisition, release retry, and stale settlement | `packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts` | +| End-to-end demand, multi-source ordered continuation, and outcome boundaries | `packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts` | +| Framework paging, final partial pages, and peek-ahead compatibility | `packages/db/tests/conformance/infinite-suite.ts` | +| Shared subset acquisition, readiness, receipt, and replay interpreter | `packages/db/tests/query/load-subset-refinement-model.property.test.ts` | +| Production-boundary refinement drivers | `packages/db/tests/query/load-subset-*-refinement-oracle.test.ts` | +| Adapter final-owner release and remount transport | Electric `electric-live-query.test.ts`; PowerSync `on-demand-sync.test.ts` | +| Query-db ownership | `packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts` | +| Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | + +### Oracle family boundary + +The shared load-subset refinement model begins after relational evaluation. Its +closed event grammar varies opaque source topology, demand relationships, +already-evaluated result contributions, public window state, settlement, +release, and teardown. It owns asynchronous demand, applied evidence, row +support, coverage, publication, source progress, and resource work. It must not +interpret query IR, weighted deltas, predicates, joins, grouping, query-level +ordering, or nested materialization. It may project already-evaluated total-order +coordinates and public window state. A new regression must reduce to this +grammar or justify a grammar change; it must not add a one-off event named after +the bug. + +The grammar composes these independent axes: + +| Axis | Values owned by this oracle family | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Source shape | One or many opaque sources; zero, one, or many already-evaluated result contributions | +| Row key domain | String or number identity; unordered membership for ownership plus shared `compareKeys` order for removal publication | +| Demand relation | Exact, shared, covered, uncovered, ordered, additional, release-pending, durably released or disposed | +| Operation | Current or superseded imperative caller; open, waiting, settled, canceled, or cleaned; zero, one, or many attached physical requests | +| Identity | Owner, operation, session, window revision, continuation task, demand, attempt, acquisition, source, transaction, row version, publication, boundary frame, failure occurrence, runtime reference slot | +| Boundary phase | Before adapter entry, inside adapter or callback entry, returned/in flight, settled, terminal listener delivery, cleanup | +| Capability | Indexed or unindexed order; expressible or opaque boundary and collation; authoritative or unknown extent | +| Evidence | Applied row keys plus `unknown`, `continues`, or `exhausted` extent; rejection or abort establishes none | +| Publication | Last complete snapshot, private replacement, failed or superseded generation, cleaned session | +| Origin | Ordinary source work or the exact ordered/additional acquisition signal lineage that authorized a row version | +| Observation | Final rows, ordered change/adapter/release/lifecycle traces, callback-time reads, readiness, boundary, canonical removals, exact errors, operation outcomes, receipts, ownership, physical starts, evidence work, ordered-path work, transient retained space, and lifetime symbol-identity entries | + +An executable history chooses values on these axes, then combines them through +the demand facts above. A logical request installs its owner before adapter +entry. It either attaches to an acquisition or starts one. Request-scoped sync +transactions make row versions visible and settle their receipts before the +acquisition can publish an outcome. Applied evidence may then establish +caller-relative coverage and row ownership. Ordered evidence may update private +window progress; only a complete publication snapshot reaches readers. +Release, truncate, replacement, restart, and cleanup change the relevant +identity or generation without changing this sequence. + +Cleanup also ends the current publication history. It must discard the +pre-sync row and virtual-property snapshots plus the recently-synced +suppression set before a new sync session starts. Synchronous publication tails +and queued microtasks remain scoped to the session that created them; after +cleanup they cannot clear or mark state owned by a restarted session. Otherwise +stale rows can classify a fresh insert as an update, or stale keys can suppress +the new session's first event. +A new-session transaction committed inside the old session's publication +callback remains in the causal queue. Its receipt stays pending until that new +row and its event are visible; the old drain cannot discard it. + +An imperative load operation is a separate caller boundary around this flow. +It owns the future requests caused while it is current, retains the promises it +already acquired after a newer operation supersedes it, and settles only after +synchronous follow-up requests have had a chance to join. Its outcome includes +caller-relative retained evidence even when coverage reuse starts no transport. +Physical acquisition count, readiness, and an operation's pending set or result +are therefore different projections. + +Logical release and durable physical release are separate transitions. A +throwing cleanup leaves a release-pending acquisition, its coverage, and row +support as retryable debt. Only accepted cleanup retires those physical facts. +Resource observations therefore count leases, acquisitions, coverage claims, +unsettled claims, retained demands, outcomes, and row-key slots separately from +transport starts. Algorithmic work is another independent observation: count +row-key copies, demand snapshots, and demand-key derivations rather than using +transport starts or retained space as a proxy for evidence computation. Ordered +source work is separate again: preserve the exact scan and cursor sequence, and +count source reads or snapshots, sorts or total-order refinements, and predicate +compilations independently. A stable result and request trace can still hide +repeated local work. + +`WindowState` takes at most one ordered source snapshot per collection state +revision. Boundary, coverage, publication, and reconciliation reads share that +snapshot; the next committed source batch invalidates it. The query predicate +is compiled once with the window and is evaluated over the shared ordered +snapshot, so another view of the same revision does not rescan, resort, or +recompile it. + +A compatible single-column built-in index walks indexed-value buckets in query +order, whether the matching view is direct or reversed. It evaluates complete +buckets until the requested filtered prefix is known, orders public keys +ascending within each bucket, and stops after the sufficient boundary bucket. +The public-key suffix does not depend on index insertion order or query +direction. Rows in worse buckets cannot add source reads or total-order +refinement work. An all-tied source is the deliberate worst case: the one +boundary bucket is the whole source and must be inspected before the public-key +suffix can choose top-K. + +An ordered bucket is a comparator-equivalence class, not an exact Map-key +bucket. Distinct values such as `null` and `undefined`, or values equated by a +custom comparator, contribute all of their public keys to the same tie class. +Reversing an index also reverses its null placement. The optimizer may reuse a +reverse index only when the requested direction and null placement describe +that reversed order; otherwise it creates a matching index or falls back to a +full `TotalOrder` refinement. A built-in index configured with a custom +comparator also keeps the full-refinement fallback: comparison metadata cannot +prove that an opaque comparator has the query's order. Public custom indexes +without lazy bucket iteration keep the same fallback. + +Runtime reference identity has a different lifetime again. Objects use weak +identity, but JavaScript symbols cannot be weak keys. Stable equality for the +same live symbol therefore retains one strong entry per distinct symbol for the +runtime identity factory's lifetime. This monotonic, usage-proportional cost is +not part of the live-demand resource bound. Eviction is not valid unless the +platform supplies weak symbol identity or another scheme proves that one live +symbol can never receive a different identity. + +Adapter entry and every result, cleanup, and listener callback are reentrancy +boundaries. Any otherwise legal event may occur before that boundary returns. +Work which has entered an adapter but has not yet returned a promise is already +pending work. A production-boundary driver must include this synchronous phase; +promise-only overlap does not reconstruct the source. + +Failures also carry boundary identity. One occurrence names its originating +options, creation order, and containing callback or acquisition frames. A +private propagation token may carry that occurrence through an authorized +nested frame, but payload equality never merges two boundaries. `undefined`, +`NaN`, primitives, and the same `Error` object can each be the payload of a +distinct occurrence. + +Publication is an ordered observation, not only a final state. Transaction and +replay laws preserve each emitted change batch, adapter invocation and release, +and the rows synchronously visible inside its callback. Terminal teardown first +reports retained failures, then emits `unsubscribed` exactly once, then clears +listeners. Reentrant teardown and cleanup retries must not duplicate or reorder +that lifecycle edge. + +Each projection may erase axes it does not own. It must preserve the identity +and cardinality of the fact it claims to check. In particular: + +- receipt laws compare each acquisition with its own applied keys; +- operation laws keep caller identity, acquired promises, first error, and + per-source/collection/generation outcomes separate from acquisition state; +- coverage laws keep caller demand separate from acquisition outcome; +- ownership laws keep logical leases separate from physical row support; +- publication laws keep demand origin, row version, and generation separate; +- work laws count physical starts separately from logical owners; +- evidence-work laws count row-key copies, demand snapshots, and demand-key + derivations separately and bound them independently of candidate count; +- ordered-work laws preserve the exact source-read and cursor sequence and + count source snapshots, sorts or total-order refinements, and predicate + compilations separately from transport and coverage-evidence work; +- space laws count each retained resource category separately; +- identity-space laws count process-lifetime symbol entries separately from + transient demand resources and preserve stable same-symbol identity; +- release laws distinguish requested, retryable, accepted, and disposed work; +- removal laws preserve the shared `compareKeys` sequence across mixed string, + number, ASCII, and non-ASCII keys rather than comparing only a set; +- trace laws preserve adapter calls, releases, emitted change batches, + callback-time reads, and terminal lifecycle events in order rather than + comparing only final state; +- error laws preserve occurrence, originating options, and report order rather + than deduplicating by payload; +- renaming laws erase names only after every allowed next-command observation + remains equal. + +Set unions, final-state equality, and settled promises are therefore supporting +views, not universal oracles. Each can hide a wrong acquisition, transient +publication, duplicate start, stale generation, or lost owner. + +The reconstruction control for a new finding is: + +1. express its source topology as already-evaluated contributions; +2. name every logical, imperative-operation, and physical identity involved; +3. state the adapter capability which makes each transport transition legal; +4. place each action at its exact boundary phase and source origin; +5. derive operation settlement, evidence, ownership, coverage, publication, + ordered observation traces, failure occurrences, canonical removal order, + evidence-path work, ordered-path work, transient retained resources, and + lifetime identity entries independently; +6. compare the first public, algorithmic-work, or retained-resource observation + that can differ; and +7. verify the same grammar admits the nearest marginal case but rejects a raw + relational or materialization problem. + +Zero-sized windows, empty sources, unknown extent, synchronous adapter results, +coverage-reused operations with no transport, superseded overlapping +operations, reentrant terminal cleanup, zero-contribution source steps, +all-tied boundaries, and mixed string/number or non-ASCII row keys are marginal +cases of this grammar, not separate families. Predicate evaluation, join +multiplicity, aggregate deltas, and nested materialization are outside it and +remain negative controls. Reclaiming live symbol-identity entries is also +outside the current platform contract; their accepted factory-lifetime cost +must remain visible. + +DBSP operator suites own incremental relational laws. The includes suites own +compiled routes and materialized nested results. A load-subset production +harness may use an eager query from those paths as its relational control, then +compare lazy demand and source progress with a small refinement projection. It +must not copy those paths into a second relational engine inside the shared +model. + +Each model law has its own projection instead of one monolithic expected-state +reducer. Each oracle identifies the first divergent checkpoint and compares +either the whole result or one exact structural difference. +Correlated-materialization scenarios use direct assertions. A boundary suite +may retain an exact expected-failure guard for a planner or ownership defect +that this graph does not own. Run the DB oracle set with `pnpm test:oracles` from `packages/db`. Broad properties use FastCheck's random seed, while structural matrices keep fixed seeds so each run covers the same named cells. Increase both corpora with -`TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve FastCheck's -reported seed and shrink path while reducing a failure. Replay a broad -campaign with `TANSTACK_DB_ORACLE_SEED= pnpm test:oracles`, then add the +`TANSTACK_DB_ORACLE_RUNS_MULTIPLIER=10 pnpm test:oracles`. Preserve the oracle +property key beside FastCheck's reported seed and shrink path while reducing a +failure. Replay one exact property with: + +```sh +TANSTACK_DB_ORACLE_PROPERTY= \ +TANSTACK_DB_ORACLE_SEED= \ +TANSTACK_DB_ORACLE_PATH= \ +pnpm test:oracles +``` + +The replay registry rejects partial or unknown coordinates and duplicate +registered names. Its static inventory must stay equal to the property helper +call sites; a missing registration fails at the helper boundary. A seed without +a property and path still runs the broad campaign. After shrinking, add the smallest case as a deterministic regression trace. The broad relationship history changes correlation keys rather than freezing diff --git a/packages/db/src/query/live/bucket-facade-adapter.ts b/packages/db/src/query/live/bucket-facade-adapter.ts index f350e88850..2d3ba91714 100644 --- a/packages/db/src/query/live/bucket-facade-adapter.ts +++ b/packages/db/src/query/live/bucket-facade-adapter.ts @@ -1,10 +1,12 @@ import { output, serializeValue } from '@tanstack/db-ivm' import { createCollection } from '../../collection/index.js' +import { runAllCallbacks } from '../../utils/callbacks.js' import { FN_SELECT_STATE, INCLUDES_ROUTING } from '../compiler/index.js' import { BUCKET_FACADE_REF } from './materialized-pipeline.js' import type { Collection } from '../../collection/index.js' import type { SyncConfig } from '../../types.js' import type { PublicationDeferral } from '../../collection/changes.js' +import type { CollectionPublicationStateSnapshot } from '../../collection/state.js' import type { BucketFacadeCompilation, BucketFacadeRef, @@ -13,6 +15,9 @@ import type { type FacadeSync = Parameters[`sync`]>[0] +const BUCKET_FACADE_PUBLIC_KEY = Symbol(`bucketFacadePublicKey`) +const BUCKET_FACADE_ORDER = Symbol(`bucketFacadeOrder`) + type PendingRow = { deletes: number inserts: number @@ -22,25 +27,27 @@ type PendingRow = { type FacadeEntry = { collection: Collection sync: FacadeSync | undefined - keys: WeakMap - order: WeakMap + readonly keys: WeakMap + readonly order: WeakMap + readonly currentOrder: Map +} + +type FacadeEntrySnapshot = { + publicationState: CollectionPublicationStateSnapshot< + Record, + string | number + > currentOrder: Map } type FacadeSnapshot = { activeBuckets: Map> entries: Map> - rows: Map< - FacadeEntry, - Array<{ - key: string | number - value: object - order: string | undefined - }> - > + entryStates: Map } export type FacadePublication = { + prepare: () => void publish: () => void rollback: () => void } @@ -51,14 +58,12 @@ export type FacadePublication = { * graph's canonical bucket-row deltas to those facades. */ export class BucketFacadeAdapter { - private readonly pending = new Map< - string, - Map> - >() - private readonly pendingActivity = new Map>() + private pending = new Map>>() + private pendingActivity = new Map>() private readonly activeBuckets = new Map>() private readonly entries = new Map>() private readonly retiredEntries = new Map>() + private readonly recoveryStates = new Map() private resolvedValues = new WeakMap() constructor( @@ -92,10 +97,18 @@ export class BucketFacadeAdapter { return this.pending.size > 0 || this.pendingActivity.size > 0 } + recover(): void { + this.recoverEntries() + } + flush(): FacadePublication { + this.recover() const snapshot = this.snapshot() + const installedPending = this.pending + const installedActivity = this.pendingActivity const deferredEntries = new Set() const publications: Array = [] + const readyEntries = new Set() const deferPublication = (entry: FacadeEntry) => { if (deferredEntries.has(entry)) return deferredEntries.add(entry) @@ -106,7 +119,7 @@ export class BucketFacadeAdapter { // their containing rows are written to the next facade. try { for (const compilation of this.compilations) { - const activity = this.pendingActivity.get(compilation.edgeId) + const activity = installedActivity.get(compilation.edgeId) const active = this.getActiveBuckets(compilation.edgeId) const newBaselines: Array = [] for (const [bucketKey, multiplicity] of activity ?? []) { @@ -116,7 +129,7 @@ export class BucketFacadeAdapter { } } - const buckets = this.pending.get(compilation.edgeId) + const buckets = installedPending.get(compilation.edgeId) for (const [bucketKey, changes] of buckets ?? []) { const existing = this.entries.get(compilation.edgeId)?.get(bucketKey) if (!active.has(bucketKey) && !existing) continue @@ -127,14 +140,26 @@ export class BucketFacadeAdapter { for (const change of changes.values()) { this.prepareChange(entry, change) } + const mayChangeVisibleOrder = + compilation.hasOrderBy && + [...changes.values()].some((change) => + this.mayChangeVisibleOrder(entry, change), + ) deferPublication(entry) - sync.begin() + // The graph is already quiescent. Install this complete child + // publication beneath any pending optimistic facade overlay instead + // of parking source progress behind that mutation. + sync.begin({ immediate: true }) for (const change of changes.values()) { - this.applyChange(entry, sync, change, compilation.hasOrderBy) + this.applyChange(entry, sync, change) + } + if (mayChangeVisibleOrder) { + sync.collection._markLayoutChange() } sync.commit() + if (entry.collection.status !== `ready`) readyEntries.add(entry) } - for (const entry of newBaselines) entry.sync?.markReady() + for (const entry of newBaselines) readyEntries.add(entry) for (const [bucketKey, multiplicity] of activity ?? []) { if (multiplicity >= 0) continue @@ -143,30 +168,68 @@ export class BucketFacadeAdapter { } } } catch (error) { - this.restore(snapshot, deferredEntries) - this.retiredEntries.clear() - for (const publication of publications) publication.discard() + try { + this.rollbackInstallation(snapshot, deferredEntries, publications) + } catch { + // Preserve the graph-install failure. A failed state restore marks its + // facade as recoverably errored before this rollback closes every + // publication handle. + } throw error } - this.pending.clear() - this.pendingActivity.clear() + // Detach, rather than clear, the deltas installed by this attempt. The + // containing root publication decides whether they commit or must be + // replayed with the next graph turn. + this.pending = new Map() + this.pendingActivity = new Map() + let prepared = false let closed = false + const prepare = () => { + if (prepared || closed) return + prepared = true + runAllCallbacks([ + ...publications.map((publication) => publication.prepare), + ...[...readyEntries].map((entry) => () => entry.sync?.markReady()), + ]) + } return { + prepare, publish: () => { if (closed) return + let firstFailure: { error: unknown } | undefined + try { + prepare() + } catch (error) { + firstFailure = { error } + } closed = true - for (const publication of publications) publication.publish() + try { + runAllCallbacks( + publications.map((publication) => publication.publish), + ) + } catch (error) { + firstFailure ??= { error } + } // Drop only the adapter's strong reference. External holders keep an // empty, ready facade; a later active interval receives a new one. this.retiredEntries.clear() + if (firstFailure) throw firstFailure.error }, rollback: () => { - if (closed) return + if (closed || prepared) return closed = true - this.restore(snapshot, deferredEntries) - this.retiredEntries.clear() - for (const publication of publications) publication.discard() + runAllCallbacks([ + () => { + this.pending = mergePendingRows(installedPending, this.pending) + this.pendingActivity = mergePendingActivity( + installedActivity, + this.pendingActivity, + ) + }, + () => + this.rollbackInstallation(snapshot, deferredEntries, publications), + ]) }, } } @@ -186,6 +249,7 @@ export class BucketFacadeAdapter { this.pending.clear() this.pendingActivity.clear() this.activeBuckets.clear() + this.recoveryStates.clear() } private accumulate( @@ -221,24 +285,24 @@ export class BucketFacadeAdapter { } private snapshot(): FacadeSnapshot { - const rows = new Map< - FacadeEntry, - Array<{ - key: string | number - value: object - order: string | undefined - }> - >() - for (const byBucket of this.entries.values()) { - for (const entry of byBucket.values()) { - rows.set( - entry, - [...entry.collection._state.syncedData].map(([key, value]) => ({ - key, - value, - order: entry.currentOrder.get(key), - })), - ) + const entryStates = new Map() + for (const [edgeId, byBucket] of this.entries) { + for (const [bucketKey, entry] of byBucket) { + const affectedKeys = new Set(entry.collection._state.syncedData.keys()) + for (const change of this.pending + .get(edgeId) + ?.get(bucketKey) + ?.values() ?? []) { + const key = change.value.publicKey + if (typeof key === `string` || typeof key === `number`) { + affectedKeys.add(key) + } + } + entryStates.set(entry, { + publicationState: + entry.collection._snapshotPublicationState(affectedKeys), + currentOrder: new Map(entry.currentOrder), + }) } } return { @@ -254,14 +318,16 @@ export class BucketFacadeAdapter { new Map(byBucket), ]), ), - rows, + entryStates, } } private restore( snapshot: FacadeSnapshot, changedEntries: Set, + failedEntries: Map, ): void { + let firstFailure: { error: unknown } | undefined const previousEntries = new Set( [...snapshot.entries.values()].flatMap((byBucket) => [ ...byBucket.values(), @@ -273,18 +339,16 @@ export class BucketFacadeAdapter { for (const entry of changedEntries) { if (!previousEntries.has(entry)) continue - const sync = entry.sync - if (!sync) continue - sync.begin() - sync.truncate() - entry.currentOrder.clear() - for (const row of snapshot.rows.get(entry) ?? []) { - entry.keys.set(row.value, row.key) - if (row.order !== undefined) entry.order.set(row.value, row.order) - entry.currentOrder.set(row.key, row.order) - sync.write({ type: `insert`, value: row.value }) + const entryState = snapshot.entryStates.get(entry) + if (!entryState) continue + try { + this.restoreEntryState(entry, entryState) + this.recoveryStates.delete(entry) + } catch (error) { + firstFailure ??= { error } + failedEntries.set(entry, error) + this.recoveryStates.set(entry, entryState) } - sync.commit() } this.entries.clear() @@ -300,6 +364,68 @@ export class BucketFacadeAdapter { for (const entry of currentEntries) { if (!previousEntries.has(entry)) void entry.collection.cleanup() } + if (firstFailure) throw firstFailure.error + } + + private restoreEntryState( + entry: FacadeEntry, + entryState: FacadeEntrySnapshot, + ): void { + try { + entry.collection._restorePublicationState(entryState.publicationState) + } finally { + entry.currentOrder.clear() + for (const [key, order] of entryState.currentOrder) { + entry.currentOrder.set(key, order) + } + } + } + + private recoverEntries(): void { + let firstFailure: { error: unknown } | undefined + const failedEntries = new Map() + for (const [entry, entryState] of this.recoveryStates) { + try { + this.restoreEntryState(entry, entryState) + this.recoveryStates.delete(entry) + } catch (error) { + firstFailure ??= { error } + failedEntries.set(entry, error) + } + } + if (!firstFailure) return + + try { + runAllCallbacks( + [...failedEntries].map(([entry, error]) => () => { + if (entry.collection.status !== `error`) entry.sync?.markError(error) + }), + ) + } catch { + // The index recovery failure remains authoritative and retryable. + } + throw firstFailure.error + } + + private rollbackInstallation( + snapshot: FacadeSnapshot, + changedEntries: Set, + publications: Array, + ): void { + const failedEntries = new Map() + runAllCallbacks([ + () => this.restore(snapshot, changedEntries, failedEntries), + () => this.retiredEntries.clear(), + ...publications.map((publication) => publication.discard), + () => + runAllCallbacks( + [...failedEntries].map(([entry, error]) => () => { + if (entry.collection.status !== `error`) { + entry.sync?.markError(error) + } + }), + ), + ]) } private accumulateActivity( @@ -337,6 +463,9 @@ export class BucketFacadeAdapter { const keys = [...entry.collection.keys()] if (sync && keys.length > 0) { deferPublication(entry) + // Route retirement precedes the root or containing-facade change that + // removed its final consumer. That later immediate transaction drains + // this earlier transaction as part of the same FIFO causal prefix. sync.begin() for (const key of keys) sync.write({ type: `delete`, key }) sync.commit() @@ -366,15 +495,16 @@ export class BucketFacadeAdapter { const collection = createCollection({ id: `__bucket-facade:${this.parentId}:${edgeId}:${bucketKey}`, getKey: (row) => { - const key = keys.get(row) ?? row?.$key + const key = + keys.get(row) ?? row?.[BUCKET_FACADE_PUBLIC_KEY] ?? row?.$key if (typeof key !== `string` && typeof key !== `number`) { throw new Error(`Bucket facade row has no public key`) } return key }, compare: (left, right) => { - const leftOrder = order.get(left) - const rightOrder = order.get(right) + const leftOrder = order.get(left) ?? left?.[BUCKET_FACADE_ORDER] + const rightOrder = order.get(right) ?? right?.[BUCKET_FACADE_ORDER] if (leftOrder === rightOrder) return 0 if (leftOrder === undefined) return 1 if (rightOrder === undefined) return -1 @@ -409,28 +539,44 @@ export class BucketFacadeAdapter { entry: FacadeEntry, sync: FacadeSync, change: PendingRow, - hasOrderBy: boolean, ): void { const key = change.value.publicKey as string | number const previousOrder = entry.currentOrder.get(key) const nextOrder = change.value.order - const orderChanged = sync.collection.has(key) && previousOrder !== nextOrder + // Graph deltas update the synced base. The public Collection view may be + // hiding that row beneath a pending optimistic delete, so it cannot tell + // us whether this delta is an insert, update, or delete of the base row. + const hasSyncedRow = entry.collection._state.syncedData.has(key) + const previousSyncedRow = entry.collection._state.syncedData.get(key) + const orderChanged = hasSyncedRow && previousOrder !== nextOrder const resolvedRow = this.resolve(change.value.value) + // Order metadata lives in a WeakMap keyed by row identity. Never attach a + // new base order to an object that may also back the optimistic overlay. const row = - orderChanged && sync.collection.get(key) === resolvedRow + orderChanged && previousSyncedRow === resolvedRow ? { ...resolvedRow } : resolvedRow entry.keys.set(row, key) + // Collection updates clone the public row. Keep its route key on an + // internal symbol so projected facade rows retain their identity. + Object.defineProperty(row, BUCKET_FACADE_PUBLIC_KEY, { + configurable: true, + value: key, + }) if (nextOrder !== undefined) { entry.order.set(row, nextOrder) + Object.defineProperty(row, BUCKET_FACADE_ORDER, { + configurable: true, + value: nextOrder, + }) } if (change.inserts > change.deletes) { sync.write({ - type: sync.collection.has(key) ? `update` : `insert`, + type: hasSyncedRow ? `update` : `insert`, value: row, }) - } else if (change.inserts === change.deletes && sync.collection.has(key)) { + } else if (change.inserts === change.deletes && hasSyncedRow) { sync.write({ type: `update`, value: row }) } else if (change.deletes > 0) { sync.write({ type: `delete`, key }) @@ -439,7 +585,27 @@ export class BucketFacadeAdapter { } entry.currentOrder.set(key, nextOrder) - if (hasOrderBy && orderChanged) sync.collection._markLayoutChange() + } + + /** Identify graph changes that can move a visible key. Collection state + * validates the final public sequence before publishing the layout signal. */ + private mayChangeVisibleOrder( + entry: FacadeEntry, + change: PendingRow, + ): boolean { + const key = change.value.publicKey as string | number + const hasSyncedRow = entry.collection._state.syncedData.has(key) + const nextHasSyncedRow = + change.inserts > change.deletes || + (change.inserts === change.deletes && hasSyncedRow) + const orderChanged = + hasSyncedRow && + nextHasSyncedRow && + entry.currentOrder.get(key) !== change.value.order + const movesBetweenBaseAndOptimisticSuffix = + hasSyncedRow !== nextHasSyncedRow && + entry.collection._state.optimisticUpserts.has(key) + return orderChanged || movesBetweenBaseAndOptimisticSuffix } /** Resolve and validate every public key before opening a sync transaction. */ @@ -502,3 +668,75 @@ function isPlainObject(value: unknown): value is Record { const prototype = Object.getPrototypeOf(value) return prototype === Object.prototype || prototype === null } + +function mergePendingRows( + earlier: Map>>, + later: Map>>, +): Map>> { + const merged = new Map>>() + + for (const [edgeId, buckets] of earlier) { + const mergedBuckets = new Map>() + merged.set(edgeId, mergedBuckets) + for (const [bucketKey, rows] of buckets) { + mergedBuckets.set( + bucketKey, + new Map( + [...rows].map(([key, change]) => [key, { ...change }] as const), + ), + ) + } + } + + for (const [edgeId, buckets] of later) { + let mergedBuckets = merged.get(edgeId) + if (!mergedBuckets) { + mergedBuckets = new Map() + merged.set(edgeId, mergedBuckets) + } + for (const [bucketKey, rows] of buckets) { + let mergedRows = mergedBuckets.get(bucketKey) + if (!mergedRows) { + mergedRows = new Map() + mergedBuckets.set(bucketKey, mergedRows) + } + for (const [key, laterChange] of rows) { + const earlierChange = mergedRows.get(key) + if (!earlierChange) { + mergedRows.set(key, { ...laterChange }) + continue + } + earlierChange.deletes += laterChange.deletes + earlierChange.inserts += laterChange.inserts + if (laterChange.inserts > 0) earlierChange.value = laterChange.value + } + } + } + + return merged +} + +function mergePendingActivity( + earlier: Map>, + later: Map>, +): Map> { + const merged = new Map( + [...earlier].map( + ([edgeId, buckets]) => [edgeId, new Map(buckets)] as const, + ), + ) + for (const [edgeId, buckets] of later) { + let mergedBuckets = merged.get(edgeId) + if (!mergedBuckets) { + mergedBuckets = new Map() + merged.set(edgeId, mergedBuckets) + } + for (const [bucketKey, multiplicity] of buckets) { + mergedBuckets.set( + bucketKey, + (mergedBuckets.get(bucketKey) ?? 0) + multiplicity, + ) + } + } + return merged +} diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index 870e370aac..80bd9b1553 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -7,9 +7,11 @@ import { import { getActivePublicationContext, transactionScopedScheduler, + withPublicationContext, } from '../../scheduler.js' import { getActiveTransaction } from '../../transactions.js' import { deepEquals } from '../../utils.js' +import { runAllCallbacks } from '../../utils/callbacks.js' import { getLoadSubsetDemandKey } from '../ir-stable-identity.js' import { isAppliedLoadSubsetOutcome } from '../load-subset-outcome.js' import { CollectionSubscriber } from './collection-subscriber.js' @@ -27,6 +29,7 @@ import type { LiveQueryInternalUtils } from './internal.js' import type { WindowOptions } from '../compiler/index.js' import type { SchedulerContextId } from '../../scheduler.js' import type { CollectionSubscription } from '../../collection/subscription.js' +import type { CollectionPublicationStateSnapshot } from '../../collection/state.js' import type { RootStreamBuilder } from '@tanstack/db-ivm' import type { OrderByOptimizationInfo } from '../compiler/order-by.js' import type { Collection } from '../../collection/index.js' @@ -52,6 +55,8 @@ import type { AllCollectionEvents } from '../../collection/events.js' export type LiveQueryCollectionUtils = UtilsRecord & { getRunCount: () => number + /** Whether this live query has observed a subset-load failure in its current sync session. */ + readonly hasSubsetError: boolean /** Most recent subset-load failure observed by this live query. */ readonly lastSubsetError: unknown | undefined /** @@ -74,6 +79,21 @@ type PendingGraphRun = { loadCallbacks: Set<() => boolean> } +function runLoadCallbacks(callbacks: Iterable<() => boolean>): boolean { + let allDone = true + let firstFailure: { error: unknown } | undefined + for (const callback of callbacks) { + try { + allDone = callback() && allDone + } catch (error) { + allDone = false + firstFailure ??= { error } + } + } + if (firstFailure) throw firstFailure.error + return allDone +} + // Global counter for auto-generated collection IDs let liveQueryCollectionCounter = 0 @@ -119,6 +139,7 @@ export class CollectionConfigBuilder< private isInErrorState = false private fatalQueryError = false private readonly erroredSourceIds = new Set() + private hasSubsetError = false private lastSubsetError: unknown | undefined // Reference to the live query collection for error state transitions @@ -282,6 +303,9 @@ export class CollectionConfigBuilder< singleResult: this.query.singleResult, utils: { getRunCount: this.getRunCount.bind(this), + get hasSubsetError() { + return builder.hasSubsetError + }, get lastSubsetError() { return builder.lastSubsetError }, @@ -302,7 +326,8 @@ export class CollectionConfigBuilder< } setWindow(options: WindowOptions): true | Promise { - if (!this.windowFn) { + const windowFn = this.windowFn + if (!windowFn) { throw new SetWindowRequiresOrderByError() } @@ -316,18 +341,38 @@ export class CollectionConfigBuilder< const operation: { failed: boolean; error?: unknown } = { failed: false } this.activeWindowOperation = operation try { - this.windowFn(options) - this.maybeRunGraphFn?.() - if (operation.failed) throw operation.error + // Window metadata is part of the synchronous publication. This also + // gives a nested operation the effective window of its immediate parent + // to restore if the nested operation fails. this.currentWindow = options + withPublicationContext(() => { + windowFn(options) + this.maybeRunGraphFn?.() + }) + if (operation.failed) throw operation.error + if (windowOperationGeneration === this.windowOperationGeneration) { + // Teardown may clear the runtime while an accepted request is still + // unwinding. Preserve that request as the desired window for the next + // sync session, but never overwrite a newer nested operation. + this.currentWindow = options + } } catch (error) { + // A rejected nested window returns ownership to its parent before the + // rollback publishes. Work caused by that publication must settle with + // the restored parent operation, not the canceled child. + loadOperation?.cancel() if ( previousWindow && + syncSession === this.syncSession && + this.currentSyncConfig !== undefined && windowOperationGeneration === this.windowOperationGeneration ) { try { - this.windowFn(previousWindow) - this.maybeRunGraphFn?.() + this.currentWindow = previousWindow + withPublicationContext(() => { + windowFn(previousWindow) + this.maybeRunGraphFn?.() + }) if (windowOperationGeneration === this.windowOperationGeneration) { this.windowOperationGeneration = previousWindowOperationGeneration } @@ -336,7 +381,6 @@ export class CollectionConfigBuilder< // window rather than replacing it with a rollback failure. } } - loadOperation?.cancel() throw error } finally { this.activeWindowOperation = previousOperation @@ -465,6 +509,7 @@ export class CollectionConfigBuilder< } recordSubsetError(error: unknown, fatalBeforeReady = false): void { + this.hasSubsetError = true this.lastSubsetError = error if (this.activeWindowOperation) { this.activeWindowOperation.failed = true @@ -559,8 +604,11 @@ export class CollectionConfigBuilder< this.isGraphRunning = true try { - const { begin, commit } = this.currentSyncConfig + const config = this.currentSyncConfig + const { begin, commit } = config const syncState = this.currentSyncState + const sessionIsActive = () => + this.currentSyncConfig === config && this.currentSyncState === syncState // Don't run if the live query is in an error state if (this.isInErrorState) { @@ -569,30 +617,34 @@ export class CollectionConfigBuilder< // Always run the graph if subscribed (eager execution) if (syncState.subscribedToAllCollections) { - let callbackCalled = false + // A window change can reach this point with no pending graph work. + // Let the loader run first so any synchronous source commit it starts + // becomes part of this same quiescence pass. + if (!syncState.graph.pendingWork()) { + callback?.() + if (!sessionIsActive()) return + } + while (syncState.graph.pendingWork()) { syncState.graph.run() + if (!sessionIsActive()) return callback?.() - callbackCalled = true + if (!sessionIsActive()) return } // Publish only after every operator has reached quiescence. A source // change can reach sibling materializations in different graph steps; // flushing between those steps would expose a mixed root snapshot. syncState.flushPendingChanges?.() - - // Ensure the callback runs at least once even when the graph has no pending work. - // This handles lazy loading scenarios where setWindow() increases the limit or - // an async loadSubset completes and we need to re-check if more data is needed. - if (!callbackCalled) { - callback?.() - } + if (!sessionIsActive()) return // On the initial run, we may need to do an empty commit to ensure that // the collection is initialized if (syncState.messagesCount === 0) { begin() + if (!sessionIsActive()) return commit() + if (!sessionIsActive()) return } // After graph processing completes, check if we should mark ready. @@ -600,7 +652,7 @@ export class CollectionConfigBuilder< // 1. All data has been processed through the graph // 2. All source collections have had a chance to send their initial data // This prevents marking ready before data is processed (fixes isReady=true with empty data) - this.updateLiveQueryStatus(this.currentSyncConfig) + this.updateLiveQueryStatus(config) } } finally { this.isGraphRunning = false @@ -758,23 +810,8 @@ export class CollectionConfigBuilder< this.incrementRunCount() - const combinedLoader = () => { - let allDone = true - let firstError: unknown - pending.loadCallbacks.forEach((loader) => { - try { - allDone = loader() && allDone - } catch (error) { - allDone = false - firstError ??= error - } - }) - if (firstError) { - throw firstError - } - // Returning false signals that callers should schedule another pass. - return allDone - } + // Returning false signals that callers should schedule another pass. + const combinedLoader = () => runLoadCallbacks(pending.loadCallbacks) this.maybeRunGraph(combinedLoader) } @@ -811,6 +848,7 @@ export class CollectionConfigBuilder< this.isInErrorState = false this.fatalQueryError = false this.erroredSourceIds.clear() + this.hasSubsetError = false this.lastSubsetError = undefined this.latestSubsetOutcomes.clear() this.lastWindowOutcomes = [] @@ -833,13 +871,13 @@ export class CollectionConfigBuilder< if (this.syncSession === syncSession) this.syncSession++ } - let firstCleanupError: unknown + let firstCleanupFailure: { error: unknown } | undefined for (const unsubscribe of syncState.unsubscribeCallbacks) { try { unsubscribe() syncState.unsubscribeCallbacks.delete(unsubscribe) } catch (error) { - firstCleanupError ??= error + firstCleanupFailure ??= { error } } } @@ -871,7 +909,7 @@ export class CollectionConfigBuilder< this.compiledAliasToCollectionId = {} } - if (firstCleanupError !== undefined) throw firstCleanupError + if (firstCleanupFailure) throw firstCleanupFailure.error tornDown = true } @@ -1007,6 +1045,10 @@ export class CollectionConfigBuilder< // transaction, avoiding duplicate key errors when joins produce multiple outputs // for the same key (e.g., first output with null, then output with joined data). let pendingChanges: Map> = new Map() + let rootNeedsReady = false + let rootRecoveryState: + | CollectionPublicationStateSnapshot + | undefined pipeline.pipe( output((data) => { @@ -1038,12 +1080,42 @@ export class CollectionConfigBuilder< return } + let rootRecoveryFailure: { error: unknown } | undefined + try { + runAllCallbacks([ + () => { + if (!rootRecoveryState) return + const stateToRecover = rootRecoveryState + try { + config.collection._restorePublicationState(stateToRecover) + rootRecoveryState = undefined + } catch (error) { + rootRecoveryFailure = { error } + throw error + } + }, + () => bucketFacades.recover(), + ]) + } catch (error) { + if (rootRecoveryFailure) { + try { + config.markError(rootRecoveryFailure.error) + } catch { + // Keep the first recovery failure authoritative and retryable. + } + } + throw error + } + let facadePublication: | ReturnType | undefined let rootPublication: | ReturnType | undefined + let rootStateSnapshot: + | CollectionPublicationStateSnapshot + | undefined try { facadePublication = bucketFacades.flush() rootPublication = hasParentChanges @@ -1065,7 +1137,13 @@ export class CollectionConfigBuilder< ) if (hasParentChanges) { - begin() + rootStateSnapshot = config.collection._snapshotPublicationState( + changesToApply.keys() as Iterable, + ) + // The graph has already reached quiescence, so this is one complete + // derived publication. Apply it beneath any pending optimistic + // overlay instead of parking source progress behind that mutation. + begin({ immediate: true }) changesToApply.forEach(this.applyChanges.bind(this, config)) if (hasOrderOnlyMove(changesToApply)) { markLayoutChange(config.collection) @@ -1073,26 +1151,62 @@ export class CollectionConfigBuilder< commit() } } catch (error) { - pendingChanges = new Map() - rootPublication?.discard() - facadePublication?.rollback() + const failedRootState = rootStateSnapshot + let rootRestoreFailure: { error: unknown } | undefined + try { + runAllCallbacks([ + ...(rootPublication ? [rootPublication.discard] : []), + ...(failedRootState + ? [ + () => { + try { + config.collection._restorePublicationState( + failedRootState, + ) + } catch (restoreError) { + rootNeedsReady = true + rootRecoveryState = failedRootState + rootRestoreFailure = { error: restoreError } + throw restoreError + } + }, + ] + : []), + ...(facadePublication ? [facadePublication.rollback] : []), + ]) + } catch { + // Preserve the graph-install failure after attempting every recovery + // step. A failed root restore remains retryable from staged deltas. + } + if (rootRestoreFailure) { + try { + config.markError(rootRestoreFailure.error) + } catch { + // The install failure remains authoritative after every graph + // participant has restored its public state. + } + } throw error } pendingChanges = new Map() - let publicationError: unknown - for (const publish of [ - rootPublication?.publish, + // Advance every participating Collection's public clocks and facade + // readiness before the first root callback. A callback failure cannot + // suppress another prepared participant's release. + runAllCallbacks([ + ...(rootPublication ? [rootPublication.prepare] : []), + facadePublication.prepare, + ...(rootNeedsReady + ? [ + () => { + rootNeedsReady = false + config.markReady() + }, + ] + : []), + ...(rootPublication ? [rootPublication.publish] : []), facadePublication.publish, - ]) { - if (!publish) continue - try { - publish() - } catch (error) { - publicationError ??= error - } - } - if (publicationError !== undefined) throw publicationError + ]) } graph.finalize() @@ -1121,6 +1235,11 @@ export class CollectionConfigBuilder< // Store the key of the result so that we can retrieve it in the // getKey function this.resultKeys.set(value, key) + const resultKey = collection.getKeyFromItem(value) + // Graph deltas update the synced base. A pending optimistic delete can + // hide that base row from the public Collection view, so collection.has() + // cannot distinguish a base update from a delete. + const hasSyncedRow = collection._state.syncedData.has(resultKey) // Store the orderBy index if it exists if (orderByIndex !== undefined) { @@ -1138,7 +1257,7 @@ export class CollectionConfigBuilder< inserts > deletes || // Just update(s) but the item is already in the collection (so // was inserted previously). - (inserts === deletes && collection.has(collection.getKeyFromItem(value))) + (inserts === deletes && hasSyncedRow) ) { write({ value, @@ -1341,10 +1460,7 @@ export class CollectionConfigBuilder< // Combine all loaders into a single callback that initiates loading more data // from any source that needs it. Returns true once all loaders have been called, // but the actual async loading may still be in progress. - const loadSubsetDataCallbacks = () => { - loaders.map((loader) => loader()) - return true - } + const loadSubsetDataCallbacks = () => runLoadCallbacks(loaders) // Mark as subscribed so the graph can start running // (graph only runs when all collections are subscribed) diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index dad987852f..68052e5bd1 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -6,7 +6,7 @@ import { import { computeOrderedLoadCursor, computeSubscriptionOrderByHints, - filterDuplicateInserts, + reconcileChangesForD2, sendChangesToInput, splitUpdates, } from './utils.js' @@ -47,10 +47,9 @@ export class CollectionSubscriber< { resolve: () => void } >() - // Track keys that have been sent to the D2 pipeline to prevent duplicate inserts - // This is necessary because different code paths (initial load, change events) - // can potentially send the same item to D2 multiple times. - private sentToD2Keys = new Set() + // Track the exact row contributed for each source key. D2 retractions must + // use that row, even when the incoming event reports a changed previous row. + private sentToD2Rows = new Map>() // Direct load tracking callback for ordered path (set during subscribeToOrderedChanges, // used by loadNextItems for subsequent requestLimitedSnapshot calls) @@ -61,6 +60,18 @@ export class CollectionSubscriber< private pendingOrderedLoadPromise: | Promise | undefined + // A sync commit can publish rows before an async loadSubset call returns its + // Promise. Block graph callbacks in that entry window; the Promise guard + // takes over as soon as requestLimitedSnapshot returns. + private orderedLoadStartInProgress = false + // Overlapping replays share one subscription, so only the latest result + // token may clear the full-source acquisition guard. + private unindexedSnapshot: + | { + subscription: CollectionSubscription + token: symbol + } + | undefined private readonly demand = new SubsetDemandController() constructor( @@ -161,6 +172,7 @@ export class CollectionSubscriber< trackLoadResult, onLoadSubsetError, ) + if (orderByInfo.limit === 0) initialSubsetPending = false } else { // Lazy sources load only the subsets demanded by the compiled graph. const includeInitialState = !this.collectionConfigBuilder.isLazySource( @@ -206,6 +218,7 @@ export class CollectionSubscriber< this.sourceId, subscription, ) + this.sentToD2Rows.clear() } // currentSyncState is always defined when subscribe() is called // (called during sync session setup) @@ -241,6 +254,11 @@ export class CollectionSubscriber< if (isInitialSync) throw error return } + if (update.releaseFailure) { + this.collectionConfigBuilder.recordSubsetError( + update.releaseFailure.error, + ) + } if (!update.changed) return if (update.empty) { @@ -280,16 +298,16 @@ export class CollectionSubscriber< callback?: () => boolean, ) { const changesArray = Array.isArray(changes) ? changes : [...changes] - const filteredChanges = filterDuplicateInserts( + const reconciledChanges = reconcileChangesForD2( changesArray, - this.sentToD2Keys, + this.sentToD2Rows, ) // currentSyncState and input are always defined when this method is called // (only called from active subscriptions during a sync session) const input = this.collectionConfigBuilder.currentSyncState!.inputs[this.sourceId]! - const sentChanges = sendChangesToInput(input, filteredChanges) + const sentChanges = sendChangesToInput(input, reconciledChanges) // Do not provide the callback that loads more data // if there's no more data to load @@ -353,6 +371,7 @@ export class CollectionSubscriber< onLoadSubsetError: (event: SubscriptionLoadSubsetErrorEvent) => void, ): CollectionSubscription { const { orderBy, offset, limit, index } = orderByInfo + this.unindexedSnapshot = undefined // Store the callback so loadNextItems can also use direct tracking. // Track in-flight ordered loads to avoid issuing redundant requests while @@ -405,14 +424,12 @@ export class CollectionSubscriber< subscriptionHolder.current = subscription this.registerSubscriptionCleanup(subscription) - // Listen for truncate events to reset cursor tracking state and sentToD2Keys - // This ensures that after a must-refetch/truncate, we don't use stale cursor data - // and allow re-inserts of previously sent keys + // Reset ordered-load state on truncate. Keep exact D2 source rows until + // their later delete/replacement batch retracts them from the live graph. const truncateUnsubscribe = this.collection.on(`truncate`, () => { this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined this.pendingOrderedLoadPromise = undefined - this.sentToD2Keys.clear() }) // Clean up truncate listener when subscription is unsubscribed @@ -421,6 +438,9 @@ export class CollectionSubscriber< subscriptionHolder.current = undefined this.lastLoadRequestKey = undefined this.lastNoProgressRequestKey = undefined + if (this.unindexedSnapshot?.subscription === subscription) { + this.unindexedSnapshot = undefined + } // Ordered continuations belong to this subscription session. A settled // load from a cleaned session must not refill through a later session. @@ -439,22 +459,18 @@ export class CollectionSubscriber< // under microtask timing (e.g., queueMicrotask delays in TanStack Query observers). if (index) { // We have an index on the first orderBy column - use lazy loading optimization - subscription.setOrderByIndex(index) + subscription.setOrderByIndex(index, orderByInfo.expandSourceOrderTies) subscription.requestLimitedSnapshot({ - limit: offset + limit, - orderBy: normalizedOrderBy, - trackLoadSubsetPromise: false, - onLoadSubsetResult: handleLoadSubsetResult, - }) - } else { - // No index available (e.g., non-ref expression): pass orderBy/limit to loadSubset - subscription.requestSnapshot({ + limit: limit === 0 ? 0 : offset + limit, orderBy: normalizedOrderBy, - limit: offset + limit, trackLoadSubsetPromise: false, onLoadSubsetResult: handleLoadSubsetResult, }) + } else if (limit > 0) { + // Without an index there is no sound cursor continuation. Load the full + // ordered source so later relational operators cannot underfill top-K. + this.requestUnindexedSnapshot(subscription, normalizedOrderBy) } return subscription @@ -472,20 +488,39 @@ export class CollectionSubscriber< return true } - const { dataNeeded, index, offset, limit } = orderByInfo + const { dataNeeded, index, offset, limit, refillFromResultDeficit } = + orderByInfo + + // The ordered subscription keeps its frozen order for later window changes, + // but an empty active window has no coordinator or continuation work. + if (limit === 0) return true + + if (!index) { + // A zero-width subscription defers this full fallback until the window + // first becomes positive. Once requested, the snapshot covers every + // later window because cursor continuation is unavailable. + this.requestUnindexedSnapshot( + subscription, + normalizeOrderByPaths(orderByInfo.orderBy, this.alias), + ) + return true + } - if (!dataNeeded || !index) { - // dataNeeded is not set when there's no index (e.g., non-ref expression - // or auto-indexing is disabled). Without an index, lazy loading can't work — - // all data was already loaded eagerly via requestSnapshot. + if (!dataNeeded) { return true } subscription.ensureOrderedWindowSize(offset + limit) - if (subscription.hasOrderedCoverageForActiveWindow) { + const missingResultRows = refillFromResultDeficit ? dataNeeded() : 0 + if ( + missingResultRows === 0 && + subscription.hasOrderedResultForActiveWindow + ) { return true } + if (this.orderedLoadStartInProgress) return true + if (this.pendingOrderedLoadPromise) { // The current window still needs the in-flight coverage. Attach it to // this operation without making an unrelated or superseded request a @@ -497,7 +532,21 @@ export class CollectionSubscriber< return true } - const n = Math.max(dataNeeded(), subscription.orderedRowsNeeded) + // A join or later predicate can discard source rows. Once the prior + // acquisition settles, grow the retained source prefix by the observed + // result deficit so already-local rows publish before another request. + // Never grow from callbacks while an acquisition is still pending: the + // same deficit can be observed more than once in that transaction. + if (missingResultRows > 0) { + subscription.ensureOrderedWindowSize( + subscription.orderedRetainedWindowSize + missingResultRows, + ) + } + if (subscription.hasOrderedResultForActiveWindow) { + return true + } + + const n = Math.max(missingResultRows, subscription.orderedRowsNeeded) const errorVersion = subscription.lastErrorVersion try { // Local rows may fill the visible window without proving its remote @@ -516,6 +565,52 @@ export class CollectionSubscriber< return true } + private requestUnindexedSnapshot( + subscription: CollectionSubscription, + orderBy: LoadSubsetOptions[`orderBy`], + ): void { + if (this.unindexedSnapshot?.subscription === subscription) return + + const requestToken = Symbol() + this.unindexedSnapshot = { + subscription, + token: requestToken, + } + try { + subscription.requestSnapshot({ + orderBy, + trackLoadSubsetPromise: false, + onLoadSubsetResult: (result, demand) => { + const token = Symbol() + this.unindexedSnapshot = { subscription, token } + if (result instanceof Promise) { + void result.catch(() => { + const current = this.unindexedSnapshot + if ( + current?.subscription === subscription && + current.token === token + ) { + this.unindexedSnapshot = undefined + } + }) + } + this.orderedLoadSubsetResult?.(result, demand) + }, + }) + } catch (error) { + const current = this.unindexedSnapshot + if ( + // requestSnapshot can reentrantly unsubscribe and clear this field. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + current?.subscription === subscription && + current.token === requestToken + ) { + this.unindexedSnapshot = undefined + } + throw error + } + } + private sendChangesToPipelineWithTracking( changes: Iterable>, subscription: CollectionSubscription, @@ -560,8 +655,10 @@ export class CollectionSubscriber< n, subscription.orderedRetainedWindowSize, subscription.orderedBoundaryKey, + subscription.orderedCoverageRevision, ) if (!cursor) { + if (subscription.settleOrderedResultAfterNoProgress()) return if (this.lastNoProgressRequestKey !== this.lastLoadRequestKey) { this.lastNoProgressRequestKey = this.lastLoadRequestKey this.collectionConfigBuilder.recordSubsetError( @@ -582,22 +679,27 @@ export class CollectionSubscriber< // Omit offset so requestLimitedSnapshot can advance based on // the number of rows already loaded (supports offset-based backends). try { - subscription.requestLimitedSnapshot({ - orderBy: cursor.normalizedOrderBy, - limit: n, - minValues: cursor.minValues, - trackLoadSubsetPromise: false, - onLoadSubsetResult: (result, demand) => { - if (result instanceof Promise) { - void result.catch(() => { - if (this.lastLoadRequestKey === loadRequestKey) { - this.lastLoadRequestKey = undefined - } - }) - } - this.orderedLoadSubsetResult?.(result, demand) - }, - }) + this.orderedLoadStartInProgress = true + try { + subscription.requestLimitedSnapshot({ + orderBy: cursor.normalizedOrderBy, + limit: n, + minValues: cursor.minValues, + trackLoadSubsetPromise: false, + onLoadSubsetResult: (result, demand) => { + if (result instanceof Promise) { + void result.catch(() => { + if (this.lastLoadRequestKey === loadRequestKey) { + this.lastLoadRequestKey = undefined + } + }) + } + this.orderedLoadSubsetResult?.(result, demand) + }, + }) + } finally { + this.orderedLoadStartInProgress = false + } } catch (error) { if (this.lastLoadRequestKey === loadRequestKey) { this.lastLoadRequestKey = undefined diff --git a/packages/db/src/query/live/subset-demand-controller.ts b/packages/db/src/query/live/subset-demand-controller.ts index a753086a9f..27c4e0f5dc 100644 --- a/packages/db/src/query/live/subset-demand-controller.ts +++ b/packages/db/src/query/live/subset-demand-controller.ts @@ -26,6 +26,7 @@ export type DemandUpdate = { changed: boolean empty: boolean ready: Promise> | true + releaseFailure?: { error: unknown } } /** @@ -57,6 +58,7 @@ export class SubsetDemandController { } const segments: Array = [] + let releaseFailure: { error: unknown } | undefined for (const segment of previous?.segments ?? []) { if (segment.state !== `failed` && intersects(segment.keys, nextKeys)) { @@ -65,10 +67,14 @@ export class SubsetDemandController { } segment.abortController.abort() - subscription.releaseSnapshot( - segment.where, - segment.abortController.signal, - ) + try { + subscription.releaseSnapshot( + segment.where, + segment.abortController.signal, + ) + } catch (error) { + releaseFailure ??= { error } + } } const coveredKeys = new Set( @@ -100,11 +106,16 @@ export class SubsetDemandController { (ready): ready is Promise => ready instanceof Promise, ) - return { + const update: DemandUpdate = { changed: true, empty: nextKeys.size === 0, ready: pending.length > 0 ? Promise.all(pending) : true, + ...(releaseFailure && { releaseFailure }), } + // A failed physical release remains cleanup debt, but it cannot leave an + // aborted segment representing current logical demand. Commit the demand + // transition first so a later incarnation acquires fresh coverage. + return update } clear(): void { diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index b70842c390..4ba81dd628 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -140,29 +140,50 @@ export function* splitUpdates< } /** - * Filter changes to prevent duplicate inserts to a D2 pipeline. - * Maintains D2 multiplicity at 1 for visible items so that deletes - * properly reduce multiplicity to 0. - * - * Mutates `sentKeys` in place: adds keys on insert, removes on delete. + * Reconcile source changes with the exact rows previously contributed to D2. + * This keeps each source key at multiplicity one and makes every retraction + * match the row identity that D2 originally received. */ -export function filterDuplicateInserts( - changes: Array>, - sentKeys: Set, -): Array> { - const filtered: Array> = [] +export function reconcileChangesForD2< + T extends object, + TKey extends string | number, +>( + changes: Array>, + sentRows: Map, +): Array> { + const reconciled: Array> = [] for (const change of changes) { if (change.type === `insert`) { - if (sentKeys.has(change.key)) { - continue // Skip duplicate - } - sentKeys.add(change.key) - } else if (change.type === `delete`) { - sentKeys.delete(change.key) + if (sentRows.has(change.key)) continue + sentRows.set(change.key, change.value) + reconciled.push(change) + continue + } + + const previousValue = sentRows.get(change.key) + if (change.type === `delete`) { + if (previousValue === undefined) continue + sentRows.delete(change.key) + reconciled.push( + previousValue === change.value + ? change + : { ...change, value: previousValue }, + ) + continue } - filtered.push(change) + + sentRows.set(change.key, change.value) + if (previousValue === undefined) { + reconciled.push({ type: `insert`, key: change.key, value: change.value }) + continue + } + reconciled.push( + previousValue === change.previousValue + ? change + : { ...change, previousValue }, + ) } - return filtered + return reconciled } /** @@ -172,15 +193,19 @@ export function filterDuplicateInserts( * * @param changes - changes to process (deletes are skipped) * @param current - the current biggest value (or undefined if none) - * @param sentKeys - set of keys already sent to D2 (for new-key detection) + * @param sentKeys - lookup of keys already sent to D2 (for new-key detection) * @param comparator - orderBy comparator * @returns `{ biggest, shouldResetLoadKey }` — the new biggest value and * whether the caller should clear its last-load-request-key */ +interface SentKeyLookup { + has: (key: string | number) => boolean +} + export function trackBiggestSentValue( changes: Array>, current: unknown | undefined, - sentKeys: Set, + sentKeys: SentKeyLookup, comparator: (a: any, b: any) => number, ): { biggest: unknown; shouldResetLoadKey: boolean } { let biggest = current @@ -259,6 +284,7 @@ export function computeOrderedLoadCursor( limit: number, demandedPrefix = limit, boundaryKey?: string | number, + progressRevision = 0, ): | { minValues: Array | undefined @@ -290,6 +316,7 @@ export function computeOrderedLoadCursor( boundaryKey: boundaryKey ?? null, offset, demandedPrefix, + progressRevision, }) if (lastLoadRequestKey === loadRequestKey) { return undefined diff --git a/packages/db/src/query/live/window-state.ts b/packages/db/src/query/live/window-state.ts index 11335e9c2c..71fb617231 100644 --- a/packages/db/src/query/live/window-state.ts +++ b/packages/db/src/query/live/window-state.ts @@ -6,6 +6,29 @@ import type { ChangeMessage } from '../../types.js' import type { BasicExpression, OrderBy } from '../ir.js' import type { TotalOrderBoundary } from '../total-order.js' +/** Compute the exact change set between two materialized publications. */ +export function diffPublications< + TRow extends object, + TKey extends string | number, +>( + publishedRows: ReadonlyMap, + desiredRows: ReadonlyMap, +): Array> { + const changes: Array> = [] + for (const [key, previousValue] of publishedRows) { + const value = desiredRows.get(key) + if (value === undefined) { + changes.push({ type: `delete`, key, value: previousValue }) + } else if (!deepEquals(previousValue, value)) { + changes.push({ type: `update`, key, value, previousValue }) + } + } + for (const [key, value] of desiredRows) { + if (!publishedRows.has(key)) changes.push({ type: `insert`, key, value }) + } + return changes +} + /** * Owns the active ordered demand and its retained local coverage. Rows outside * the retained prefix stay in the source collection until a later window @@ -22,6 +45,8 @@ export class WindowState< private hasFullCoverage = false private needsFullRefinement = false private needsPrefixRefresh = false + private locallySettledSize = 0 + private hasOutcomeFreeSettlement = false private hasInitialCoverage = false private hasUnsettledInitialMutation = false private revision = 0 @@ -29,12 +54,19 @@ export class WindowState< private readonly candidateKeys = new Set() private readonly provenanceKeys = new Set() private readonly admittedKeys = new Set() + private sourceSnapshot: + | { + revision: number + rows: Array> + } + | undefined constructor( private readonly collection: CollectionImpl, orderBy: OrderBy, - private readonly where: BasicExpression | undefined, + where: BasicExpression | undefined, targetSize: number, + private readonly expandSourceOrderTies = false, ) { this.totalOrder = new TotalOrder(orderBy, collection) const evaluateWhere = where && compileSingleRowExpression(where) @@ -46,6 +78,25 @@ export class WindowState< } ensureSize(size: number): void { + if ( + size > this.activeSize && + this.hasOutcomeFreeSettlement && + this.locallySettledSize === this.activeSize + ) { + // The same numerical window can recur after a shrink. Give that growth + // a new request generation so an old no-progress key cannot suppress + // the required refresh from the start. + this.revision++ + } + if ( + size < this.activeSize && + this.hasOutcomeFreeSettlement && + this.localPrefixSize >= size + ) { + // A shrink can reuse the already-published local prefix. Keep the + // settlement exact to the smaller window so any later growth refreshes. + this.locallySettledSize = size + } this.activeSize = size this.retainedSize = Math.max(this.retainedSize, size) } @@ -71,6 +122,13 @@ export class WindowState< return this.hasFullCoverage || this.coveredSize >= this.activeSize } + /** Whether this caller may stop loading its exact active window. */ + get satisfiesActiveWindow(): boolean { + return ( + this.coversActiveWindow || this.locallySettledSize === this.activeSize + ) + } + get coveredPrefixSize(): number { return this.coveredSize } @@ -103,6 +161,8 @@ export class WindowState< this.hasFullCoverage = false this.needsFullRefinement = false this.needsPrefixRefresh = false + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = false this.hasUnsettledInitialMutation = false this.candidateKeys.clear() @@ -114,6 +174,8 @@ export class WindowState< rowKeys: ReadonlyArray | undefined, exhausted: boolean, ): void { + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = true if (exhausted) { this.hasUnsettledInitialMutation = false @@ -146,6 +208,8 @@ export class WindowState< requestedPrefix: number, requestRevision: number, ): void { + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.hasInitialCoverage = true if (exhausted) { this.establishFullCoverage() @@ -192,6 +256,7 @@ export class WindowState< * proof. */ recordLocalRequestSatisfaction(requestedPrefix: number): void { + this.hasOutcomeFreeSettlement = true this.candidateKeys.clear() this.provenanceKeys.clear() this.admittedKeys.clear() @@ -199,20 +264,29 @@ export class WindowState< this.admittedKeys.add(change.key) } // Outcome-free completions (`true` and Promise) do not prove - // exhaustion. Only count rows that are now present, so a short synchronous - // page can request another pass until the active prefix is actually filled. - this.coveredSize = Math.min(requestedPrefix, this.admittedKeys.size) + // exhaustion. Keep their exact settled request separate from the applied + // row count so this caller can publish without creating reusable evidence. + if (this.admittedKeys.size >= requestedPrefix) { + this.locallySettledSize = requestedPrefix + } this.needsFullRefinement = false this.needsPrefixRefresh = true } + /** Stop a legacy outcome-free request only after its boundary stops moving. */ + settleLocalRequestAfterNoProgress(): boolean { + if (!this.hasOutcomeFreeSettlement) return false + this.locallySettledSize = this.activeSize + return true + } + admitChanges(changes: ReadonlyArray>): void { if (this.hasFullCoverage) return // Initial applied rows remain candidates until their boundary equivalence // class is refined. Live source changes during that request still belong // to the same ordered prefix and must survive its later settlement. - if (this.admittedKeys.size === 0) { + if (this.admittedKeys.size === 0 && this.locallySettledSize === 0) { if (this.hasInitialCoverage) { if (this.updateKnownPrefix(this.candidateKeys, changes)) { this.revision++ @@ -251,6 +325,8 @@ export class WindowState< this.provenanceKeys.clear() this.needsFullRefinement = false this.needsPrefixRefresh = true + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false } } @@ -352,19 +428,7 @@ export class WindowState< } } - const changes: Array> = [] - for (const [key, previousValue] of publishedRows) { - const value = desired.get(key) - if (value === undefined) { - changes.push({ type: `delete`, key, value: previousValue }) - } else if (!deepEquals(previousValue, value)) { - changes.push({ type: `update`, key, value, previousValue }) - } - } - for (const [key, value] of desired) { - if (!publishedRows.has(key)) changes.push({ type: `insert`, key, value }) - } - return changes + return diffPublications(publishedRows, desired) } private readPrefix(): Array> { @@ -379,6 +443,8 @@ export class WindowState< this.hasFullCoverage = true this.needsFullRefinement = false this.needsPrefixRefresh = false + this.locallySettledSize = 0 + this.hasOutcomeFreeSettlement = false this.coveredSize = Number.POSITIVE_INFINITY this.candidateKeys.clear() this.provenanceKeys.clear() @@ -389,28 +455,66 @@ export class WindowState< allowedKeys: ReadonlySet | undefined, limit?: number, ): Array> { - const rows = this.collection.currentStateAsChanges({ - ...(this.where && { where: this.where }), - orderBy: this.totalOrder.orderBy, - }) as Array> | undefined + const rows = this.readSourceSnapshot().filter(({ value }) => + this.matchesWhere(value), + ) const allowed = allowedKeys === undefined - ? (rows ?? []) - : (rows ?? []).filter((change) => allowedKeys.has(change.key)) - return limit === undefined ? allowed : allowed.slice(0, limit) + ? rows + : rows.filter((change) => allowedKeys.has(change.key)) + if (limit === undefined) return allowed + return this.expandSourceOrderTies + ? this.prefixThroughTieClass(allowed, limit) + : allowed.slice(0, limit) + } + + /** + * A provider orders only by the source-owned query terms. Keep the complete + * boundary equivalence class so D2 can apply the local key tie-breaker and + * any later joined or derived order terms without missing candidates. + */ + private prefixThroughTieClass( + rows: Array>, + limit: number, + ): Array> { + if (limit <= 0 || rows.length <= limit) return rows.slice(0, limit) + + const boundary = rows[limit - 1]! + let end = limit + while ( + end < rows.length && + this.totalOrder.compareRows(boundary.value, rows[end]!.value) === 0 + ) { + end++ + } + return rows.slice(0, end) } private readSourceRows( allowedKeys: ReadonlySet | undefined, limit?: number, ): Array> { - const rows = this.collection.currentStateAsChanges({ - orderBy: this.totalOrder.orderBy, - }) as Array> | undefined + const rows = this.readSourceSnapshot() const allowed = allowedKeys === undefined - ? (rows ?? []) - : (rows ?? []).filter((change) => allowedKeys.has(change.key)) + ? rows + : rows.filter((change) => allowedKeys.has(change.key)) return limit === undefined ? allowed : allowed.slice(0, limit) } + + private readSourceSnapshot(): Array> { + const revision = this.collection._stateRevision + if ( + this.sourceSnapshot !== undefined && + this.sourceSnapshot.revision === revision + ) { + return this.sourceSnapshot.rows + } + + const rows = this.collection.currentStateAsChanges({ + orderBy: this.totalOrder.orderBy, + }) as Array> + this.sourceSnapshot = { revision, rows } + return rows + } } diff --git a/packages/db/src/query/load-subset-options.ts b/packages/db/src/query/load-subset-options.ts index 92d56c8951..691f41cdaa 100644 --- a/packages/db/src/query/load-subset-options.ts +++ b/packages/db/src/query/load-subset-options.ts @@ -1,4 +1,17 @@ +import { + isUint8ArrayCandidate, + readDateTimestamp, + snapshotTemporalEqualityValue, + snapshotUint8ArrayBytes, +} from '../utils/comparison.js' +import { isTemporal } from '../utils.js' import { Func, PropRef, Value } from './ir.js' +import { + assertSnapshotCapableStructuralValue, + getExpressionArgumentValueContext, + snapshotMembershipCandidateValues, +} from './expression-value-context.js' +import type { ExpressionValueContext } from './expression-value-context.js' import type { BasicExpression } from './ir.js' import type { LoadSubsetOptions } from '../types.js' @@ -44,14 +57,9 @@ export function snapshotLoadSubsetDemand( return demand } -type ExpressionCloneContext = - | `exact-output` - | `equality-operand` - | `ordering-operand` - function cloneBasicExpression( expression: BasicExpression, - context: ExpressionCloneContext = `exact-output`, + context: ExpressionValueContext = `exact-output`, ): BasicExpression { switch (expression.type) { case `ref`: @@ -60,58 +68,62 @@ function cloneBasicExpression( return new Value( context === `equality-operand` ? snapshotEqualityValue(expression.value) - : context === `ordering-operand` - ? snapshotStructuralValue(expression.value) - : expression.value, + : context === `membership-candidates` + ? snapshotMembershipCandidates(expression.value) + : context === `ordering-operand` + ? snapshotStructuralOperand(expression.value) + : context === `structural-operand` + ? snapshotStructuralOperand(expression.value) + : expression.value, ) case `func`: return new Func( expression.name, expression.args.map((arg, index) => { - if ( - expression.name === `in` && - index === 1 && - arg.type === `val` && - Array.isArray(arg.value) - ) { - return new Value( - arg.value.map((value) => snapshotEqualityValue(value)), - ) - } - - const argumentContext: ExpressionCloneContext = - expression.name === `eq` - ? `equality-operand` - : isOrderingFunction(expression.name) - ? `ordering-operand` - : `exact-output` + const argumentContext = getExpressionArgumentValueContext( + expression.name, + index, + expression.args.length, + context, + ) return cloneBasicExpression(arg, argumentContext) }), ) } } -function isOrderingFunction(name: string): boolean { - return name === `gt` || name === `gte` || name === `lt` || name === `lte` +function snapshotStructuralOperand(value: T): T { + assertSnapshotCapableStructuralValue(value) + return snapshotStructuralValue(value) } function snapshotEqualityValue(value: T): T { if (value instanceof Date) { - return new Date(value.getTime()) as T + return new Date(readDateTimestamp(value)) as T } if (typeof Buffer !== `undefined` && value instanceof Buffer) { - return Buffer.from(value) as T + return Buffer.from(snapshotUint8ArrayBytes(value)) as T + } + + if (isUint8ArrayCandidate(value)) { + return snapshotUint8ArrayBytes(value) as T } - if (value instanceof Uint8Array) { - return value.slice() as T + if (isTemporal(value)) { + return snapshotTemporalEqualityValue(value) as T } // Other objects use reference equality in predicate identity and comparison. return value } +function snapshotMembershipCandidates(value: T): T { + const candidates = snapshotMembershipCandidateValues(value) + if (candidates === undefined) return value + return candidates.map((candidate) => snapshotEqualityValue(candidate)) as T +} + function snapshotStructuralValue( value: T, seen: WeakMap = new WeakMap(), @@ -155,10 +167,15 @@ function snapshotStructuralValue( } if (Array.isArray(value)) { - const result: Array = [] + const result: Array = new Array(value.length) seen.set(value, result) - for (const item of value) { - result.push(snapshotStructuralValue(item, seen)) + for (const key of Object.keys(value)) { + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + writable: true, + value: snapshotStructuralValue(value[Number(key)], seen), + }) } return result as T } @@ -194,10 +211,15 @@ function snapshotStructuralValue( const result = Object.create(prototype) as Record seen.set(value, result) for (const key of Object.keys(value)) { - result[key] = snapshotStructuralValue( - (value as Record)[key], - seen, - ) + Object.defineProperty(result, key, { + configurable: true, + enumerable: true, + writable: true, + value: snapshotStructuralValue( + (value as Record)[key], + seen, + ), + }) } return result as T } diff --git a/packages/db/src/query/load-subset-outcome.ts b/packages/db/src/query/load-subset-outcome.ts index f95d72b94d..fb5fc16003 100644 --- a/packages/db/src/query/load-subset-outcome.ts +++ b/packages/db/src/query/load-subset-outcome.ts @@ -13,6 +13,23 @@ const loadSubsetResultDemandMatchers = new WeakMap< (options: LoadSubsetOptions) => boolean >() +function snapshotAppliedRowKeys( + appliedRowKeys: ReadonlyArray | undefined, +): ReadonlyArray | undefined { + if (appliedRowKeys === undefined) return undefined + + const snapshot: Array = [] + for (const key of appliedRowKeys) { + if (typeof key !== `string` && typeof key !== `number`) { + throw new TypeError( + `loadSubset appliedRowKeys must contain only string or number keys`, + ) + } + snapshot.push(key) + } + return Object.freeze(snapshot) +} + export function recordLoadSubsetPromiseDemandMatcher( promise: Promise, matches: (options: LoadSubsetOptions) => boolean, @@ -27,8 +44,13 @@ export function recordLoadSubsetResultDemandMatcher( if (typeof result !== `object`) return result // Give each physical acquisition its own result identity. A source may reuse - // one result object across calls with different demands. - const retainedResult = { ...result } + // one result object across calls with different demands. Snapshot nested + // source evidence here too, before the caller publishes coverage from it. + const appliedRowKeys = snapshotAppliedRowKeys(result.appliedRowKeys) + const retainedResult: LoadSubsetResult = { + hasMore: result.hasMore, + ...(appliedRowKeys === undefined ? {} : { appliedRowKeys }), + } loadSubsetResultDemandMatchers.set(retainedResult, matches) return retainedResult } @@ -54,7 +76,7 @@ export function createAppliedLoadSubsetOutcome( generation: number, sourceResult: void | LoadSubsetResult, ): AppliedLoadSubsetOutcome { - const appliedRowKeys = sourceResult?.appliedRowKeys + const appliedRowKeys = snapshotAppliedRowKeys(sourceResult?.appliedRowKeys) return { collectionId, demand, @@ -65,9 +87,7 @@ export function createAppliedLoadSubsetOutcome( : sourceResult?.hasMore === false ? `exhausted` : `unknown`, - ...(appliedRowKeys === undefined - ? {} - : { appliedRowKeys: Object.freeze([...appliedRowKeys]) }), + ...(appliedRowKeys === undefined ? {} : { appliedRowKeys }), } } diff --git a/packages/db/src/query/predicate-utils.ts b/packages/db/src/query/predicate-utils.ts index 3241f9e55d..921e017661 100644 --- a/packages/db/src/query/predicate-utils.ts +++ b/packages/db/src/query/predicate-utils.ts @@ -380,15 +380,10 @@ export function minusWherePredicates( ) } - // If from is undefined then we are asking for all data - // so we need to load all data minus what we already loaded - // i.e. we need to load NOT(subtractPredicate) if (fromPredicate === undefined) { - return { - type: `func`, - name: `not`, - args: [subtractPredicate], - } as BasicExpression + // NOT(subtractPredicate) would also filter UNKNOWN rows under three-valued + // logic, even though those rows were not loaded. Fall back to the full request. + return null } // Check if fromPredicate is entirely contained in subtractPredicate @@ -404,21 +399,25 @@ export function minusWherePredicates( ) if (commonConditions.length > 0) { // Extract predicates without common conditions - const fromWithoutCommon = removeConditions(fromPredicate, commonConditions) - const subtractWithoutCommon = removeConditions( + const fromRemoval = removeConditions(fromPredicate, commonConditions) + const subtractRemoval = removeConditions( subtractPredicate, commonConditions, ) - // Recursively compute difference on simplified predicates - const simplifiedDifference = minusWherePredicates( - fromWithoutCommon, - subtractWithoutCommon, - ) + // Recurse only when both operands lost at least one flattened AND term. + // This strict decrease is the termination measure for common-condition + // simplification. + if (fromRemoval.removed && subtractRemoval.removed) { + const simplifiedDifference = minusWherePredicates( + fromRemoval.predicate, + subtractRemoval.predicate, + ) - if (simplifiedDifference !== null) { - // Combine the simplified difference with common conditions - return combineConditions([...commonConditions, simplifiedDifference]) + if (simplifiedDifference !== null) { + // Combine the simplified difference with common conditions + return combineConditions([...commonConditions, simplifiedDifference]) + } } } @@ -1037,35 +1036,40 @@ function extractAllConditions( /** * Remove specified conditions from a predicate. - * Returns the predicate with the specified conditions removed, or undefined if all conditions are removed. + * Reports whether removal made progress and returns the remaining predicate, + * or undefined when all conditions were removed. */ function removeConditions( predicate: BasicExpression, conditionsToRemove: Array>, -): BasicExpression | undefined { - if (predicate.type === `func` && predicate.name === `and`) { - const remainingArgs = predicate.args.filter( - (arg) => - !conditionsToRemove.some((cond) => - areExpressionsEqual(arg as BasicExpression, cond), - ), +): { + predicate: BasicExpression | undefined + removed: boolean +} { + const conditions = extractAllConditions(predicate) + const remainingConditions = [...conditions] + + // Consume one occurrence per common term so duplicate predicates remain. + for (const conditionToRemove of conditionsToRemove) { + const matchingIndex = remainingConditions.findIndex((condition) => + areExpressionsEqual(condition, conditionToRemove), ) - - if (remainingArgs.length === 0) { - return undefined - } else if (remainingArgs.length === 1) { - return remainingArgs[0]! - } else { - return { - type: `func`, - name: `and`, - args: remainingArgs, - } as BasicExpression + if (matchingIndex !== -1) { + remainingConditions.splice(matchingIndex, 1) } } - // For non-AND predicates, don't remove anything - return predicate + if (remainingConditions.length === conditions.length) { + return { predicate, removed: false } + } + + return { + predicate: + remainingConditions.length === 0 + ? undefined + : combineConditions(remainingConditions), + removed: true, + } } /** diff --git a/packages/db/src/query/subset-dedupe.ts b/packages/db/src/query/subset-dedupe.ts index f342382e7e..a2673c4474 100644 --- a/packages/db/src/query/subset-dedupe.ts +++ b/packages/db/src/query/subset-dedupe.ts @@ -24,11 +24,30 @@ type SharedAbortLease = { dispose: () => void } -type InflightCall = { +type LogicalLoadReservation = { + generation: number + invalidatesCoverage: boolean + active: boolean + acquisition?: AcquisitionOwnership +} + +type AcquisitionOwnership = { + matchesPhysicalRequest: (options: LoadSubsetOptions) => boolean + generation: number + reservations: Set +} + +type InflightCall = AcquisitionOwnership & { options: LoadSubsetOptions promise: Promise lease: SharedAbortLease - matchesPhysicalRequest: (options: LoadSubsetOptions) => boolean + trackable: boolean +} + +function isInflightCall( + acquisition: AcquisitionOwnership, +): acquisition is InflightCall { + return `lease` in acquisition } /** @@ -78,11 +97,22 @@ export class DeduplicatedLoadSubset { // Each entry also owns the shared cancellation lease for its requesters. private inflightCalls: Array = [] + // Retain exact acquisition ownership after settlement so a later exact + // owner can share its evidence until the final logical lease releases. + private exactAcquisitions: Array = [] + // Generation counter to invalidate in-flight requests after reset() // When reset() is called, this increments, and any in-flight completion handlers // check if their captured generation matches before updating tracking state private generation = 0 + // Core releases the exact options object that it passed to loadSubset. + // A queue preserves that identity when one object is reused across calls. + private ownerReservations = new WeakMap< + LoadSubsetOptions, + Array + >() + constructor(opts: { loadSubset: LoadSubsetFn onDeduplicate?: (options: LoadSubsetOptions) => void @@ -104,6 +134,39 @@ export class DeduplicatedLoadSubset { loadSubset = ( options: LoadSubsetOptions, ): true | Promise => { + const reservation = this.reserveOwner(options, options.limit !== 0) + try { + // A zero-width window has no rows to acquire and establishes no coverage. + // Keep only its logical reservation so reused option objects still + // release in invocation order without invalidating another request. + if (options.limit === 0) { + this.onDeduplicate?.(options) + return true + } + + return this.loadSubsetRequest(options, reservation) + } catch (error) { + this.removeOwnerReservation(options, reservation) + throw error + } + } + + private loadSubsetRequest( + options: LoadSubsetOptions, + reservation: LogicalLoadReservation, + ): true | Promise { + const exactAcquisition = this.exactAcquisitions.find( + (acquisition) => + acquisition.generation === this.generation && + acquisition.matchesPhysicalRequest(options), + ) + if (exactAcquisition) { + exactAcquisition.reservations.add(reservation) + reservation.acquisition = exactAcquisition + this.onDeduplicate?.(options) + return true + } + // If we've loaded all data, everything is covered if (this.hasLoadedAllData) { this.onDeduplicate?.(options) @@ -140,6 +203,8 @@ export class DeduplicatedLoadSubset { ) if (matchingInflight !== undefined) { + matchingInflight.reservations.add(reservation) + reservation.acquisition = matchingInflight matchingInflight.lease.attach(options.signal) // An in-flight call will load data that covers this request // Every requester shares the physical work and cancellation lease. A @@ -168,10 +233,7 @@ export class DeduplicatedLoadSubset { ...options, signal: lease.signal, }) - const loadOptions = cloneLoadSubsetOptions({ - ...options, - signal: lease.signal, - }) + const loadOptions = cloneLoadSubsetOptions(trackingOptions) if ( this.unlimitedWhere !== undefined && options.limit === undefined && @@ -191,6 +253,7 @@ export class DeduplicatedLoadSubset { isLoadSubsetRequestSubsumedBy(physicalRequest, candidate) // Call underlying loadSubset to load the missing data + const requestGeneration = this.generation let resultPromise: true | Promise try { resultPromise = this._loadSubset(loadOptions) @@ -201,43 +264,94 @@ export class DeduplicatedLoadSubset { // Handle both sync (true) and async (Promise) return values if (resultPromise === true) { - if (!lease.aborted) this.updateTracking(trackingOptions) + if ( + requestGeneration === this.generation && + reservation.active && + !lease.aborted + ) { + this.updateTracking(trackingOptions) + const acquisition: AcquisitionOwnership = { + matchesPhysicalRequest, + generation: requestGeneration, + reservations: new Set([reservation]), + } + reservation.acquisition = acquisition + this.exactAcquisitions.push(acquisition) + } lease.dispose() return true } else { - // Async return - track the promise and update tracking after it resolves - - // Capture the current generation - this lets us detect if reset() was called - // while this request was in-flight, so we can skip updating tracking state - const capturedGeneration = this.generation - - // We need to create a reference to the in-flight entry so we can remove it later - const inflightEntry = { - options: trackingOptions, - lease, - matchesPhysicalRequest, - promise: resultPromise + const ownsRequestAtAdapterReturn = + requestGeneration === this.generation && reservation.active + // Assimilate foreign Promise implementations before installing stateful + // handlers. Calling their `then` now preserves reentrant adapter effects, + // while the native bridge defers fulfillment, rejection, and thrown errors + // until the entry below exists. + const normalizedResultPromise = new Promise( + (resolve, reject) => { + resultPromise.then(resolve, reject) + }, + ) + const installation: { entry: InflightCall | undefined } = { + entry: undefined, + } + let observedPromise: Promise + try { + observedPromise = normalizedResultPromise .then((result) => { + // Retain every fallible adapter result field before publishing + // coverage. A rejected caller must never leave reusable evidence. + const retainedResult = recordLoadSubsetResultDemandMatcher( + result, + matchesPhysicalRequest, + ) // Only update tracking if this request is still from the current generation // If reset() was called, the generation will have incremented and we should // not repopulate the state that was just cleared - if (capturedGeneration === this.generation && !lease.aborted) { + if ( + installation.entry?.trackable && + installation.entry.generation === this.generation && + !lease.aborted + ) { this.updateTracking(trackingOptions) + this.exactAcquisitions.push(installation.entry) } - return recordLoadSubsetResultDemandMatcher( - result, - matchesPhysicalRequest, - ) + return retainedResult }) .finally(() => { // Always remove from in-flight array on completion OR rejection // This ensures failed requests can be retried instead of being cached forever - const index = this.inflightCalls.indexOf(inflightEntry) - if (index !== -1) { - this.inflightCalls.splice(index, 1) + if (installation.entry) { + const index = this.inflightCalls.indexOf(installation.entry) + if (index !== -1) { + this.inflightCalls.splice(index, 1) + } } lease.dispose() - }), + }) + } catch (error) { + lease.dispose() + throw error + } + const inflightEntry: InflightCall = { + options: trackingOptions, + lease, + matchesPhysicalRequest, + generation: requestGeneration, + trackable: ownsRequestAtAdapterReturn, + reservations: ownsRequestAtAdapterReturn + ? new Set([reservation]) + : new Set(), + promise: observedPromise, + } + installation.entry = inflightEntry + const ownsRequestAfterHandlerInstallation = + requestGeneration === this.generation && reservation.active + inflightEntry.trackable = ownsRequestAfterHandlerInstallation + if (ownsRequestAfterHandlerInstallation) { + reservation.acquisition = inflightEntry + } else { + inflightEntry.reservations.clear() } recordLoadSubsetPromiseDemandMatcher( @@ -246,7 +360,9 @@ export class DeduplicatedLoadSubset { ) // Store the in-flight entry so concurrent subset calls can wait for it - this.inflightCalls.push(inflightEntry) + if (ownsRequestAfterHandlerInstallation) { + this.inflightCalls.push(inflightEntry) + } return projectLoadSubsetResultForCaller( inflightEntry.promise, options, @@ -264,15 +380,27 @@ export class DeduplicatedLoadSubset { * across live-query lifetimes must return this method as their unloadSubset * callback. * - * The reset is intentionally conservative. One released request may clear - * evidence still useful to another owner, causing a later refetch, but it can - * never reuse evidence for rows that core no longer retains. Until adapters - * report which retained rows came from which demand, the settled-case cost is - * bounded to one new physical request for each distinct demand revisited - * before deduplication state is rebuilt. + * Settled exact evidence and in-flight work are tracked by logical owner, so + * a late release cannot retire a newer generation or work that another owner + * still needs. Broader inferred coverage remains conservative. Core must + * release the same options object that it passed to loadSubset; unmatched + * releases are no-ops. */ - unloadSubset = (_options: LoadSubsetOptions): void => { - this.reset() + unloadSubset = (options: LoadSubsetOptions): void => { + const reservation = this.shiftOwnerReservation(options) + // A synchronous adapter throw never established helper state. Core may + // still release that logical demand later, but it must not invalidate a + // newer request that happens to use equivalent options. + if (!reservation || reservation.generation !== this.generation) return + if (!reservation.invalidatesCoverage) return + + const acquisition = reservation.acquisition + if (acquisition) { + acquisition.reservations.delete(reservation) + if (acquisition.reservations.size > 0) return + this.retireAcquisition(acquisition) + } + this.clearInferredTracking() } /** @@ -284,15 +412,79 @@ export class DeduplicatedLoadSubset { * state after the reset. This prevents old requests from repopulating cleared state. */ reset(): void { - this.unlimitedWhere = undefined - this.hasLoadedAllData = false - this.limitedCalls = [] + this.clearLoadedTracking() + for (const inflight of this.inflightCalls) { + inflight.trackable = false + inflight.lease.dispose() + } this.inflightCalls = [] // Increment generation to invalidate any in-flight completion handlers // This ensures requests that were started before reset() don't repopulate the state this.generation++ } + private reserveOwner( + options: LoadSubsetOptions, + invalidatesCoverage: boolean, + ): LogicalLoadReservation { + const reservation = { + generation: this.generation, + invalidatesCoverage, + active: true, + } + const reservations = this.ownerReservations.get(options) + if (reservations) reservations.push(reservation) + else this.ownerReservations.set(options, [reservation]) + return reservation + } + + private shiftOwnerReservation( + options: LoadSubsetOptions, + ): LogicalLoadReservation | undefined { + const reservations = this.ownerReservations.get(options) + const reservation = reservations?.shift() + if (reservation) reservation.active = false + if (reservations?.length === 0) this.ownerReservations.delete(options) + return reservation + } + + private removeOwnerReservation( + options: LoadSubsetOptions, + reservation: LogicalLoadReservation, + ): void { + reservation.active = false + const reservations = this.ownerReservations.get(options) + const reservationIndex = reservations?.indexOf(reservation) ?? -1 + if (reservationIndex !== -1) reservations!.splice(reservationIndex, 1) + if (reservations?.length === 0) this.ownerReservations.delete(options) + + const acquisition = reservation.acquisition + if (!acquisition) return + acquisition.reservations.delete(reservation) + if (acquisition.reservations.size > 0) return + this.retireAcquisition(acquisition) + } + + private retireAcquisition(acquisition: AcquisitionOwnership): void { + const exactIndex = this.exactAcquisitions.indexOf(acquisition) + if (exactIndex !== -1) this.exactAcquisitions.splice(exactIndex, 1) + if (!isInflightCall(acquisition)) return + acquisition.trackable = false + const inflightIndex = this.inflightCalls.indexOf(acquisition) + if (inflightIndex !== -1) this.inflightCalls.splice(inflightIndex, 1) + } + + private clearLoadedTracking(): void { + this.clearInferredTracking() + this.exactAcquisitions = [] + } + + private clearInferredTracking(): void { + this.unlimitedWhere = undefined + this.hasLoadedAllData = false + this.limitedCalls = [] + } + private updateTracking(options: LoadSubsetOptions): void { // Update tracking based on whether this was a limited or unlimited call if (options.limit === undefined && options.cursor === undefined) { diff --git a/packages/db/src/query/total-order.ts b/packages/db/src/query/total-order.ts index 5a4629d697..0b720e6e86 100644 --- a/packages/db/src/query/total-order.ts +++ b/packages/db/src/query/total-order.ts @@ -69,14 +69,21 @@ export class TotalOrder< return { key, values: this.values(row) } } + /** Compare only the query-visible order terms, without the local key tie-breaker. */ + compareRows(left: TRow, right: TRow): number { + for (const { extract, compare } of this.terms) { + const result = compare(extract(left), extract(right)) + if (result !== 0) return result + } + return 0 + } + compareEntries( left: readonly [TKey, TRow], right: readonly [TKey, TRow], ): number { - for (const { extract, compare } of this.terms) { - const result = compare(extract(left[1]), extract(right[1])) - if (result !== 0) return result - } + const result = this.compareRows(left[1], right[1]) + if (result !== 0) return result return compareKeys(left[0], right[0]) } diff --git a/packages/db/src/scheduler.ts b/packages/db/src/scheduler.ts index d87ac05319..aaf8475ce5 100644 --- a/packages/db/src/scheduler.ts +++ b/packages/db/src/scheduler.ts @@ -1,3 +1,5 @@ +import { runAllCallbacks } from './utils/callbacks.js' + /** * Identifier used to scope scheduled work. Maps to a transaction id for live queries. */ @@ -187,8 +189,9 @@ export class Scheduler { /** Clear all scheduled jobs for a context. */ clear(contextId: SchedulerContextId): void { this.contexts.delete(contextId) - // Notify listeners that this context was cleared - this.clearListeners.forEach((listener) => listener(contextId)) + runAllCallbacks( + [...this.clearListeners].map((listener) => () => listener(contextId)), + ) } /** Register a listener to be notified when a context is cleared. */ @@ -221,7 +224,12 @@ export class Scheduler { export const transactionScopedScheduler = new Scheduler() -let activePublicationContext: SchedulerContextId | undefined +type ActivePublication = { + contextId: SchedulerContextId + failure?: { error: unknown } +} + +let activePublication: ActivePublication | undefined /** * Returns the Collection publication that currently owns synchronous change @@ -229,7 +237,19 @@ let activePublicationContext: SchedulerContextId | undefined * observe one committed batch. */ export function getActivePublicationContext(): SchedulerContextId | undefined { - return activePublicationContext + return activePublication?.contextId +} + +/** + * Retains the first failure produced by a nested publication effect. The + * outer publication surfaces it only after all work already queued in the + * shared context has run. + */ +export function deferPublicationFailure(error: unknown): void { + if (!activePublication) { + throw new Error(`Cannot defer a failure outside a publication context`) + } + activePublication.failure ??= { error } } /** @@ -238,18 +258,33 @@ export function getActivePublicationContext(): SchedulerContextId | undefined { * only after every subscriber to the original committed batch has observed it. */ export function withPublicationContext(publish: () => T): T { - if (activePublicationContext !== undefined) return publish() + if (activePublication) return publish() const contextId = Symbol(`collection-publication`) - activePublicationContext = contextId + const publication: ActivePublication = { contextId } + activePublication = publication try { const result = publish() - transactionScopedScheduler.flush(contextId) + let graphFailure: { error: unknown } | undefined + try { + transactionScopedScheduler.flush(contextId) + } catch (error) { + graphFailure = { error } + } + if (publication.failure) { + throw publication.failure.error + } + if (graphFailure) throw graphFailure.error return result } catch (error) { - transactionScopedScheduler.clear(contextId) + try { + transactionScopedScheduler.clear(contextId) + } catch { + // Clearing is cleanup for an already failed publication. Its own failure + // cannot replace the exact publication or graph failure that caused it. + } throw error } finally { - activePublicationContext = undefined + activePublication = undefined } } diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index c30ee78f05..194af4f435 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -535,6 +535,7 @@ class Transaction> { if (this.state === `completed`) { throw new TransactionAlreadyCompletedRollbackError() } + if (this.state === `failed`) return this this.setState(`failed`) @@ -636,11 +637,26 @@ class Transaction> { transaction: this as unknown as TransactionWithMutations, }) + // Rollback can win while mutationFn is in flight. Its failed state and + // rejected persistence receipt are terminal for this commit attempt. + // TypeScript keeps the entry-state narrowing across the await, although + // rollback may reenter and change it while mutationFn is pending. + if ((this.state as TransactionState) !== `persisting`) { + return this + } + this.setState(`completed`) this.touchCollection() this.isPersisted.resolve(this) } catch (error) { + // A manual or cascading rollback can also win while mutationFn is in + // flight. Its terminal outcome owns this commit attempt, so a late + // rejection cannot run rollback again or affect newer transactions. + if ((this.state as TransactionState) !== `persisting`) { + return this + } + // Preserve the original error for rethrowing const originalError = error instanceof Error ? error : new Error(String(error)) diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index e652087419..925baece40 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -222,6 +222,10 @@ const temporalTypes = new Set([ `Temporal.ZonedDateTime`, ]) +/** + * A Temporal value. Objects that claim a Temporal tag are expected to obey the + * Temporal contract, including immutable value semantics. + */ export interface TemporalLike { [Symbol.toStringTag]: string toString: () => string diff --git a/packages/db/src/utils/callbacks.ts b/packages/db/src/utils/callbacks.ts new file mode 100644 index 0000000000..1b0aba4859 --- /dev/null +++ b/packages/db/src/utils/callbacks.ts @@ -0,0 +1,12 @@ +/** Attempt every callback, then rethrow the first exact failure value. */ +export function runAllCallbacks(callbacks: Iterable<() => void>): void { + let firstFailure: { error: unknown } | undefined + for (const callback of callbacks) { + try { + callback() + } catch (error) { + firstFailure ??= { error } + } + } + if (firstFailure) throw firstFailure.error +} diff --git a/packages/db/src/utils/comparison.ts b/packages/db/src/utils/comparison.ts index 26aa399433..61d52b8a3e 100644 --- a/packages/db/src/utils/comparison.ts +++ b/packages/db/src/utils/comparison.ts @@ -1,5 +1,6 @@ import { isTemporal } from '../utils' import type { CompareOptions } from '../query/builder/types' +import type { TemporalLike } from '../utils' // WeakMap to store stable IDs for objects const objectIds = new WeakMap() @@ -28,17 +29,65 @@ function getObjectId(obj: object): number { export function isUnorderable(value: any): boolean { return ( (typeof value === `number` && Number.isNaN(value)) || - (value instanceof Date && Number.isNaN(value.getTime())) + (value instanceof Date && Number.isNaN(readDateTimestamp(value))) ) } +/** Read a Date's internal timestamp without invoking an instance override. */ +export function readDateTimestamp(value: Date): number { + return Reflect.apply(Date.prototype.getTime, value, []) +} + +const typedArrayTagGetter = Object.getOwnPropertyDescriptor( + Object.getPrototypeOf(Uint8Array.prototype), + Symbol.toStringTag, +)?.get + +/** Whether a value has intrinsic Uint8Array slots, independent of its realm. */ +export function hasIntrinsicUint8ArraySlots( + value: unknown, +): value is Uint8Array { + return ( + ArrayBuffer.isView(value) && + typedArrayTagGetter !== undefined && + Reflect.apply(typedArrayTagGetter, value, []) === `Uint8Array` + ) +} + +/** + * Whether a value must use binary equality semantics. Local prototype claims + * enter this path so slot-less proxies are rejected instead of becoming opaque. + */ +export function isUint8ArrayCandidate(value: unknown): value is Uint8Array { + return value instanceof Uint8Array || hasIntrinsicUint8ArraySlots(value) +} + +/** Copy a Uint8Array's internal bytes without invoking custom iteration. */ +export function snapshotUint8ArrayBytes(value: Uint8Array): Uint8Array { + if (!hasIntrinsicUint8ArraySlots(value)) { + throw new TypeError( + `Cannot snapshot binary equality value without intrinsic typed-array slots`, + ) + } + return new Uint8Array(value) +} + /** * Universal comparison function for all data types * Handles null/undefined, strings, arrays, dates, objects, and primitives * Always sorts null/undefined values first */ -export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { - const { nulls } = opts +const compareAscending = ( + a: any, + b: any, + opts: CompareOptions, + invertNulls: boolean, +): number => { + const nulls = invertNulls + ? opts.nulls === `first` + ? `last` + : `first` + : opts.nulls // Handle null/undefined if (a == null && b == null) return 0 @@ -65,19 +114,22 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { // if a and b are both arrays, compare them element by element if (Array.isArray(a) && Array.isArray(b)) { - for (let i = 0; i < Math.min(a.length, b.length); i++) { - const result = ascComparator(a[i], b[i], opts) + const aLength = a.length + const bLength = b.length + const commonLength = Math.min(aLength, bLength) + for (let i = 0; i < commonLength; i++) { + const result = compareAscending(a[i], b[i], opts, invertNulls) if (result !== 0) { return result } } // All elements are equal up to the minimum length - return a.length - b.length + return aLength - bLength } // If both are dates, compare them if (a instanceof Date && b instanceof Date) { - return a.getTime() - b.getTime() + return readDateTimestamp(a) - readDateTimestamp(b) } // If both are Temporal objects, use compareTemporalValues for correct semantic ordering @@ -108,6 +160,9 @@ export const ascComparator = (a: any, b: any, opts: CompareOptions): number => { return 0 } +export const ascComparator = (a: any, b: any, opts: CompareOptions): number => + compareAscending(a, b, opts, false) + /** * Descending comparator function for ordering values * Handles null/undefined as largest values (opposite of ascending) @@ -116,12 +171,7 @@ export const descComparator = ( a: unknown, b: unknown, opts: CompareOptions, -): number => { - return ascComparator(b, a, { - ...opts, - nulls: opts.nulls === `first` ? `last` : `first`, - }) -} +): number => compareAscending(b, a, opts, true) export function makeComparator( opts: CompareOptions, @@ -146,35 +196,98 @@ export const defaultComparator = makeComparator({ * Compare two Uint8Arrays for content equality */ function areUint8ArraysEqual(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) { + const aBytes = snapshotUint8ArrayBytes(a) + const bBytes = snapshotUint8ArrayBytes(b) + if (aBytes.byteLength !== bBytes.byteLength) { return false } - for (let i = 0; i < a.byteLength; i++) { - if (a[i] !== b[i]) { + for (let i = 0; i < aBytes.byteLength; i++) { + if (aBytes[i] !== bBytes[i]) { return false } } return true } -/** - * Threshold for normalizing Uint8Arrays to string representations. - * Arrays larger than this will use reference equality to avoid memory overhead. - * 128 bytes is enough for common ID formats (ULIDs are 16 bytes, UUIDs are 16 bytes) - * while avoiding excessive string allocation for large binary data. - */ -const UINT8ARRAY_NORMALIZE_THRESHOLD = 128 - /** * Sentinel value representing undefined in normalized form. * This allows distinguishing between "start from beginning" (undefined parameter) * and "start from the key undefined" (actual undefined value in the tree). */ -export const UNDEFINED_SENTINEL = `__TS_DB_BTREE_UNDEFINED_VALUE__` +const NORMALIZED_KEY_PREFIX = `\u0000tanstack-db:` + +function normalizedKey(kind: string, value: string): string { + return `${NORMALIZED_KEY_PREFIX}${kind}:${value}` +} + +export const UNDEFINED_SENTINEL = normalizedKey(`undefined`, ``) const UNORDERABLE_BTREE_SENTINEL = Object.freeze({ kind: `tanstack-db-unorderable`, }) +/** Clone a Temporal equality value without trusting mutable brand lookalikes. */ +export function snapshotTemporalEqualityValue( + value: TemporalLike, +): TemporalLike { + const tag = value[Symbol.toStringTag] + const prototype = Object.getPrototypeOf(value) + const constructorDescriptor = + prototype === null + ? undefined + : Object.getOwnPropertyDescriptor(prototype, `constructor`) + const constructor = constructorDescriptor?.value + const fromDescriptor = + typeof constructor === `function` + ? Object.getOwnPropertyDescriptor(constructor, `from`) + : undefined + const toStringDescriptor = + prototype === null + ? undefined + : Object.getOwnPropertyDescriptor(prototype, `toString`) + const brandAccessorName = TEMPORAL_BRAND_ACCESSORS[tag] + const brandAccessorDescriptor = + prototype === null || brandAccessorName === undefined + ? undefined + : Object.getOwnPropertyDescriptor(prototype, brandAccessorName) + if ( + typeof constructor !== `function` || + typeof fromDescriptor?.value !== `function` || + typeof toStringDescriptor?.value !== `function` || + typeof brandAccessorDescriptor?.get !== `function` + ) { + throw new TypeError(`Cannot snapshot ${tag} equality value`) + } + + // Temporal accessors brand-check their receiver's internal slots. A tag plus + // constructor-shaped methods is not enough to establish a genuine value. + Reflect.apply(brandAccessorDescriptor.get, value, []) + const serialized = Reflect.apply(toStringDescriptor.value, value, []) + const snapshot = Reflect.apply(fromDescriptor.value, constructor, [ + serialized, + ]) + if ( + snapshot === value || + !isTemporal(snapshot) || + snapshot[Symbol.toStringTag] !== tag || + Object.getPrototypeOf(snapshot) !== prototype + ) { + throw new TypeError(`Cannot snapshot ${tag} equality value`) + } + Reflect.apply(brandAccessorDescriptor.get, snapshot, []) + return snapshot +} + +const TEMPORAL_BRAND_ACCESSORS: Readonly> = { + 'Temporal.Duration': `years`, + 'Temporal.Instant': `epochNanoseconds`, + 'Temporal.PlainDate': `year`, + 'Temporal.PlainDateTime': `year`, + 'Temporal.PlainMonthDay': `day`, + 'Temporal.PlainTime': `hour`, + 'Temporal.PlainYearMonth': `year`, + 'Temporal.ZonedDateTime': `epochNanoseconds`, +} + /** * Normalize a value for comparison and Map key usage * Converts values that can't be directly compared or used as Map keys @@ -184,33 +297,39 @@ const UNORDERABLE_BTREE_SENTINEL = Object.freeze({ * for BTree index operations that need to distinguish undefined values. */ export function normalizeValue(value: any): any { + // Internal normalized keys occupy a reserved string domain. Escape user + // strings in that domain so a literal cannot equal a binary or Temporal key. + if (typeof value === `string`) { + return value.startsWith(NORMALIZED_KEY_PREFIX) + ? normalizedKey(`string`, value) + : value + } + if (typeof value !== `object` || value === null) { return value } if (value instanceof Date) { - return value.getTime() + return readDateTimestamp(value) } if (isTemporal(value)) { - return `__temporal__${value[Symbol.toStringTag]}__${value.toString()}` + return normalizedKey( + `temporal`, + `${value[Symbol.toStringTag]}:${value.toString()}`, + ) } // Normalize Uint8Arrays/Buffers to a string representation for Map key usage // This enables content-based equality for binary data like ULIDs - const isUint8Array = - (typeof Buffer !== `undefined` && value instanceof Buffer) || - value instanceof Uint8Array - - if (isUint8Array) { - // Only normalize small arrays to avoid memory overhead for large binary data - if (value.byteLength <= UINT8ARRAY_NORMALIZE_THRESHOLD) { - // Convert to a string representation that can be used as a Map key - // Use a special prefix to avoid collisions with user strings - return `__u8__${Array.from(value).join(`,`)}` - } - // For large arrays, fall back to reference equality - // Users working with large binary data should use a derived key if needed + if (isUint8ArrayCandidate(value)) { + // Convert to a string representation that can be used as a Map key. + // Equality compares every binary value by content, so index keys must not + // switch to reference identity at an arbitrary byte length. + return normalizedKey( + `binary`, + Array.from(snapshotUint8ArrayBytes(value)).join(`,`), + ) } return value @@ -320,12 +439,8 @@ export function areValuesEqual(a: any, b: any): boolean { } // Check for Uint8Array/Buffer comparison - const aIsUint8Array = - (typeof Buffer !== `undefined` && a instanceof Buffer) || - a instanceof Uint8Array - const bIsUint8Array = - (typeof Buffer !== `undefined` && b instanceof Buffer) || - b instanceof Uint8Array + const aIsUint8Array = isUint8ArrayCandidate(a) + const bIsUint8Array = isUint8ArrayCandidate(b) // If both are Uint8Arrays, compare by content if (aIsUint8Array && bIsUint8Array) { diff --git a/packages/db/tests/collection-auto-index.test.ts b/packages/db/tests/collection-auto-index.test.ts index 4fdaac0127..b4e25fdd8f 100644 --- a/packages/db/tests/collection-auto-index.test.ts +++ b/packages/db/tests/collection-auto-index.test.ts @@ -252,11 +252,15 @@ describe(`Collection Auto-Indexing`, () => { it(`should create auto-indexes for transformed fields of subqueries when autoIndex is "eager"`, async () => {}) - it(`should not create duplicate auto-indexes for the same field`, async () => { + it(`should not create duplicate auto-indexes when locale options are omitted`, async () => { const autoIndexCollection = createCollection({ getKey: (item) => item.id, autoIndex: `eager`, defaultIndexType: BTreeIndex, + defaultStringCollation: { + stringSort: `locale`, + localeOptions: { sensitivity: undefined }, + }, startSync: true, sync: { sync: ({ begin, write, commit, markReady }) => { diff --git a/packages/db/tests/collection-change-events.test.ts b/packages/db/tests/collection-change-events.test.ts index 085af31f08..dee88c6d59 100644 --- a/packages/db/tests/collection-change-events.test.ts +++ b/packages/db/tests/collection-change-events.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' -import { currentStateAsChanges } from '../src/collection/change-events.js' +import { + createFilterFunctionFromExpression, + currentStateAsChanges, +} from '../src/collection/change-events.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' import { BTreeIndex } from '../src/indexes/btree-index.js' @@ -13,6 +16,26 @@ interface TestUser { status: `active` | `inactive` } +it(`treats predicate evaluation failures as nonmatches`, () => { + const filter = createFilterFunctionFromExpression( + new Func(`eq`, [new PropRef([`status`]), new Value(`active`)]), + ) + const row = { + id: `1`, + name: `Ada`, + age: 36, + score: 100, + status: `active`, + } as TestUser + Object.defineProperty(row, `status`, { + get: () => { + throw new Error(`predicate evaluation failed`) + }, + }) + + expect(filter(row)).toBe(false) +}) + describe(`currentStateAsChanges`, () => { let mockSync: ReturnType diff --git a/packages/db/tests/collection-indexes.test.ts b/packages/db/tests/collection-indexes.test.ts index bd5f4868c7..6931e69a51 100644 --- a/packages/db/tests/collection-indexes.test.ts +++ b/packages/db/tests/collection-indexes.test.ts @@ -15,6 +15,9 @@ import { } from '../src/query/builder/functions' import { PropRef } from '../src/query/ir' import { BTreeIndex } from '../src/indexes/btree-index.js' +import { DEFAULT_COMPARE_OPTIONS } from '../src/utils.js' +import { findIndexForField } from '../src/utils/index-optimization.js' +import { makeComparator } from '../src/utils/comparison.js' import { expectIndexUsage, stripVirtualProps, withIndexTracking } from './utils' import type { Collection } from '../src/collection/index.js' import type { MutationFn, PendingMutation } from '../src/types' @@ -161,6 +164,81 @@ describe(`Collection Indexes`, () => { expect(index.indexedKeysSet.size).toBe(5) }) + it(`should match compare options by collation semantics`, () => { + const index = collection.createIndex((row) => row.status) + + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + locale: undefined, + localeOptions: undefined, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + localeOptions: { sensitivity: undefined }, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + direction: `desc`, + nulls: `last`, + }), + ).toBe(true) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + direction: `desc`, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + locale: `de-DE`, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + localeOptions: { sensitivity: `base` }, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + nulls: `last`, + }), + ).toBe(false) + expect( + index.matchesCompareOptions({ + ...DEFAULT_COMPARE_OPTIONS, + stringSort: `lexical`, + }), + ).toBe(false) + }) + + it(`should reuse an index for equivalent locale identifiers`, () => { + const indexCompareOptions = { + ...DEFAULT_COMPARE_OPTIONS, + locale: `en-us`, + } + const index = collection.createIndex((row) => row.name, { + options: { + compareOptions: indexCompareOptions, + compareFn: makeComparator(indexCompareOptions), + }, + }) + + expect( + findIndexForField(collection, [`name`], { + ...DEFAULT_COMPARE_OPTIONS, + locale: `en-US`, + }), + ).toBe(index) + }) + it(`should create multiple indexes`, () => { const statusIndex = collection.createIndex((row) => row.status) const ageIndex = collection.createIndex((row) => row.age) diff --git a/packages/db/tests/collection-lifecycle.test.ts b/packages/db/tests/collection-lifecycle.test.ts index a5cf03f19e..1c84158cfc 100644 --- a/packages/db/tests/collection-lifecycle.test.ts +++ b/packages/db/tests/collection-lifecycle.test.ts @@ -1,11 +1,27 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { createCollection } from '../src/collection/index.js' import { CleanupQueue } from '../src/collection/cleanup-queue.js' +import { InvalidCollectionStatusTransitionError } from '../src/errors.js' +import { + getActivePublicationContext, + transactionScopedScheduler, + withPublicationContext, +} from '../src/scheduler.js' // Mock setTimeout and clearTimeout for testing GC behavior const originalSetTimeout = global.setTimeout const originalClearTimeout = global.clearTimeout +function getChangesManager(collection: object): { + emitEmptyReadyEvent: () => void +} { + return ( + collection as unknown as { + _changes: { emitEmptyReadyEvent: () => void } + } + )._changes +} + describe(`Collection Lifecycle Management`, () => { let mockSetTimeout: ReturnType let mockClearTimeout: ReturnType @@ -511,6 +527,919 @@ describe(`Collection Lifecycle Management`, () => { subscription.unsubscribe() }) + it(`freezes first-ready callback membership before delivery`, async () => { + let markReadyCallback: (() => void) | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `first-ready-membership-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const subscription = collection.subscribeChanges(() => {}) + let removeLater = () => {} + + collection.onFirstReady(() => { + calls.push(`first`) + removeLater() + collection.onFirstReady(() => calls.push(`nested`)) + }) + removeLater = collection.onFirstReady(() => calls.push(`later`)) + + try { + markReadyCallback!() + + expect(calls).toEqual([`first`, `nested`, `later`]) + + collection.onFirstReady(() => calls.push(`after`)) + expect(calls).toEqual([`first`, `nested`, `later`, `after`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + expect(calls).toEqual([`first`, `nested`, `later`, `after`]) + } + }) + + it.each([ + { + from: `ready`, + expectedStatus: `ready`, + expectedFirstReadyCalls: 1, + invalid: false, + }, + { + from: `error`, + expectedStatus: `ready`, + expectedFirstReadyCalls: 1, + invalid: false, + }, + { + from: `idle`, + expectedStatus: `idle`, + expectedFirstReadyCalls: 0, + invalid: true, + }, + { + from: `cleaned-up`, + expectedStatus: `cleaned-up`, + expectedFirstReadyCalls: 0, + invalid: true, + }, + ] as const)( + `defines the $from -> ready transition`, + async ({ from, expectedStatus, expectedFirstReadyCalls, invalid }) => { + const syncFailure = new Error(`sync failed before recovery`) + let firstReadyCalls = 0 + let recoveryFirstReadyCalls = 0 + const collection = createCollection<{ id: string; name: string }>({ + id: `mark-ready-from-${from}`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + collection.onFirstReady(() => { + firstReadyCalls++ + }) + + if (from === `ready` || from === `error`) { + collection._lifecycle.setStatus(`loading`) + collection._lifecycle.markReady() + } + if (from === `error`) { + collection._lifecycle.markError(syncFailure) + expect(collection._lifecycle.getSyncError()).toBe(syncFailure) + } else if (from === `cleaned-up`) { + collection._lifecycle.setStatus(`cleaned-up`) + } + expect(collection.status).toBe(from) + + if (from === `error`) { + collection.onFirstReady(() => { + recoveryFirstReadyCalls++ + }) + expect(recoveryFirstReadyCalls).toBe(1) + } + + const transitionTrace: Array< + | { + kind: `status` + previousStatus: string + status: string + syncError: unknown + } + | { + kind: `dependent-ready` + status: string + syncError: unknown + } + > = [] + collection.on(`status:change`, ({ previousStatus, status }) => { + transitionTrace.push({ + kind: `status`, + previousStatus, + status, + syncError: collection._lifecycle.getSyncError(), + }) + }) + const changes = getChangesManager(collection) + const originalEmitEmptyReadyEvent = + changes.emitEmptyReadyEvent.bind(changes) + vi.spyOn(changes, `emitEmptyReadyEvent`).mockImplementation(() => { + transitionTrace.push({ + kind: `dependent-ready`, + status: collection.status, + syncError: collection._lifecycle.getSyncError(), + }) + originalEmitEmptyReadyEvent() + }) + + let didThrow = false + let thrown: unknown + try { + collection._lifecycle.markReady() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(invalid) + if (invalid) { + expect(thrown).toBeInstanceOf(InvalidCollectionStatusTransitionError) + expect((thrown as Error).message).toBe( + `Invalid collection status transition from "${from}" to "ready" for collection "mark-ready-from-${from}"`, + ) + } + expect(collection.status).toBe(expectedStatus) + expect(firstReadyCalls).toBe(expectedFirstReadyCalls) + expect(recoveryFirstReadyCalls).toBe(from === `error` ? 1 : 0) + expect(transitionTrace).toEqual( + from === `error` + ? [ + { + kind: `status`, + previousStatus: `error`, + status: `ready`, + syncError: undefined, + }, + { + kind: `dependent-ready`, + status: `ready`, + syncError: undefined, + }, + ] + : [], + ) + expect(collection._lifecycle.getSyncError()).toBeUndefined() + + await collection.cleanup() + }, + ) + + it(`does not resume ready effects after a status listener cleans up`, () => { + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-cleanup-test`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) + const firstReadyStatuses: Array = [] + collection.onFirstReady(() => { + firstReadyStatuses.push(collection.status) + }) + collection.on(`status:ready`, () => { + void collection.cleanup() + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + + expect(collection.status).toBe(`cleaned-up`) + expect(collection._lifecycle.hasBeenReady).toBe(false) + expect(firstReadyStatuses).toEqual([`ready`]) + expect(readyEvent).not.toHaveBeenCalled() + + const laterFirstReady = vi.fn() + const removeLater = collection.onFirstReady(laterFirstReady) + expect(laterFirstReady).not.toHaveBeenCalled() + removeLater() + }) + + it(`does not resume ready effects after a status listener enters error`, async () => { + const failure = new Error(`ready listener failed the sync`) + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-error-test`, + getKey: (item) => item.id, + startSync: false, + sync: { sync: () => {} }, + }) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) + const firstReady = vi.fn() + collection.onFirstReady(firstReady) + collection.on(`status:ready`, () => { + collection._lifecycle.markError(failure) + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + + expect(collection.status).toBe(`error`) + expect(collection._lifecycle.getSyncError()).toBe(failure) + expect(collection._lifecycle.hasBeenReady).toBe(false) + expect(firstReady).not.toHaveBeenCalled() + expect(readyEvent).not.toHaveBeenCalled() + await collection.cleanup() + }) + + it(`does not resume an outer ready transition after a synchronous restart`, async () => { + let syncStarts = 0 + let restartedPreload: Promise | undefined + let restartOnce = true + let lateSubscription: { unsubscribe: () => void } | undefined + const lateReadyBatches: Array> = [] + const firstReadyStatuses: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-listener-aba-test`, + getKey: (item) => item.id, + startSync: false, + sync: { + sync: ({ markReady }) => { + syncStarts++ + markReady() + }, + }, + }) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) + collection.onFirstReady(() => { + firstReadyStatuses.push(collection.status) + }) + collection.on(`status:ready`, () => { + if (!restartOnce) return + restartOnce = false + void collection.cleanup() + restartedPreload = collection.preload() + lateSubscription = collection.subscribeChanges((batch) => { + lateReadyBatches.push(batch) + }) + }) + collection._lifecycle.setStatus(`loading`) + + collection._lifecycle.markReady() + await restartedPreload + + expect(syncStarts).toBe(1) + expect(collection.status).toBe(`ready`) + expect(firstReadyStatuses).toEqual([`ready`]) + expect(lateReadyBatches).toEqual([]) + expect(readyEvent).toHaveBeenCalledOnce() + lateSubscription!.unsubscribe() + await collection.cleanup() + }) + + it(`starts a fresh first-ready cycle after cleanup of a failed ready effect`, async () => { + const readyCallbacks: Array<() => void> = [] + const firstFailure = new Error(`first ready cycle failed exactly`) + const trace: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-effect-restart-test`, + getKey: (item) => item.id, + startSync: false, + sync: { + sync: ({ markReady }) => { + readyCallbacks.push(markReady) + }, + }, + }) + const readyEvent = vi.spyOn( + getChangesManager(collection), + `emitEmptyReadyEvent`, + ) + collection.onFirstReady(() => { + trace.push(`first failure:${collection.status}`) + throw firstFailure + }) + collection.onFirstReady(() => { + trace.push(`first later:${collection.status}`) + }) + const firstPreload = collection.preload() + let firstPreloadSettled = false + void firstPreload.then(() => { + firstPreloadSettled = true + }) + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(firstPreloadSettled).toBe(false) + + let thrown: unknown + try { + readyCallbacks[0]!() + } catch (error) { + thrown = error + } + expect(thrown).toBe(firstFailure) + await expect(firstPreload).resolves.toBeUndefined() + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + expect(readyEvent).toHaveBeenCalledOnce() + + await collection.cleanup() + expect(collection.status).toBe(`cleaned-up`) + expect(collection._lifecycle.hasBeenReady).toBe(false) + + collection.onFirstReady(() => { + trace.push(`second:${collection.status}`) + }) + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + + const secondPreload = collection.preload() + let secondPreloadSettled = false + void secondPreload.then(() => { + secondPreloadSettled = true + }) + expect(secondPreload).not.toBe(firstPreload) + expect(readyCallbacks).toHaveLength(2) + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(secondPreloadSettled).toBe(false) + + readyCallbacks[0]!() + await Promise.resolve() + expect(collection.status).toBe(`loading`) + expect(secondPreloadSettled).toBe(false) + expect(trace).toEqual([`first failure:ready`, `first later:ready`]) + expect(readyEvent).toHaveBeenCalledOnce() + + readyCallbacks[1]!() + await expect(secondPreload).resolves.toBeUndefined() + expect(secondPreloadSettled).toBe(true) + + expect(trace).toEqual([ + `first failure:ready`, + `first later:ready`, + `second:ready`, + ]) + expect(readyEvent).toHaveBeenCalledTimes(2) + + await collection.cleanup() + }) + + it(`attempts every first-ready effect before rethrowing the first failure`, async () => { + let markReadyCallback: (() => void) | undefined + const readyBatches: Array> = [] + const readyTrace: Array = [] + const laterFailure = new Error(`later first-ready failure`) + const laterCallback = vi.fn(() => { + readyTrace.push(`later:${collection.status}`) + throw laterFailure + }) + let preloadSettled = false + + const collection = createCollection<{ id: string; name: string }>({ + id: `first-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + const subscription = collection.subscribeChanges((batch) => { + readyTrace.push(`dependent:${collection.status}`) + readyBatches.push(batch) + }) + collection.onFirstReady(() => { + readyTrace.push(`first:${collection.status}`) + throw undefined + }) + collection.onFirstReady(laterCallback) + void collection.preload().then(() => { + preloadSettled = true + }) + + try { + let didThrow = false + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + didThrow = true + thrown = error + } + await Promise.resolve() + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(laterCallback).toHaveBeenCalledOnce() + expect(preloadSettled).toBe(true) + expect(readyBatches).toEqual([[]]) + expect(readyTrace).toEqual([ + `first:ready`, + `later:ready`, + `dependent:ready`, + ]) + expect(collection.status).toBe(`ready`) + + expect(() => markReadyCallback!()).not.toThrow() + expect(laterCallback).toHaveBeenCalledOnce() + expect(readyBatches).toEqual([[]]) + expect(readyTrace).toEqual([ + `first:ready`, + `later:ready`, + `dependent:ready`, + ]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`does not classify synchronous first-ready callback failures as sync failures`, async () => { + const laterFailure = new Error(`later synchronous first-ready failure`) + const callbackTrace: Array = [] + let syncContinued = false + + const collection = createCollection<{ id: string; name: string }>({ + id: `synchronous-first-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + syncContinued = true + }, + }, + }) + collection.onFirstReady(() => { + callbackTrace.push(`first`) + throw undefined + }) + collection.onFirstReady(() => { + callbackTrace.push(`later`) + throw laterFailure + }) + + try { + let didThrow = false + let thrown: unknown + try { + collection._sync.startSync() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(syncContinued).toBe(true) + expect(callbackTrace).toEqual([`first`, `later`]) + expect(collection.status).toBe(`ready`) + await expect(collection.preload()).resolves.toBeUndefined() + } finally { + await collection.cleanup() + } + }) + + it(`rejects a pending preload when the adapter fails after marking ready`, async () => { + const adapterFailure = new Error(`adapter failed after ready`) + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-then-adapter-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + throw adapterFailure + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + await expect(collection.preload()).rejects.toBe(adapterFailure) + expect(collection.status).toBe(`error`) + } finally { + await collection.cleanup() + } + }) + + it(`ends the synchronous sync-entry boundary after an adapter failure`, async () => { + const adapterFailure = new Error(`adapter entry failed`) + let markReadyCallback: (() => void) | undefined + const collection = createCollection<{ id: string; name: string }>({ + id: `failed-sync-entry-boundary-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + throw adapterFailure + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + expect(() => collection._sync.startSync()).toThrow(adapterFailure) + expect(collection.status).toBe(`error`) + + let didThrow = false + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(collection.status).toBe(`ready`) + } finally { + await collection.cleanup() + } + }) + + it(`attempts every dependent ready listener before rethrowing`, async () => { + let markReadyCallback: (() => void) | undefined + const firstFailure = new Error(`first dependent failed`) + const firstBatches: Array> = [] + const secondBatches: Array> = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + const first = collection.subscribeChanges((batch) => { + firstBatches.push(batch) + throw firstFailure + }) + const second = collection.subscribeChanges((batch) => { + secondBatches.push(batch) + }) + + try { + let thrown: unknown + try { + markReadyCallback!() + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(firstBatches).toEqual([[]]) + expect(secondBatches).toEqual([[]]) + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`flushes work queued by a ready listener when a sibling throws`, async () => { + let markReadyCallback: (() => void) | undefined + const firstFailure = new Error(`dependent failed after sibling queued`) + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-scheduler-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + expect(contextId).toBeDefined() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw firstFailure + }) + + try { + expect(() => markReadyCallback!()).toThrow(firstFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`flushes ready work before rethrowing at an outer publication boundary`, async () => { + let markReadyCallback: (() => void) | undefined + const listenerFailure = new Error(`nested dependent failed`) + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-dependent-ready-scheduler-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(() => + withPublicationContext(() => markReadyCallback!()), + ).toThrow(listenerFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`preserves a falsy ready failure through a nested publication`, async () => { + let markReadyCallback: (() => void) | undefined + const scheduledJob = vi.fn() + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-falsy-ready-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw undefined + }) + + try { + let didThrow = false + let thrown: unknown + try { + withPublicationContext(() => markReadyCallback!()) + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + expect(scheduledJob).toHaveBeenCalledOnce() + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`surfaces a ready graph failure after running its job`, async () => { + let markReadyCallback: (() => void) | undefined + const graphFailure = new Error(`ready graph failed`) + const scheduledJob = vi.fn(() => { + throw graphFailure + }) + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-graph-failure-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const subscription = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + + try { + expect(() => markReadyCallback!()).toThrow(graphFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`keeps the ready listener failure when its queued graph job also fails`, async () => { + let markReadyCallback: (() => void) | undefined + const listenerFailure = new Error(`ready listener failed first`) + const graphFailure = new Error(`ready graph also failed`) + const scheduledJob = vi.fn(() => { + throw graphFailure + }) + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-failure-priority-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const second = collection.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(() => markReadyCallback!()).toThrow(listenerFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + expect(collection.status).toBe(`ready`) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`resolves a pending preload after a ready callback failure alone`, async () => { + let syncContinued = false + const collection = createCollection<{ id: string; name: string }>({ + id: `ready-callback-preload-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReady() + syncContinued = true + }, + }, + }) + collection.onFirstReady(() => { + throw undefined + }) + + try { + await expect(collection.preload()).resolves.toBeUndefined() + expect(syncContinued).toBe(true) + expect(collection.status).toBe(`ready`) + } finally { + await collection.cleanup() + } + }) + + it(`delivers ready to the subscription snapshot when one listener unsubscribes another`, async () => { + let markReadyCallback: (() => void) | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-membership-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + calls.push(`first`) + second.unsubscribe() + }) + const second = collection.subscribeChanges(() => { + calls.push(`second`) + }) + + try { + markReadyCallback!() + expect(calls).toEqual([`first`, `second`]) + } finally { + first.unsubscribe() + second.unsubscribe() + await collection.cleanup() + } + }) + + it(`excludes a dependent added during ready delivery until the next batch`, async () => { + let beginCallback: (() => void) | undefined + let writeCallback: + | ((message: { + type: `insert` + value: { id: string; name: string } + }) => void) + | undefined + let commitCallback: (() => void) | undefined + let markReadyCallback: (() => void) | undefined + let added: { unsubscribe: () => void } | undefined + const calls: Array = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `dependent-ready-addition-test`, + getKey: (item) => item.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + beginCallback = begin + writeCallback = write + commitCallback = () => { + commit() + } + markReadyCallback = markReady + }, + }, + }) + const first = collection.subscribeChanges(() => { + calls.push(`first`) + added ??= collection.subscribeChanges(() => calls.push(`added`)) + }) + const second = collection.subscribeChanges(() => calls.push(`second`)) + + try { + markReadyCallback!() + expect(calls).toEqual([`first`, `second`]) + + beginCallback!() + writeCallback!({ + type: `insert`, + value: { id: `one`, name: `One` }, + }) + commitCallback!() + expect(calls).toEqual([`first`, `second`, `first`, `second`, `added`]) + } finally { + first.unsubscribe() + second.unsubscribe() + added?.unsubscribe() + await collection.cleanup() + } + }) + + it(`notifies a dependent added during the first-ready fan-out`, async () => { + let markReadyCallback: (() => void) | undefined + let dependent: { unsubscribe: () => void } | undefined + const readyBatches: Array> = [] + const collection = createCollection<{ id: string; name: string }>({ + id: `nested-dependent-ready-test`, + getKey: (item) => item.id, + sync: { + sync: ({ markReady }) => { + markReadyCallback = markReady as () => void + }, + }, + }) + collection.onFirstReady(() => { + dependent = collection.subscribeChanges((batch) => { + readyBatches.push(batch) + }) + }) + const preload = collection.preload() + + try { + markReadyCallback!() + await preload + expect(readyBatches).toEqual([[]]) + } finally { + dependent?.unsubscribe() + await collection.cleanup() + } + }) + it(`should fire status:change event with 'cleaned-up' status before clearing event handlers`, () => { const collection = createCollection<{ id: string; name: string }>({ id: `cleanup-event-test`, diff --git a/packages/db/tests/collection-metadata-publication-oracle.property.test.ts b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts new file mode 100644 index 0000000000..0586c142c5 --- /dev/null +++ b/packages/db/tests/collection-metadata-publication-oracle.property.test.ts @@ -0,0 +1,828 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { createDeferred } from '../src/deferred.js' +import { SyncTransactionAbortedError } from '../src/errors.js' +import { createLiveQueryCollection } from '../src/query/index.js' +import { createTransaction } from '../src/transactions.js' +import { oraclePropertyOptions } from './oracle-config.js' +import type { Collection } from '../src/collection/index.js' +import type { ChangeMessage, SyncConfig } from '../src/types.js' + +type PublicationRow = { + id: number + position: number +} + +type SyncActions = Parameters[`sync`]>[0] + +type MetadataOperation = { type: `set`; value: unknown } | { type: `delete` } + +type MetadataEntryState = { present: false } | { present: true; value: unknown } + +type MetadataWrite = { key: number } & MetadataOperation + +type PublicationRound = { + key: number + delta: number + metadata: ReadonlyArray + outcome: `commit` | `abort` +} + +type ReadablePublicationCollection = { + values: () => IterableIterator + cleanup: () => Promise +} + +type PublicationHarness = { + rows: Collection + liveRows: ReadablePublicationCollection + batches: Array>> + unsubscribe: () => void + getSync: () => SyncActions +} + +type PublishedPublicationRow = PublicationRow & { + $collectionId: string + $key: number + $origin: `local` | `remote` + $synced: boolean +} + +const metadataValueArbitrary = fc.oneof( + fc.constant(undefined), + fc.constant(null), + fc.constant(false), + fc.constant(true), + fc.constant(0), + fc.constant(Number.NaN), + fc.constant(``), + fc.integer(), + fc.string(), + fc.record({ nested: fc.integer() }), +) + +const metadataOperationArbitrary: fc.Arbitrary = fc.oneof( + metadataValueArbitrary.map((value) => ({ type: `set` as const, value })), + fc.constant({ type: `delete` as const }), +) + +const metadataEntryStateArbitrary: fc.Arbitrary = fc.oneof( + fc.constant({ present: false as const }), + metadataValueArbitrary.map((value) => ({ + present: true as const, + value, + })), +) + +const metadataWriteArbitrary = fc + .tuple(fc.integer({ min: 0, max: 2 }), metadataOperationArbitrary) + .map(([key, operation]) => ({ key, ...operation })) + +const publicationRoundArbitrary: fc.Arbitrary = fc + .record({ + key: fc.integer({ min: 0, max: 2 }), + delta: fc.constantFrom(-2, -1, 1, 2), + extraMetadata: fc.array(metadataWriteArbitrary, { maxLength: 2 }), + outcome: fc.constantFrom(`commit` as const, `abort` as const), + primaryMetadata: metadataOperationArbitrary, + }) + .map(({ key, delta, extraMetadata, outcome, primaryMetadata }) => ({ + key, + delta, + outcome, + metadata: [{ key, ...primaryMetadata }, ...extraMetadata], + })) + +const metadataCancellationArbitrary = fc.record({ + canceledKeys: fc.uniqueArray(fc.integer({ min: 0, max: 2 }), { + minLength: 1, + maxLength: 3, + }), + retainedKeys: fc.uniqueArray(fc.integer({ min: 0, max: 2 }), { + minLength: 1, + maxLength: 3, + }), + canceledOperation: metadataOperationArbitrary, + retainedOperation: metadataOperationArbitrary, + canceledFirst: fc.boolean(), + initialMetadata: fc.array(metadataEntryStateArbitrary, { + minLength: 3, + maxLength: 3, + }), +}) + +const metadataRollbackCaseArbitrary = fc.record({ + initialMetadata: metadataEntryStateArbitrary, + pendingOperation: metadataOperationArbitrary, +}) + +const metadataRollbackArbitrary = fc + .record({ + sourceKey: fc.integer({ min: 0, max: 2 }), + metadataKeyOffset: fc.constantFrom(1, 2), + sourceDelta: fc.integer({ min: 1, max: 10 }), + metadataCase: metadataRollbackCaseArbitrary, + }) + .map(({ sourceKey, metadataKeyOffset, sourceDelta, metadataCase }) => ({ + ...metadataCase, + sourceKey, + metadataKey: (sourceKey + metadataKeyOffset) % 3, + sourceDelta, + })) + +let nextMetadataRollbackHarnessId = 0 + +async function createPublicationHarness(): Promise { + let sync!: SyncActions + const rows = createCollection({ + id: `metadata-publication-source`, + getKey: (row) => row.id, + startSync: true, + sync: { + sync: (actions) => { + sync = actions + actions.begin() + for (let id = 0; id < 3; id++) { + actions.write({ type: `insert`, value: { id, position: id } }) + } + actions.commit() + actions.markReady() + }, + }, + }) + const liveRows = createLiveQueryCollection((query) => + query.from({ row: rows }), + ) + await liveRows.preload() + + const batches: Array>> = + [] + const subscription = rows.subscribeChanges((changes) => { + batches.push(changes) + }) + return { + rows, + liveRows, + batches, + unsubscribe: () => subscription.unsubscribe(), + getSync: () => sync, + } +} + +function expectUniqueBatchKeys( + batches: ReadonlyArray< + ReadonlyArray> + >, +): void { + for (const batch of batches) { + const keys = batch.map((change) => change.key) + expect(keys).toEqual([...new Set(keys)]) + } +} + +function selectPublishedRow( + row: PublicationRow | undefined, +): PublishedPublicationRow | undefined { + if (row === undefined) return undefined + const published = row as PublishedPublicationRow + return { + id: published.id, + position: published.position, + $collectionId: published.$collectionId, + $key: published.$key, + $origin: published.$origin, + $synced: published.$synced, + } +} + +function selectPublishedChange( + change: ChangeMessage, +) { + return { + type: change.type, + key: change.key, + value: selectPublishedRow(change.value), + previousValue: selectPublishedRow(change.previousValue), + } +} + +function expectPublishedRows( + harness: PublicationHarness, + model: ReadonlyMap, +): void { + const expected = [...model.values()].sort((a, b) => a.id - b.id) + const selectBaseRows = (collection: ReadablePublicationCollection) => + [...collection.values()] + .map((row) => ({ id: row.id, position: row.position })) + .sort((a, b) => a.id - b.id) + + expect(selectBaseRows(harness.rows)).toEqual(expected) + expect(selectBaseRows(harness.liveRows)).toEqual(expected) +} + +async function applyRound( + harness: PublicationHarness, + round: PublicationRound, + model: Map, + metadataModel: Map, +): Promise { + const previous = model.get(round.key)! + const next = { ...previous, position: previous.position + round.delta } + const batchCountBefore = harness.batches.length + const keyWasPreviouslyPublished = harness.batches.some((batch) => + batch.some((change) => change.key === round.key), + ) + const sync = harness.getSync() + const transaction = createTransaction({ + mutationFn: async () => { + sync.begin({ immediate: true }) + sync.write({ type: `update`, value: next }) + sync.commit() + + sync.begin() + for (const write of round.metadata) { + if (write.type === `set`) { + sync.metadata!.row.set(write.key, write.value) + } else { + sync.metadata!.row.delete(write.key) + } + } + if (round.outcome === `commit`) { + sync.commit() + } else { + const controller = new AbortController() + const receipt = sync.commit(controller.signal) + controller.abort() + if (receipt !== true) { + await receipt.catch((error: unknown) => { + if (!(error instanceof SyncTransactionAbortedError)) throw error + }) + } + } + }, + }) + transaction.mutate(() => { + harness.rows.update(round.key, (draft) => { + draft.position = next.position + }) + }) + await transaction.isPersisted.promise + + model.set(round.key, next) + if (round.outcome === `commit`) { + for (const write of round.metadata) { + if (write.type === `set`) { + metadataModel.set(write.key, write.value) + } else { + metadataModel.delete(write.key) + } + } + } + await Promise.resolve() + const virtualRow = ( + row: PublicationRow, + synced: boolean, + ): PublishedPublicationRow => ({ + ...row, + $collectionId: harness.rows.id, + $key: row.id, + $origin: `local`, + $synced: synced, + }) + const expectedOptimisticChange = keyWasPreviouslyPublished + ? { + type: `update`, + key: round.key, + value: virtualRow(next, false), + previousValue: virtualRow(previous, true), + } + : { + type: `insert`, + key: round.key, + value: virtualRow(next, false), + previousValue: undefined, + } + expect( + harness.batches + .slice(batchCountBefore) + .map((batch) => batch.map(selectPublishedChange)), + ).toEqual([ + [expectedOptimisticChange], + [ + { + type: `update`, + key: round.key, + value: virtualRow(next, true), + previousValue: virtualRow(next, false), + }, + ], + ]) + expectUniqueBatchKeys(harness.batches) + expectPublishedRows(harness, model) + const byKey = ( + [a]: readonly [number, unknown], + [b]: readonly [number, unknown], + ) => a - b + expect([...harness.rows._state.syncedMetadata.entries()].sort(byKey)).toEqual( + [...metadataModel.entries()].sort(byKey), + ) + expect(harness.rows._state.preSyncVisibleState.size).toBe(0) + expect(harness.rows._state.preSyncVirtualState.size).toBe(0) + expect(harness.rows._state.recentlySyncedKeys.size).toBe(0) +} + +async function runPublicationHistory( + rounds: ReadonlyArray, +): Promise { + const harness = await createPublicationHarness() + const model = new Map( + [0, 1, 2].map((id) => [id, { id, position: id }] as const), + ) + const metadataModel = new Map() + try { + for (const round of rounds) { + await applyRound(harness, round, model, metadataModel) + } + } finally { + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +} + +async function expectMetadataCancellationOwnership( + canceledKeys: ReadonlyArray, + retainedKeys: ReadonlyArray, + canceledOperation: MetadataOperation, + retainedOperation: MetadataOperation, + canceledFirst: boolean, + initialMetadataState: ReadonlyArray, +): Promise { + const harness = await createPublicationHarness() + const initialMetadata = new Map() + for (const [key, state] of initialMetadataState.entries()) { + if (state.present) initialMetadata.set(key, state.value) + } + const initialSync = harness.getSync() + initialSync.begin() + for (const [key, value] of initialMetadata) { + initialSync.metadata!.row.set(key, value) + } + initialSync.commit() + await Promise.resolve() + + const persistence = createDeferred() + const heldTransaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + heldTransaction.mutate(() => { + harness.rows.insert({ id: 99, position: 99 }) + }) + expect(heldTransaction.state).toBe(`persisting`) + + const stageMetadata = ( + keys: ReadonlyArray, + operation: MetadataOperation, + ) => { + const sync = harness.getSync() + sync.begin() + for (const key of keys) { + if (operation.type === `set`) { + sync.metadata!.row.set(key, operation.value) + } else { + sync.metadata!.row.delete(key) + } + } + const receipt = sync.commit() + if (receipt === true) { + throw new Error(`Persisting optimistic work did not hold metadata sync`) + } + const transaction = harness.rows._state.pendingSyncedTransactions.at(-1)! + void receipt.catch(() => undefined) + return { receipt, transaction } + } + + const first = canceledFirst + ? stageMetadata(canceledKeys, canceledOperation) + : stageMetadata(retainedKeys, retainedOperation) + const second = canceledFirst + ? stageMetadata(retainedKeys, retainedOperation) + : stageMetadata(canceledKeys, canceledOperation) + const canceled = canceledFirst ? first : second + const retained = canceledFirst ? second : first + const expectedVirtualSnapshots = (keys: ReadonlyArray) => + new Map( + [...new Set(keys)].map((key) => [ + key, + { + $collectionId: harness.rows.id, + $key: key, + $origin: `remote`, + $synced: true, + }, + ]), + ) + + try { + harness.rows._state.capturePreSyncVisibleState() + const expectedBefore = new Set([...canceledKeys, ...retainedKeys]) + expect(harness.rows._state.recentlySyncedKeys).toEqual(expectedBefore) + expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( + expectedBefore, + ) + expect(harness.rows._state.preSyncVirtualState).toEqual( + expectedVirtualSnapshots([...canceledKeys, ...retainedKeys]), + ) + const batchCountBefore = harness.batches.length + + harness.rows._state.cancelPendingSyncedTransaction(canceled.transaction) + + const expectedAfter = new Set(retainedKeys) + expect(harness.rows._state.pendingSyncedTransactions).toEqual([ + retained.transaction, + ]) + expect(retained.transaction.rowMetadataWrites).toEqual( + new Map(retainedKeys.map((key) => [key, retainedOperation])), + ) + expect(harness.rows._state.recentlySyncedKeys).toEqual(expectedAfter) + expect(new Set(harness.rows._state.preSyncVisibleState.keys())).toEqual( + expectedAfter, + ) + expect(harness.rows._state.preSyncVirtualState).toEqual( + expectedVirtualSnapshots(retainedKeys), + ) + expect(harness.batches).toHaveLength(batchCountBefore) + expect(harness.rows._state.syncedMetadata).toEqual(initialMetadata) + await expect(canceled.receipt).rejects.toBeInstanceOf( + SyncTransactionAbortedError, + ) + + persistence.resolve() + await heldTransaction.isPersisted.promise + await expect(retained.receipt).resolves.toBeUndefined() + expect(harness.rows._state.preSyncVisibleState.size).toBe(0) + expect(harness.rows._state.preSyncVirtualState.size).toBe(0) + expect(harness.rows._state.recentlySyncedKeys.size).toBe(0) + const expectedMetadata = new Map(initialMetadata) + for (const key of retainedKeys) { + if (retainedOperation.type === `set`) { + expectedMetadata.set(key, retainedOperation.value) + } else { + expectedMetadata.delete(key) + } + } + expect(harness.rows._state.syncedMetadata).toEqual(expectedMetadata) + } finally { + if (retained.transaction.applied.isPending()) { + harness.rows._state.cancelPendingSyncedTransaction(retained.transaction) + await retained.receipt.catch(() => undefined) + } + persistence.resolve() + await heldTransaction.isPersisted.promise.catch(() => undefined) + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +} + +async function expectMetadataRollbackRecovery({ + sourceKey, + metadataKey, + sourceDelta, + initialMetadata, + pendingOperation, + additionalMetadata = [], + separatePendingTransactions = false, +}: { + sourceKey: number + metadataKey: number + sourceDelta: number + initialMetadata: MetadataEntryState + pendingOperation: MetadataOperation + additionalMetadata?: ReadonlyArray<{ + key: number + initialMetadata: MetadataEntryState + pendingOperation: MetadataOperation + }> + separatePendingTransactions?: boolean +}): Promise { + const harnessId = nextMetadataRollbackHarnessId++ + const source = await createPublicationHarness() + const { rows, getSync } = source + const derived = createLiveQueryCollection({ + id: `metadata-rollback-derived-${harnessId}`, + query: (query) => + query.from({ row: rows }).select(({ row }) => ({ + id: row.id, + position: row.position, + })), + getKey: (row) => row.id, + }) + await derived.preload() + + const metadataCases = [ + { key: metadataKey, initialMetadata, pendingOperation }, + ...additionalMetadata, + ] + const stageMetadata = ( + writes: ReadonlyArray<{ key: number; operation: MetadataOperation }>, + ) => { + const applied = createDeferred() + void applied.promise.catch(() => undefined) + const transaction = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map( + writes.map(({ key, operation }) => [key, operation]), + ), + collectionMetadataWrites: new Map(), + applied, + } + derived._state.pendingSyncedTransactions.push(transaction) + return transaction + } + + const initialWrites = metadataCases.flatMap( + ({ key, initialMetadata: state }) => + state.present + ? [ + { + key, + operation: { + type: `set` as const, + value: state.value, + }, + }, + ] + : [], + ) + if (initialWrites.length > 0) { + stageMetadata(initialWrites) + derived._state.commitPendingTransactions() + } + + const pendingWrites = metadataCases.map( + ({ key, pendingOperation: operation }) => ({ key, operation }), + ) + const pendingTransactions = separatePendingTransactions + ? pendingWrites.map((write) => stageMetadata([write])) + : [stageMetadata(pendingWrites)] + const sourceRowsBefore = [...rows.values()].map((row) => ({ ...row })) + const rowsBefore = [...derived.values()].map((row) => ({ ...row })) + const originBefore = new Map(derived._state.rowOrigins) + const hydrationSeedsBefore = new Set(derived._state.hydrationSeedKeys) + const hydratedBefore = new Set(derived._state.hydratedKeys) + const syncedBefore = new Set(derived._state.syncedKeys) + const preSyncBefore = new Map(derived._state.preSyncVisibleState) + const preSyncVirtualBefore = new Map(derived._state.preSyncVirtualState) + const recentlySyncedBefore = new Set(derived._state.recentlySyncedKeys) + const published: Array< + ReadonlyArray> + > = [] + const subscription = derived.subscribeChanges((changes) => { + published.push(changes) + }) + + const publicationFailure = new Error(`metadata rollback publication failed`) + const commitPendingTransactions = derived._state.commitPendingTransactions + let shouldFail = true + derived._state.commitPendingTransactions = () => { + commitPendingTransactions() + if (shouldFail) { + shouldFail = false + throw publicationFailure + } + } + + try { + const previousSourceRow = rows.get(sourceKey)! + let thrown: unknown + try { + getSync().begin() + getSync().write({ + type: `update`, + value: { + ...previousSourceRow, + position: previousSourceRow.position + sourceDelta, + }, + }) + getSync().commit() + } catch (error) { + thrown = error + } + expect(thrown).toBe(publicationFailure) + + expect(rows.get(sourceKey)?.position).toBe( + previousSourceRow.position + sourceDelta, + ) + expect([...rows.values()].map((row) => ({ ...row }))).toEqual( + sourceRowsBefore.map((row) => + row.id === sourceKey + ? { ...row, position: row.position + sourceDelta } + : row, + ), + ) + expect([...derived.values()].map((row) => ({ ...row }))).toEqual(rowsBefore) + expect(derived._state.syncedMetadata).toEqual( + new Map( + metadataCases.flatMap(({ key, initialMetadata: state }) => + state.present ? [[key, state.value]] : [], + ), + ), + ) + expect(derived._state.pendingSyncedTransactions).toEqual( + pendingTransactions, + ) + for (const pending of pendingTransactions) { + expect(pending.applicationStarted).toBe(false) + expect(pending.applied.isPending()).toBe(true) + } + expect(derived._state.rowOrigins).toEqual(originBefore) + expect(derived._state.hydrationSeedKeys).toEqual(hydrationSeedsBefore) + expect(derived._state.hydratedKeys).toEqual(hydratedBefore) + expect(derived._state.syncedKeys).toEqual(syncedBefore) + expect(derived._state.preSyncVisibleState).toEqual(preSyncBefore) + expect(derived._state.preSyncVirtualState).toEqual(preSyncVirtualBefore) + expect(derived._state.recentlySyncedKeys).toEqual(recentlySyncedBefore) + expect(published).toEqual([]) + } finally { + derived._state.commitPendingTransactions = commitPendingTransactions + for (const pending of pendingTransactions) { + derived._state.cancelPendingSyncedTransaction(pending) + } + subscription.unsubscribe() + source.unsubscribe() + await Promise.all([ + derived.cleanup(), + source.liveRows.cleanup(), + rows.cleanup(), + ]) + } +} + +it(`publishes one event per key when metadata-only sync retires optimistic work`, async () => { + await runPublicationHistory([ + { + key: 1, + delta: 1, + metadata: [{ key: 1, type: `set`, value: false }], + outcome: `commit`, + }, + { + key: 1, + delta: 1, + metadata: [{ key: 1, type: `delete` }], + outcome: `commit`, + }, + ]) +}) + +it(`includes metadata-only keys in a publication snapshot`, async () => { + const harness = await createPublicationHarness() + const applied = createDeferred() + void applied.promise.catch(() => undefined) + const transaction = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map([[1, { type: `set` as const, value: false }]]), + collectionMetadataWrites: new Map(), + applied, + } + harness.rows._state.pendingSyncedTransactions.push(transaction) + + try { + const snapshot = harness.rows._state.snapshotPublicationState([]) + expect([...snapshot.keys.keys()]).toEqual([1]) + expect(snapshot.keys.get(1)?.syncedMetadata).toEqual({ + present: false, + value: undefined, + }) + expect(snapshot.pendingSyncedTransactions).toEqual([transaction]) + } finally { + harness.rows._state.cancelPendingSyncedTransaction(transaction) + harness.unsubscribe() + await Promise.all([harness.liveRows.cleanup(), harness.rows.cleanup()]) + } +}) + +it(`releases only canceled metadata keys while another sync remains pending`, async () => { + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `delete` }, + { type: `set`, value: false }, + true, + [ + { present: true, value: undefined }, + { present: true, value: false }, + { present: true, value: null }, + ], + ) +}) + +it(`does not apply canceled metadata to an absent base key`, async () => { + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `set`, value: `canceled` }, + { type: `set`, value: `retained` }, + true, + [{ present: false }, { present: false }, { present: false }], + ) +}) + +it(`settles an older metadata owner after canceling the newer owner`, async () => { + await expectMetadataCancellationOwnership( + [0, 1], + [1, 2], + { type: `delete` }, + { type: `set`, value: `retained` }, + false, + [ + { present: true, value: undefined }, + { present: false }, + { present: true, value: false }, + ], + ) +}) + +it(`restores pending metadata when a derived publication fails`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + initialMetadata: { present: true, value: false }, + pendingOperation: { type: `delete` }, + }) +}) + +it(`restores an existing metadata value after a failed replacement`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + initialMetadata: { present: true, value: `before` }, + pendingOperation: { type: `set`, value: `after` }, + }) +}) + +it(`restores every metadata key after one failed publication`, async () => { + await expectMetadataRollbackRecovery({ + sourceKey: 0, + metadataKey: 1, + sourceDelta: 1, + initialMetadata: { present: true, value: `before` }, + pendingOperation: { type: `set`, value: `after` }, + separatePendingTransactions: true, + additionalMetadata: [ + { + key: 2, + initialMetadata: { present: true, value: false }, + pendingOperation: { type: `delete` }, + }, + ], + }) +}) + +fcTest.prop( + [fc.array(publicationRoundArbitrary, { minLength: 1, maxLength: 8 })], + oraclePropertyOptions(50, `collection-publication.metadata-only`), +)( + `keeps metadata-only optimistic settlement a valid keyed diff across histories`, + runPublicationHistory, +) + +fcTest.prop( + [metadataCancellationArbitrary], + oraclePropertyOptions(50, `collection-publication.metadata-cancellation`), +)( + `keeps metadata suppression owned by the remaining pending transactions`, + ({ + canceledKeys, + retainedKeys, + canceledOperation, + retainedOperation, + canceledFirst, + initialMetadata, + }) => + expectMetadataCancellationOwnership( + canceledKeys, + retainedKeys, + canceledOperation, + retainedOperation, + canceledFirst, + initialMetadata, + ), +) +fcTest.prop( + [metadataRollbackArbitrary], + oraclePropertyOptions(30, `collection-publication.metadata-rollback`), +)( + `restores metadata-only state after failed derived publications`, + expectMetadataRollbackRecovery, +) diff --git a/packages/db/tests/collection-state-retention-oracle.property.test.ts b/packages/db/tests/collection-state-retention-oracle.property.test.ts new file mode 100644 index 0000000000..0bc0e6a4a2 --- /dev/null +++ b/packages/db/tests/collection-state-retention-oracle.property.test.ts @@ -0,0 +1,791 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { DuplicateKeySyncError } from '../src/errors.js' +import { createTransaction } from '../src/transactions.js' +import { oraclePropertyOptions } from './oracle-config.js' +import type { Collection } from '../src/collection/index.js' +import type { SyncConfig, TransactionState } from '../src/types.js' + +type RetainedRow = { + id: number + value: number +} + +type SyncActions = Parameters[`sync`]>[0] + +type RetentionAction = + | { type: `insert`; row: RetainedRow } + | { type: `update`; row: RetainedRow } + | { type: `delete`; key: number } + | { type: `replace`; rows: ReadonlyArray } + | { type: `restart` } + | { + type: `reentrantRestart` + row: RetainedRow + commitPhase: `insideListener` | `afterOldReturn` + } + +type RetentionHarness = { + collection: Collection + sync: SyncActions +} + +const retainedRowArbitrary = fc.record({ + id: fc.integer({ min: 0, max: 3 }), + value: fc.integer({ min: -2, max: 2 }), +}) + +function snapshotRetainedRow(row: RetainedRow): RetainedRow { + return { id: row.id, value: row.value } +} + +const retentionActionArbitrary: fc.Arbitrary = fc.oneof( + { + weight: 4, + arbitrary: retainedRowArbitrary.map((row) => ({ + type: `insert` as const, + row, + })), + }, + { + weight: 4, + arbitrary: retainedRowArbitrary.map((row) => ({ + type: `update` as const, + row, + })), + }, + { + weight: 4, + arbitrary: fc + .integer({ min: 0, max: 3 }) + .map((key) => ({ type: `delete` as const, key })), + }, + { + weight: 2, + arbitrary: fc + .uniqueArray(retainedRowArbitrary, { + selector: (row) => row.id, + maxLength: 4, + }) + .map((rows) => ({ type: `replace` as const, rows })), + }, + { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, + { + // Keep each phase at least as likely as the original unsplit restart arm. + weight: 3, + arbitrary: fc + .tuple( + retainedRowArbitrary, + fc.constantFrom(`insideListener` as const, `afterOldReturn` as const), + ) + .map(([row, commitPhase]) => ({ + type: `reentrantRestart` as const, + row, + commitPhase, + })), + }, +) + +function createRetentionHarness(): RetentionHarness { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + return { + collection, + get sync() { + return sync + }, + } +} + +function applyAction( + action: RetentionAction, + model: Map, + sync: SyncActions, +): void { + sync.begin() + switch (action.type) { + case `insert`: { + const previous = model.get(action.row.id) + if (previous !== undefined && previous.value !== action.row.value) { + expect(() => + sync.write({ + type: `insert`, + value: snapshotRetainedRow(action.row), + }), + ).toThrow(DuplicateKeySyncError) + break + } + const expectedRow = snapshotRetainedRow(action.row) + sync.write({ type: `insert`, value: snapshotRetainedRow(action.row) }) + model.set(expectedRow.id, expectedRow) + break + } + case `update`: { + const expectedRow = snapshotRetainedRow(action.row) + sync.write({ type: action.type, value: snapshotRetainedRow(action.row) }) + model.set(expectedRow.id, expectedRow) + break + } + case `delete`: + sync.write({ type: `delete`, key: action.key }) + model.delete(action.key) + break + case `replace`: + sync.truncate() + model.clear() + for (const row of action.rows) { + const expectedRow = snapshotRetainedRow(row) + sync.write({ type: `insert`, value: snapshotRetainedRow(row) }) + model.set(expectedRow.id, expectedRow) + } + break + case `restart`: + case `reentrantRestart`: + throw new Error(`Restart actions require the lifecycle driver`) + } + expect(sync.commit()).toBe(true) +} + +function expectRetainedState( + collection: Collection, + model: ReadonlyMap, +): void { + const expectedRows = [...model.entries()].sort(([a], [b]) => a - b) + const retainedRows = [...collection._state.syncedData.entries()].sort( + ([a], [b]) => a - b, + ) + + expect(retainedRows).toEqual(expectedRows) + expect([...collection._state.syncedKeys].sort((a, b) => a - b)).toEqual( + expectedRows.map(([key]) => key), + ) + expect( + [...collection._state.rowOrigins.keys()] + .filter((key) => !model.has(key)) + .sort((a, b) => a - b), + ).toEqual([]) + expect( + [...collection.state.entries()] + .map(([key, row]) => [key, { id: row.id, value: row.value }] as const) + .sort(([a], [b]) => a - b), + ).toEqual(expectedRows) +} + +async function runRetentionHistory( + actions: ReadonlyArray, +): Promise { + const harness = createRetentionHarness() + const { collection } = harness + const model = new Map() + try { + expectRetainedState(collection, model) + for (const action of actions) { + if (action.type === `restart`) { + await collection.cleanup() + collection.startSyncImmediate() + model.clear() + } else if (action.type === `reentrantRestart`) { + const oldSync = harness.sync + const triggerType = model.has(action.row.id) ? `update` : `insert` + const triggerRow = { + id: action.row.id, + value: (model.get(action.row.id)?.value ?? action.row.value) + 1, + } + const expectedTriggerRow = snapshotRetainedRow(triggerRow) + const restartedRow = { + id: (action.row.id + 1) % 4, + value: action.row.value + 1, + } + const expectedRestartedRow = snapshotRetainedRow(restartedRow) + const retainedMarker = { id: -1, value: action.row.value } + const expectedRetainedMarker = snapshotRetainedRow(retainedMarker) + let cleanup: Promise | undefined + let restarted = false + let restartedSync: SyncActions | undefined + let restartedReceipt: true | Promise | undefined + let restartedReceiptOutcome: Promise | undefined + let restartedReceiptSettled = false + const settlementTimeline: Array< + `checkpoint` | `publication` | `receipt` + > = [] + const batches: Array<{ + changes: Array<{ + type: string + key: string | number + row: RetainedRow + previousRow: RetainedRow | undefined + }> + rows: Array + }> = [] + const subscription = collection.subscribeChanges( + (changes) => { + batches.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + row: { id: value.id, value: value.value }, + previousRow: + previousValue === undefined + ? undefined + : { + id: previousValue.id, + value: previousValue.value, + }, + })), + rows: [...collection.values()] + .map(({ id, value }) => ({ id, value })) + .sort((left, right) => left.id - right.id), + }) + if (changes.some(({ key }) => key === expectedRestartedRow.id)) { + queueMicrotask(() => settlementTimeline.push(`publication`)) + } + if (restarted) return + restarted = true + cleanup = collection.cleanup() + collection.startSyncImmediate() + restartedSync = harness.sync + restartedSync.begin() + restartedSync.write({ + type: `insert`, + value: snapshotRetainedRow(restartedRow), + }) + if (action.commitPhase === `insideListener`) { + restartedReceipt = restartedSync.commit() + if (restartedReceipt !== true) { + restartedReceiptOutcome = restartedReceipt.then((value) => { + settlementTimeline.push(`receipt`) + restartedReceiptSettled = true + return value + }) + } + queueMicrotask(() => settlementTimeline.push(`checkpoint`)) + } else { + // Synthetic generation canary: seed restarted-session + // publication state so the old publication tail cannot clear it. + // The batch assertions below exercise the public restart path. + collection._state.preSyncVisibleState.set(-1, retainedMarker) + collection._state.recentlySyncedKeys.add(expectedRestartedRow.id) + } + }, + { includeInitialState: false }, + ) + + oldSync.begin() + oldSync.write({ + type: `update`, + value: snapshotRetainedRow(triggerRow), + }) + expect(oldSync.commit()).toBe(true) + expect(restarted).toBe(true) + expect(restartedSync).toBeDefined() + if (restartedSync === undefined) { + throw new Error(`restarted sync session was not captured`) + } + if (action.commitPhase === `insideListener`) { + expect(restartedReceipt).toBeDefined() + expect(restartedReceipt).not.toBe(true) + expect(restartedReceipt).toBeInstanceOf(Promise) + expect(restartedReceiptSettled).toBe(false) + expect(settlementTimeline).toEqual([]) + if (restartedReceipt === undefined || restartedReceipt === true) { + throw new Error(`restarted sync receipt was not parked`) + } + expect(restartedReceiptOutcome).toBeDefined() + await expect(restartedReceiptOutcome).resolves.toBeUndefined() + expect(restartedReceiptSettled).toBe(true) + expect(settlementTimeline).toEqual([ + `checkpoint`, + `publication`, + `receipt`, + ]) + } else { + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, expectedRetainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([expectedRestartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + await Promise.resolve() + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[-1, expectedRetainedMarker]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([expectedRestartedRow.id]), + ) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + expect(restartedSync.commit()).toBe(true) + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + expect(collection._state.recentlySyncedKeys).toEqual( + new Set([expectedRestartedRow.id]), + ) + await Promise.resolve() + expect(collection._state.recentlySyncedKeys.size).toBe(0) + } + const triggerRows = new Map(model) + triggerRows.set(expectedTriggerRow.id, expectedTriggerRow) + expect(batches).toEqual([ + { + changes: [ + { + type: triggerType, + key: expectedTriggerRow.id, + row: expectedTriggerRow, + previousRow: model.get(expectedTriggerRow.id), + }, + ], + rows: [...triggerRows.values()].sort( + (left, right) => left.id - right.id, + ), + }, + { + changes: [], + rows: [], + }, + { + changes: [ + { + type: `insert`, + key: expectedRestartedRow.id, + row: expectedRestartedRow, + previousRow: undefined, + }, + ], + rows: [expectedRestartedRow], + }, + ]) + subscription.unsubscribe() + + await cleanup + model.clear() + model.set(expectedRestartedRow.id, expectedRestartedRow) + } else { + applyAction(action, model, harness.sync) + } + expectRetainedState(collection, model) + } + } finally { + await collection.cleanup() + } +} + +it(`retains only keys in the authoritative synced state`, async () => { + await runRetentionHistory([ + { type: `insert`, row: { id: 1, value: 1 } }, + { type: `insert`, row: { id: 2, value: 2 } }, + { type: `delete`, key: 1 }, + { type: `update`, row: { id: 1, value: -1 } }, + { type: `replace`, rows: [{ id: 3, value: 0 }] }, + { type: `delete`, key: 3 }, + ]) +}) + +it(`retains a missing row introduced by a sync update`, async () => { + await runRetentionHistory([{ type: `update`, row: { id: 1, value: 1 } }]) +}) + +it.each( + ([`insert`, `update`] as const).flatMap((triggerType) => + ([`insideListener`, `afterOldReturn`] as const).map( + (commitPhase) => [triggerType, commitPhase] as const, + ), + ), +)( + `retains an old-session %s and a restarted row committed %s`, + async (triggerType, commitPhase) => { + await runRetentionHistory([ + ...(triggerType === `update` + ? ([{ type: `insert`, row: { id: 1, value: 1 } }] as const) + : []), + { + type: `reentrantRestart`, + row: { id: 1, value: 1 }, + commitPhase, + }, + ]) + }, +) + +it(`releases retained keys after long unique-key churn`, async () => { + const keyCount = 1_000 + const actions: Array = [] + for (let key = 0; key < keyCount; key++) { + actions.push({ type: `insert`, row: { id: key, value: key } }) + actions.push({ type: `delete`, key }) + } + + await runRetentionHistory(actions) +}) + +it(`starts a new sync session without retained publication state`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + const events: Array<{ type: string; key: string | number }> = [] + let subscription: ReturnType | undefined + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + subscription = collection.subscribeChanges( + (changes) => { + events.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + })), + ) + }, + { includeInitialState: false }, + ) + + sync.begin() + sync.write({ type: `update`, value: { id: 1, value: 2 } }) + collection._state.capturePreSyncVisibleState() + expect(collection._state.preSyncVisibleState.size).toBe(1) + expect(collection._state.recentlySyncedKeys).toEqual(new Set([1])) + + const cleanup = collection.cleanup() + const retainedAfterCleanup = { + visibleRows: collection._state.preSyncVisibleState.size, + virtualRows: collection._state.preSyncVirtualState.size, + recentKeys: collection._state.recentlySyncedKeys.size, + } + await cleanup + + events.length = 0 + collection.startSyncImmediate() + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 3 } }) + expect(sync.commit()).toBe(true) + + expect({ retainedAfterCleanup, events }).toEqual({ + retainedAfterCleanup: { visibleRows: 0, virtualRows: 0, recentKeys: 0 }, + events: [{ type: `insert`, key: 1 }], + }) + } finally { + subscription?.unsubscribe() + await collection.cleanup() + } +}) + +it(`keeps a restarted session's publication state after the old listener returns`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + let cleanup: Promise | undefined + let restarted = false + const subscription = collection.subscribeChanges( + () => { + if (restarted) return + restarted = true + cleanup = collection.cleanup() + collection.startSyncImmediate() + collection._state.preSyncVisibleState.set(2, { id: 2, value: 2 }) + collection._state.recentlySyncedKeys.add(2) + }, + { includeInitialState: false }, + ) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + expect(restarted).toBe(true) + expect(collection._state.preSyncVisibleState).toEqual( + new Map([[2, { id: 2, value: 2 }]]), + ) + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + expect(collection._state.hasReceivedFirstCommit).toBe(false) + + sync.begin() + sync.write({ type: `insert`, value: { id: 3, value: 3 } }) + expect(sync.commit()).toBe(true) + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.preSyncVirtualState.size).toBe(0) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.recentlySyncedKeys.size).toBe(0) + await cleanup + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`does not let an old publication microtask clear restarted sync state`, async () => { + let sync!: SyncActions + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + actions.markReady() + }, + }, + }) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + const cleanup = collection.cleanup() + collection.startSyncImmediate() + sync.begin() + sync.write({ type: `insert`, value: { id: 2, value: 2 } }) + collection._state.capturePreSyncVisibleState() + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + + await Promise.resolve() + + expect(collection._state.recentlySyncedKeys).toEqual(new Set([2])) + + expect(sync.commit()).toBe(true) + expect(collection._state.hasReceivedFirstCommit).toBe(true) + await Promise.resolve() + expect(collection._state.preSyncVisibleState.size).toBe(0) + expect(collection._state.preSyncVirtualState.size).toBe(0) + expect(collection._state.recentlySyncedKeys.size).toBe(0) + await cleanup + } finally { + await collection.cleanup() + } +}) + +it(`publishes a virtual-state update when a restarted optimistic row is confirmed`, async () => { + let sync!: SyncActions + let syncSession = 0 + let releaseMutation!: () => void + const mutationHold = new Promise((resolve) => { + releaseMutation = resolve + }) + const collection = createCollection({ + getKey: (row) => row.id, + startSync: true, + sync: { + rowUpdateMode: `full`, + sync: (actions) => { + sync = actions + syncSession++ + if (syncSession === 1) actions.markReady() + }, + }, + }) + type ObservedRow = RetainedRow & { + $collectionId: string + $key: number + $origin: `local` | `remote` + $synced: boolean + } + type ObservedChange = { + type: string + key: string | number + value: ObservedRow + previousValue?: ObservedRow + } + const snapshotRow = (row: ObservedRow): ObservedRow => ({ + id: row.id, + value: row.value, + $collectionId: row.$collectionId, + $key: row.$key, + $origin: row.$origin, + $synced: row.$synced, + }) + const publications: Array<{ + changes: Array + rows: Array + }> = [] + const restartStatuses: Array = [] + const settlementTimeline: Array<`publication` | `receipt`> = [] + let restarted = false + let readMutationState: (() => TransactionState) | undefined + let rollbackMutation: (() => void) | undefined + let mutationCommit: Promise | undefined + let syncReceipt: ReturnType | undefined + let syncReceiptOutcome: Promise | undefined + let syncReceiptSettled = false + const subscription = collection.subscribeChanges( + (changes) => { + publications.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotRow(value), + ...(previousValue === undefined + ? {} + : { previousValue: snapshotRow(previousValue) }), + })), + rows: [...collection.state.values()].map(snapshotRow), + }) + if (changes.some(({ type, key }) => type === `update` && key === 2)) { + queueMicrotask(() => settlementTimeline.push(`publication`)) + } + if (restarted || !changes.some(({ key }) => key === 1)) return + + restarted = true + restartStatuses.push(collection.status) + void collection.cleanup() + restartStatuses.push(collection.status) + collection.startSyncImmediate() + restartStatuses.push(collection.status) + sync.markReady() + restartStatuses.push(collection.status) + + const transaction = createTransaction({ + autoCommit: false, + mutationFn: () => mutationHold, + }) + readMutationState = () => transaction.state + rollbackMutation = () => transaction.rollback() + void transaction.isPersisted.promise.catch(() => undefined) + transaction.mutate(() => collection.insert({ id: 2, value: 2 })) + mutationCommit = transaction.commit() + + sync.begin() + sync.write({ type: `insert`, value: { id: 2, value: 2 } }) + syncReceipt = sync.commit() + if (syncReceipt !== true) { + syncReceiptOutcome = syncReceipt.then((value) => { + settlementTimeline.push(`receipt`) + syncReceiptSettled = true + return value + }) + } + }, + { includeInitialState: false }, + ) + + try { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 1 } }) + expect(sync.commit()).toBe(true) + + const remoteRow = (id: number): ObservedRow => ({ + id, + value: id, + $collectionId: collection.id, + $key: id, + $origin: `remote`, + $synced: true, + }) + const localRow = (id: number): ObservedRow => ({ + id, + value: id, + $collectionId: collection.id, + $key: id, + $origin: `local`, + $synced: false, + }) + const expectedPublications = [ + { + changes: [{ type: `insert`, key: 1, value: remoteRow(1) }], + rows: [remoteRow(1)], + }, + { changes: [], rows: [] }, + { + changes: [{ type: `insert`, key: 2, value: localRow(2) }], + rows: [localRow(2)], + }, + { + changes: [ + { + type: `update`, + key: 2, + value: remoteRow(2), + previousValue: localRow(2), + }, + ], + rows: [remoteRow(2)], + }, + ] + expect(publications).toEqual(expectedPublications.slice(0, 3)) + expect([...collection.state.keys()]).toEqual([2]) + expect(restartStatuses).toEqual([`ready`, `cleaned-up`, `loading`, `ready`]) + expect(collection.status).toBe(`ready`) + + expect(syncReceipt).toBeDefined() + expect(syncReceipt).not.toBe(true) + expect(syncReceiptSettled).toBe(false) + if (syncReceipt === undefined || syncReceipt === true) { + throw new Error(`restarted sync receipt was not parked`) + } + expect(syncReceipt).toBeInstanceOf(Promise) + expect(syncReceiptOutcome).toBeDefined() + expect(rollbackMutation).toBeDefined() + await Promise.resolve() + expect(syncReceiptSettled).toBe(false) + expect(settlementTimeline).toEqual([]) + + rollbackMutation?.() + expect(publications).toEqual(expectedPublications) + expect(syncReceiptSettled).toBe(false) + await expect(syncReceiptOutcome).resolves.toBeUndefined() + expect(syncReceiptSettled).toBe(true) + expect(settlementTimeline).toEqual([`publication`, `receipt`]) + expect(publications).toEqual(expectedPublications) + expect([...collection.state.values()].map(snapshotRow)).toEqual([ + remoteRow(2), + ]) + + releaseMutation() + await mutationCommit + expect(readMutationState?.()).toBe(`failed`) + expect(publications).toEqual(expectedPublications) + expect([...collection.state.values()].map(snapshotRow)).toEqual([ + remoteRow(2), + ]) + } finally { + releaseMutation() + await mutationCommit + subscription.unsubscribe() + await collection.cleanup() + } +}) + +fcTest.prop( + [fc.array(retentionActionArbitrary, { minLength: 1, maxLength: 20 })], + oraclePropertyOptions(100, `collection-state.retention`), +)( + `matches retained authoritative state without optimistic overlays after every committed sync history`, + async (actions) => { + await runRetentionHistory(actions) + }, +) diff --git a/packages/db/tests/collection-subscribe-changes.test.ts b/packages/db/tests/collection-subscribe-changes.test.ts index 08dce91992..2a7eb9e288 100644 --- a/packages/db/tests/collection-subscribe-changes.test.ts +++ b/packages/db/tests/collection-subscribe-changes.test.ts @@ -2340,6 +2340,8 @@ describe(`Virtual properties`, () => { ) expect(optimisticInsert).toBeDefined() expect(optimisticInsert!.value.$synced).toBe(false) + expect(collection._state.pendingLocalOrigins.has(`row-1`)).toBe(true) + expect(collection._state.pendingOptimisticUpserts.has(`row-1`)).toBe(true) changes.length = 0 @@ -2361,6 +2363,8 @@ describe(`Virtual properties`, () => { expect(confirmedUpdate).toBeDefined() expect(confirmedUpdate!.value.$synced).toBe(true) expect(confirmedUpdate!.previousValue?.$synced).toBe(false) + expect(collection._state.pendingLocalOrigins.size).toBe(0) + expect(collection._state.pendingOptimisticUpserts.size).toBe(0) subscription.unsubscribe() }) diff --git a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts index 8b9d6be57c..6627383e2b 100644 --- a/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts +++ b/packages/db/tests/collection-subscriber-duplicate-inserts.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' import { BTreeIndex } from '../src/indexes/btree-index.js' import { createLiveQueryCollection, eq } from '../src/query/index.js' +import { reconcileChangesForD2 } from '../src/query/live/utils.js' import { mockSyncCollectionOptions } from './utils.js' import type { ChangeMessage } from '../src/types.js' @@ -15,8 +16,8 @@ import type { ChangeMessage } from '../src/types.js' * If duplicate inserts reach D2, multiplicity becomes > 1, and deletes won't * properly remove items (multiplicity goes from 2 to 1, not triggering removal). * - * The fix: CollectionSubscriber tracks keys sent to D2 (sentToD2Keys) and - * filters out duplicate inserts before they reach the pipeline. + * The source boundary tracks the exact row sent for each key. It filters + * duplicate inserts and uses the stored row for later D2 retractions. * * Additionally, for JOIN queries with lazy sources: * - The includeInitialState fix ensures internal lazy-loading subscriptions @@ -40,6 +41,37 @@ type Order = { } describe(`CollectionSubscriber duplicate insert prevention`, () => { + it(`retracts the exact row previously contributed for a source key`, () => { + const sentRows = new Map>() + const inserted = { id: `1`, status: `draft` } + const changed = { id: `1`, status: `published` } + + reconcileChangesForD2( + [{ type: `insert`, key: `1`, value: inserted }], + sentRows, + ) + const reconciled = reconcileChangesForD2( + [ + { + type: `update`, + key: `1`, + value: changed, + previousValue: changed, + }, + ], + sentRows, + ) + + expect(reconciled).toEqual([ + { + type: `update`, + key: `1`, + value: changed, + previousValue: inserted, + }, + ]) + }) + it(`should properly delete items from live query with orderBy + limit`, async () => { // This test verifies that items can be properly deleted from a live query // with orderBy + limit. If duplicate inserts reach D2, the delete won't work. diff --git a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts index ecec052497..48f9b03361 100644 --- a/packages/db/tests/collection-subscription-replay-oracle.property.test.ts +++ b/packages/db/tests/collection-subscription-replay-oracle.property.test.ts @@ -5,6 +5,7 @@ import { createDeferred } from '../src/deferred.js' import { BTreeIndex } from '../src/indexes/btree-index.js' import { ReverseIndex } from '../src/indexes/reverse-index.js' import { attachLoadSubsetRequestSignal } from '../src/load-subset-request-provenance.js' +import { getStableExpressionHash } from '../src/query/ir-stable-identity.js' import { Func, PropRef, Value } from '../src/query/ir.js' import { DeduplicatedLoadSubset } from '../src/query/subset-dedupe.js' import { createTransaction } from '../src/transactions.js' @@ -294,7 +295,7 @@ async function exerciseReplayCallbackCleanup({ }, unloadSubset: (options) => { unloads.push(options) - if (cleanupArmed && options.where === whereB && !failedB) { + if (cleanupArmed && sameWhere(options.where, whereB) && !failedB) { failedB = true throw nestedFailure } @@ -629,8 +630,8 @@ function expectSameSubsetRequest( actual: LoadSubsetOptions, expected: LoadSubsetOptions, ): void { - expect(actual.where).toBe(expected.where) - expect(actual.orderBy).toBe(expected.orderBy) + expect(sameWhere(actual.where, expected.where)).toBe(true) + expect(actual.orderBy).toEqual(expected.orderBy) expect(actual.limit).toBe(expected.limit) expect(actual.cursor).toEqual(expected.cursor) expect(actual.offset).toBe(expected.offset) @@ -641,13 +642,23 @@ function expectReplayRequestToRestart( stored: LoadSubsetOptions, expectedOffset = 0, ): void { - expect(actual.where).toBe(stored.where) - expect(actual.orderBy).toBe(stored.orderBy) + expect(sameWhere(actual.where, stored.where)).toBe(true) + expect(actual.orderBy).toEqual(stored.orderBy) expect(actual.limit).toBe(stored.limit) expect(actual.cursor).toBeUndefined() expect(actual.offset).toBe(expectedOffset) } +function sameWhere( + actual: LoadSubsetOptions[`where`], + expected: LoadSubsetOptions[`where`], +): boolean { + if (actual === undefined || expected === undefined) { + return actual === expected + } + return getStableExpressionHash(actual) === getStableExpressionHash(expected) +} + async function runReplayScenario(scenario: ReplayScenario): Promise { let begin!: () => void let write!: ( @@ -673,10 +684,12 @@ async function runReplayScenario(scenario: ReplayScenario): Promise { new Func(`eq`, [new PropRef([`id`]), new Value(demandId)]), ]), ) - const demandIdByWhere = new Map< - NonNullable, - ReplayDemandId - >([...demandWheres].map(([demandId, where]) => [where, demandId])) + const demandIdByWhereHash = new Map( + [...demandWheres].map(([demandId, where]) => [ + getStableExpressionHash(where), + demandId, + ]), + ) const requestByDemand = new Map() const activeDemandIds = new Set(scenario.demandIds) @@ -744,7 +757,9 @@ async function runReplayScenario(scenario: ReplayScenario): Promise { const demandId = options.where === undefined ? undefined - : demandIdByWhere.get(options.where) + : demandIdByWhereHash.get( + getStableExpressionHash(options.where), + ) if (demandId === undefined) { throw new Error(`Subset request did not preserve its demand`) } @@ -1794,7 +1809,7 @@ async function runOptimisticReplayScenario( } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, ...oracleReplay } = readOracleRunConfig() const generatedRuns = 30 * multiplier const generatedTimeout = 5_000 * multiplier @@ -2742,6 +2757,145 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`keeps an ordinary same-key write authoritative while an unordered request is pending`, async () => { + type Row = { id: `a` | `x`; rank: number } + type Outcome = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + type Phase = `initial` | `replay` | `additional` | `probe` + + const replayFailure = new Error(`sibling replay failed`) + const additionalLoad = createDeferred() + const loads: Array<{ phase: Phase; options: LoadSubsetOptions }> = [] + let phase: Phase = `initial` + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + + const collection = createCollection({ + id: `ordinary-write-during-unordered-request`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + + const apply = ( + row: Row, + signal: AbortSignal | undefined, + ): Outcome => { + begin() + write({ type: `insert`, value: row }) + commit(signal) + return { hasMore: false, appliedRowKeys: [row.id] } + } + + return { + loadSubset: (options) => { + loads.push({ phase, options }) + if (phase === `initial`) { + return options.orderBy + ? Promise.resolve(apply({ id: `a`, rank: 1 }, options.signal)) + : Promise.resolve({ + hasMore: false, + appliedRowKeys: [] as const, + }) + } + if (phase === `replay`) { + return options.orderBy + ? Promise.resolve(apply({ id: `x`, rank: 0 }, options.signal)) + : Promise.reject(replayFailure) + } + if (phase === `additional`) return additionalLoad.promise + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [] as const, + }) + }, + unloadSubset: () => {}, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderBy: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ] + const seedWhere = new Func(`eq`, [new PropRef([`id`]), new Value(`a`)]) + const additionalWhere = new Func(`eq`, [ + new PropRef([`id`]), + new Value(`x`), + ]) + const visible = new Set() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.add(key) + } + }) + subscription.setOrderByIndex(index) + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + subscription.requestSnapshot({ where: seedWhere }) + await flushPromises() + expect([...visible]).toEqual([`a`]) + + phase = `replay` + begin() + truncate() + commit() + await flushPromises() + await flushPromises() + expect(subscription.lastError).toBe(replayFailure) + expect(subscription.orderedBoundaryKey).toBe(`a`) + expect([...visible]).toEqual([`a`]) + + phase = `additional` + subscription.requestSnapshot({ where: additionalWhere }) + await flushPromises() + expect(loads.at(-1)).toMatchObject({ phase: `additional` }) + expect(loads.at(-1)?.options.orderBy).toBeUndefined() + + begin() + write({ type: `update`, value: { id: `x`, rank: -1 } }) + commit() + expect(subscription.orderedBoundaryKey).toBe(`x`) + expect([...visible].sort()).toEqual([`a`, `x`]) + + additionalLoad.reject(new Error(`sibling acquisition failed`)) + await flushPromises() + subscription.releaseSnapshot(additionalWhere) + expect(subscription.orderedBoundaryKey).toBe(`x`) + expect([...visible].sort()).toEqual([`a`, `x`]) + + phase = `probe` + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + await flushPromises() + expect(loads.at(-1)).toMatchObject({ + phase: `probe`, + options: { cursor: { lastKey: `x` } }, + }) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it.each([ `sync`, `async`, @@ -3429,17 +3583,17 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereX) { + if (sameWhere(options.where, whereX)) { begin() write({ type: `insert`, value: { id: `x`, value: 3 } }) return commit(options.signal) } if (replaying) { - return options.where === whereA + return sameWhere(options.where, whereA) ? replayA.promise : replayB.promise } - const id = options.where === whereA ? `a` : `b` + const id = sameWhere(options.where, whereA) ? `a` : `b` begin() write({ type: `insert`, @@ -3620,7 +3774,7 @@ describe(`CollectionSubscription replay oracle`, () => { } await flushPromises() - if (replacementResult === `return` || replacementResult === `resolve`) { + if (replacementResult === `resolve`) { expect(errorObservations).toEqual([]) expect([...visible.keys()]).toEqual([`y`]) expect(subscription.orderedBoundaryKey).toBe(`y`) @@ -3631,6 +3785,12 @@ describe(`CollectionSubscription replay oracle`, () => { await flushPromises() expect([...visible.keys()]).toEqual([`z`]) expect(subscription.orderedBoundaryKey).toBe(`z`) + } else if (replacementResult === `return`) { + // A synchronous outcome-free result can settle this acquisition, + // but cannot prove that the replay is a complete replacement. + expect(errorObservations).toEqual([]) + expect([...visible.keys()]).toEqual([]) + expect(subscription.orderedBoundaryKey).toBe(`y`) } else { expect(errorObservations).toEqual([[`x`]]) expect([...visible.keys()]).toEqual([`x`]) @@ -3694,11 +3854,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions.push(options) throw startError } - if (options.where === whereNestedSecond) { + if (sameWhere(options.where, whereNestedSecond)) { nestedOptions.push(options) throw secondStartError } @@ -3835,11 +3995,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { outerLoadCount++ if ( originContext === `replay-entry` && @@ -3854,7 +4014,7 @@ describe(`CollectionSubscription replay oracle`, () => { requestInner() } } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { if (propagation === `async`) { return (async () => { requestInner() @@ -3868,7 +4028,7 @@ describe(`CollectionSubscription replay oracle`, () => { unloadSubset: (options) => { if ( originContext === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { requestMiddle() } @@ -4002,12 +4162,12 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions = options throw startError } if ( - options.where === whereOuter || + sameWhere(options.where, whereOuter) || options.orderBy !== undefined ) { outerLoadCount++ @@ -4027,7 +4187,7 @@ describe(`CollectionSubscription replay oracle`, () => { unloadSubset: (options) => { if ( cleanupArmed && - options.where === whereCleanup && + sameWhere(options.where, whereCleanup) && cleanupThrowCount === 0 ) { cleanupThrowCount++ @@ -4766,7 +4926,12 @@ describe(`CollectionSubscription replay oracle`, () => { value: { id: `b`, rank: 2, version: 2 }, }) commit(options.signal) - return settlement === `sync` ? true : Promise.resolve() + return settlement === `sync` + ? true + : Promise.resolve({ + hasMore: false, + appliedRowKeys: [`b`] as const, + }) }, unloadSubset: (options) => { unloads.push(options) @@ -4847,8 +5012,7 @@ describe(`CollectionSubscription replay oracle`, () => { await flushPromises() const publishesReplacement = - callback === `none` || - (callback === `cleanup-succeed` && settlement === `sync`) + settlement === `async` && callback === `none` expect(subscription.status).toBe(`ready`) expect(escapedCallbackError).toBeUndefined() expect( @@ -5028,6 +5192,479 @@ describe(`CollectionSubscription replay oracle`, () => { } }) + it(`compiles an additional-demand predicate once per logical demand`, async () => { + type Row = { id: string; rank: number } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: () => void + const collection = createCollection({ + id: `additional-demand-predicate-compilation`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => true, + unloadSubset: () => {}, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderBy: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `first` }, + }, + ] + let expressionReads = 0 + // Compilation reads the IR node type; the compiled evaluator does not. + // Count those reads without exposing test instrumentation in production. + const expression = new Proxy( + new Func(`eq`, [new PropRef([`id`]), new Value(`sibling`)]), + { + get(target, property, receiver) { + if (property === `type`) expressionReads++ + return Reflect.get(target, property, receiver) + }, + }, + ) + const subscription = collection.subscribeChanges(() => {}) + subscription.setOrderByIndex(index) + + const publish = (value: Row) => { + begin() + write({ type: `insert`, value }) + commit() + } + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + subscription.requestSnapshot({ where: expression }) + const firstDemandReads = expressionReads + expect(firstDemandReads).toBeGreaterThan(0) + + publish({ id: `sibling`, rank: 2 }) + publish({ id: `ordered`, rank: 1 }) + expect(expressionReads).toBe(firstDemandReads) + + subscription.releaseSnapshot(expression) + const beforeReplacement = expressionReads + subscription.requestSnapshot({ where: expression }) + expect(expressionReads).toBeGreaterThan(beforeReplacement) + const replacementDemandReads = expressionReads + + publish({ id: `later`, rank: 0 }) + expect(expressionReads).toBe(replacementDemandReads) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`snapshots a logical demand before caller-owned predicate mutation`, async () => { + type Row = { id: `a` | `b`; other: `a` | `b` } + type Outcome = { + hasMore: false + appliedRowKeys: ReadonlyArray + } + const rows: ReadonlyArray = [ + { id: `a`, other: `b` }, + { id: `b`, other: `a` }, + ] + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + const replay = createDeferred() + const loads: Array = [] + const unloads: Array = [] + const collection = createCollection({ + id: `logical-demand-predicate-snapshot`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + begin() + for (const row of rows) write({ type: `insert`, value: row }) + commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + if (loads.length === 1) { + // Adapter code owns only this acquisition copy. Mutating it + // must not rewrite the private demand used by later replay. + ;((options.where as Func).args[0] as PropRef).path[0] = `other` + return true + } + return replay.promise + }, + unloadSubset: (options) => { + unloads.push(options) + }, + } + }, + }, + }) + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + const ref = new PropRef([`id`]) + const where = new Func(`eq`, [ref, new Value(`a`)]) + + try { + subscription.requestSnapshot({ where }) + expect([...visible.keys()]).toEqual([`a`]) + + ref.path[0] = `other` + begin() + truncate() + commit() + await flushPromises() + expect(loads).toHaveLength(2) + + begin() + for (const row of rows) write({ type: `insert`, value: row }) + const receipt = commit(loads[1]?.signal) + if (receipt !== true) await receipt + replay.resolve({ hasMore: false, appliedRowKeys: [`a`, `b`] }) + await flushPromises() + + expect(((loads[1]?.where as Func).args[0] as PropRef).path).toEqual([ + `id`, + ]) + expect([...visible.keys()]).toEqual([`a`]) + + subscription.releaseSnapshot(where) + expect(unloads.at(-1)).toBe(loads[1]) + } finally { + replay.resolve({ hasMore: false, appliedRowKeys: [] }) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`snapshots mutable values beneath output-producing predicate functions`, async () => { + type Row = { id: `row` } + type Outcome = { + hasMore: false + appliedRowKeys: ReadonlyArray + } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + const replay = createDeferred() + const loads: Array = [] + const collection = createCollection({ + id: `logical-demand-value-snapshot`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + begin() + write({ type: `insert`, value: { id: `row` } }) + commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + return loads.length === 1 ? true : replay.promise + }, + } + }, + }, + }) + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + const bytes = Buffer.from([65]) + const where = new Func(`eq`, [ + new Func(`concat`, [new Value(bytes)]), + new Value(`A`), + ]) + + try { + subscription.requestSnapshot({ where }) + expect([...visible.keys()]).toEqual([`row`]) + + bytes[0] = 66 + begin() + truncate() + commit() + await flushPromises() + + begin() + write({ type: `insert`, value: { id: `row` } }) + const receipt = commit(loads[1]?.signal) + if (receipt !== true) await receipt + replay.resolve({ hasMore: false, appliedRowKeys: [`row`] }) + await flushPromises() + + expect([...visible.keys()]).toEqual([`row`]) + } finally { + replay.resolve({ hasMore: false, appliedRowKeys: [] }) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it.each([`reference-path`, `direction`] as const)( + `snapshots ordered demand state before %s mutation`, + async (mutation) => { + type Row = { + id: `a` | `b` + rank: number + other: number + version: number + } + let begin!: () => void + let write!: ( + message: ChangeMessageOrDeleteKeyMessage, + ) => void + let commit!: () => void + const collection = createCollection({ + id: `ordered-demand-snapshot-${mutation}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + begin() + write({ + type: `insert`, + value: { id: `a`, rank: 1, other: 2, version: 0 }, + }) + write({ + type: `insert`, + value: { id: `b`, rank: 2, other: 1, version: 0 }, + }) + commit() + params.markReady() + return { loadSubset: () => true } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const orderRef = new PropRef([`rank`]) + const compareOptions: OrderBy[number][`compareOptions`] = { + direction: `asc`, + nulls: `first`, + stringSort: `lexical`, + } + const orderBy: OrderBy = [{ expression: orderRef, compareOptions }] + const visible = new Map() + const subscription = collection.subscribeChanges((changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }) + subscription.setOrderByIndex(index) + + try { + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + expect([...visible.keys()]).toEqual([`a`]) + + if (mutation === `reference-path`) orderRef.path[0] = `other` + else compareOptions.direction = `desc` + + begin() + write({ + type: `update`, + value: { id: `b`, rank: 2, other: 1, version: 1 }, + }) + commit() + + expect([...visible.keys()]).toEqual([`a`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + + it.each([`before-first-request`, `after-first-publication`] as const)( + `keeps one ordered machine when caller state mutates %s`, + async (timing) => { + type Row = { + id: `a` | `b` + group: `keep` | `drop` + alternate: `keep` | `drop` + rank: number + other: number + } + const loads: Array = [] + const collection = createCollection({ + id: `ordered-machine-${timing}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + params.begin() + params.write({ + type: `insert`, + value: { + id: `a`, + group: `keep`, + alternate: `drop`, + rank: 1, + other: 2, + }, + }) + params.write({ + type: `insert`, + value: { + id: `b`, + group: `drop`, + alternate: `keep`, + rank: 2, + other: 1, + }, + }) + params.commit() + params.markReady() + return { + loadSubset: (options) => { + loads.push(options) + return true + }, + } + }, + }, + }) + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + }) + const whereRef = new PropRef([`group`]) + const where = new Func(`eq`, [ + whereRef, + new Value(`keep`), + ]) + const orderRef = new PropRef([`rank`]) + const compareOptions: OrderBy[number][`compareOptions`] = { + direction: `asc`, + nulls: `first`, + stringSort: `lexical`, + } + const orderBy: OrderBy = [{ expression: orderRef, compareOptions }] + const visible = new Map() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + const key = change.key as Row[`id`] + if (change.type === `delete`) visible.delete(key) + else visible.set(key, change.value) + } + }, + { whereExpression: where }, + ) + subscription.setOrderByIndex(index) + const mutateCallerState = () => { + whereRef.path[0] = `alternate` + if (timing === `after-first-publication`) { + orderRef.path[0] = `other` + compareOptions.direction = `desc` + } + } + + try { + if (timing === `before-first-request`) mutateCallerState() + subscription.requestLimitedSnapshot({ orderBy, limit: 1 }) + + if (timing === `after-first-publication`) { + expect([...visible.keys()]).toEqual([`a`]) + mutateCallerState() + subscription.requestLimitedSnapshot({ orderBy, limit: 2 }) + } + + const lastLoad = loads.at(-1)! + const loadedWhere = lastLoad.where as Func + const loadedOrder = lastLoad.orderBy![0]! + expect((loadedWhere.args[0] as PropRef).path).toEqual([`group`]) + expect((loadedOrder.expression as PropRef).path).toEqual([`rank`]) + expect(loadedOrder.compareOptions.direction).toBe(`asc`) + expect([...visible.keys()]).toEqual([`a`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + + it(`rejects unsupported structural demand constants before adapter entry`, async () => { + type Row = { id: string } + let loadCount = 0 + const collection = createCollection({ + id: `unsupported-structural-demand`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + params.markReady() + return { + loadSubset: () => { + loadCount++ + return true + }, + } + }, + }, + }) + const subscription = collection.subscribeChanges(() => {}) + const value = { [Symbol.toPrimitive]: () => `A` } + const where = new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(`A`), + ]) + + try { + expect(() => subscription.requestSnapshot({ where })).toThrow( + /snapshot structural expression value/i, + ) + expect(loadCount).toBe(0) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`releases ordered publication authority before retrying failed adapter cleanup`, async () => { type Row = { id: string; rank: number } let begin!: () => void @@ -6215,7 +6852,11 @@ describe(`CollectionSubscription replay oracle`, () => { }, unloadSubset: (options) => { unloads.push(options) - if (cleanupArmed && options.where === whereC && !cleanupFailed) { + if ( + cleanupArmed && + sameWhere(options.where, whereC) && + !cleanupFailed + ) { cleanupFailed = true throw cleanupFailure } @@ -6468,15 +7109,17 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { nestedOptions = options throw startFailure } - if (replaying && options.where === whereA) throw replayFailure + if (replaying && sameWhere(options.where, whereA)) { + throw replayFailure + } return true }, unloadSubset: (options) => { - if (options.where === whereC && !cleanupFailed) { + if (sameWhere(options.where, whereC) && !cleanupFailed) { cleanupFailed = true throw cleanupFailure } @@ -6581,12 +7224,12 @@ describe(`CollectionSubscription replay oracle`, () => { loadSubset: (options) => { loads.push(options) if (!replaying) return true - return options.where === whereA + return sameWhere(options.where, whereA) ? replayA.promise : replayB.promise }, unloadSubset: (options) => { - if (options.where !== whereA) return + if (!sameWhere(options.where, whereA)) return unloadAttempts++ if (unloadAttempts <= 2) { throw cleanupFailure @@ -6688,7 +7331,7 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereNested) { + if (sameWhere(options.where, whereNested)) { failedOptions = options throw failure } @@ -6764,7 +7407,7 @@ describe(`CollectionSubscription replay oracle`, () => { loadSubset: () => true, unloadSubset: (options) => { unloads.push(options) - if (armed && options.where === whereB && !failed) { + if (armed && sameWhere(options.where, whereB) && !failed) { failed = true failedOptions = options throw failure @@ -6892,31 +7535,31 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereAfterTeardown) { + if (sameWhere(options.where, whereAfterTeardown)) { postTeardownLoads++ } - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { failedOptions = options throw failure } if ( replaying && activeFrame === `adapter-entry` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { failWithinBoundary(options) } if ( replaying && activeFrame === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { subscription.releaseSnapshot(whereCleanup) } return true }, unloadSubset: (options) => { - if (options.where !== whereCleanup) return + if (!sameWhere(options.where, whereCleanup)) return cleanupUnloads.push(options) if (replaying && activeFrame === `cleanup`) { failWithinBoundary(options) @@ -7042,14 +7685,14 @@ describe(`CollectionSubscription replay oracle`, () => { if ( replaying && activeFrame === `adapter-entry` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { startTeardown() } if ( replaying && activeFrame === `cleanup` && - options.where === whereOuter + sameWhere(options.where, whereOuter) ) { subscription.releaseSnapshot(whereActiveCleanup) } @@ -7059,11 +7702,11 @@ describe(`CollectionSubscription replay oracle`, () => { if ( replaying && activeFrame === `cleanup` && - options.where === whereActiveCleanup + sameWhere(options.where, whereActiveCleanup) ) { startTeardown() } - if (options.where !== whereTeardownCleanup) return + if (!sameWhere(options.where, whereTeardownCleanup)) return teardownCleanupOptions = options teardownCleanupUnloads++ if (!teardownCleanupFailed) { @@ -7174,11 +7817,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { subscription.releaseSnapshot(whereCleanup) return true } - if (options.where !== whereOuter) return true + if (!sameWhere(options.where, whereOuter)) return true outerLoads++ if (!replaying || outerLoads !== 2) return true @@ -7197,7 +7840,7 @@ describe(`CollectionSubscription replay oracle`, () => { throw outerFailure }, unloadSubset: (options) => { - if (options.where !== whereCleanup) return + if (!sameWhere(options.where, whereCleanup)) return cleanupUnloads++ cleanupOptions ??= options if (!cleanupFailed) { @@ -7337,7 +7980,7 @@ describe(`CollectionSubscription replay oracle`, () => { return { loadSubset: (options) => { loads.push(options) - if (options.where === whereB) { + if (sameWhere(options.where, whereB)) { nestedOptions = options throw failure } @@ -7345,7 +7988,7 @@ describe(`CollectionSubscription replay oracle`, () => { }, unloadSubset: (options) => { unloads.push(options) - if (options.where === whereA) { + if (sameWhere(options.where, whereA)) { try { owner.current!.requestSnapshot({ where: whereB }) } catch { @@ -7408,11 +8051,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { return (async () => { owner.current!.requestSnapshot({ where: whereInner }) await Promise.resolve() @@ -7421,7 +8064,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, @@ -7501,11 +8144,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { try { owner.current!.requestSnapshot({ where: whereInner }) } catch (error) { @@ -7514,8 +8157,8 @@ describe(`CollectionSubscription replay oracle`, () => { return true } if ( - options.where === whereLater || - (demandKind === `ordered` && options.orderBy === orderBy) + sameWhere(options.where, whereLater) || + (demandKind === `ordered` && options.orderBy !== undefined) ) { laterOptions = options if (laterFailure === `throw`) throw retainedCarrier @@ -7524,7 +8167,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, @@ -7597,11 +8240,11 @@ describe(`CollectionSubscription replay oracle`, () => { params.markReady() return { loadSubset: (options) => { - if (options.where === whereInner) { + if (sameWhere(options.where, whereInner)) { innerOptions = options throw failure } - if (options.where === whereMiddle) { + if (sameWhere(options.where, whereMiddle)) { middleOptions = options return (async () => { await Promise.resolve() @@ -7611,7 +8254,7 @@ describe(`CollectionSubscription replay oracle`, () => { return true }, unloadSubset: (options) => { - if (options.where === whereOuter) { + if (sameWhere(options.where, whereOuter)) { owner.current!.requestSnapshot({ where: whereMiddle }) } }, @@ -8330,7 +8973,8 @@ describe(`CollectionSubscription replay oracle`, () => { direction === `asc` ? ([`three`, `four`] as const) : ([`four`, `three`] as const) - const succeeds = delivery === `return` || delivery === `resolve` + const sourceSucceeded = delivery === `return` || delivery === `resolve` + const publishesReplacement = delivery === `resolve` const expectedIds = identity === `changed` ? replacementIds : initialIds try { @@ -8397,14 +9041,12 @@ describe(`CollectionSubscription replay oracle`, () => { } await flushPromises() expect(collection.toArray.map(({ id }) => id).sort()).toEqual( - succeeds ? [...expectedIds].sort() : [], + sourceSucceeded ? [...expectedIds].sort() : [], ) expect(publicationSnapshots).toEqual( delivery === `resolve` ? [[initialIds[0]], [...expectedIds].sort()] - : delivery === `return` && identity === `changed` - ? [[initialIds[0]], [expectedIds[0]]] - : [[initialIds[0]]], + : [[initialIds[0]]], ) const loadCountBeforeWiden = loadOptions.length @@ -8413,13 +9055,17 @@ describe(`CollectionSubscription replay oracle`, () => { limit: 1, minValues: [direction === `asc` ? 2 : 1], }) - if (succeeds) { + if (publishesReplacement) { expect(loadOptions).toHaveLength(loadCountBeforeWiden) } else { - expect(loadOptions[loadCountBeforeWiden]).toMatchObject({ - offset: 1, - cursor: { lastKey: initialIds[0] }, - }) + expect(loadOptions[loadCountBeforeWiden]).toMatchObject( + delivery === `return` + ? { offset: 1, cursor: undefined } + : { + offset: 1, + cursor: { lastKey: initialIds[0] }, + }, + ) } } finally { subscription.unsubscribe() @@ -8451,20 +9097,24 @@ describe(`CollectionSubscription replay oracle`, () => { { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `ordered`, rows: [], }, { type: `commitPublication`, publicationId: `initial` }, { type: `requestDemand`, + sourceId: `source`, ownerId: `other-owner`, sessionId: `session`, demandId: `other`, + attemptId: `other-attempt`, alreadyAborted: false, }, { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `other`, rows: [], }, @@ -8472,6 +9122,7 @@ describe(`CollectionSubscription replay oracle`, () => { ] const expectedBoundary = () => projectAtomicOrderedPublicationState(history, { + sourceId: `source`, demandId: `ordered`, direction: `asc`, initialWindowSize: 1, @@ -8547,7 +9198,10 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `beginReplacement`, publicationId: `replacement`, - demandIds: [`ordered`, `other`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `other` }, + ], }) const orderedReplay = replayLoads.find(({ options }) => options.orderBy) @@ -8565,6 +9219,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `stagePublicationRows`, publicationId: `replacement`, + sourceId: `source`, demandId: `ordered`, rows: [{ key: `new-ordered`, orderValue: 1 }], }) @@ -8575,6 +9230,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `ordered`, outcome: `success`, extent: `exhausted`, @@ -8592,6 +9248,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `other`, outcome: `success`, extent: `exhausted`, @@ -8601,6 +9258,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `replacement`, + sourceId: `source`, demandId: `other`, outcome: `failure`, }) @@ -8625,7 +9283,10 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `beginReplacement`, publicationId: `failed-replacement`, - demandIds: [`ordered`, `other`], + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `other` }, + ], }) const nextOrderedReplay = nextReplayLoads.find( ({ options }) => options.orderBy, @@ -8640,6 +9301,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `ordered`, outcome: `failure`, }) @@ -8650,6 +9312,7 @@ describe(`CollectionSubscription replay oracle`, () => { history.push({ type: `settleReplacement`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `other`, outcome: `success`, extent: `exhausted`, @@ -8892,7 +9555,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.ownership`, + ), )( `matches replay and ownership laws for a random or replayed seed`, runReplayScenario, @@ -8901,7 +9568,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sequentialReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.sequential`, + ), )( `matches synchronous, asynchronous, and partial-failure replay laws`, runSequentialReplayScenario, @@ -8924,7 +9595,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [replayCompletionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.completion`, + ), )( `preserves replay completion authority for a random or replayed seed`, runReplayCompletionScenario, @@ -8941,7 +9616,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [cleanupRestartScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.restart`, + ), )( `isolates cleanup and restart sessions for a random or replayed seed`, runCleanupRestartScenario, @@ -8959,7 +9638,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [sharedSubscriptionScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.shared`, + ), )( `keeps shared transport and logical ownership distinct for a random or replayed seed`, runSharedSubscriptionScenario, @@ -8977,7 +9660,11 @@ describe(`CollectionSubscription replay oracle`, () => { fcTest.prop( [optimisticReplayScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + oracleReplay, + `subscription-replay.optimistic`, + ), )( `preserves optimistic overlays across replay outcomes for a random or replayed seed`, runOptimisticReplayScenario, diff --git a/packages/db/tests/collection-sync-reentrancy.test.ts b/packages/db/tests/collection-sync-reentrancy.test.ts index a0239bf772..946a456374 100644 --- a/packages/db/tests/collection-sync-reentrancy.test.ts +++ b/packages/db/tests/collection-sync-reentrancy.test.ts @@ -13,6 +13,17 @@ type Row = { type SyncOps = Parameters[`sync`]>[0] +type OrderedRow = Row & { rank: number } +type OrderedSync = Parameters[`sync`]>[0] + +type LayoutCallback = { + changes: Array + keys: Array + values: Array + markedReceiptSettled: boolean + revision: number +} + type ListenerAction = `commit` | `abort` type ListenerScenario = { @@ -92,6 +103,20 @@ function stageInsert( sync.write({ type: `insert`, value: row }) } +function installInitialOrderedRows(sync: OrderedSync): void { + sync.begin({ immediate: true }) + sync.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + sync.commit() + sync.markReady() +} + async function runListenerScenario(scenario: ListenerScenario): Promise { const harness = createSyncHarness( `generated-listener-sync-${generatedHarnessId++}`, @@ -190,10 +215,820 @@ async function runListenerScenario(scenario: ListenerScenario): Promise { } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() const generatedRuns = 30 * multiplier describe(`sync publication reentrancy`, () => { + it.each([`open`, `prepared`, `published`] as const)( + `starts a second publication cycle with the first cycle %s`, + async (firstCycleState) => { + const harness = createSyncHarness(`publication-cycle-${firstCycleState}`) + const { collection } = harness + const callbacks: Array<{ + changes: Array + visibleValue: string + revision: number + }> = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map((change) => change.value.value), + visibleValue: collection.get(1)!.value, + revision: collection._stateRevision, + }) + }, + { includeInitialState: false }, + ) + const initialRevision = collection._stateRevision + const write = (type: `insert` | `update`, value: string) => { + harness.sync.begin({ immediate: true }) + harness.sync.write({ type, value: { id: 1, value } }) + harness.sync.commit() + } + + try { + const firstPublication = collection._deferPublication() + write(`insert`, `first`) + + if (firstCycleState === `open`) { + const secondPublication = collection._deferPublication() + write(`update`, `second`) + firstPublication.prepare() + secondPublication.prepare() + firstPublication.publish() + secondPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`, `second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } else if (firstCycleState === `prepared`) { + firstPublication.prepare() + expect(() => collection._deferPublication()).toThrow( + `Cannot start a publication cycle while another is prepared`, + ) + firstPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + ]) + } else { + firstPublication.prepare() + firstPublication.publish() + const secondPublication = collection._deferPublication() + write(`update`, `second`) + secondPublication.prepare() + secondPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + { + changes: [`second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + + it(`lets a publication callback start the next publication cycle`, async () => { + const harness = createSyncHarness(`publication-cycle-from-callback`) + const { collection } = harness + const callbacks: Array<{ + changes: Array + visibleValue: string + revision: number + }> = [] + const initialRevision = collection._stateRevision + const write = (type: `insert` | `update`, value: string) => { + harness.sync.begin({ immediate: true }) + harness.sync.write({ type, value: { id: 1, value } }) + harness.sync.commit() + } + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map((change) => change.value.value), + visibleValue: collection.get(1)!.value, + revision: collection._stateRevision, + }) + + if (changes[0]?.value.value === `first`) { + const secondPublication = collection._deferPublication() + write(`update`, `second`) + secondPublication.prepare() + secondPublication.publish() + } + }, + { includeInitialState: false }, + ) + + try { + const firstPublication = collection._deferPublication() + write(`insert`, `first`) + firstPublication.prepare() + firstPublication.publish() + + expect(callbacks).toEqual([ + { + changes: [`first`], + visibleValue: `first`, + revision: initialRevision + 1, + }, + { + changes: [`second`], + visibleValue: `second`, + revision: initialRevision + 2, + }, + ]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`publishes an internal layout swap with unchanged endpoints`, async () => { + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-middle-swap`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + ops.begin({ immediate: true }) + for (let id = 1; id <= 5; id++) { + ops.write({ + type: `insert`, + value: { id, value: `value-${id}`, rank: id }, + }) + } + ops.commit() + ops.markReady() + }, + }, + }) + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: false, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + + try { + const revisionBeforeSwap = collection._layoutRevision + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 3, value: `value-3`, rank: 4 }, + }) + sync.write({ + type: `update`, + value: { id: 4, value: `value-4`, rank: 3 }, + }) + sync.collection._markLayoutChange() + expect(sync.commit()).toBe(true) + + expect([...collection.keys()]).toEqual([1, 2, 4, 3, 5]) + expect(collection._layoutRevision).toBe(revisionBeforeSwap + 1) + expect(callbacks).toEqual([ + { + changes: [3, 4], + keys: [1, 2, 4, 3, 5], + values: [`value-1`, `value-2`, `value-4`, `value-3`, `value-5`], + markedReceiptSettled: false, + revision: revisionBeforeSwap + 1, + }, + ]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`compares layout with the public state before an immediate prefix drain`, async () => { + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + ops.begin({ immediate: true }) + ops.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 0 }, + }) + ops.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + ops.commit() + ops.markReady() + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + const callbacks: Array<{ + changes: Array + keys: Array + values: Array + }> = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(1, (draft) => { + draft.value = `optimistic-one` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.collection._markLayoutChange() + const secondReceipt = sync.commit() + + expect([...collection.keys()]).toEqual([1, 2, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-one`, + `two`, + `optimistic-three`, + ]) + expect(callbacks).toEqual([]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain) + await Promise.all( + [firstReceipt, secondReceipt] + .filter((receipt) => receipt !== true) + .map((receipt) => receipt), + ) + + updatePersistence.resolve() + insertPersistence.resolve() + await Promise.all([ + update.isPersisted.promise, + insert.isPersisted.promise, + ]) + } finally { + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it(`uses the post-removal public layout before an unmarked prefix drain`, async () => { + const updatePersistence = createDeferred() + const deletePersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-removal-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onDelete: () => deletePersistence.promise, + }) + const callbacks: Array = [] + let parkedReceiptSettled = false + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: parkedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + let deletion: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const parkedReceipt = sync.commit() + expect(parkedReceipt).not.toBe(true) + if (parkedReceipt !== true) { + void parkedReceipt.then(() => { + parkedReceiptSettled = true + }) + } + + deletion = collection.delete(1) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + await Promise.resolve() + expect(parkedReceiptSettled).toBe(false) + expect([...collection.keys()]).toEqual([2]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 2, value: `server-two`, rank: 1 }, + }) + const drainReceipt = sync.commit() + + expect(drainReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-two`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain) + expect(callbacks).toEqual([]) + if (parkedReceipt !== true) await parkedReceipt + expect(parkedReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + deletePersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await deletion?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`publishes a parked layout mark when optimistic persistence drains it`, async () => { + const updatePersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-normal-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + }) + let markedReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const receipt = sync.commit() + expect(receipt).not.toBe(true) + if (receipt !== true) { + void receipt.then(() => { + markedReceiptSettled = true + }) + } + + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + await Promise.resolve() + expect(markedReceiptSettled).toBe(false) + expect([...collection.keys()]).toEqual([1, 2]) + + updatePersistence.resolve() + await update.isPersisted.promise + if (receipt !== true) await receipt + + expect([...collection.keys()]).toEqual([2, 1]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `two`, + `one`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [1, 2], + keys: [2, 1], + values: [`two`, `one`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + expect(markedReceiptSettled).toBe(true) + } finally { + updatePersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await collection.cleanup() + } + }) + + it.each([`first`, `middle`, `last`, `first-and-middle`] as const)( + `honors %s layout marks in an immediate causal prefix`, + async (markPosition) => { + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-immediate-${markPosition}`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + let firstReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: firstReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(2, (draft) => { + draft.value = `optimistic-two` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: + markPosition === `first` || markPosition === `first-and-middle` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-a`, rank: 1 }, + }) + if (markPosition === `first` || markPosition === `first-and-middle`) { + sync.collection._markLayoutChange() + } + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + if (firstReceipt !== true) { + void firstReceipt.then(() => { + firstReceiptSettled = true + }) + } + await Promise.resolve() + expect(firstReceiptSettled).toBe(false) + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + + sync.begin() + sync.write({ + type: `update`, + value: + markPosition === `middle` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-b`, rank: 1 }, + }) + if (markPosition === `middle` || markPosition === `first-and-middle`) { + sync.collection._markLayoutChange() + } + const middleReceipt = sync.commit() + expect(middleReceipt).not.toBe(true) + + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: + markPosition === `last` + ? { id: 1, value: `one`, rank: 2 } + : { id: 2, value: `server-two-c`, rank: 1 }, + }) + if (markPosition === `last`) sync.collection._markLayoutChange() + const lastReceipt = sync.commit() + + expect(lastReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2, 1, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `optimistic-two`, + `one`, + `optimistic-three`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [1], + keys: [2, 1, 3], + values: [`optimistic-two`, `one`, `optimistic-three`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + + await Promise.all( + [firstReceipt, middleReceipt] + .filter((receipt) => receipt !== true) + .map((receipt) => receipt), + ) + expect(firstReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }, + ) + + it(`honors a parked layout mark when truncate drains its causal prefix`, async () => { + const updatePersistence = createDeferred() + const insertPersistence = createDeferred() + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-prefix-truncate-drain`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + onUpdate: () => updatePersistence.promise, + onInsert: () => insertPersistence.promise, + }) + let markedReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled, + revision: collection._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + const update = collection.update(1, (draft) => { + draft.value = `optimistic-one` + }) + let insert: ReturnType | undefined + + try { + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + const firstReceipt = sync.commit() + expect(firstReceipt).not.toBe(true) + if (firstReceipt !== true) { + void firstReceipt.then(() => { + markedReceiptSettled = true + }) + } + + insert = collection.insert({ + id: 3, + value: `optimistic-three`, + rank: 3, + }) + callbacks.length = 0 + const revisionBeforeDrain = collection._layoutRevision + expect([...collection.keys()]).toEqual([1, 2, 3]) + + sync.begin() + sync.truncate() + sync.write({ + type: `insert`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.write({ + type: `insert`, + value: { id: 2, value: `two`, rank: 1 }, + }) + const truncateReceipt = sync.commit() + + expect(truncateReceipt).toBe(true) + expect([...collection.keys()]).toEqual([2, 1, 3]) + expect(collection.toArray.map(({ value }) => value)).toEqual([ + `two`, + `optimistic-one`, + `optimistic-three`, + ]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1) + expect(callbacks).toEqual([ + { + changes: [2, 1, 3, 1, 3, 1, 2], + keys: [2, 1, 3], + values: [`two`, `optimistic-one`, `optimistic-three`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + ]) + if (firstReceipt !== true) await firstReceipt + expect(markedReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + updatePersistence.resolve() + insertPersistence.resolve() + await update.isPersisted.promise.catch(() => undefined) + await insert?.isPersisted.promise.catch(() => undefined) + await collection.cleanup() + } + }) + + it(`captures a fresh layout boundary for each reentrant causal prefix`, async () => { + let sync!: OrderedSync + const collection = createCollection({ + id: `layout-reentrant-prefixes`, + getKey: (row) => row.id, + compare: (left, right) => left.rank - right.rank, + startSync: true, + sync: { + sync: (ops) => { + sync = ops + installInitialOrderedRows(ops) + }, + }, + }) + let listenerDepth = 0 + let maxListenerDepth = 0 + let queuedRestore = false + let innerReceipt: Promise | undefined + let innerReceiptSettled = false + const callbacks: Array = [] + const subscription = collection.subscribeChanges( + (changes) => { + listenerDepth++ + maxListenerDepth = Math.max(maxListenerDepth, listenerDepth) + callbacks.push({ + changes: changes.map(({ key }) => key as number), + keys: [...collection.keys()], + values: collection.toArray.map(({ value }) => value), + markedReceiptSettled: innerReceiptSettled, + revision: collection._layoutRevision, + }) + + if (!queuedRestore) { + queuedRestore = true + sync.begin() + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 0 }, + }) + sync.collection._markLayoutChange() + const receipt = sync.commit() + if (receipt === true) { + throw new Error(`Expected listener-created work to queue`) + } + innerReceipt = receipt + void receipt.then(() => { + innerReceiptSettled = true + }) + } + + listenerDepth-- + }, + { includeInitialState: false }, + ) + + try { + const revisionBeforeDrain = collection._layoutRevision + sync.begin({ immediate: true }) + sync.write({ + type: `update`, + value: { id: 1, value: `one`, rank: 2 }, + }) + sync.collection._markLayoutChange() + expect(sync.commit()).toBe(true) + + expect([...collection.keys()]).toEqual([1, 2]) + expect(collection._layoutRevision).toBe(revisionBeforeDrain + 2) + expect(callbacks).toEqual([ + { + changes: [1], + keys: [2, 1], + values: [`two`, `one`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 1, + }, + { + changes: [1], + keys: [1, 2], + values: [`one`, `two`], + markedReceiptSettled: false, + revision: revisionBeforeDrain + 2, + }, + ]) + expect(maxListenerDepth).toBe(1) + expect(innerReceipt).toBeDefined() + await innerReceipt + expect(innerReceiptSettled).toBe(true) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }) + it(`preserves sync work opened by a listener until it is committed`, async () => { const harness = createSyncHarness(`listener-opened-sync-work`) const { collection } = harness @@ -609,7 +1444,11 @@ describe(`sync publication reentrancy`, () => { fcTest.prop( [listenerScenarioArbitrary], - oracleRandomParameters(generatedRuns, replaySeed), + oracleRandomParameters( + generatedRuns, + replay, + `collection-sync.reentrant-drain`, + ), )( `matches the reentrant drain laws for a random or replayed seed`, runListenerScenario, diff --git a/packages/db/tests/collection.test.ts b/packages/db/tests/collection.test.ts index 3ff8ede815..c167a3a689 100644 --- a/packages/db/tests/collection.test.ts +++ b/packages/db/tests/collection.test.ts @@ -2329,4 +2329,84 @@ describe(`Collection isLoadingSubset property`, () => { expect(result).toBe(true) expect(collection.isLoadingSubset).toBe(false) }) + + it(`rejects an already-aborted subset request before the adapter branch`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-subset-request`, + getKey: (item) => item.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + await collection.cleanup() + }) + + it(`rejects an already-aborted subset request before the eager return`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-eager-subset-request`, + getKey: (item) => item.id, + syncMode: `eager`, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + await collection.cleanup() + }) + + it(`rejects an already-aborted subset request before deferred start`, async () => { + const loadSubset = vi.fn(() => true as const) + const collection = createCollection<{ id: string; value: string }>({ + id: `already-aborted-deferred-subset-request`, + getKey: (item) => item.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { loadSubset } + }, + }, + }) + expect(collection._deferSyncStart()).toBe(true) + const request = new AbortController() + request.abort() + + await expect( + collection._sync.loadSubset({ signal: request.signal }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(loadSubset).not.toHaveBeenCalled() + expect(collection.isLoadingSubset).toBe(false) + collection._resumeSyncStart() + expect(loadSubset).not.toHaveBeenCalled() + await collection.cleanup() + }) }) diff --git a/packages/db/tests/comparison.property.test.ts b/packages/db/tests/comparison.property.test.ts index dd62790011..2cff29760f 100644 --- a/packages/db/tests/comparison.property.test.ts +++ b/packages/db/tests/comparison.property.test.ts @@ -375,36 +375,56 @@ describe(`normalizeValue property-based tests`, () => { }) fcTest.prop([fc.uint8Array({ minLength: 0, maxLength: 128 })])( - `small Uint8Arrays normalize to string representation`, + `small Uint8Arrays normalize to a stable key`, (arr) => { const normalized = normalizeValue(arr) expect(typeof normalized).toBe(`string`) - expect(normalized).toMatch(/^__u8__/) + expect(normalized).toBe(normalizeValue(new Uint8Array(arr))) }, ) fcTest.prop([fc.uint8Array({ minLength: 129, maxLength: 200 })])( - `large Uint8Arrays are not normalized`, + `large Uint8Arrays normalize to a stable key`, (arr) => { const normalized = normalizeValue(arr) - expect(normalized).toBe(arr) + expect(typeof normalized).toBe(`string`) + expect(normalized).toBe(normalizeValue(new Uint8Array(arr))) }, ) - fcTest.prop([fc.string()])(`strings pass through unchanged`, (str) => { - expect(normalizeValue(str)).toBe(str) - }) + fcTest.prop([fc.string()])( + `strings preserve equality after normalization`, + (str) => { + expect(normalizeValue(str)).toBe(normalizeValue(`${str}`)) + }, + ) fcTest.prop([fc.integer()])(`integers pass through unchanged`, (n) => { expect(normalizeValue(n)).toBe(n) }) fcTest.prop([fc.uint8Array({ minLength: 0, maxLength: 128 })])( - `normalization is idempotent for Uint8Arrays`, + `binary keys cannot collide with user strings`, (arr) => { - const normalized1 = normalizeValue(arr) - // For strings (which small arrays become), normalizing again should be identity - expect(normalizeValue(normalized1)).toBe(normalized1) + const normalized = normalizeValue(arr) + expect(normalizeValue(normalized)).not.toBe(normalized) + }, + ) + + fcTest( + `reads binary keys from intrinsic bytes instead of custom iteration`, + () => { + const bytes = new Uint8Array([2]) + Object.defineProperty(bytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + + expect(normalizeValue(bytes)).toBe(normalizeValue(new Uint8Array([2]))) + expect(normalizeValue(bytes)).not.toBe( + normalizeValue(new Uint8Array([1])), + ) }, ) }) diff --git a/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts new file mode 100644 index 0000000000..a9d5d93478 --- /dev/null +++ b/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts @@ -0,0 +1,715 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { + createCollection, + createLiveQueryCollection, + eq, +} from '../src/index.js' +import { BTreeIndex } from '../src/indexes/btree-index.js' +import { createEffect } from '../src/query/effect.js' +import { reconcileChangesForD2 } from '../src/query/live/utils.js' +import { oraclePropertyOptions } from './oracle-config.js' +import { flushPromises } from './utils.js' +import type { ChangeMessage, SyncConfig } from '../src/types.js' + +type SourceRow = { + id: number + revision: number + value: number +} + +type SourceSyncActions = Parameters[`sync`]>[0] + +type SourceKey = string | number + +type SourceOperation = + | { + type: `upsert` + key: SourceKey + row: SourceRow + reportedPreviousValue: SourceRow + } + | { + type: `rawUpdate` + key: SourceKey + row: SourceRow + reportedPreviousValue: SourceRow + } + | { type: `replay`; key: SourceKey } + | { type: `delete`; key: SourceKey; reportedValue: SourceRow } + +type ReconciliationStep = + | { type: `batch`; operations: ReadonlyArray } + | { type: `truncate` } + | { type: `teardown` } + | { type: `restart` } + +type ReconciliationModel = { + sourceRows: Map + sentRows: Map + relation: Map + graphActive: boolean +} + +const sourceRowArbitrary = fc.record({ + id: fc.integer({ min: 0, max: 3 }), + revision: fc.integer({ min: 0, max: 4 }), + value: fc.integer({ min: -2, max: 2 }), +}) + +const sourceKeyArbitrary: fc.Arbitrary = fc.oneof( + fc.integer({ min: 0, max: 2 }), + fc.constantFrom(`0`, `1`, `source`), +) + +const sourceOperationArbitrary: fc.Arbitrary = fc.oneof( + fc + .record({ + key: sourceKeyArbitrary, + row: sourceRowArbitrary, + reportedPreviousValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `upsert` as const, ...operation })), + fc + .record({ + key: sourceKeyArbitrary, + row: sourceRowArbitrary, + reportedPreviousValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `rawUpdate` as const, ...operation })), + sourceKeyArbitrary.map((key) => ({ type: `replay` as const, key })), + fc + .record({ + key: sourceKeyArbitrary, + reportedValue: sourceRowArbitrary, + }) + .map((operation) => ({ type: `delete` as const, ...operation })), +) + +const reconciliationStepArbitrary: fc.Arbitrary = fc.oneof( + { + weight: 8, + arbitrary: fc + .array(sourceOperationArbitrary, { minLength: 1, maxLength: 5 }) + .map((operations) => ({ type: `batch` as const, operations })), + }, + { weight: 1, arbitrary: fc.constant({ type: `truncate` as const }) }, + { weight: 1, arbitrary: fc.constant({ type: `teardown` as const }) }, + { weight: 1, arbitrary: fc.constant({ type: `restart` as const }) }, +) + +const reconciliationHistoryArbitrary = fc.array(reconciliationStepArbitrary, { + minLength: 1, + maxLength: 30, +}) + +function rowIdentity(row: SourceRow): string { + return `${row.id}:${row.revision}:${row.value}` +} + +function expectedWeightedRowIdentity(key: SourceKey, row: SourceRow): string { + const sourceIdentity = [typeof key, String(key)].join(`:`) + const payloadIdentity = [row.id, row.revision, row.value] + .map(String) + .join(`:`) + return `${sourceIdentity}|${payloadIdentity}` +} + +function addWeight( + relation: Map, + key: SourceKey, + row: SourceRow, + weight: 1 | -1, +): void { + const identity = `${typeof key}:${String(key)}|${rowIdentity(row)}` + const nextWeight = (relation.get(identity) ?? 0) + weight + if (nextWeight === 0) relation.delete(identity) + else relation.set(identity, nextWeight) +} + +function applyToRelation( + relation: Map, + changes: ReadonlyArray>, +): void { + for (const change of changes) { + if (change.type === `insert`) { + addWeight(relation, change.key, change.value, 1) + } else if (change.type === `update`) { + addWeight(relation, change.key, change.previousValue!, -1) + addWeight(relation, change.key, change.value, 1) + } else { + addWeight(relation, change.key, change.value, -1) + } + } +} + +function sourceChangesFor( + operations: ReadonlyArray, + sourceRows: Map, +): Array> { + const changes: Array> = [] + for (const operation of operations) { + if (operation.type === `upsert`) { + const previousValue = sourceRows.get(operation.key) + changes.push( + previousValue === undefined + ? { type: `insert`, key: operation.key, value: operation.row } + : { + type: `update`, + key: operation.key, + value: operation.row, + previousValue: operation.reportedPreviousValue, + }, + ) + sourceRows.set(operation.key, operation.row) + } else if (operation.type === `rawUpdate`) { + changes.push({ + type: `update`, + key: operation.key, + value: operation.row, + previousValue: operation.reportedPreviousValue, + }) + sourceRows.set(operation.key, operation.row) + } else if (operation.type === `replay`) { + const row = sourceRows.get(operation.key) + if (row !== undefined) { + changes.push({ type: `insert`, key: operation.key, value: row }) + } + } else { + changes.push({ + type: `delete`, + key: operation.key, + value: operation.reportedValue, + }) + sourceRows.delete(operation.key) + } + } + return changes +} + +function expectTrackerMatchesSource( + sourceRows: ReadonlyMap, + sentRows: ReadonlyMap, +): void { + const compareEntries = ( + [a]: readonly [SourceKey, SourceRow], + [b]: readonly [SourceKey, SourceRow], + ) => `${typeof a}:${String(a)}`.localeCompare(`${typeof b}:${String(b)}`) + expect([...sentRows.entries()].sort(compareEntries)).toEqual( + [...sourceRows.entries()].sort(compareEntries), + ) +} + +function expectWeightedRelationMatchesSource( + sourceRows: ReadonlyMap, + relation: ReadonlyMap, +): void { + expect( + [...relation.entries()].sort(([a], [b]) => a.localeCompare(b)), + ).toEqual( + [...sourceRows.entries()] + .map(([key, row]) => [expectedWeightedRowIdentity(key, row), 1] as const) + .sort(([a], [b]) => a.localeCompare(b)), + ) +} + +function createReconciliationModel(): ReconciliationModel { + return { + sourceRows: new Map(), + sentRows: new Map(), + relation: new Map(), + graphActive: true, + } +} + +function applyReconciliationStep( + model: ReconciliationModel, + step: ReconciliationStep, +): void { + if (step.type === `truncate`) { + // Truncate is only an early lifecycle signal. Its later source batch + // still needs the retained exact rows to retract the active graph. + } else if (step.type === `teardown`) { + model.sentRows.clear() + model.relation.clear() + model.graphActive = false + } else if (step.type === `restart`) { + if (!model.graphActive) { + const replay = [...model.sourceRows].map(([key, value]) => ({ + type: `insert` as const, + key, + value, + })) + applyToRelation( + model.relation, + reconcileChangesForD2(replay, model.sentRows), + ) + model.graphActive = true + } + } else { + const changes = sourceChangesFor(step.operations, model.sourceRows) + if (model.graphActive) { + const reconciled = reconcileChangesForD2(changes, model.sentRows) + applyToRelation(model.relation, reconciled) + } + } + + if (model.graphActive) { + expectTrackerMatchesSource(model.sourceRows, model.sentRows) + expectWeightedRelationMatchesSource(model.sourceRows, model.relation) + } else { + expect(model.sentRows.size).toBe(0) + expect(model.relation.size).toBe(0) + } +} + +function upsert( + key: SourceKey, + row: SourceRow, + reportedPreviousValue: SourceRow = row, +): ReconciliationStep { + return { + type: `batch`, + operations: [{ type: `upsert`, key, row, reportedPreviousValue }], + } +} + +function createOrderedSourceHarness(id: string) { + let sync!: SourceSyncActions + let loadSubsetCalls = 0 + const contributed = { id: 1, revision: 1, value: 1 } + const staleDelete = { id: 1, revision: 2, value: 1 } + const replacement = { id: 1, revision: 3, value: 2 } + const source = createCollection({ + id, + getKey: (row) => row.id, + startSync: true, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (actions) => { + sync = actions + actions.markReady() + return { + loadSubset: async () => { + loadSubsetCalls++ + if (loadSubsetCalls > 1) await new Promise(() => {}) + return { + hasMore: false as const, + appliedRowKeys: [contributed.id], + } + }, + } + }, + }, + }) + sync.begin() + sync.write({ type: `insert`, value: contributed }) + expect(sync.commit()).toBe(true) + + let sourceCallback: Parameters[0] | undefined + let suppressSourceChanges = false + const subscribeChanges = source.subscribeChanges.bind(source) + source.subscribeChanges = ((callback, options) => { + sourceCallback = callback + return subscribeChanges((changes) => { + if (!suppressSourceChanges) callback(changes) + }, options) + }) as typeof source.subscribeChanges + + return { + contributed, + replacement, + source, + staleDelete, + suppressSourceChanges: () => { + suppressSourceChanges = true + }, + publish: (changes: Array>) => { + if (sourceCallback === undefined) { + throw new Error(`Query did not subscribe to its source`) + } + const publish = sourceCallback as unknown as ( + messages: Array>, + ) => void + publish(changes) + }, + truncate: () => { + sync.begin() + sync.truncate() + expect(sync.commit()).toBe(true) + }, + } +} + +it(`ignores unknown deletes and inserts unknown updates at the D2 boundary`, () => { + const sentRows = new Map() + const stale = { id: 1, revision: 1, value: 1 } + const current = { id: 2, revision: 2, value: 2 } + + expect( + reconcileChangesForD2( + [{ type: `delete`, key: `row`, value: stale }], + sentRows, + ), + ).toEqual([]) + expect( + reconcileChangesForD2( + [ + { + type: `update`, + key: `row`, + previousValue: stale, + value: current, + }, + ], + sentRows, + ), + ).toEqual([{ type: `insert`, key: `row`, value: current }]) + expect(sentRows).toEqual(new Map([[`row`, current]])) +}) + +it(`retracts the exact Effect source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-effect-truncate-reconciliation`, + ) + const { contributed, source, staleDelete } = harness + const events: Array<{ + type: string + value: { id: number; revision: number; value: number } + }> = [] + const effect = createEffect({ + query: (query) => + query + .from({ row: source }) + .where(({ row }) => eq(row.revision, contributed.revision)) + .orderBy(({ row }) => row.value) + .limit(1), + onBatch: (batch) => { + events.push(...batch) + }, + }) + try { + await flushPromises() + expect(events).toHaveLength(1) + expect(events[0]).toMatchObject({ + type: `enter`, + key: 1, + value: contributed, + }) + const publishedValue = events[0]!.value + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(events).toEqual([{ type: `enter`, key: 1, value: publishedValue }]) + + harness.publish([{ type: `delete`, key: 1, value: staleDelete }]) + await flushPromises() + expect(events).toEqual([ + { type: `enter`, key: 1, value: publishedValue }, + { type: `exit`, key: 1, value: publishedValue }, + ]) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + +it(`retracts the exact live-query source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-live-query-truncate-reconciliation`, + ) + const { contributed, source, staleDelete } = harness + const live = createLiveQueryCollection({ + id: `d2-live-query-truncate-result`, + query: (query) => + query + .from({ row: source }) + .where(({ row }) => eq(row.revision, contributed.revision)) + .orderBy(({ row }) => row.value) + .limit(1), + startSync: true, + }) + + try { + await live.preload() + expect(live.get(contributed.id)).toMatchObject(contributed) + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(live.get(contributed.id)).toMatchObject(contributed) + + harness.publish([{ type: `delete`, key: 1, value: staleDelete }]) + await flushPromises() + expect(live.get(contributed.id)).toBeUndefined() + } finally { + await live.cleanup() + await source.cleanup() + } +}) + +it(`replaces the retained Effect source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness(`d2-effect-truncate-replacement`) + const { contributed, replacement, source, staleDelete } = harness + const batches: Array< + Array<{ + type: string + value: SourceRow + previousValue?: SourceRow + }> + > = [] + const effect = createEffect({ + query: (query) => + query + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + onBatch: (batch) => { + batches.push(batch) + }, + }) + + try { + await flushPromises() + expect(batches).toHaveLength(1) + expect(batches[0]).toHaveLength(1) + expect(batches[0]![0]).toMatchObject({ + type: `enter`, + key: 1, + value: contributed, + }) + const publishedValue = batches[0]![0]!.value + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(batches).toHaveLength(1) + + harness.publish([ + { + type: `update`, + key: 1, + previousValue: staleDelete, + value: replacement, + }, + ]) + await flushPromises() + expect(batches).toHaveLength(2) + expect(batches[1]).toHaveLength(1) + expect(batches[1]![0]).toMatchObject({ + type: `update`, + key: 1, + value: replacement, + }) + expect(batches[1]![0]!.previousValue).toBe(publishedValue) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + +it(`replaces the retained live-query source row after an ordered truncate`, async () => { + const harness = createOrderedSourceHarness( + `d2-live-query-truncate-replacement`, + ) + const { contributed, replacement, source, staleDelete } = harness + const live = createLiveQueryCollection({ + id: `d2-live-query-truncate-replacement-result`, + query: (query) => + query + .from({ row: source }) + .orderBy(({ row }) => row.value) + .limit(1), + startSync: true, + }) + const batches: Array>> = [] + + try { + await live.preload() + expect(live.get(contributed.id)).toMatchObject(contributed) + const publishedValue = live.get(contributed.id) + const subscription = live.subscribeChanges( + (changes) => batches.push(changes), + { includeInitialState: false }, + ) + + harness.suppressSourceChanges() + harness.truncate() + await flushPromises() + expect(batches).toEqual([]) + expect(live.get(contributed.id)).toBe(publishedValue) + + harness.publish([ + { + type: `update`, + key: 1, + previousValue: staleDelete, + value: replacement, + }, + ]) + await flushPromises() + expect(batches).toHaveLength(1) + expect(batches[0]).toHaveLength(1) + expect(batches[0]![0]).toMatchObject({ + type: `update`, + key: 1, + value: replacement, + }) + expect(batches[0]![0]!.previousValue).toEqual(publishedValue) + expect(live.get(replacement.id)).toMatchObject(replacement) + subscription.unsubscribe() + } finally { + await live.cleanup() + await source.cleanup() + } +}) + +it(`keeps revision and value in weighted row identity`, () => { + const key = `row` + const base = { id: 1, revision: 1, value: 1 } + const differentRevision = { id: 1, revision: 2, value: 1 } + const differentValue = { id: 1, revision: 1, value: 2 } + const relation = new Map() + + addWeight(relation, key, base, 1) + addWeight(relation, key, differentRevision, 1) + addWeight(relation, key, differentValue, 1) + + expect(relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(key, base), 1], + [expectedWeightedRowIdentity(key, differentRevision), 1], + [expectedWeightedRowIdentity(key, differentValue), 1], + ]), + ) +}) + +it(`keeps numeric and string source keys distinct across restart`, () => { + const model = createReconciliationModel() + const row = { id: 1, revision: 1, value: 1 } + const keys = [0, `0`] as const + + applyReconciliationStep(model, { + type: `batch`, + operations: keys.map((key) => ({ + type: `upsert` as const, + key, + row, + reportedPreviousValue: row, + })), + }) + expect(model.sourceRows).toEqual( + new Map([ + [0, row], + [`0`, row], + ]), + ) + expect(model.sentRows).toEqual(model.sourceRows) + expect(model.relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(0, row), 1], + [expectedWeightedRowIdentity(`0`, row), 1], + ]), + ) + + applyReconciliationStep(model, { type: `teardown` }) + applyReconciliationStep(model, { type: `restart` }) + expect(model.sourceRows).toEqual( + new Map([ + [0, row], + [`0`, row], + ]), + ) + expect(model.sentRows).toEqual(model.sourceRows) + expect(model.relation).toEqual( + new Map([ + [expectedWeightedRowIdentity(0, row), 1], + [expectedWeightedRowIdentity(`0`, row), 1], + ]), + ) +}) + +it(`preserves external source rows across graph teardown and restart`, () => { + const model = createReconciliationModel() + const row = { id: 1, revision: 1, value: 1 } + + applyReconciliationStep(model, upsert(`row`, row)) + applyReconciliationStep(model, { type: `teardown` }) + expect(model.graphActive).toBe(false) + expect(model.sourceRows).toEqual(new Map([[`row`, row]])) + expect(model.sentRows).toEqual(new Map()) + expect(model.relation).toEqual(new Map()) + + applyReconciliationStep(model, { type: `restart` }) + expect(model.graphActive).toBe(true) + expect(model.sourceRows).toEqual(new Map([[`row`, row]])) + expect(model.sentRows).toEqual(new Map([[`row`, row]])) + expect(model.relation).toEqual( + new Map([[expectedWeightedRowIdentity(`row`, row), 1]]), + ) +}) + +it(`replays external source changes made while the graph is down`, () => { + const model = createReconciliationModel() + const first = { id: 1, revision: 1, value: 1 } + const replacement = { id: 1, revision: 2, value: 2 } + + applyReconciliationStep(model, upsert(`row`, first)) + applyReconciliationStep(model, { type: `teardown` }) + applyReconciliationStep(model, upsert(`row`, replacement, first)) + expect(model.sourceRows).toEqual(new Map([[`row`, replacement]])) + expect(model.sentRows).toEqual(new Map()) + expect(model.relation).toEqual(new Map()) + + applyReconciliationStep(model, { type: `restart` }) + expect(model.sourceRows).toEqual(new Map([[`row`, replacement]])) + expect(model.sentRows).toEqual(new Map([[`row`, replacement]])) + expect(model.relation).toEqual( + new Map([[expectedWeightedRowIdentity(`row`, replacement), 1]]), + ) +}) + +it(`generates teardown, down-state source changes, and restart`, () => { + const histories = fc.sample(reconciliationHistoryArbitrary, { + seed: 1780, + numRuns: 500, + }) + + expect( + histories.some((steps) => { + let graphActive = true + let sawTeardown = false + let sawDownStateSourceChange = false + for (const step of steps) { + if (step.type === `teardown`) { + graphActive = false + sawTeardown = true + } else if (step.type === `restart`) { + if (!graphActive && sawTeardown && sawDownStateSourceChange) { + return true + } + graphActive = true + } else if (step.type === `batch` && !graphActive) { + sawDownStateSourceChange = true + } + } + return false + }), + ).toBe(true) +}) + +fcTest.prop( + [reconciliationHistoryArbitrary], + oraclePropertyOptions(200, `d2-source.exact-retractions`), +)( + `keeps one exact D2 contribution per source key across batched histories`, + (steps) => { + const model = createReconciliationModel() + for (const step of steps) { + applyReconciliationStep(model, step) + } + }, +) diff --git a/packages/db/tests/db-client.test.ts b/packages/db/tests/db-client.test.ts index fe8a0db0e2..6fdf08c344 100644 --- a/packages/db/tests/db-client.test.ts +++ b/packages/db/tests/db-client.test.ts @@ -1207,6 +1207,21 @@ describe(`DbClient`, () => { expect(() => adapterWrite({ id: `1`, name: `adapter` })).not.toThrow() expect(collection.get(`1`)?.name).toBe(`adapter`) + expect(collection._state.hydrationSeedKeys.has(`1`)).toBe(false) + expect(collection._state.hydratedKeys.has(`1`)).toBe(false) + + client.hydrate({ + collections: [ + { + collectionId: `ready-hydration-seed`, + rows: [{ key: `1`, value: { id: `1`, name: `late hydration` } }], + }, + ], + }) + + expect(collection.get(`1`)?.name).toBe(`adapter`) + expect(collection._state.hydrationSeedKeys.has(`1`)).toBe(false) + expect(collection._state.hydratedKeys.has(`1`)).toBe(false) }) it(`does not let a late stream chunk overwrite adapter rows or metadata`, async () => { diff --git a/packages/db/tests/effect.test.ts b/packages/db/tests/effect.test.ts index 7e5c12f676..8c9c644466 100644 --- a/packages/db/tests/effect.test.ts +++ b/packages/db/tests/effect.test.ts @@ -8,6 +8,7 @@ import { } from './utils.js' import type { DeltaEvent, + LoadSubsetOptions, SubscriptionLoadSubsetErrorEvent, } from '../src/index.js' @@ -677,6 +678,8 @@ describe(`createEffect`, () => { it(`reports one in-progress cleanup failure to every disposer`, async () => { const failure = new Error(`source release failed`) + let unloadCount = 0 + let shouldFail = true let resolveHandler!: () => void const handlerPending = new Promise((resolve) => { resolveHandler = resolve @@ -696,7 +699,8 @@ describe(`createEffect`, () => { return true }, unloadSubset: () => { - throw failure + unloadCount++ + if (shouldFail) throw failure }, } }, @@ -710,12 +714,73 @@ describe(`createEffect`, () => { await flushPromises() const firstDispose = effect.dispose() const secondDispose = effect.dispose() + expect(secondDispose).toBe(firstDispose) resolveHandler() await expect(firstDispose).rejects.toBe(failure) await expect(secondDispose).rejects.toBe(failure) + expect(unloadCount).toBe(1) + + shouldFail = false + const retry = effect.dispose() + expect(retry).not.toBe(firstDispose) + await retry + expect(unloadCount).toBe(2) await source.cleanup() }) + + it.each([ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, + ])(`retries a falsy cleanup failure: $name`, async ({ name, failure }) => { + let unloadCount = 0 + const source = createCollection<{ id: number }>({ + id: `effect-falsy-cleanup-${name}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => { + unloadCount++ + if (unloadCount === 1) throw failure + }, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => q.from({ source }), + onBatch: () => {}, + }) + + try { + await flushPromises() + let didReject = false + let rejection: unknown + try { + await effect.dispose() + } catch (error) { + didReject = true + rejection = error + } + expect(didReject).toBe(true) + expect(Object.is(rejection, failure)).toBe(true) + expect(unloadCount).toBe(1) + + await effect.dispose() + expect(unloadCount).toBe(2) + } finally { + await effect.dispose() + await source.cleanup() + } + }) }) describe(`auto-generated IDs`, () => { @@ -1390,6 +1455,173 @@ describe(`createEffect`, () => { ) } + it(`refills a joined result window after source rows are rejected`, async () => { + type Parent = { id: number; rank: number; groupId: number } + type Child = { id: number; groupId: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] + const delivered = new Set() + let requestCount = 0 + const parents = createCollection({ + id: `effect-joined-underfill-parents`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: () => { + const requestNumber = ++requestCount + const requested = requestNumber === 1 ? rows.slice(0, 2) : rows + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit() + return Promise.resolve(receipt).then(() => ({ + hasMore: requestNumber === 1, + appliedRowKeys: requested.map(({ id }) => id), + })) + }, + } + }, + }, + }) + const children = createCollection( + mockSyncCollectionOptions({ + id: `effect-joined-underfill-children`, + getKey: (row) => row.id, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ], + }), + ) + const visible = new Set() + const effect = createEffect<{ id: number }, string | number>({ + query: (q) => + q + .from({ parent: parents }) + .innerJoin({ child: children }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + onEnter: ({ value }) => { + visible.add(value.id) + }, + onExit: ({ value }) => { + visible.delete(value.id) + }, + }) + + try { + await flushPromises() + expect([...visible]).toEqual([2, 3]) + expect(requestCount).toBe(2) + } finally { + await effect.dispose() + await Promise.all([parents.cleanup(), children.cleanup()]) + } + }) + + it(`loads the full joined ordered source without an index`, async () => { + type Parent = { id: number; rank: number; groupId: number } + type Child = { id: number; groupId: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] + const delivered = new Set() + const requests: Array = [] + const parents = createCollection({ + id: `effect-no-index-underfill-parents`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options) => { + requests.push(options) + const requested = + options.limit === undefined + ? rows + : rows.slice(0, options.limit) + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit() + return Promise.resolve(receipt).then(() => ({ + hasMore: requested.length < rows.length, + appliedRowKeys: requested.map(({ id }) => id), + })) + }, + } + }, + }, + }) + const children = createCollection( + mockSyncCollectionOptions({ + id: `effect-no-index-underfill-children`, + getKey: (row) => row.id, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ], + }), + ) + const visible = new Set() + const effect = createEffect<{ id: number }, string | number>({ + query: (q) => + q + .from({ parent: parents }) + .innerJoin({ child: children }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + onEnter: ({ value }) => { + visible.add(value.id) + }, + onExit: ({ value }) => { + visible.delete(value.id) + }, + }) + + try { + await flushPromises() + expect([...visible]).toEqual([2, 3]) + expect(requests).toHaveLength(1) + expect(requests[0]?.limit).toBeUndefined() + } finally { + await effect.dispose() + await Promise.all([parents.cleanup(), children.cleanup()]) + } + }) + it(`should load more data when pipeline filters items from the orderBy window`, async () => { // 6 users, ordered by name asc, limit 3 // But we filter on active=true, and Bob/Dave are inactive @@ -1627,25 +1859,36 @@ describe(`createEffect`, () => { it(`releases every source when one unsubscriber throws`, async () => { const failure = new Error(`first source unload failed`) + let leftShouldFail = true + let leftUnloadCount = 0 + let rightUnloadCount = 0 const createSource = (id: string, unloadSubset: () => void) => createCollection<{ id: number }>({ id, getKey: (row) => row.id, syncMode: `on-demand`, sync: { - sync: ({ markReady }) => { + sync: ({ begin, write, commit, markReady }) => { markReady() return { - loadSubset: () => true, + loadSubset: () => { + begin() + write({ type: `insert`, value: { id: 1 } }) + commit() + return true + }, unloadSubset, } }, }, }) const left = createSource(`effect-cleanup-left`, () => { - throw failure + leftUnloadCount++ + if (leftShouldFail) throw failure + }) + const right = createSource(`effect-cleanup-right`, () => { + rightUnloadCount++ }) - const right = createSource(`effect-cleanup-right`, () => {}) const effect = createEffect({ query: (q) => q @@ -1662,6 +1905,13 @@ describe(`createEffect`, () => { await expect(effect.dispose()).rejects.toBe(failure) expect(left.subscriberCount).toBe(0) expect(right.subscriberCount).toBe(0) + expect(leftUnloadCount).toBe(1) + expect(rightUnloadCount).toBe(1) + + leftShouldFail = false + await effect.dispose() + expect(leftUnloadCount).toBe(2) + expect(rightUnloadCount).toBe(1) await Promise.all([left.cleanup(), right.cleanup()]) }) @@ -1899,6 +2149,86 @@ describe(`createEffect`, () => { } }) + it(`reports failed obsolete-demand release without failing the source commit`, async () => { + const failure = new Error(`obsolete effect demand release failed`) + const users = createCollection( + mockSyncCollectionOptions({ + id: `effect-obsolete-release-users`, + getKey: (user) => user.id, + initialData: [sampleUsers[0]!], + }), + ) + let loadCount = 0 + let unloadCount = 0 + const consoleError = vi + .spyOn(console, `error`) + .mockImplementation(() => {}) + const issues = createCollection({ + id: `effect-obsolete-release-issues`, + getKey: (issue) => issue.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + loadCount++ + return true + }, + unloadSubset: () => { + unloadCount++ + if (unloadCount <= 2) throw failure + }, + } + }, + }, + }) + const sourceErrors: Array = [] + const effect = createEffect({ + query: (q) => + q + .from({ user: users }) + .leftJoin({ issue: issues }, ({ user, issue }) => + eq(user.id, issue.userId), + ) + .select(({ user, issue }) => ({ + id: user.id, + issueId: issue.id, + })), + onBatch: () => {}, + onSourceError: (error) => sourceErrors.push(error), + }) + + try { + await flushPromises() + expect(loadCount).toBe(1) + + let commitError: unknown + try { + users.utils.begin() + users.utils.write({ type: `delete`, value: sampleUsers[0]! }) + users.utils.commit() + } catch (error) { + commitError = error + } + await flushPromises() + + expect(commitError).toBeUndefined() + expect(sourceErrors).toEqual([failure]) + expect(effect.disposed).toBe(true) + expect(unloadCount).toBe(2) + + await effect.dispose() + expect(unloadCount).toBe(3) + } finally { + await effect.dispose() + await Promise.all([users.cleanup(), issues.cleanup()]) + consoleError.mockRestore() + } + }) + it(`reports a rejected ordered subset load and disposes the effect`, async () => { const failure = new Error(`ordered subset failed`) let loadCount = 0 @@ -1991,7 +2321,7 @@ describe(`createEffect`, () => { }, unloadSubset: () => { unloadCount++ - throw cleanupFailure + if (unloadCount <= 2) throw cleanupFailure }, } }, @@ -2027,7 +2357,8 @@ describe(`createEffect`, () => { cleanupFailure, cleanupFailure, ]) - await expect(effect.dispose()).rejects.toBe(cleanupError) + await effect.dispose() + expect(unloadCount).toBe(4) } finally { consoleErrorSpy.mockRestore() await users.cleanup() diff --git a/packages/db/tests/integration/uint8array-id-comparison.test.ts b/packages/db/tests/integration/uint8array-id-comparison.test.ts index 7b13c04f63..481b7d465c 100644 --- a/packages/db/tests/integration/uint8array-id-comparison.test.ts +++ b/packages/db/tests/integration/uint8array-id-comparison.test.ts @@ -79,8 +79,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { expect(resultByName?.name).toBe(makeItemName(selectedItemIndex)) }) - it(`should use reference equality for large Uint8Arrays (> 128 bytes)`, async () => { - // Create a large Uint8Array (> 128 bytes) that should use reference equality + it(`should use content equality for large Uint8Arrays`, async () => { const largeId = new Uint8Array(200).fill(42) interface LargeItem { @@ -102,7 +101,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { }), ) - // Query with the exact same reference - this should work + // The same reference works. const queryWithSameRef = createLiveQueryCollection((q) => q .from({ item: collection }) @@ -113,12 +112,10 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { await queryWithSameRef.preload() const resultWithSameRef = Array.from(queryWithSameRef.entries())[0]?.[1] - // Should find the item because we're using the same reference expect(resultWithSameRef).toBeDefined() expect(resultWithSameRef?.name).toBe(`Large Item`) - // Query with a different instance but same content - this will NOT work - // because large arrays use reference equality + // A different instance with the same bytes has the same value. const differentInstance = new Uint8Array(200).fill(42) const queryWithDifferentRef = createLiveQueryCollection((q) => q @@ -132,8 +129,7 @@ describe(`Uint8Array ID comparison (user reproduction)`, () => { queryWithDifferentRef.entries(), )[0]?.[1] - // Should NOT find the item because large arrays use reference equality - // This is expected behavior to avoid memory overhead - expect(resultWithDifferentRef).toBeUndefined() + expect(resultWithDifferentRef).toBeDefined() + expect(resultWithDifferentRef?.name).toBe(`Large Item`) }) }) diff --git a/packages/db/tests/live-query-order-only-move.test.ts b/packages/db/tests/live-query-order-only-move.test.ts index 5d6bf5181d..ddcf06ae54 100644 --- a/packages/db/tests/live-query-order-only-move.test.ts +++ b/packages/db/tests/live-query-order-only-move.test.ts @@ -43,7 +43,7 @@ async function makeOrderedByAge(source: ReturnType) { const flush = () => new Promise((r) => setTimeout(r, 0)) -describe(`order-only move (RFC #1623 phase 4)`, () => { +describe(`order-only move publication`, () => { it(`republishes the ordered result when a row moves but its value is unchanged`, async () => { const source = makeSource() const lq = await makeOrderedByAge(source) @@ -117,7 +117,7 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { observer.dispose() }) - it(`refreshes a detached observer when an order-only sync is parked`, async () => { + it(`refreshes a detached observer while a separate mutation persists`, async () => { const source = makeSource() const persist = createDeferred() const lq = createLiveQueryCollection({ @@ -134,9 +134,14 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { { id: string; name: string }, string >(lq as any) + const publications: Array> = [] + const subscription = lq.subscribeChanges( + (changes) => publications.push(changes), + { includeInitialState: false }, + ) const before = observer.getSnapshot() - const collectionLayoutRevisionBefore = lq._layoutRevision + const layoutRevisionBeforeMutation = lq._layoutRevision expect((before.data as Array).map((row) => row.id)).toEqual([ `2`, `1`, @@ -149,6 +154,9 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { (draft) => void (draft.name = `Pending`), ) expect(mutation.state).toBe(`persisting`) + expect(lq._layoutRevision).toBe(layoutRevisionBeforeMutation) + expect(publications).toEqual([]) + const layoutRevisionBeforeSourceCommit = lq._layoutRevision source.utils.begin() source.utils.write({ @@ -156,15 +164,16 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { value: { id: `2`, name: `Bob`, age: 99 }, }) source.utils.commit() - await flush() - const parked = observer.getSnapshot() - expect((parked.data as Array).map((row) => row.id)).toEqual([ - `2`, + const whilePersisting = observer.getSnapshot() + expect((whilePersisting.data as Array).map((row) => row.id)).toEqual([ `1`, `3`, + `2`, ]) - expect(lq._layoutRevision).toBe(collectionLayoutRevisionBefore) + expect(lq._layoutRevision).toBe(layoutRevisionBeforeSourceCommit + 1) + expect(publications).toEqual([[]]) + const publishedLayoutRevision = lq._layoutRevision persist.resolve() await mutation.isPersisted.promise @@ -176,7 +185,9 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { `3`, `2`, ]) - expect(lq._layoutRevision).toBeGreaterThan(collectionLayoutRevisionBefore) + expect(lq._layoutRevision).toBe(publishedLayoutRevision) + expect(publications).toEqual([[]]) + subscription.unsubscribe() observer.dispose() }) @@ -211,6 +222,48 @@ describe(`order-only move (RFC #1623 phase 4)`, () => { observer.dispose() }) + it(`does not publish a move whose only crossed peer is optimistically deleted`, async () => { + const source = makeSource() + const persist = createDeferred() + const lq = createLiveQueryCollection({ + getKey: (row) => row.id, + query: (q) => + q + .from({ p: source }) + .orderBy(({ p }) => p.age, `asc`) + .select(({ p }) => ({ id: p.id, name: p.name })), + onDelete: () => persist.promise, + }) + await lq.preload() + const publications: Array> = [] + const subscription = lq.subscribeChanges( + (changes) => publications.push(changes), + { includeInitialState: false }, + ) + const mutation = lq.delete(`2`) + + expect(mutation.state).toBe(`persisting`) + expect(lq.toArray.map(({ id }) => id)).toEqual([`1`, `3`]) + publications.length = 0 + const revisionBeforeSource = lq._layoutRevision + + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: `1`, name: `Alice`, age: 10 }, + }) + source.utils.commit() + + expect(lq.toArray.map(({ id }) => id)).toEqual([`1`, `3`]) + expect(publications).toEqual([]) + expect(lq._layoutRevision).toBe(revisionBeforeSource) + + persist.resolve() + await mutation.isPersisted.promise + subscription.unsubscribe() + await Promise.all([lq.cleanup(), source.cleanup()]) + }) + it(`does not publish when multiple moves cancel within one transaction`, async () => { const source = makeSource() const lq = await makeOrderedByAge(source) diff --git a/packages/db/tests/load-subset-full-flow-model.ts b/packages/db/tests/load-subset-full-flow-model.ts index 33f43c6d17..699bba65ea 100644 --- a/packages/db/tests/load-subset-full-flow-model.ts +++ b/packages/db/tests/load-subset-full-flow-model.ts @@ -1,6 +1,24 @@ +/** + * A shared event vocabulary for small, independent refinement projections. + * + * This is deliberately not a second implementation of the Collection state + * machine. Each projector owns one law and ignores unrelated events. The + * lifecycle command model generates legal acquisition/release histories; + * boundary suites compare these projections with public Collection + * observations at the points where planes meet. + */ export type FullFlowOwnerId = string export type FullFlowSessionId = string export type FullFlowDemandId = string +export type FullFlowAttemptId = string +export type FullFlowSourceId = string +export type FullFlowTransactionId = string +export type FullFlowAcquisitionId = string +export type FullFlowVersionedRow = { + sourceId: FullFlowSourceId + rowKey: string + version: number +} export type FullFlowPublicationId = string export type FullFlowPublishedOrderRow = { @@ -8,6 +26,11 @@ export type FullFlowPublishedOrderRow = { orderValue: number } +export type FullFlowSourceDemand = { + sourceId: FullFlowSourceId + demandId: FullFlowDemandId +} + export type OrderedContinuationEvidencePage = { requestedPrefix: number appliedKeys: ReadonlyArray @@ -22,6 +45,68 @@ export type OrderedContinuationEvidence = { rowsNeeded: number } +export type OrderedSourceStep = { + sourceKey: string + resultKeys: ReadonlyArray + demandKeys: ReadonlyArray +} + +export type OrderedSourceProgress = { + visibleResultKeys: ReadonlyArray + scannedSourceKeys: ReadonlyArray + sourceCursorKeys: ReadonlyArray + demandedKeys: ReadonlyArray + rowsNeeded: number + sourceExhausted: boolean +} + +/** + * Projects the smallest forward source scan that fills a result window. Each + * step contains result contributions already evaluated by the owning DBSP + * oracle or an eager production control. This model owns source progress only; + * it does not interpret predicates, joins, grouping, ordering, or includes. + */ +export function projectOrderedSourceProgress(options: { + sourceSteps: ReadonlyArray + offset: number + limit: number +}): OrderedSourceProgress { + const scannedSourceKeys: Array = [] + const resultKeys: Array = [] + const demandedKeys: Array = [] + const seenDemandKeys = new Set() + const targetSize = options.limit === 0 ? 0 : options.offset + options.limit + + for (const step of options.sourceSteps) { + if (resultKeys.length >= targetSize) break + + scannedSourceKeys.push(step.sourceKey) + for (const demandKey of step.demandKeys) { + if (!seenDemandKeys.has(demandKey)) { + seenDemandKeys.add(demandKey) + demandedKeys.push(demandKey) + } + } + resultKeys.push(...step.resultKeys) + } + + const visibleResultKeys = resultKeys.slice( + options.offset, + options.offset + options.limit, + ) + + return { + visibleResultKeys, + scannedSourceKeys, + sourceCursorKeys: scannedSourceKeys.map((_, index) => + index === 0 ? undefined : scannedSourceKeys[index - 1], + ), + demandedKeys, + rowsNeeded: Math.max(0, options.limit - visibleResultKeys.length), + sourceExhausted: scannedSourceKeys.length === options.sourceSteps.length, + } +} + /** * Projects ordered evidence from request receipts alone. Requested size and * source progress are independent inputs; only eligible applied rows count @@ -106,37 +191,51 @@ export type LoadSubsetFullFlowEvent = type: `requestDemand` ownerId: FullFlowOwnerId sessionId: FullFlowSessionId + sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId alreadyAborted: boolean } | { type: `applyAuthoritativeRows` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId rowKeys: ReadonlyArray } + | { + type: `settleDemandWithoutEvidence` + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + } | { type: `applyUnprovenRows` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId rowKeys: ReadonlyArray } | { type: `rejectDemand` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId + attemptId: FullFlowAttemptId } | { type: `truncateSource` sessionId: FullFlowSessionId + sourceId: FullFlowSourceId } | { type: `releaseDemand` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId demandId: FullFlowDemandId - rowKeys: ReadonlyArray - finalRowOwner: boolean - invalidatesAdapterEvidence: boolean + attemptId: FullFlowAttemptId } | { type: `restartSession` @@ -162,9 +261,98 @@ export type LoadSubsetFullFlowEvent = type: `runContinuation` taskId: string } + | { + type: `stageSyncTransaction` + transactionId: FullFlowTransactionId + sourceId: FullFlowSourceId + rowKeys: ReadonlyArray + } + | { + type: `commitSyncTransaction` + transactionId: FullFlowTransactionId + parked: boolean + signalAborted: boolean + } + | { + type: `enterSyncApplication` + transactionId: FullFlowTransactionId + } + | { + type: `abortSyncTransaction` + transactionId: FullFlowTransactionId + } + | { + type: `publishSyncTransaction` + transactionId: FullFlowTransactionId + } + | { + type: `settleSyncReceipt` + transactionId: FullFlowTransactionId + } + | { + type: `establishPublication` + sourceId: FullFlowSourceId + rows: ReadonlyArray + } + | { + type: `startReplay` + attemptId: string + sourceId: FullFlowSourceId + } + | { + type: `writeReplayRows` + attemptId: string + rows: ReadonlyArray + acceptedByCore: boolean + } + | { + type: `settleReplay` + attemptId: string + outcome: `resolve` | `reject` + } + | { + type: `registerSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + } + | { + type: `settleSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + outcome: `resolve` | `reject` + } + | { + type: `retireSourceDemand` + sessionId: FullFlowSessionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + } + | { + type: `startAcquisition` + acquisitionId: FullFlowAcquisitionId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + } + | { + type: `attachAcquisitionOwner` + acquisitionId: FullFlowAcquisitionId + ownerId: FullFlowOwnerId + } + | { + type: `settleAcquisition` + acquisitionId: FullFlowAcquisitionId + outcome: `resolve` | `reject` + rowKeys: ReadonlyArray + } | { type: `stagePublicationRows` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId rows: ReadonlyArray } @@ -175,17 +363,19 @@ export type LoadSubsetFullFlowEvent = | { type: `beginReplacement` publicationId: FullFlowPublicationId - demandIds: ReadonlyArray + demands: ReadonlyArray } | { type: `settleReplacement` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId outcome: `failure` | `abort` } | { type: `settleReplacement` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId demandId: FullFlowDemandId outcome: `success` extent: `exhausted` | `continues` @@ -193,15 +383,189 @@ export type LoadSubsetFullFlowEvent = | { type: `establishReplacementCoverage` publicationId: FullFlowPublicationId + sourceId: FullFlowSourceId + demandId: FullFlowDemandId } | { type: `resizeOrderedWindow` + sourceId: FullFlowSourceId + demandId: FullFlowDemandId size: number } export type ExpectedAdapterLifecycleEvent = { type: `invoke` | `release` ownerId: FullFlowOwnerId + sourceId: FullFlowSourceId + attemptId: FullFlowAttemptId +} + +type ScopedIdentity = string +type ActiveDemandAttempts = Map> +type AcquisitionAttempts = Map> + +function scopedIdentity(...parts: ReadonlyArray): ScopedIdentity { + return parts.map((part) => `${part.length}:${part}`).join(`|`) +} + +function sourceDemandIdentity( + sourceId: FullFlowSourceId, + demandId: FullFlowDemandId, +): ScopedIdentity { + return scopedIdentity(sourceId, demandId) +} + +function sourceAttemptIdentity( + sourceId: FullFlowSourceId, + attemptId: FullFlowAttemptId, +): ScopedIdentity { + return scopedIdentity(sourceId, attemptId) +} + +function sourceDemandAttemptIdentity( + sourceId: FullFlowSourceId, + demandId: FullFlowDemandId, + attemptId: FullFlowAttemptId, +): ScopedIdentity { + return scopedIdentity(sourceId, demandId, attemptId) +} + +function sourceRowIdentity( + sourceId: FullFlowSourceId, + rowKey: string, +): ScopedIdentity { + return scopedIdentity(sourceId, rowKey) +} + +function belongsToSource( + identity: ScopedIdentity, + sourceId: FullFlowSourceId, +): boolean { + return identity.startsWith(`${sourceId.length}:${sourceId}|`) +} + +function addActiveDemandAttempt( + activeAttempts: ActiveDemandAttempts, + demandId: ScopedIdentity, + attemptId: ScopedIdentity, +): void { + let attempts = activeAttempts.get(demandId) + if (!attempts) { + attempts = new Set() + activeAttempts.set(demandId, attempts) + } + attempts.add(attemptId) +} + +function releaseActiveDemandAttempt( + activeAttempts: ActiveDemandAttempts, + demandId: ScopedIdentity, + attemptId: ScopedIdentity, +): boolean { + const attempts = activeAttempts.get(demandId) + if (!attempts?.delete(attemptId)) return false + if (attempts.size > 0) return false + activeAttempts.delete(demandId) + return true +} + +function addAcquisitionAttempt( + acquisitionAttempts: AcquisitionAttempts, + acquisitionId: ScopedIdentity, + attemptId: ScopedIdentity, +): void { + let attempts = acquisitionAttempts.get(acquisitionId) + if (!attempts) { + attempts = new Set() + acquisitionAttempts.set(acquisitionId, attempts) + } + attempts.add(attemptId) +} + +function releaseAcquisitionAttempt( + acquisitionAttempts: AcquisitionAttempts, + acquisitionId: ScopedIdentity, + attemptId: ScopedIdentity, +): boolean { + const attempts = acquisitionAttempts.get(acquisitionId) + if (!attempts?.delete(attemptId) || attempts.size > 0) return false + acquisitionAttempts.delete(acquisitionId) + return true +} + +type DemandAttemptRecord = { + ownerId: FullFlowOwnerId + demandId: FullFlowDemandId + settled: boolean + released: boolean +} + +/** Reject histories that cannot name logical demand attempts unambiguously. */ +function assertWellFormedDemandAttempts( + history: ReadonlyArray, +): void { + const attempts = new Map() + + for (const event of history) { + if (event.type === `requestDemand`) { + const attemptKey = sourceAttemptIdentity(event.sourceId, event.attemptId) + if (attempts.has(attemptKey)) { + throw new Error( + `Demand attempt "${event.attemptId}" was requested more than once`, + ) + } + attempts.set(attemptKey, { + ownerId: event.ownerId, + demandId: event.demandId, + settled: false, + released: false, + }) + continue + } + + const usesDemandAttempt = + event.type === `applyAuthoritativeRows` || + event.type === `applyUnprovenRows` || + event.type === `rejectDemand` || + event.type === `settleDemandWithoutEvidence` || + event.type === `releaseDemand` + if (!usesDemandAttempt) continue + + const attempt = attempts.get( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) + if (!attempt) { + throw new Error( + `Demand attempt "${event.attemptId}" was used before it was requested`, + ) + } + if (attempt.demandId !== event.demandId) { + throw new Error( + `Demand attempt "${event.attemptId}" changed its demand identity`, + ) + } + if (`ownerId` in event && attempt.ownerId !== event.ownerId) { + throw new Error( + `Demand attempt "${event.attemptId}" changed its owner identity`, + ) + } + + if (event.type === `releaseDemand`) { + if (attempt.released) { + throw new Error( + `Demand attempt "${event.attemptId}" was released more than once`, + ) + } + attempt.released = true + } else { + if (attempt.settled) { + throw new Error( + `Demand attempt "${event.attemptId}" settled more than once`, + ) + } + attempt.settled = true + } + } } /** @@ -214,16 +578,34 @@ export type ExpectedAdapterLifecycleEvent = { export function projectAdapterLifecycle( history: ReadonlyArray, ): Array { - const invokedOwners = new Set() + assertWellFormedDemandAttempts(history) + const invokedAttempts = new Set() const projected: Array = [] for (const event of history) { if (event.type === `requestDemand` && !event.alreadyAborted) { - invokedOwners.add(event.ownerId) - projected.push({ type: `invoke`, ownerId: event.ownerId }) + invokedAttempts.add( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) + projected.push({ + type: `invoke`, + ownerId: event.ownerId, + sourceId: event.sourceId, + attemptId: event.attemptId, + }) } - if (event.type === `releaseDemand` && invokedOwners.delete(event.ownerId)) { - projected.push({ type: `release`, ownerId: event.ownerId }) + if ( + event.type === `releaseDemand` && + invokedAttempts.delete( + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) + ) { + projected.push({ + type: `release`, + ownerId: event.ownerId, + sourceId: event.sourceId, + attemptId: event.attemptId, + }) } } @@ -233,50 +615,133 @@ export function projectAdapterLifecycle( /** * Projects physical transport work from adapter evidence lifetime. * - * Request settlement alone is not evidence. Only an applied authoritative row - * publication makes the exact demand reusable, and an unload that invalidates - * that evidence forces the next owner to fetch again. + * Concurrent owners attach to one in-flight exact demand. Settlement alone is + * not reusable evidence: only an applied authoritative row publication makes + * the demand reusable, and an unload that invalidates that evidence forces the + * next owner to fetch again. */ export function projectTransportLoads( history: ReadonlyArray, ): number { - const reusableDemands = new Set() - const requestEpochs = new Map() - let sourceEpoch = 0 + assertWellFormedDemandAttempts(history) + const reusableAcquisitions = new Map() + const inFlightAcquisitions = new Map() + const attemptAcquisitions = new Map() + const acquisitionAttempts: AcquisitionAttempts = new Map() let loads = 0 for (const event of history) { switch (event.type) { - case `requestDemand`: - if (!event.alreadyAborted && !reusableDemands.has(event.demandId)) { + case `requestDemand`: { + if (event.alreadyAborted) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + let acquisitionId = + inFlightAcquisitions.get(demandKey) ?? + reusableAcquisitions.get(demandKey) + if (acquisitionId === undefined) { loads++ + acquisitionId = attemptKey + inFlightAcquisitions.set(demandKey, acquisitionId) } - if (!event.alreadyAborted) { - requestEpochs.set(event.ownerId, sourceEpoch) - } + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) break - case `applyAuthoritativeRows`: - if (requestEpochs.get(event.ownerId) === sourceEpoch) { - reusableDemands.add(event.demandId) + } + case `applyAuthoritativeRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId === undefined || + inFlightAcquisitions.get(demandKey) !== acquisitionId + ) { + break } + inFlightAcquisitions.delete(demandKey) + reusableAcquisitions.set(demandKey, acquisitionId) break + } case `truncateSource`: - sourceEpoch++ - reusableDemands.clear() + for (const demandKey of reusableAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + reusableAcquisitions.delete(demandKey) + } + } + for (const demandKey of inFlightAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + inFlightAcquisitions.delete(demandKey) + } + } break case `applyUnprovenRows`: case `rejectDemand`: + case `settleDemandWithoutEvidence`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + inFlightAcquisitions.get(demandKey) === acquisitionId + ) { + inFlightAcquisitions.delete(demandKey) + } break - case `releaseDemand`: - if (event.invalidatesAdapterEvidence) { - reusableDemands.delete(event.demandId) + } + case `releaseDemand`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + attemptKey, + ) + ) { + if (reusableAcquisitions.get(demandKey) === acquisitionId) { + reusableAcquisitions.delete(demandKey) + } + if (inFlightAcquisitions.get(demandKey) === acquisitionId) { + inFlightAcquisitions.delete(demandKey) + } } break + } case `restartSession`: case `cleanupSession`: case `advanceWindowRevision`: case `scheduleContinuation`: case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: case `stagePublicationRows`: case `commitPublication`: case `beginReplacement`: @@ -345,10 +810,27 @@ export function projectAuthorizedContinuationStarts( break } case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: case `applyUnprovenRows`: case `rejectDemand`: case `truncateSource`: case `releaseDemand`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: case `stagePublicationRows`: case `commitPublication`: case `beginReplacement`: @@ -362,42 +844,126 @@ export function projectAuthorizedContinuationStarts( return starts } -/** Projects reusable demand evidence without using registry state. */ -export function projectReusableDemands( +export type ExpectedReusableDemand = { + sourceId: FullFlowSourceId + demandId: FullFlowDemandId +} + +/** Projects source-qualified reusable demand evidence without registry state. */ +export function projectReusableSourceDemands( history: ReadonlyArray, -): Array { - const reusableDemands = new Set() - const requestEpochs = new Map() - let sourceEpoch = 0 +): Array { + assertWellFormedDemandAttempts(history) + const activeAttempts: ActiveDemandAttempts = new Map() + const currentAcquisitions = new Map() + const reusableAcquisitions = new Map< + ScopedIdentity, + { acquisitionId: ScopedIdentity; demand: ExpectedReusableDemand } + >() + const attemptAcquisitions = new Map() + const acquisitionAttempts: AcquisitionAttempts = new Map() for (const event of history) { switch (event.type) { case `requestDemand`: if (!event.alreadyAborted) { - requestEpochs.set(event.ownerId, sourceEpoch) + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + addActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + const acquisitionId = + currentAcquisitions.get(demandKey) ?? + reusableAcquisitions.get(demandKey)?.acquisitionId ?? + attemptKey + currentAcquisitions.set(demandKey, acquisitionId) + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) } break - case `applyAuthoritativeRows`: - if (requestEpochs.get(event.ownerId) === sourceEpoch) { - reusableDemands.add(event.demandId) + case `applyAuthoritativeRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + reusableAcquisitions.set(demandKey, { + acquisitionId, + demand: { sourceId: event.sourceId, demandId: event.demandId }, + }) + currentAcquisitions.delete(demandKey) } break + } case `truncateSource`: - sourceEpoch++ - reusableDemands.clear() + for (const demandKey of currentAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + currentAcquisitions.delete(demandKey) + } + } + for (const demandKey of reusableAcquisitions.keys()) { + if (belongsToSource(demandKey, event.sourceId)) { + reusableAcquisitions.delete(demandKey) + } + } break - case `releaseDemand`: - if (event.invalidatesAdapterEvidence) { - reusableDemands.delete(event.demandId) + case `releaseDemand`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + releaseActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + attemptKey, + ) + ) { + if (currentAcquisitions.get(demandKey) === acquisitionId) { + currentAcquisitions.delete(demandKey) + } + if ( + reusableAcquisitions.get(demandKey)?.acquisitionId === acquisitionId + ) { + reusableAcquisitions.delete(demandKey) + } } break + } case `applyUnprovenRows`: case `rejectDemand`: + case `settleDemandWithoutEvidence`: case `restartSession`: case `cleanupSession`: case `advanceWindowRevision`: case `scheduleContinuation`: case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: case `stagePublicationRows`: case `commitPublication`: case `beginReplacement`: @@ -408,7 +974,20 @@ export function projectReusableDemands( } } - return [...reusableDemands].sort() + return [...reusableAcquisitions.values()] + .map(({ demand }) => demand) + .sort((left, right) => + left.sourceId === right.sourceId + ? left.demandId.localeCompare(right.demandId) + : left.sourceId.localeCompare(right.sourceId), + ) +} + +/** Single-source convenience projection retained for existing controls. */ +export function projectReusableDemands( + history: ReadonlyArray, +): Array { + return projectReusableSourceDemands(history).map(({ demandId }) => demandId) } /** @@ -419,6 +998,7 @@ export function projectReusableDemands( export function projectOrderedPublicationBoundary( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` prefixSize: number @@ -426,8 +1006,9 @@ export function projectOrderedPublicationBoundary( ): FullFlowPublishedOrderRow | undefined { const staged = new Map< FullFlowPublicationId, - Map> + Map> >() + const targetDemand = sourceDemandIdentity(options.sourceId, options.demandId) let committedRows: ReadonlyArray = [] for (const event of history) { @@ -437,13 +1018,16 @@ export function projectOrderedPublicationBoundary( publication = new Map() staged.set(event.publicationId, publication) } - publication.set(event.demandId, event.rows) + publication.set( + sourceDemandIdentity(event.sourceId, event.demandId), + event.rows, + ) continue } if (event.type === `commitPublication`) { const publication = staged.get(event.publicationId) - if (publication?.has(options.demandId)) { - committedRows = publication.get(options.demandId) ?? [] + if (publication?.has(targetDemand)) { + committedRows = publication.get(targetDemand) ?? [] } } } @@ -475,6 +1059,7 @@ export function projectOrderedPublicationBoundary( export function projectAtomicOrderedPublications( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` initialWindowSize: number @@ -504,31 +1089,35 @@ export type AtomicOrderedPublicationProjection = { export function projectAtomicOrderedPublicationState( history: ReadonlyArray, options: { + sourceId: FullFlowSourceId demandId: FullFlowDemandId direction: `asc` | `desc` initialWindowSize: number }, ): AtomicOrderedPublicationProjection { + assertWellFormedDemandAttempts(history) const staged = new Map< FullFlowPublicationId, - Map> + Map> >() const attempts = new Map< FullFlowPublicationId, Map< - FullFlowDemandId, + ScopedIdentity, | { outcome: `success`; publishable: boolean } | { outcome: `failure` | `abort`; publishable: false } | undefined > >() - const activeAdditionalDemands = new Set() + const activeAdditionalDemands: ActiveDemandAttempts = new Map() const publications: Array> = [] let currentPublication: AtomicOrderedPublicationState | undefined let retainsPreviousPublication = false let currentReplacement: FullFlowPublicationId | undefined + let currentPublicationId: FullFlowPublicationId | undefined let retainedSize = options.initialWindowSize let closed = false + const targetDemand = sourceDemandIdentity(options.sourceId, options.demandId) const sortRows = (rows: ReadonlyArray) => [...rows].sort((left, right) => { @@ -543,14 +1132,15 @@ export function projectAtomicOrderedPublicationState( const publicationState = ( publicationId: FullFlowPublicationId, + orderedPrefixSize = retainedSize, ): AtomicOrderedPublicationState | undefined => { const publication = staged.get(publicationId) - const orderedRows = publication?.get(options.demandId) + const orderedRows = publication?.get(targetDemand) if (!publication || !orderedRows) return undefined - const orderedPrefix = sortRows(orderedRows).slice(0, retainedSize) + const orderedPrefix = sortRows(orderedRows).slice(0, orderedPrefixSize) const desired = new Map(orderedPrefix.map((row) => [row.key, row] as const)) - for (const demandId of activeAdditionalDemands) { + for (const demandId of activeAdditionalDemands.keys()) { for (const row of publication.get(demandId) ?? []) { desired.set(row.key, row) } @@ -562,12 +1152,16 @@ export function projectAtomicOrderedPublicationState( } } - const publish = (publicationId: FullFlowPublicationId) => { - const next = publicationState(publicationId) + const publish = ( + publicationId: FullFlowPublicationId, + orderedPrefixSize?: number, + ) => { + const next = publicationState(publicationId, orderedPrefixSize) if (!next) return const previous = publications.at(-1) if (previous === undefined && next.rows.length === 0) { currentPublication = next + currentPublicationId = publicationId return } if ( @@ -579,10 +1173,12 @@ export function projectAtomicOrderedPublicationState( ) ) { currentPublication = next + currentPublicationId = publicationId return } publications.push(next.rows) currentPublication = next + currentPublicationId = publicationId } const finishCurrentReplacement = () => { @@ -596,8 +1192,8 @@ export function projectAtomicOrderedPublicationState( } const current = attempts.get(currentReplacement) - const ordered = current?.get(options.demandId) - const activeDemandFailed = [...activeAdditionalDemands].some( + const ordered = current?.get(targetDemand) + const activeDemandFailed = [...activeAdditionalDemands.keys()].some( (demandId) => current?.get(demandId)?.outcome !== `success`, ) if (ordered?.outcome !== `success` || activeDemandFailed) { @@ -623,7 +1219,10 @@ export function projectAtomicOrderedPublicationState( publication = new Map() staged.set(event.publicationId, publication) } - publication.set(event.demandId, event.rows) + publication.set( + sourceDemandIdentity(event.sourceId, event.demandId), + event.rows, + ) break } case `commitPublication`: { @@ -635,19 +1234,31 @@ export function projectAtomicOrderedPublicationState( case `beginReplacement`: attempts.set( event.publicationId, - new Map(event.demandIds.map((demandId) => [demandId, undefined])), + new Map( + event.demands.map(({ sourceId, demandId }) => [ + sourceDemandIdentity(sourceId, demandId), + undefined, + ]), + ), ) currentReplacement = event.publicationId retainsPreviousPublication = true break case `resizeOrderedWindow`: + if ( + event.sourceId !== options.sourceId || + event.demandId !== options.demandId + ) { + break + } retainedSize = Math.max(retainedSize, event.size) break case `settleReplacement`: { const attempt = attempts.get(event.publicationId) - if (!attempt?.has(event.demandId)) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + if (!attempt?.has(demandKey)) break attempt.set( - event.demandId, + demandKey, event.outcome === `success` ? { outcome: `success`, @@ -659,8 +1270,14 @@ export function projectAtomicOrderedPublicationState( break } case `establishReplacementCoverage`: { - if (event.publicationId !== currentReplacement) break - const ordered = attempts.get(event.publicationId)?.get(options.demandId) + if ( + event.publicationId !== currentReplacement || + event.sourceId !== options.sourceId || + event.demandId !== options.demandId + ) { + break + } + const ordered = attempts.get(event.publicationId)?.get(targetDemand) if (ordered?.outcome === `success`) { ordered.publishable = true finishCurrentReplacement() @@ -668,8 +1285,16 @@ export function projectAtomicOrderedPublicationState( break } case `requestDemand`: - if (!event.alreadyAborted && event.demandId !== options.demandId) { - activeAdditionalDemands.add(event.demandId) + if ( + !event.alreadyAborted && + (event.sourceId !== options.sourceId || + event.demandId !== options.demandId) + ) { + addActiveDemandAttempt( + activeAdditionalDemands, + sourceDemandIdentity(event.sourceId, event.demandId), + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) } break case `applyAuthoritativeRows`: @@ -677,7 +1302,24 @@ export function projectAtomicOrderedPublicationState( case `rejectDemand`: break case `releaseDemand`: - activeAdditionalDemands.delete(event.demandId) + if ( + releaseActiveDemandAttempt( + activeAdditionalDemands, + sourceDemandIdentity(event.sourceId, event.demandId), + sourceAttemptIdentity(event.sourceId, event.attemptId), + ) && + currentPublicationId !== undefined + ) { + // A private replacement may have grown the target window. Releasing + // another demand filters the last complete public prefix; it cannot + // expose rows known only to the private replacement. + publish( + currentPublicationId, + currentReplacement === undefined + ? retainedSize + : currentPublication?.orderedPrefixSize, + ) + } break case `truncateSource`: case `restartSession`: @@ -702,24 +1344,766 @@ export function projectAtomicOrderedPublicationState( } } -/** Derives visible row identity without consulting Collection implementation. */ +/** Derives source-qualified row identity without consulting Collection state. */ +export function projectRetainedSourceRows( + history: ReadonlyArray, +): Array { + assertWellFormedDemandAttempts(history) + const activeAttempts: ActiveDemandAttempts = new Map() + const activeAttemptIds = new Set() + const currentAcquisitions = new Map() + const reusableRows = new Map< + ScopedIdentity, + { acquisitionId: ScopedIdentity; rows: Set } + >() + const attemptAcquisitions = new Map() + const acquisitionAttempts: AcquisitionAttempts = new Map() + const rowClaims = new Map< + ScopedIdentity, + { row: ExpectedPublicRow; attempts: Set } + >() + const attemptRows = new Map>() + + const claimRows = ( + attemptKey: ScopedIdentity, + sourceId: FullFlowSourceId, + rowKeys: Iterable, + ) => { + let claimed = attemptRows.get(attemptKey) + if (!claimed) { + claimed = new Set() + attemptRows.set(attemptKey, claimed) + } + for (const rowKey of rowKeys) { + const rowIdentity = sourceRowIdentity(sourceId, rowKey) + claimed.add(rowIdentity) + let claim = rowClaims.get(rowIdentity) + if (!claim) { + claim = { row: { sourceId, rowKey }, attempts: new Set() } + rowClaims.set(rowIdentity, claim) + } + claim.attempts.add(attemptKey) + } + } + + const releaseRows = (attemptKey: ScopedIdentity) => { + for (const rowIdentity of attemptRows.get(attemptKey) ?? []) { + const claim = rowClaims.get(rowIdentity) + claim?.attempts.delete(attemptKey) + if (claim?.attempts.size === 0) rowClaims.delete(rowIdentity) + } + attemptRows.delete(attemptKey) + } + + for (const event of history) { + switch (event.type) { + case `requestDemand`: { + if (event.alreadyAborted) break + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + addActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + activeAttemptIds.add(attemptKey) + const retained = reusableRows.get(demandKey) + const acquisitionId = + currentAcquisitions.get(demandKey) ?? + retained?.acquisitionId ?? + attemptKey + currentAcquisitions.set(demandKey, acquisitionId) + attemptAcquisitions.set(attemptKey, acquisitionId) + addAcquisitionAttempt(acquisitionAttempts, acquisitionId, attemptKey) + if (retained) claimRows(attemptKey, event.sourceId, retained.rows) + break + } + case `applyAuthoritativeRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + const rows = new Set(event.rowKeys) + reusableRows.set(demandKey, { acquisitionId, rows }) + currentAcquisitions.delete(demandKey) + } + for (const participant of participants) { + if (activeAttemptIds.has(participant)) { + claimRows(participant, event.sourceId, event.rowKeys) + } + } + break + } + case `applyUnprovenRows`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + currentAcquisitions.delete(demandKey) + } + const participants = + acquisitionId === undefined + ? [] + : (acquisitionAttempts.get(acquisitionId) ?? []) + for (const participant of participants) { + if (activeAttemptIds.has(participant)) { + claimRows(participant, event.sourceId, event.rowKeys) + } + } + break + } + case `rejectDemand`: + case `settleDemandWithoutEvidence`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + const acquisitionId = attemptAcquisitions.get(attemptKey) + if ( + acquisitionId !== undefined && + currentAcquisitions.get(demandKey) === acquisitionId + ) { + currentAcquisitions.delete(demandKey) + } + break + } + case `truncateSource`: + for (const scope of currentAcquisitions.keys()) { + if (belongsToSource(scope, event.sourceId)) { + currentAcquisitions.delete(scope) + } + } + for (const scope of reusableRows.keys()) { + if (belongsToSource(scope, event.sourceId)) { + reusableRows.delete(scope) + } + } + for (const rowIdentity of rowClaims.keys()) { + if (belongsToSource(rowIdentity, event.sourceId)) { + rowClaims.delete(rowIdentity) + for (const rows of attemptRows.values()) rows.delete(rowIdentity) + } + } + break + case `releaseDemand`: { + const demandKey = sourceDemandIdentity(event.sourceId, event.demandId) + const attemptKey = sourceAttemptIdentity( + event.sourceId, + event.attemptId, + ) + activeAttemptIds.delete(attemptKey) + const acquisitionId = attemptAcquisitions.get(attemptKey) + releaseActiveDemandAttempt(activeAttempts, demandKey, attemptKey) + releaseRows(attemptKey) + if ( + acquisitionId !== undefined && + releaseAcquisitionAttempt( + acquisitionAttempts, + acquisitionId, + attemptKey, + ) + ) { + if (currentAcquisitions.get(demandKey) === acquisitionId) { + currentAcquisitions.delete(demandKey) + } + if (reusableRows.get(demandKey)?.acquisitionId === acquisitionId) { + reusableRows.delete(demandKey) + } + } + break + } + default: + break + } + } + + return sortPublicRows([...rowClaims.values()].map(({ row }) => row)) +} + +/** Single-source convenience projection retained for existing controls. */ export function projectRetainedRowKeys( history: ReadonlyArray, ): Array { - const retainedRows = new Set() + return projectRetainedSourceRows(history).map(({ rowKey }) => rowKey) +} + +export type ExpectedSyncReceiptState = `pending` | `resolved` | `rejected` + +export type ExpectedPublicRow = { + sourceId: FullFlowSourceId + rowKey: string +} + +export type ExpectedSyncTransactionObservation = { + visibleRows: Array + publishedBatches: Array> + callbackReads: Array> + receipts: Array<{ + transactionId: FullFlowTransactionId + state: ExpectedSyncReceiptState + }> +} + +type SyncTransactionState = + | `staged` + | `committed` + | `parked` + | `applying` + | `published` + | `resolved` + | `rejected` + +type ProjectedSyncTransaction = { + sourceId: FullFlowSourceId + rowKeys: ReadonlyArray + state: SyncTransactionState +} + +function sortPublicRows( + rows: Iterable, +): Array { + return [...rows].sort((left, right) => + left.sourceId === right.sourceId + ? left.rowKey.localeCompare(right.rowKey) + : left.sourceId.localeCompare(right.sourceId), + ) +} + +/** + * Projects the sync transaction's public contract without consulting the + * collection queue. Abort can still win while work is staged, committed, or + * parked. Once application starts, publication is irrevocable. A receipt does + * not resolve until the published batch and callback-time reads are visible. + */ +export function projectSyncTransactions( + history: ReadonlyArray, +): ExpectedSyncTransactionObservation { + const transactions = new Map< + FullFlowTransactionId, + ProjectedSyncTransaction + >() + const visibleRows = new Map() + const publishedBatches: Array> = [] + const callbackReads: Array> = [] for (const event of history) { - if ( - event.type === `applyAuthoritativeRows` || - event.type === `applyUnprovenRows` - ) { - event.rowKeys.forEach((key) => retainedRows.add(key)) + switch (event.type) { + case `stageSyncTransaction`: + transactions.set(event.transactionId, { + sourceId: event.sourceId, + rowKeys: event.rowKeys, + state: `staged`, + }) + break + case `commitSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if (!transaction || transaction.state !== `staged`) break + transaction.state = event.signalAborted + ? `rejected` + : event.parked + ? `parked` + : `committed` + break + } + case `enterSyncApplication`: { + const transaction = transactions.get(event.transactionId) + if ( + transaction?.state === `committed` || + transaction?.state === `parked` + ) { + transaction.state = `applying` + } + break + } + case `abortSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if ( + transaction?.state === `staged` || + transaction?.state === `committed` || + transaction?.state === `parked` + ) { + transaction.state = `rejected` + } + break + } + case `publishSyncTransaction`: { + const transaction = transactions.get(event.transactionId) + if (transaction?.state !== `applying`) break + const batch = transaction.rowKeys.map((rowKey) => ({ + sourceId: transaction.sourceId, + rowKey, + })) + for (const row of batch) { + visibleRows.set(`${row.sourceId}\u0000${row.rowKey}`, row) + } + transaction.state = `published` + publishedBatches.push(sortPublicRows(batch)) + callbackReads.push(sortPublicRows(visibleRows.values())) + break + } + case `settleSyncReceipt`: { + const transaction = transactions.get(event.transactionId) + if (transaction?.state === `published`) { + transaction.state = `resolved` + } + break + } + case `requestDemand`: + case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: + case `releaseDemand`: + case `restartSession`: + case `cleanupSession`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: + break } - if (event.type === `truncateSource`) retainedRows.clear() - if (event.type === `releaseDemand` && event.finalRowOwner) { - event.rowKeys.forEach((key) => retainedRows.delete(key)) + } + + return { + visibleRows: sortPublicRows(visibleRows.values()), + publishedBatches, + callbackReads, + receipts: [...transactions] + .map(([transactionId, transaction]) => { + const state = + transaction.state === `resolved` + ? `resolved` + : transaction.state === `rejected` + ? `rejected` + : `pending` + return { transactionId, state } as const + }) + .sort((left, right) => + left.transactionId.localeCompare(right.transactionId), + ), + } +} + +export type ExpectedVersionedChange = { + type: `insert` | `update` | `delete` + row: FullFlowVersionedRow + previousVersion?: number +} + +export type ExpectedReplayObservation = { + coreRows: Array + visibleRows: Array + publishedBatches: Array> + callbackReads: Array> +} + +type ProjectedReplayAttempt = { + outcome?: `resolve` | `reject` +} + +type ProjectedReplaySession = { + sourceId: FullFlowSourceId + currentAttemptId: string + attempts: Map + baseline: Map +} + +function versionedRowIdentity(row: FullFlowVersionedRow): string { + return `${row.sourceId}\u0000${row.rowKey}` +} + +function sortVersionedRows( + rows: Iterable, +): Array { + return [...rows].sort((left, right) => + versionedRowIdentity(left).localeCompare(versionedRowIdentity(right)), + ) +} + +function versionedPublicationDiff( + baseline: ReadonlyMap, + replacement: ReadonlyMap, +): Array { + const changes: Array = [] + for (const [identity, previous] of baseline) { + const next = replacement.get(identity) + if (!next) { + changes.push({ type: `delete`, row: previous }) + } else if (next.version !== previous.version) { + changes.push({ + type: `update`, + row: next, + previousVersion: previous.version, + }) + } + } + for (const [identity, row] of replacement) { + if (!baseline.has(identity)) changes.push({ type: `insert`, row }) + } + return changes.sort((left, right) => + versionedRowIdentity(left.row).localeCompare( + versionedRowIdentity(right.row), + ), + ) +} + +/** + * Projects truncate replay as a replacement protocol. Core rows and last-good + * publication are independent domains: truncate clears core immediately, but + * public rows change only after every overlapping attempt settles and the + * newest attempt succeeds. + */ +export function projectReplayPublication( + history: ReadonlyArray, +): ExpectedReplayObservation { + const coreRows = new Map() + const visibleRows = new Map() + const publishedBatches: Array> = [] + const callbackReads: Array> = [] + const sessions = new Map() + const attemptSessions = new Map() + + for (const event of history) { + switch (event.type) { + case `establishPublication`: { + const batch: Array = [] + for (const row of event.rows) { + const identity = versionedRowIdentity(row) + coreRows.set(identity, row) + visibleRows.set(identity, row) + batch.push({ type: `insert`, row }) + } + if (batch.length > 0) { + publishedBatches.push(batch) + callbackReads.push(sortVersionedRows(visibleRows.values())) + } + break + } + case `startReplay`: { + let session = sessions.get(event.sourceId) + if (!session) { + session = { + sourceId: event.sourceId, + currentAttemptId: event.attemptId, + attempts: new Map(), + baseline: new Map( + [...visibleRows].filter( + ([, row]) => row.sourceId === event.sourceId, + ), + ), + } + sessions.set(event.sourceId, session) + } + session.currentAttemptId = event.attemptId + session.attempts.set(event.attemptId, {}) + attemptSessions.set(event.attemptId, session) + for (const [identity, row] of coreRows) { + if (row.sourceId === event.sourceId) coreRows.delete(identity) + } + break + } + case `writeReplayRows`: + if (event.acceptedByCore) { + for (const row of event.rows) { + coreRows.set(versionedRowIdentity(row), row) + } + } + break + case `settleReplay`: { + const session = attemptSessions.get(event.attemptId) + const attempt = session?.attempts.get(event.attemptId) + if (!session || !attempt) break + attempt.outcome = event.outcome + if ([...session.attempts.values()].some(({ outcome }) => !outcome)) { + break + } + + const current = session.attempts.get(session.currentAttemptId) + if (current?.outcome === `resolve`) { + const replacement = new Map( + [...coreRows].filter( + ([, row]) => row.sourceId === session.sourceId, + ), + ) + const changes = versionedPublicationDiff( + session.baseline, + replacement, + ) + for (const [identity, row] of visibleRows) { + if (row.sourceId === session.sourceId) visibleRows.delete(identity) + } + for (const [identity, row] of replacement) { + visibleRows.set(identity, row) + } + if (changes.length > 0) { + publishedBatches.push(changes) + callbackReads.push(sortVersionedRows(visibleRows.values())) + } + } + sessions.delete(session.sourceId) + for (const attemptId of session.attempts.keys()) { + attemptSessions.delete(attemptId) + } + break + } + case `requestDemand`: + case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: + case `releaseDemand`: + case `restartSession`: + case `cleanupSession`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: + break + } + } + + return { + coreRows: sortVersionedRows(coreRows.values()), + visibleRows: sortVersionedRows(visibleRows.values()), + publishedBatches, + callbackReads, + } +} + +export type ExpectedSourceReadiness = { + status: `loading` | `ready` | `error` | `cleaned-up` + pendingSources: Array + failedSources: Array +} + +/** Projects initial live-query readiness across every reachable source. */ +export function projectSourceReadiness( + history: ReadonlyArray, +): ExpectedSourceReadiness { + const demands = new Map< + string, + { + sourceId: FullFlowSourceId + demandId: FullFlowDemandId + attemptId: FullFlowAttemptId + state: `pending` | `resolved` | `rejected` + } + >() + let currentSession: FullFlowSessionId | undefined + let cleanedUp = false + + for (const event of history) { + switch (event.type) { + case `registerSourceDemand`: + currentSession ??= event.sessionId + if (event.sessionId !== currentSession) break + cleanedUp = false + demands.set( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + { + sourceId: event.sourceId, + demandId: event.demandId, + attemptId: event.attemptId, + state: `pending`, + }, + ) + break + case `settleSourceDemand`: { + if (event.sessionId !== currentSession) break + const demand = demands.get( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + ) + if (demand) + demand.state = event.outcome === `resolve` ? `resolved` : `rejected` + break + } + case `retireSourceDemand`: + if (event.sessionId !== currentSession) break + demands.delete( + sourceDemandAttemptIdentity( + event.sourceId, + event.demandId, + event.attemptId, + ), + ) + break + case `cleanupSession`: + if (event.sessionId === currentSession) { + cleanedUp = true + demands.clear() + } + break + case `restartSession`: + currentSession = event.nextSessionId + cleanedUp = false + demands.clear() + break + case `requestDemand`: + case `applyAuthoritativeRows`: + case `settleDemandWithoutEvidence`: + case `releaseDemand`: + case `advanceWindowRevision`: + case `scheduleContinuation`: + case `runContinuation`: + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + case `establishPublication`: + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + case `startAcquisition`: + case `attachAcquisitionOwner`: + case `settleAcquisition`: + break + } + } + + const currentDemands = [...demands.values()] + const pendingSources = [ + ...new Set( + currentDemands + .filter(({ state }) => state === `pending`) + .map(({ sourceId }) => sourceId), + ), + ].sort() + const failedSources = [ + ...new Set( + currentDemands + .filter(({ state }) => state === `rejected`) + .map(({ sourceId }) => sourceId), + ), + ].sort() + + return { + status: cleanedUp + ? `cleaned-up` + : failedSources.length > 0 + ? `error` + : pendingSources.length > 0 || currentDemands.length === 0 + ? `loading` + : `ready`, + pendingSources, + failedSources, + } +} + +export type ExpectedAcquisitionObservation = { + physicalStarts: Array + owners: Array<{ + ownerId: FullFlowOwnerId + state: `pending` | `resolved` | `rejected` + rowKeys: Array + }> + visibleRowKeys: Array +} + +/** + * Projects the semantic result of physical acquisition sharing. + * + * A physical acquisition may serve one or many logical owners. Sharing may + * reduce transport starts, but it cannot change any owner's settlement or the + * rows made visible by successful work. + */ +export function projectAcquisitionSettlement( + history: ReadonlyArray, +): ExpectedAcquisitionObservation { + const acquisitions = new Map< + FullFlowAcquisitionId, + { + owners: Set + state: `pending` | `resolved` | `rejected` + rowKeys: Array + } + >() + const physicalStarts: Array = [] + const visibleRowKeys = new Set() + + for (const event of history) { + switch (event.type) { + case `startAcquisition`: + if (!acquisitions.has(event.acquisitionId)) { + acquisitions.set(event.acquisitionId, { + owners: new Set(), + state: `pending`, + rowKeys: [], + }) + physicalStarts.push(event.acquisitionId) + } + break + case `attachAcquisitionOwner`: + acquisitions.get(event.acquisitionId)?.owners.add(event.ownerId) + break + case `settleAcquisition`: { + const acquisition = acquisitions.get(event.acquisitionId) + if (!acquisition || acquisition.state !== `pending`) break + acquisition.state = + event.outcome === `resolve` ? `resolved` : `rejected` + acquisition.rowKeys = [...new Set(event.rowKeys)].sort() + if (acquisition.state === `resolved`) { + acquisition.rowKeys.forEach((rowKey) => visibleRowKeys.add(rowKey)) + } + break + } + default: + break } } - return [...retainedRows].sort() + return { + physicalStarts, + owners: [...acquisitions.values()] + .flatMap((acquisition) => + [...acquisition.owners].map((ownerId) => ({ + ownerId, + state: acquisition.state, + rowKeys: acquisition.state === `resolved` ? acquisition.rowKeys : [], + })), + ) + .sort((left, right) => left.ownerId.localeCompare(right.ownerId)), + visibleRowKeys: [...visibleRowKeys].sort(), + } } diff --git a/packages/db/tests/load-subset-outcome.test.ts b/packages/db/tests/load-subset-outcome.test.ts index cfe048df06..65da4d0e10 100644 --- a/packages/db/tests/load-subset-outcome.test.ts +++ b/packages/db/tests/load-subset-outcome.test.ts @@ -935,7 +935,7 @@ describe(`loadSubset outcomes`, () => { }, ) - it(`tracks opaque demand values by runtime reference`, async () => { + it(`tracks opaque equality demand values by runtime reference`, async () => { const loadSubset = vi.fn((_options: LoadSubsetOptions) => Promise.resolve({ hasMore: false }), ) @@ -958,20 +958,6 @@ describe(`loadSubset outcomes`, () => { const createDemands = (value: unknown): Array => [ { where: new Func(`eq`, [field, new Value(value)]) }, { where: new Func(`in`, [field, new Value([value])]) }, - { - orderBy: [ - { - expression: new Func(`coalesce`, [field, new Value(value)]), - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - }, - { - cursor: { - whereFrom: new Func(`gt`, [field, new Value(value)]), - whereCurrent: new Func(`eq`, [field, new Value(value)]), - }, - }, ] const demands = [ ...createDemands(() => `opaque`), @@ -1323,7 +1309,7 @@ describe(`loadSubset outcomes`, () => { }, ) - it(`scopes source extent to a narrowed physical acquisition`, async () => { + it(`preserves source extent for a conservative full acquisition`, async () => { const adapterCalls: Array = [] const deduplicated = new DeduplicatedLoadSubset({ loadSubset: (options) => { @@ -1362,8 +1348,8 @@ describe(`loadSubset outcomes`, () => { const outcome = collection._sync.loadSubset({}) expect(adapterCalls).toHaveLength(2) - expect(adapterCalls[1]?.where).toBeDefined() - await expect(outcome).resolves.toMatchObject({ extent: `unknown` }) + expect(adapterCalls[1]).toEqual({}) + await expect(outcome).resolves.toMatchObject({ extent: `exhausted` }) } finally { await collection.cleanup() } diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 2a0375432a..04546a3d62 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -1,8 +1,134 @@ type OracleEnvironment = Record +const staticOracleProperties = [ + `collection-sync.reentrant-drain`, + `collection-state.retention`, + `collection-publication.metadata-cancellation`, + `collection-publication.metadata-only`, + `collection-publication.metadata-rollback`, + `coverage-registry.claim-churn`, + `coverage-registry.state-machine`, + `d2-source.exact-retractions`, + `includes-collection.layout-swap`, + `includes-collection.optimistic-child-history`, + `includes-collection.public-key-order`, + `includes-collection.relationship-history`, + `includes-cross-formulation.equivalence`, + `includes-cross-formulation.ordered-window`, + `includes-optimistic.ancestor-rollback`, + `includes-optimistic.confirm-different-route`, + `includes-optimistic.confirm-same-route`, + `includes-optimistic.descendant-rollback`, + `includes-optimistic.rekey-detach`, + `includes-optimistic.rekey-rollback`, + `includes-optimistic.repeated-history`, + `includes-optimistic.sibling-route-rollback`, + `includes-publication.atomic-parent-replacement`, + `includes-publication.child-scalar`, + `includes-publication.optimistic-rollback`, + `includes-publication.parent-route`, + `includes-temporal.release-reentry`, + `includes-temporal.demand-scheduling`, + `includes.alpha-renaming`, + `includes.incremental-history`, + `includes.nested-scalar-materialization`, + `includes.optimistic-convergence`, + `includes.scenario-statistics`, + `load-subset-full-flow.atomic-replacement`, + `load-subset-full-flow.automatic-progress`, + `load-subset-full-flow.boundary-provenance`, + `load-subset-full-flow.consumer-parity`, + `load-subset-full-flow.continuation-evidence`, + `load-subset-full-flow.continuation-statistics`, + `load-subset-full-flow.multi-source-ordered`, + `load-subset-full-flow.multi-source-statistics`, + `load-subset-full-flow.truncate-evidence`, + `load-subset-lifecycle.state-machine`, + `load-subset-projection.state-equivalence`, + `load-subset.async-settlement`, + `load-subset.changing-predicate`, + `load-subset.concurrent-dedupe`, + `load-subset.coverage`, + `load-subset.distinct-window-predicate`, + `load-subset.ordered-window`, + `load-subset.rejected-waiter`, + `ordered-work.forward-exhaustion`, + `ordered-work.forward-prefix`, + `ordered-work.custom-comparator-fallback`, + `ordered-work.public-key-suffix`, + `ordered-work.reverse-exhaustion`, + `ordered-work.reverse-prefix`, + `ordered-work.snapshot-reuse`, + `pagination.async-cursor`, + `pagination.multi-order`, + `pagination.nullable-cursor`, + `pagination.ordered-window`, + `pagination.pending-history`, + `pagination.pending-mutation`, + `pagination.window-transition`, + `predicate-subtraction.duplicate-terms`, + `predicate-subtraction.finite-world`, + `predicate-subtraction.unbounded`, + `subscription-replay.completion`, + `subscription-replay.optimistic`, + `subscription-replay.ownership`, + `subscription-replay.restart`, + `subscription-replay.sequential`, + `subscription-replay.shared`, +] as const + +const publicationProperties = [ + `parent-scalar`, + `parent-then-child`, + `optimistic-before-confirm`, + `optimistic-after-confirm`, +].flatMap((law) => + [`direct`, `joined`].flatMap((q1Shape) => + [`passThrough`, `where`, `orderBy`, `select`].map( + (q2Shape) => `includes-publication.${law}.${q1Shape}.${q2Shape}`, + ), + ), +) + +const refinementProperties = Array.from( + { length: 11 }, + (_, index) => `load-subset-refinement.${1_779_001 + index}`, +) + +export function validateOraclePropertyRegistry( + properties: ReadonlyArray, +): ReadonlySet { + const registry = new Set() + for (const property of properties) { + if (registry.has(property)) { + throw new Error(`duplicate oracle property: ${property}`) + } + registry.add(property) + } + return registry +} + +const registeredOracleProperties = validateOraclePropertyRegistry([ + ...staticOracleProperties, + ...publicationProperties, + ...refinementProperties, +]) + +function assertRegisteredOracleProperty(property: string): void { + if (!registeredOracleProperties.has(property)) { + throw new Error(`unknown oracle property: ${property}`) + } +} + +export type OracleReplayConfig = { + replaySeed: number | undefined + replayPath: string | undefined + replayProperty: string | undefined +} + export function readOracleRunConfig( environment: OracleEnvironment = process.env, -): { multiplier: number; replaySeed: number | undefined } { +): OracleReplayConfig & { multiplier: number } { const multiplierValue = environment.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER ?? `1` const multiplier = Number(multiplierValue) if ( @@ -16,23 +142,79 @@ export function readOracleRunConfig( } const seedValue = environment.TANSTACK_DB_ORACLE_SEED - if (seedValue === undefined) return { multiplier, replaySeed: undefined } + const replayPath = environment.TANSTACK_DB_ORACLE_PATH + const replayProperty = environment.TANSTACK_DB_ORACLE_PROPERTY + if (seedValue === undefined) { + if (replayPath !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_SEED`, + ) + } + if (replayProperty !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PROPERTY requires TANSTACK_DB_ORACLE_PATH`, + ) + } + return { + multiplier, + replaySeed: undefined, + replayPath: undefined, + replayProperty: undefined, + } + } const replaySeed = Number(seedValue) if (seedValue.trim() === `` || !Number.isSafeInteger(replaySeed)) { throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) } - return { multiplier, replaySeed } + if (replayPath === undefined) { + if (replayProperty !== undefined) { + throw new Error( + `TANSTACK_DB_ORACLE_PROPERTY requires TANSTACK_DB_ORACLE_PATH`, + ) + } + return { + multiplier, + replaySeed, + replayPath: undefined, + replayProperty: undefined, + } + } + if (replayPath.trim() === ``) { + throw new Error(`TANSTACK_DB_ORACLE_PATH must be non-empty`) + } + if (!/^\d+(?::\d+)*$/.test(replayPath)) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH must contain colon-separated nonnegative integers`, + ) + } + if (replayProperty === undefined || replayProperty.trim() === ``) { + throw new Error( + `TANSTACK_DB_ORACLE_PATH requires TANSTACK_DB_ORACLE_PROPERTY`, + ) + } + assertRegisteredOracleProperty(replayProperty) + return { multiplier, replaySeed, replayPath, replayProperty } } export function oracleRandomParameters( numRuns: number, - replaySeed: number | undefined, -): { numRuns: number; seed?: number } { - return replaySeed === undefined ? { numRuns } : { numRuns, seed: replaySeed } + replay: OracleReplayConfig, + property: string, +): { numRuns: number; seed?: number; path?: string } { + assertRegisteredOracleProperty(property) + const { replaySeed, replayPath, replayProperty } = replay + if (replaySeed === undefined) return { numRuns } + return { + numRuns, + seed: replaySeed, + ...(replayPath !== undefined && replayProperty === property + ? { path: replayPath } + : {}), + } } -const { multiplier, replaySeed: seed } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() /** Keeps ordinary CI bounded while allowing long randomized oracle campaigns. */ export function oracleRuns(baseRuns: number): number { @@ -40,12 +222,13 @@ export function oracleRuns(baseRuns: number): number { } /** Replays broad randomized properties when a campaign seed is supplied. */ -export function oraclePropertyOptions(baseRuns: number): { +export function oraclePropertyOptions( + baseRuns: number, + property: string, +): { numRuns: number seed?: number + path?: string } { - return { - numRuns: oracleRuns(baseRuns), - ...(seed === undefined ? {} : { seed }), - } + return oracleRandomParameters(oracleRuns(baseRuns), replay, property) } diff --git a/packages/db/tests/query/bucket-facade-adapter.test.ts b/packages/db/tests/query/bucket-facade-adapter.test.ts index 4b969e2114..f5d223d86c 100644 --- a/packages/db/tests/query/bucket-facade-adapter.test.ts +++ b/packages/db/tests/query/bucket-facade-adapter.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' import { eq } from '../../src/query/builder/functions.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' import { BucketFacadeAdapter } from '../../src/query/live/bucket-facade-adapter.js' import { CollectionConfigBuilder } from '../../src/query/live/collection-config-builder.js' import { BUCKET_FACADE_REF } from '../../src/query/live/materialized-pipeline.js' @@ -17,30 +18,90 @@ import type { Context } from '../../src/query/builder/types.js' type FacadeSync = Parameters>[`sync`]>[0] +class ThrowingBuildIndex extends BasicIndex { + throwBeforeBuild = false + throwOnBuild = false + + override build(entries: Iterable<[number, unknown]>): void { + if (this.throwBeforeBuild) { + throw new Error(`facade index rebuild failed`) + } + super.build(entries) + if (this.throwOnBuild) { + throw new Error(`facade index rebuild failed`) + } + } +} + describe(`BucketFacadeAdapter`, () => { - it(`restores facade state when a flush fails after writing`, async () => { + it(`moves a row when the graph reuses its object for a new order`, async () => { + const graph = new D2() + const rows = graph.newInput<[string, BucketRow]>() + const activeBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-order-parent`, + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: true }], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const moving = { id: 1, value: `moving` } + const fixed = { id: 2, value: `fixed` } + activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + rows.sendData( + new MultiSet([ + [[bucketKey, { publicKey: moving.id, value: moving, order: `0` }], 1], + [[bucketKey, { publicKey: fixed.id, value: fixed, order: `1` }], 1], + ]), + ) + graph.run() + adapter.flush().publish() + + const facadeRef: BucketFacadeRef = { + [BUCKET_FACADE_REF]: { edgeId: `children`, bucketKey }, + } + const facade = adapter.resolve(facadeRef) as unknown as Collection< + typeof moving, + number + > + expect(facade.toArray.map(({ id }) => id)).toEqual([1, 2]) + + rows.sendData( + new MultiSet([ + [[bucketKey, { publicKey: moving.id, value: moving, order: `0` }], -1], + [[bucketKey, { publicKey: moving.id, value: moving, order: `2` }], 1], + ]), + ) + graph.run() + adapter.flush().publish() + + expect(facade.toArray.map(({ id }) => id)).toEqual([2, 1]) + await adapter.cleanup() + }) + + it(`restores facade state without public effects when a flush fails`, async () => { const graph = new D2() const rows = graph.newInput<[string, BucketRow]>() const activeBuckets = graph.newInput<[string, true]>() const adapter = new BucketFacadeAdapter( `facade-rollback-parent`, - [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: false }], + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: true }], () => {}, ) graph.finalize() const bucketKey = `group-1` const original = { id: 1, value: `original` } + const fixed = { id: 3, value: `fixed` } activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) rows.sendData( new MultiSet([ [ - [ - bucketKey, - { publicKey: original.id, value: original, order: undefined }, - ], + [bucketKey, { publicKey: original.id, value: original, order: `0` }], 1, ], + [[bucketKey, { publicKey: fixed.id, value: fixed, order: `1` }], 1], ]), ) graph.run() @@ -53,11 +114,25 @@ describe(`BucketFacadeAdapter`, () => { typeof original, number > - expect(facade.toArray.map(stripVirtualProps)).toEqual([original]) + expect(facade.toArray.map(stripVirtualProps)).toEqual([original, fixed]) const publications: Array = [] const subscription = facade.subscribeChanges((changes) => { publications.push(changes) }) + let layoutPublications = 0 + const unsubscribeLayout = facade._subscribeLayoutChanges(() => { + layoutPublications++ + }) + let statusChanges = 0 + const unsubscribeStatus = facade.on(`status:change`, () => { + statusChanges++ + }) + let truncates = 0 + const unsubscribeTruncate = facade.on(`truncate`, () => { + truncates++ + }) + const stateRevision = facade._stateRevision + const layoutRevision = facade._layoutRevision const entries = ( adapter as unknown as { @@ -78,6 +153,319 @@ describe(`BucketFacadeAdapter`, () => { } const replacement = { id: 1, value: `replacement` } + const added = { id: 2, value: `added` } + rows.sendData( + new MultiSet([ + [ + [bucketKey, { publicKey: original.id, value: original, order: `0` }], + -1, + ], + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: `2`, + }, + ], + 1, + ], + [[bucketKey, { publicKey: added.id, value: added, order: `3` }], 1], + ]), + ) + graph.run() + + expect(() => adapter.flush()).toThrow(`facade flush failed`) + expect(facade.toArray.map(stripVirtualProps)).toEqual([original, fixed]) + expect([ + ...( + entries.get(`children`)?.get(bucketKey) as unknown as { + currentOrder: Map + } + ).currentOrder, + ]).toEqual([ + [original.id, `0`], + [fixed.id, `1`], + ]) + expect(publications).toEqual([]) + expect(layoutPublications).toBe(0) + expect(statusChanges).toBe(0) + expect(truncates).toBe(0) + expect(facade._stateRevision).toBe(stateRevision) + expect(facade._layoutRevision).toBe(layoutRevision) + expect(facade.status).toBe(`ready`) + + const restoredOriginal = facade.get(original.id) + const restoredFixed = facade.get(fixed.id) + if (!restoredOriginal || !restoredFixed) { + throw new Error(`Missing restored facade rows`) + } + expect(facade.getKeyFromItem(restoredOriginal)).toBe(original.id) + expect(facade.getKeyFromItem(restoredFixed)).toBe(fixed.id) + + adapter.flush().publish() + + expect(facade.toArray.map(stripVirtualProps)).toEqual([ + fixed, + replacement, + added, + ]) + expect(layoutPublications).toBe(0) + expect(publications).toHaveLength(1) + expect(publications[0]).toHaveLength(2) + expect(statusChanges).toBe(0) + expect(truncates).toBe(0) + expect(facade._stateRevision).toBe(stateRevision + 1) + expect(facade._layoutRevision).toBe(layoutRevision + 1) + expect(facade.toArray.map((row) => facade.getKeyFromItem(row))).toEqual([ + fixed.id, + replacement.id, + added.id, + ]) + expect([ + ...( + entries.get(`children`)?.get(bucketKey) as unknown as { + currentOrder: Map + } + ).currentOrder, + ]).toEqual([ + [original.id, `2`], + [fixed.id, `1`], + [added.id, `3`], + ]) + + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: `2`, + }, + ], + -1, + ], + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: `0`, + }, + ], + 1, + ], + ]), + ) + graph.run() + adapter.flush().publish() + + expect(facade.toArray.map(stripVirtualProps)).toEqual([ + replacement, + fixed, + added, + ]) + expect(layoutPublications).toBe(1) + expect(publications).toHaveLength(2) + expect(publications[1]).toEqual([]) + expect(facade._stateRevision).toBe(stateRevision + 1) + expect(facade._layoutRevision).toBe(layoutRevision + 2) + + unsubscribeTruncate() + unsubscribeStatus() + unsubscribeLayout() + subscription.unsubscribe() + await adapter.cleanup() + }) + + it(`publishes fresh facade readiness only after every install succeeds`, async () => { + const graph = new D2() + const firstRows = graph.newInput<[string, BucketRow]>() + const firstActiveBuckets = graph.newInput<[string, true]>() + const secondRows = graph.newInput<[string, BucketRow]>() + const secondActiveBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-ready-parent`, + [ + { + edgeId: `first`, + rows: firstRows, + activeBuckets: firstActiveBuckets, + hasOrderBy: false, + }, + { + edgeId: `second`, + rows: secondRows, + activeBuckets: secondActiveBuckets, + hasOrderBy: false, + }, + ], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const firstFacade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `first`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + { id: number; value: string }, + number + > + const secondFacade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `second`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + { id: number; value: string }, + number + > + const firstStatuses: Array = [] + const secondStatuses: Array = [] + const unsubscribeFirst = firstFacade.on(`status:change`, ({ status }) => { + firstStatuses.push(status) + }) + const unsubscribeSecond = secondFacade.on(`status:change`, ({ status }) => { + secondStatuses.push(status) + }) + + const first = { id: 1, value: `first` } + const second = { id: 2, value: `second` } + firstActiveBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + secondActiveBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + firstRows.sendData( + new MultiSet([ + [ + [bucketKey, { publicKey: first.id, value: first, order: undefined }], + 1, + ], + ]), + ) + secondRows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { publicKey: second.id, value: second, order: undefined }, + ], + 1, + ], + ]), + ) + graph.run() + + const entries = ( + adapter as unknown as { + entries: Map> + } + ).entries + const secondSync = entries.get(`second`)?.get(bucketKey)?.sync + if (!secondSync) throw new Error(`Missing second facade sync`) + const commit = secondSync.commit + let shouldThrow = true + secondSync.commit = () => { + const applied = commit() + if (shouldThrow) { + shouldThrow = false + throw new Error(`second facade failed`) + } + return applied + } + + expect(() => adapter.flush()).toThrow(`second facade failed`) + expect(firstFacade.status).toBe(`loading`) + expect(secondFacade.status).toBe(`loading`) + expect(firstStatuses).toEqual([]) + expect(secondStatuses).toEqual([]) + expect(firstFacade.toArray).toEqual([]) + expect(secondFacade.toArray).toEqual([]) + + const retry = adapter.flush() + expect(firstFacade.status).toBe(`loading`) + expect(secondFacade.status).toBe(`loading`) + retry.prepare() + expect(firstFacade.status).toBe(`ready`) + expect(secondFacade.status).toBe(`ready`) + retry.publish() + expect(firstFacade.toArray.map(stripVirtualProps)).toEqual([first]) + expect(secondFacade.toArray.map(stripVirtualProps)).toEqual([second]) + expect(firstStatuses).toEqual([`ready`]) + expect(secondStatuses).toEqual([`ready`]) + + unsubscribeFirst() + unsubscribeSecond() + await adapter.cleanup() + }) + + it(`closes publication state when facade index restore fails`, async () => { + const graph = new D2() + const rows = graph.newInput<[string, BucketRow]>() + const activeBuckets = graph.newInput<[string, true]>() + const adapter = new BucketFacadeAdapter( + `facade-index-rollback-parent`, + [{ edgeId: `children`, rows, activeBuckets, hasOrderBy: false }], + () => {}, + ) + graph.finalize() + + const bucketKey = `group-1` + const original = { id: 1, value: `original` } + activeBuckets.sendData(new MultiSet([[[bucketKey, true], 1]])) + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { publicKey: original.id, value: original, order: undefined }, + ], + 1, + ], + ]), + ) + graph.run() + adapter.flush().publish() + + const facade = adapter.resolve({ + [BUCKET_FACADE_REF]: { edgeId: `children`, bucketKey }, + } satisfies BucketFacadeRef) as unknown as Collection< + typeof original, + number + > + const index = facade.createIndex((row) => row.value, { + indexType: ThrowingBuildIndex, + }) as ThrowingBuildIndex + const publications: Array = [] + const subscription = facade.subscribeChanges( + (changes) => { + publications.push(changes) + }, + { includeInitialState: false }, + ) + const revision = facade._stateRevision + + const entry = ( + adapter as unknown as { + entries: Map> + } + ).entries + .get(`children`) + ?.get(bucketKey) + const sync = entry?.sync + if (!sync) throw new Error(`Missing facade sync`) + const commit = sync.commit + let shouldThrow = true + sync.commit = () => { + const applied = commit() + if (shouldThrow) { + shouldThrow = false + throw new Error(`facade flush failed`) + } + return applied + } + + const replacement = { id: 1, value: `replacement` } + index.throwBeforeBuild = true rows.sendData( new MultiSet([ [ @@ -103,14 +491,54 @@ describe(`BucketFacadeAdapter`, () => { graph.run() expect(() => adapter.flush()).toThrow(`facade flush failed`) - expect(facade.toArray.map(stripVirtualProps)).toEqual([original]) + expect(facade.status).toBe(`error`) + expect(facade._state.syncedData.get(original.id)).toMatchObject(original) + expect(publications).toEqual([]) + expect(facade._stateRevision).toBe(revision) + + const final = { id: 1, value: `final` } + rows.sendData( + new MultiSet([ + [ + [ + bucketKey, + { + publicKey: replacement.id, + value: replacement, + order: undefined, + }, + ], + -1, + ], + [ + [bucketKey, { publicKey: final.id, value: final, order: undefined }], + 1, + ], + ]), + ) + graph.run() + expect(() => adapter.flush()).toThrow(`facade index rebuild failed`) + expect(facade.status).toBe(`error`) + expect(facade._state.syncedData.get(original.id)).toMatchObject(original) expect(publications).toEqual([]) + index.throwBeforeBuild = false + adapter.flush().publish() + expect(facade.status).toBe(`ready`) + expect(facade.toArray.map(stripVirtualProps)).toEqual([final]) + expect(publications).toHaveLength(2) + expect(publications[0]).toEqual([]) + expect(publications[1]).toHaveLength(1) + expect(facade._stateRevision).toBe(revision + 1) + expect(index.lookup(`eq`, `original`)).toEqual(new Set()) + expect(index.lookup(`eq`, `replacement`)).toEqual(new Set()) + expect(index.lookup(`eq`, `final`)).toEqual(new Set([original.id])) + subscription.unsubscribe() await adapter.cleanup() }) - it(`drops pending parent changes when facade flushing fails`, async () => { + it(`retries pending parent changes when facade flushing fails`, async () => { type Parent = { id: number; groupId: number } type Child = { id: number; groupId: number } const parents = createCollection( @@ -169,7 +597,7 @@ describe(`BucketFacadeAdapter`, () => { throw new Error(`Missing live query sync state`) } syncState.flushPendingChanges() - expect(live.has(2)).toBe(false) + expect(live.has(2)).toBe(true) } finally { CollectionConfigBuilder.prototype.getConfig = originalGetConfig vi.restoreAllMocks() diff --git a/packages/db/tests/query/coverage-registry-oracle.property.test.ts b/packages/db/tests/query/coverage-registry-oracle.property.test.ts index 8c2e8fc253..7ca042ce85 100644 --- a/packages/db/tests/query/coverage-registry-oracle.property.test.ts +++ b/packages/db/tests/query/coverage-registry-oracle.property.test.ts @@ -6,6 +6,7 @@ import { createLoadSubsetCoverageRegistry, } from '../../src/query/coverage-registry.js' import { oraclePropertyOptions } from '../oracle-config.js' +import type { CoverageRegistryResourceCounts } from '../../src/query/coverage-registry.js' import type { AppliedLoadSubsetOutcome } from '../../src/types.js' import type { Command } from 'fast-check' @@ -468,6 +469,22 @@ function createReleaseProbe(failFirst: boolean): ReleaseProbe { return probe } +function expectRegistryResourceBounds( + resourceCounts: CoverageRegistryResourceCounts, +): void { + // One logical lease may own several physical attempts. Bound each retained + // slot by claims, not by the number of unique lease tokens. + expect(resourceCounts.claims).toBeLessThanOrEqual( + resourceCounts.retainedDemands + resourceCounts.unsettledClaims, + ) + expect(resourceCounts.retainedDemands).toBeLessThanOrEqual( + resourceCounts.claims, + ) + expect(resourceCounts.retainedOutcomes).toBeLessThanOrEqual( + resourceCounts.claims, + ) +} + function expectReleaseFailure(release: () => unknown): void { let threw = false try { @@ -573,15 +590,7 @@ function assertRegistryModel(model: RegistryModel, real: RegistryReal): void { } const resourceCounts = real.registry.resourceCounts() expect(resourceCounts).toEqual(expectedResourceCounts) - expect(resourceCounts.claims).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) - expect(resourceCounts.retainedDemands).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) - expect(resourceCounts.retainedOutcomes).toBeLessThanOrEqual( - resourceCounts.liveLeases + resourceCounts.unsettledClaims, - ) + expectRegistryResourceBounds(resourceCounts) for (const row of modelRows) { expect(real.registry.rowOwnerCount(row)).toBe( model.acquisitions.filter( @@ -1090,6 +1099,38 @@ class DisposeCommand implements Command { } describe(`coverage registry oracle`, () => { + it(`bounds evidence when one lease owns parallel physical acquisitions`, () => { + const registry = createPrefixRegistry() + const lease = registry.addLease(1) + const acquisitions = [ + addPrefixAcquisition(registry, { + generation: 1, + leases: [lease], + release: vi.fn(), + prefix: 1, + }), + addPrefixAcquisition(registry, { + generation: 1, + leases: [lease], + release: vi.fn(), + prefix: 1, + }), + ] + + acquisitions.forEach((acquisition) => + registry.settleLease(acquisition, lease), + ) + + expect(registry.resourceCounts()).toMatchObject({ + liveLeases: 1, + acquisitions: 2, + claims: 2, + unsettledClaims: 0, + retainedDemands: 2, + }) + expectRegistryResourceBounds(registry.resourceCounts()) + }) + it(`fences old evidence while retaining its physical release obligation`, () => { const registry = createPrefixRegistry() const oldRelease = vi.fn() @@ -1329,7 +1370,7 @@ describe(`coverage registry oracle`, () => { fcTest.prop( [claimChurnArbitrary, fc.integer({ min: 1, max: 8 })], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `coverage-registry.claim-churn`), )(`bounds long claim churn for a random or replayed seed`, runClaimChurn) it(`restores a compacted narrower fact when the wider acquisition retires`, () => { @@ -1985,7 +2026,7 @@ describe(`coverage registry oracle`, () => { maxCommands: 40, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions(100, `coverage-registry.state-machine`), )( `matches the lease, retry, settlement, publication, ownership, and disposal state machine`, (commands) => { diff --git a/packages/db/tests/query/includes-collection-oracle.property.test.ts b/packages/db/tests/query/includes-collection-oracle.property.test.ts index 5d8c2f45a2..6b31803218 100644 --- a/packages/db/tests/query/includes-collection-oracle.property.test.ts +++ b/packages/db/tests/query/includes-collection-oracle.property.test.ts @@ -1,5 +1,11 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect } from 'vitest' +import { describe, expect, vi } from 'vitest' +import { createDeferred } from '../../src/deferred.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' +import { createOptimisticAction } from '../../src/optimistic-action.js' +import { BucketFacadeAdapter } from '../../src/query/live/bucket-facade-adapter.js' +import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' import { add, caseWhen, @@ -34,6 +40,119 @@ type ChildRow = { value: number } +type LayoutSwapScenario = { + length: number + swapIndex: number +} + +type FacadeCandidateScanScenario = { + candidatePosition: `first` | `last` + finalLayout: `moved` | `restored` +} + +class ThrowingUpdateIndex extends BasicIndex { + updateFailure: { error: unknown } | undefined + buildFailure: { error: unknown; stage: `before` | `after` } | undefined + buildCalls = 0 + + override update(key: number, oldItem: unknown, newItem: unknown): void { + super.update(key, oldItem, newItem) + if (this.updateFailure) throw this.updateFailure.error + } + + override build(entries: Iterable<[number, unknown]>): void { + this.buildCalls += 1 + if (this.buildFailure?.stage === `before`) { + throw this.buildFailure.error + } + super.build(entries) + if (this.buildFailure?.stage === `after`) { + throw this.buildFailure.error + } + } +} + +function captureFailure(callback: () => void): { error: unknown } | undefined { + try { + callback() + return undefined + } catch (error) { + return { error } + } +} + +const exhaustiveLayoutSwapScenarios: Array = Array.from( + { length: 9 }, + (_, offset) => offset + 4, +).flatMap((length) => + Array.from({ length: length - 3 }, (_, offset) => ({ + length, + swapIndex: offset + 1, + })), +) + +const layoutSwapScenarioArbitrary: fc.Arbitrary = fc + .integer({ min: 4, max: 12 }) + .chain((length) => + fc.integer({ min: 1, max: length - 3 }).map((swapIndex) => ({ + length, + swapIndex, + })), + ) + +const facadeCandidateScanScenarios: ReadonlyArray = + [ + { candidatePosition: `first`, finalLayout: `moved` }, + { candidatePosition: `first`, finalLayout: `restored` }, + { candidatePosition: `last`, finalLayout: `moved` }, + { candidatePosition: `last`, finalLayout: `restored` }, + ] + +type ProjectedChildChange = { + type: `insert` | `update` | `delete` + key: number + value: ChildRow + previousValue?: ChildRow +} + +function projectChildChange( + change: ChangeMessage, +): ProjectedChildChange { + const projectRow = ({ id, parentGroup, value }: ChildRow): ChildRow => ({ + id, + parentGroup, + value, + }) + return { + type: change.type, + key: Number(change.key), + value: projectRow(change.value), + ...(change.previousValue + ? { previousValue: projectRow(change.previousValue) } + : {}), + } +} + +type ProjectedValueChange = { + type: `insert` | `update` | `delete` + key: number + value: number + previousValue?: number +} + +function projectValueChange( + change: ChangeMessage<{ value: number }, string | number>, +): ProjectedValueChange { + return { + type: change.type, + key: Number(change.key), + value: change.value.value, + ...(change.previousValue + ? { previousValue: change.previousValue.value } + : {}), + } +} + type CollectionAction = | { type: `putParent`; row: ParentRow } | { type: `deleteParent`; id: number } @@ -84,6 +203,243 @@ function expectedMaterializations(rows: ReadonlyArray) { } } +async function expectRootAndFacadeLayoutSwap({ + length, + swapIndex, +}: LayoutSwapScenario): Promise { + type OrderedChild = ChildRow & { position: number } + const parents = createControlledCollection(`layout-swap-parents`, [ + { id: 1, group: 1 }, + ]) + const initialRows: Array = Array.from( + { length }, + (_, index) => ({ + id: index + 1, + parentGroup: 1, + value: index + 1, + position: index, + }), + ) + const children = createControlledCollection( + `layout-swap-children`, + initialRows, + ) + const root = createLiveQueryCollection((q) => + q + .from({ child: children.collection }) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ id: child.id, value: child.value })), + ) + const nested = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ id: child.id, value: child.value })), + })), + ) + let rootSubscription: { unsubscribe: () => void } | undefined + let facadeSubscription: { unsubscribe: () => void } | undefined + + try { + await Promise.all([root.preload(), nested.preload()]) + const facade = nested.get(1)!.children + const rootRevision = root._layoutRevision + const facadeRevision = facade._layoutRevision + const rootPublicationSizes: Array = [] + const facadePublicationSizes: Array = [] + const rootCallbackKeys: Array> = [] + const facadeCallbackKeys: Array> = [] + rootSubscription = root.subscribeChanges( + (changes) => { + rootPublicationSizes.push(changes.length) + rootCallbackKeys.push(root.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + facadeSubscription = facade.subscribeChanges( + (changes) => { + facadePublicationSizes.push(changes.length) + facadeCallbackKeys.push(facade.toArray.map(({ id }) => id)) + }, + { includeInitialState: false }, + ) + const expectedKeys = initialRows.map(({ id }) => id) + ;[expectedKeys[swapIndex], expectedKeys[swapIndex + 1]] = [ + expectedKeys[swapIndex + 1]!, + expectedKeys[swapIndex]!, + ] + const first = initialRows[swapIndex]! + const second = initialRows[swapIndex + 1]! + + children.writeBatch([ + { + type: `update`, + value: { ...first, position: second.position }, + }, + { + type: `update`, + value: { ...second, position: first.position }, + }, + ]) + + expect(root.toArray.map(({ id }) => id)).toEqual(expectedKeys) + expect(facade.toArray.map(({ id }) => id)).toEqual(expectedKeys) + expect(root._layoutRevision).toBe(rootRevision + 1) + expect(facade._layoutRevision).toBe(facadeRevision + 1) + expect(rootPublicationSizes).toEqual([0]) + expect(facadePublicationSizes).toEqual([0]) + expect(rootCallbackKeys).toEqual([expectedKeys]) + expect(facadeCallbackKeys).toEqual([expectedKeys]) + } finally { + rootSubscription?.unsubscribe() + facadeSubscription?.unsubscribe() + await Promise.all([ + root.cleanup(), + nested.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } +} + +async function expectFacadeCandidateScan({ + candidatePosition, + finalLayout, +}: FacadeCandidateScanScenario): Promise { + type OrderedChild = ChildRow & { position: number } + const parents = createControlledCollection(`candidate-scan-parents`, [ + { id: 1, group: 1 }, + ]) + const initialRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + { id: 30, parentGroup: 1, value: 30, position: 2 }, + ] + const children = createControlledCollection( + `candidate-scan-children`, + initialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + let subscription: { unsubscribe: () => void } | undefined + let restoreFacadeGetKey: (() => void) | undefined + + try { + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const callbackValues: Array> = [] + subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const revision = facade._layoutRevision + const candidateRow = initialRows[candidatePosition === `first` ? 0 : 1]! + const valueRow = initialRows[candidatePosition === `first` ? 1 : 0]! + const changedKeyOrder: Array = [] + const originalGetKey = facade.config.getKey + facade.config.getKey = (row) => { + const key = Number(originalGetKey(row)) + if ( + (key === candidateRow.id || key === valueRow.id) && + !changedKeyOrder.includes(key) + ) { + changedKeyOrder.push(key) + } + return key + } + restoreFacadeGetKey = () => { + facade.config.getKey = originalGetKey + } + const valueUpdate = { + type: `update` as const, + value: { ...valueRow, value: valueRow.value + 1 }, + } + const orderUpdates = [ + { + type: `update` as const, + value: { ...candidateRow, position: 3 }, + }, + ...(finalLayout === `restored` + ? [ + { + type: `update` as const, + value: candidateRow, + }, + ] + : []), + ] + + children.writeBatch( + candidatePosition === `first` + ? [...orderUpdates, valueUpdate] + : [valueUpdate, ...orderUpdates], + ) + + const expectedKeys = + finalLayout === `moved` + ? initialRows + .filter(({ id }) => id !== candidateRow.id) + .map(({ id }) => id) + .concat(candidateRow.id) + : initialRows.map(({ id }) => id) + const expectedValues = expectedKeys.map((id) => + id === valueRow.id ? valueRow.value + 1 : id, + ) + if (finalLayout === `moved`) { + expect(changedKeyOrder).toEqual([10, 20]) + expect(changedKeyOrder[candidatePosition === `first` ? 0 : 1]).toBe( + candidateRow.id, + ) + } else { + expect(changedKeyOrder).toEqual([valueRow.id]) + } + expect(keys()).toEqual(expectedKeys) + expect(values()).toEqual(expectedValues) + expect(publications).toEqual([ + [ + { + type: `update`, + key: valueRow.id, + value: valueRow.value + 1, + previousValue: valueRow.value, + }, + ], + ]) + expect(callbackKeys).toEqual([expectedKeys]) + expect(callbackValues).toEqual([expectedValues]) + expect(facade._layoutRevision).toBe( + revision + (finalLayout === `moved` ? 1 : 0), + ) + } finally { + restoreFacadeGetKey?.() + subscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } +} + function createCollectionQuery( parents: Collection, children: Collection, @@ -325,8 +681,128 @@ const exhaustiveActions: ReadonlyArray = [ { type: `deleteChild`, id: 10 }, ] +type PendingFacadeOperation = `insert` | `update` | `delete` +type PendingFacadeOptimisticOperation = Exclude< + PendingFacadeOperation, + `insert` +> +type PendingFacadeKeyRelation = `disjoint-key` | `same-key` +type PendingFacadeShape = `unordered` | `ordered` + +const pendingFacadeOptimisticOperations = [`update`, `delete`] as const +const pendingFacadeSourceOperations = [`insert`, `update`, `delete`] as const +const pendingFacadeSettlements = [`resolve`, `reject`] as const +const pendingFacadeKeyRelations = [`disjoint-key`, `same-key`] as const +const pendingFacadeShapes = [`unordered`, `ordered`] as const +const pendingFacadeInitialRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, +] + +function pendingOptimisticFacadeRow( + operation: PendingFacadeOptimisticOperation, +): ChildRow { + if (operation === `update`) { + return { id: 10, parentGroup: 1, value: 11 } + } + return { id: 10, parentGroup: 1, value: 10 } +} + +function pendingSourceFacadeRow( + operation: PendingFacadeOperation, + keyRelation: PendingFacadeKeyRelation, +): ChildRow { + if (operation === `insert`) { + return { id: 40, parentGroup: 1, value: 40 } + } + if (keyRelation === `same-key`) { + return { + id: 10, + parentGroup: 1, + value: operation === `update` ? 21 : 10, + } + } + if (operation === `update`) { + return { id: 20, parentGroup: 1, value: 21 } + } + return { id: 20, parentGroup: 1, value: 20 } +} + +function applyPendingFacadeOperation( + rows: Map, + operation: PendingFacadeOperation, + row: ChildRow, +): void { + if (operation === `delete`) rows.delete(row.id) + else rows.set(row.id, { ...row }) +} + +function expectedPendingFacadeRows( + rows: ReadonlyMap, + shape: PendingFacadeShape = `unordered`, + orderRows: ReadonlyMap = rows, +): Array { + return [...rows.values()] + .map((row) => ({ ...row })) + .sort((left, right) => { + if (shape === `unordered`) return left.id - right.id + const leftOrder = orderRows.get(left.id)?.value + const rightOrder = orderRows.get(right.id)?.value + if (leftOrder === rightOrder) return left.id - right.id + if (leftOrder === undefined) return 1 + if (rightOrder === undefined) return -1 + return leftOrder - rightOrder + }) +} + +function projectPendingFacadeRows( + rows: ReadonlyArray, + shape: PendingFacadeShape, +): Array { + const projected = rows.map(({ id, parentGroup, value }) => ({ + id, + parentGroup, + value, + })) + return shape === `ordered` + ? projected + : projected.sort((left, right) => left.id - right.id) +} + +function expectedPendingFacadeChange( + before: ReadonlyMap, + after: ReadonlyMap, + key: number, +): ProjectedChildChange | undefined { + const previousValue = before.get(key) + const value = after.get(key) + if ( + previousValue?.id === value?.id && + previousValue?.parentGroup === value?.parentGroup && + previousValue?.value === value?.value + ) { + return undefined + } + if (!previousValue && value) { + return { type: `insert`, key, value: { ...value } } + } + if (previousValue && !value) { + return { type: `delete`, key, value: { ...previousValue } } + } + if (!previousValue || !value) return undefined + return { + type: `update`, + key, + value: { ...value }, + previousValue: { ...previousValue }, + } +} + describe(`Collection-valued includes oracle`, () => { - fcTest.prop([collectionScenarioArbitrary], oraclePropertyOptions(30))( + fcTest.prop( + [collectionScenarioArbitrary], + oraclePropertyOptions(30, `includes-collection.relationship-history`), + )( `keeps Collection, toArray, and materialize equivalent across generated relationship histories`, ({ parentGroup, childValue, actions }) => runTrace({ @@ -624,9 +1100,16 @@ describe(`Collection-valued includes oracle`, () => { group: 1, value: 1, } + const initialSibling: NodeRow = { + id: 20, + kind: `child`, + group: 1, + value: 2, + } const nodes = createControlledCollection(`rollback-nodes`, [ initialParent, initialChild, + initialSibling, ]) const live = createLiveQueryCollection((q) => q @@ -638,30 +1121,79 @@ describe(`Collection-valued includes oracle`, () => { children: q .from({ child: nodes.collection }) .where(({ child }) => eq(child.kind, `child`)) - .where(({ child }) => eq(child.group, parent.group)), + .where(({ child }) => eq(child.group, parent.group)) + .orderBy(({ child }) => child.value), })), ) await live.preload() const facade = live.get(1)!.children + const rootIndex = live.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex + const pendingApplied = createDeferred() + void pendingApplied.promise.catch(() => undefined) + const pendingFacadeSync = { + committed: true, + applicationStarted: false, + layoutChanged: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map([ + [initialChild.id, { type: `set` as const, value: `pending` }], + ]), + collectionMetadataWrites: new Map(), + applied: pendingApplied, + } + facade._state.pendingSyncedTransactions.push(pendingFacadeSync) + facade._state.capturePreSyncVisibleState() + const recentlySyncedBeforeFailure = new Set( + facade._state.recentlySyncedKeys, + ) + const preSyncVirtualBeforeFailure = new Map( + facade._state.preSyncVirtualState, + ) + expect([...preSyncVirtualBeforeFailure.keys()]).toEqual([initialChild.id]) const rootPublications: Array = [] const childPublications: Array = [] + const childReceiptStates: Array = [] + const rootCallbackFacadeSnapshots: Array<{ + rows: Array<{ id: number; value: number }> + stateRevision: number + layoutRevision: number + }> = [] const rootSubscription = live.subscribeChanges( - (batch) => rootPublications.push(...batch), + (batch) => { + rootPublications.push(...batch) + rootCallbackFacadeSnapshots.push({ + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + stateRevision: facade._stateRevision, + layoutRevision: facade._layoutRevision, + }) + }, { includeInitialState: false }, ) const childSubscription = facade.subscribeChanges( - (batch) => childPublications.push(...batch), + (batch) => { + childPublications.push(...batch) + childReceiptStates.push(pendingApplied.isPending()) + }, { includeInitialState: false }, ) - const originalGetKey = live.config.getKey - live.config.getKey = (row) => { - if (row.value === 2) throw new Error(`root key failed`) - return originalGetKey(row) - } + const childObserver = createLiveQueryObserver(facade) + let observerNotifications = 0 + childObserver.subscribe(() => observerNotifications++) + observerNotifications = 0 + const observerBeforeFailure = childObserver.getSnapshot() + const rootStateRevisionBeforeFailure = live._stateRevision + const rootLayoutRevisionBeforeFailure = live._layoutRevision + const childStateRevisionBeforeFailure = facade._stateRevision + const childLayoutRevisionBeforeFailure = facade._layoutRevision + const rootFailure = new Error(`root index failed`) + rootIndex.updateFailure = { error: rootFailure } try { - expect(() => + const failure = captureFailure(() => nodes.writeBatch([ { type: `update`, @@ -669,32 +1201,92 @@ describe(`Collection-valued includes oracle`, () => { }, { type: `update`, - value: { ...initialChild, value: 2 }, + value: { ...initialChild, value: 3 }, + }, + { + type: `update`, + value: { ...initialSibling, value: 0 }, }, ]), - ).toThrow(`root key failed`) + ) + expect(failure?.error).toBe(rootFailure) expect(live.get(1)!.value).toBe(1) - expect(facade.get(10)!.value).toBe(1) + expect([...rootIndex.equalityLookup(1)]).toEqual([1]) + expect([...rootIndex.equalityLookup(2)]).toEqual([]) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + { id: 20, value: 2 }, + ]) expect(rootPublications).toEqual([]) expect(childPublications).toEqual([]) + expect(childReceiptStates).toEqual([]) + expect(rootCallbackFacadeSnapshots).toEqual([]) + expect(live._stateRevision).toBe(rootStateRevisionBeforeFailure) + expect(live._layoutRevision).toBe(rootLayoutRevisionBeforeFailure) + expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure) + expect(facade._layoutRevision).toBe(childLayoutRevisionBeforeFailure) + expect(childObserver.getSnapshot()).toBe(observerBeforeFailure) + expect(observerNotifications).toBe(0) + expect(facade._state.pendingSyncedTransactions).toHaveLength(1) + expect(facade._state.pendingSyncedTransactions[0]).toBe( + pendingFacadeSync, + ) + expect( + facade._state.pendingSyncedTransactions[0]!.applied.isPending(), + ).toBe(true) + expect(facade._state.recentlySyncedKeys).toEqual( + recentlySyncedBeforeFailure, + ) + expect(facade._state.preSyncVirtualState).toEqual( + preSyncVirtualBeforeFailure, + ) + await Promise.resolve() + expect(facade._state.recentlySyncedKeys).toEqual( + recentlySyncedBeforeFailure, + ) + expect(facade._state.preSyncVirtualState).toEqual( + preSyncVirtualBeforeFailure, + ) - live.config.getKey = originalGetKey - nodes.writeBatch([ - { - type: `update`, - value: { ...initialParent, value: 3 }, - }, + rootIndex.updateFailure = undefined + // Only the root changes on retry. The child deltas consumed by the + // failed graph turn must remain staged until the whole publication + // commits; the source will not emit them again. + nodes.write(`update`, { ...initialParent, value: 3 }) + expect(pendingApplied.isPending()).toBe(false) + await pendingApplied.promise + expect(facade._state.syncedMetadata.get(initialChild.id)).toBe( + `pending`, + ) + expect(live.get(1)!.value).toBe(3) + expect([...rootIndex.equalityLookup(1)]).toEqual([]) + expect([...rootIndex.equalityLookup(3)]).toEqual([1]) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 20, value: 0 }, + { id: 10, value: 3 }, + ]) + expect(rootPublications).toHaveLength(1) + expect(childPublications).toHaveLength(2) + expect(childReceiptStates).toEqual([true]) + expect(rootCallbackFacadeSnapshots).toEqual([ { - type: `update`, - value: { ...initialChild, value: 3 }, + rows: [ + { id: 20, value: 0 }, + { id: 10, value: 3 }, + ], + stateRevision: childStateRevisionBeforeFailure + 1, + layoutRevision: childLayoutRevisionBeforeFailure + 1, }, ]) - expect(live.get(1)!.value).toBe(3) - expect(facade.get(10)!.value).toBe(3) - expect(rootPublications).toHaveLength(1) - expect(childPublications).toHaveLength(1) + expect(facade._stateRevision).toBe(childStateRevisionBeforeFailure + 1) + expect(facade._layoutRevision).toBe( + childLayoutRevisionBeforeFailure + 1, + ) + expect(childObserver.getSnapshot()).not.toBe(observerBeforeFailure) + expect(observerNotifications).toBe(1) } finally { - live.config.getKey = originalGetKey + rootIndex.updateFailure = undefined + childObserver.dispose() rootSubscription.unsubscribe() childSubscription.unsubscribe() await Promise.all([live.cleanup(), nodes.collection.cleanup()]) @@ -703,56 +1295,2623 @@ describe(`Collection-valued includes oracle`, () => { ) fcTest( - `child-only changes flush the facade without republishing the parent`, + `root and facade recovery retry together after a failed graph install`, async () => { - const parents = createControlledCollection(`facade-only-parents`, [ - { id: 1, group: 1 }, - ]) - const children = createControlledCollection(`facade-only-children`, [ - { id: 10, parentGroup: 1, value: 1 }, - ]) + type NodeRow = { + id: number + kind: `parent` | `child` + group: number + value: number + } + const initialParent: NodeRow = { + id: 1, + kind: `parent`, + group: 1, + value: 1, + } + const initialChild: NodeRow = { + id: 10, + kind: `child`, + group: 1, + value: 1, + } + const initialSibling: NodeRow = { + id: 11, + kind: `child`, + group: 1, + value: 10, + } + const nodes = createControlledCollection( + `root-restore-failure-nodes`, + [initialParent, initialChild, initialSibling], + ) const live = createLiveQueryCollection((q) => - q.from({ parent: parents.collection }).select(({ parent }) => ({ - id: parent.id, - children: q - .from({ child: children.collection }) - .where(({ child }) => eq(child.parentGroup, parent.group)), - })), + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.group, parent.group)), + })), ) await live.preload() const facade = live.get(1)!.children - const rootPublications: Array = [] - const childPublications: Array = [] + const rootIndex = live.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex + const facadeIndex = facade.createIndex((row) => row.value, { + indexType: ThrowingUpdateIndex, + }) as ThrowingUpdateIndex + type ProjectedNodeChange = { + type: `insert` | `update` | `delete` + key: number + value: Pick + previousValue?: Pick + } + const projectNodeRow = ({ id, kind, group, value }: NodeRow) => ({ + id, + kind, + group, + value, + }) + const projectNodeChange = ( + change: ChangeMessage, + ): ProjectedNodeChange => ({ + type: change.type, + key: Number(change.key), + value: projectNodeRow(change.value), + ...(change.previousValue + ? { previousValue: projectNodeRow(change.previousValue) } + : {}), + }) + type ProjectedRootChange = { + type: `insert` | `update` | `delete` + key: number + value: { id: number; value: number; preservesFacade: boolean } + previousValue?: { + id: number + value: number + preservesFacade: boolean + } + } + const projectRootChange = ( + change: ChangeMessage< + { id: number; value: number; children: typeof facade }, + string | number + >, + ): ProjectedRootChange => ({ + type: change.type, + key: Number(change.key), + value: { + id: change.value.id, + value: change.value.value, + preservesFacade: change.value.children === facade, + }, + ...(change.previousValue + ? { + previousValue: { + id: change.previousValue.id, + value: change.previousValue.value, + preservesFacade: change.previousValue.children === facade, + }, + } + : {}), + }) + const rootPublications: Array> = [] + const childPublications: Array> = [] const rootSubscription = live.subscribeChanges( - (batch) => rootPublications.push(...batch), + (batch) => rootPublications.push(batch.map(projectRootChange)), { includeInitialState: false }, ) const childSubscription = facade.subscribeChanges( - (batch) => childPublications.push(...batch), + (batch) => childPublications.push(batch.map(projectNodeChange)), { includeInitialState: false }, ) - - try { - children.write(`update`, { - id: 10, - parentGroup: 1, - value: 2, + const errorSnapshots: Array<{ + root: number + children: Array<{ id: number; value: number }> + indexKeys: { + one: Array + two: Array + ten: Array + twenty: Array + } + }> = [] + const unsubscribeError = live.on(`status:error`, () => { + errorSnapshots.push({ + root: live.get(1)!.value, + children: facade.toArray.map(({ id, value }) => ({ id, value })), + indexKeys: { + one: [...facadeIndex.equalityLookup(1)], + two: [...facadeIndex.equalityLookup(2)], + ten: [...facadeIndex.equalityLookup(10)], + twenty: [...facadeIndex.equalityLookup(20)], + }, }) + }) + const readinessOrder: Array<`facade` | `root`> = [] + const unsubscribeRootReady = live.on(`status:ready`, () => { + readinessOrder.push(`root`) + }) + const unsubscribeFacadeReady = facade.on(`status:ready`, () => { + readinessOrder.push(`facade`) + }) + const rootRevision = live._stateRevision + const childRevision = facade._stateRevision + const installFailure = new Error(`root index failed`) + rootIndex.updateFailure = { error: installFailure } + rootIndex.buildFailure = { error: false, stage: `before` } + facadeIndex.buildFailure = { error: undefined, stage: `before` } + try { + const failedInstall = captureFailure(() => + nodes.writeBatch([ + { + type: `update`, + value: { ...initialParent, value: 2 }, + }, + { + type: `update`, + value: { ...initialChild, value: 2 }, + }, + { + type: `update`, + value: { ...initialSibling, value: 20 }, + }, + ]), + ) + expect(failedInstall?.error).toBe(installFailure) + expect(live.status).toBe(`error`) + expect(live.get(1)!.value).toBe(1) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ]) expect(rootPublications).toEqual([]) - expect(childPublications).toHaveLength(1) - expect(live.get(1)!.children).toBe(facade) - expect( - [...facade.values()].map(({ id, parentGroup, value }) => ({ + expect(childPublications).toEqual([]) + expect(live._stateRevision).toBe(rootRevision) + expect(facade._stateRevision).toBe(childRevision) + expect(errorSnapshots).toEqual([ + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, + }, + ]) + + rootIndex.updateFailure = undefined + const rootBuildCalls = rootIndex.buildCalls + const facadeBuildCalls = facadeIndex.buildCalls + const simultaneousRecoveryFailure = captureFailure(() => + nodes.write(`update`, { ...initialParent, value: 3 }), + ) + expect(simultaneousRecoveryFailure).toEqual({ error: false }) + expect(rootIndex.buildCalls).toBe(rootBuildCalls + 1) + expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 1) + expect(live.status).toBe(`error`) + expect(live.get(1)!.value).toBe(1) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(errorSnapshots).toEqual([ + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, + }, + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, + }, + ]) + + facadeIndex.buildFailure = undefined + const facadeRecoveryFailure = captureFailure(() => + nodes.write(`update`, { ...initialParent, value: 4 }), + ) + expect(facadeRecoveryFailure).toEqual({ error: false }) + expect(rootIndex.buildCalls).toBe(rootBuildCalls + 2) + expect(facadeIndex.buildCalls).toBe(facadeBuildCalls + 2) + expect(errorSnapshots).toEqual([ + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, + }, + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [], two: [10], ten: [], twenty: [11] }, + }, + { + root: 1, + children: [ + { id: 10, value: 1 }, + { id: 11, value: 10 }, + ], + indexKeys: { one: [10], two: [], ten: [11], twenty: [] }, + }, + ]) + expect([...facadeIndex.equalityLookup(1)]).toEqual([10]) + expect([...facadeIndex.equalityLookup(2)]).toEqual([]) + expect([...facadeIndex.equalityLookup(10)]).toEqual([11]) + expect([...facadeIndex.equalityLookup(20)]).toEqual([]) + + rootIndex.buildFailure = undefined + nodes.write(`update`, { ...initialParent, value: 5 }) + + expect(live.status).toBe(`ready`) + expect(facade.status).toBe(`ready`) + expect(readinessOrder).toEqual([`facade`, `root`]) + expect(live.get(1)!.value).toBe(5) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 2 }, + { id: 11, value: 20 }, + ]) + const rootPublicationsAfterRecovery: Array> = + [ + [], + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 5, preservesFacade: true }, + previousValue: { id: 1, value: 1, preservesFacade: true }, + }, + ], + ] + const childPublicationsAfterRecovery: Array< + Array + > = [ + [], + [ + { + type: `update`, + key: 10, + value: { ...initialChild, value: 2 }, + previousValue: initialChild, + }, + { + type: `update`, + key: 11, + value: { ...initialSibling, value: 20 }, + previousValue: initialSibling, + }, + ], + ] + expect(rootPublications).toEqual(rootPublicationsAfterRecovery) + expect(childPublications).toEqual(childPublicationsAfterRecovery) + expect(live._stateRevision).toBe(rootRevision + 1) + expect(facade._stateRevision).toBe(childRevision + 1) + expect([...rootIndex.equalityLookup(2)]).toEqual([]) + expect([...rootIndex.equalityLookup(3)]).toEqual([]) + expect([...rootIndex.equalityLookup(4)]).toEqual([]) + expect([...rootIndex.equalityLookup(5)]).toEqual([1]) + + const facadeRevisionAfterRecovery = facade._stateRevision + const facadeBuildCallsAfterRecovery = facadeIndex.buildCalls + const pendingChecks: Array = [] + const hasPendingChanges = + BucketFacadeAdapter.prototype.hasPendingChanges + const pendingSpy = vi + .spyOn(BucketFacadeAdapter.prototype, `hasPendingChanges`) + .mockImplementation(function (this: BucketFacadeAdapter) { + const result = hasPendingChanges.call(this) + pendingChecks.push(result) + return result + }) + try { + nodes.write(`update`, { ...initialParent, value: 6 }) + } finally { + pendingSpy.mockRestore() + } + expect(pendingChecks).toEqual([false]) + expect(facadeIndex.buildCalls).toBe(facadeBuildCallsAfterRecovery) + expect(live.get(1)!.value).toBe(6) + expect(facade.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 10, value: 2 }, + { id: 11, value: 20 }, + ]) + expect([...facadeIndex.equalityLookup(2)]).toEqual([10]) + expect([...facadeIndex.equalityLookup(20)]).toEqual([11]) + expect(rootPublications).toEqual([ + ...rootPublicationsAfterRecovery, + [ + { + type: `update`, + key: 1, + value: { id: 1, value: 6, preservesFacade: true }, + previousValue: { id: 1, value: 5, preservesFacade: true }, + }, + ], + ]) + expect(childPublications).toEqual(childPublicationsAfterRecovery) + expect(facade._stateRevision).toBe(facadeRevisionAfterRecovery) + expect(readinessOrder).toEqual([`facade`, `root`]) + } finally { + rootIndex.updateFailure = undefined + rootIndex.buildFailure = undefined + facadeIndex.updateFailure = undefined + facadeIndex.buildFailure = undefined + unsubscribeError() + unsubscribeRootReady() + unsubscribeFacadeReady() + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + + fcTest( + `child-only changes flush the facade without republishing the parent`, + async () => { + const parents = createControlledCollection(`facade-only-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`facade-only-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootPublications: Array = [] + const childPublications: Array = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(...batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => childPublications.push(...batch), + { includeInitialState: false }, + ) + + try { + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 2, + }) + + expect(rootPublications).toEqual([]) + expect(childPublications).toHaveLength(1) + expect(live.get(1)!.children).toBe(facade) + expect( + [...facade.values()].map(({ id, parentGroup, value }) => ({ id, parentGroup, value, })), - ).toEqual([{ id: 10, parentGroup: 1, value: 2 }]) + ).toEqual([{ id: 10, parentGroup: 1, value: 2 }]) + } finally { + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + for (const { throwingParentId, position, failure } of [ + { throwingParentId: 1, position: `first`, failure: `error` }, + { throwingParentId: 2, position: `middle`, failure: `undefined` }, + { throwingParentId: 3, position: `last`, failure: `null` }, + ] as const) { + fcTest( + `a throwing ${position} facade callback does not suppress sibling publications`, + async () => { + const parents = createControlledCollection(`callback-error-parents`, [ + { id: 1, group: 1 }, + { id: 2, group: 2 }, + { id: 3, group: 3 }, + ]) + const children = createControlledCollection(`callback-error-children`, [ + { id: 10, parentGroup: 1, value: 1 }, + { id: 20, parentGroup: 2, value: 2 }, + { id: 30, parentGroup: 3, value: 3 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.id) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + + await live.preload() + const facades = [1, 2, 3].map((parentId) => ({ + parentId, + collection: live.get(parentId)!.children, + })) + const callbackParentIds: Array = [] + const callbackError = + failure === `error` + ? new Error(`facade ${throwingParentId} callback failed`) + : failure === `undefined` + ? undefined + : null + const subscriptions = facades.map(({ parentId, collection }) => + collection.subscribeChanges( + () => { + callbackParentIds.push(parentId) + if (parentId === throwingParentId) throw callbackError + if (position === `first` && parentId === 3) { + throw new Error(`later facade callback failed`) + } + }, + { includeInitialState: false }, + ), + ) + + try { + let didThrow = false + let publicationError: unknown + try { + children.writeBatch([ + { + type: `update`, + value: { id: 10, parentGroup: 1, value: 11 }, + }, + { + type: `update`, + value: { id: 20, parentGroup: 2, value: 12 }, + }, + { + type: `update`, + value: { id: 30, parentGroup: 3, value: 13 }, + }, + ]) + } catch (error) { + didThrow = true + publicationError = error + } + + expect(didThrow).toBe(true) + expect(publicationError).toBe(callbackError) + expect(callbackParentIds).toEqual([1, 2, 3]) + expect( + facades.map(({ collection }) => collection.toArray[0]!.value), + ).toEqual([11, 12, 13]) + } finally { + for (const subscription of subscriptions) subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + fcTest( + `cleanup during root publication suppresses a prepared facade callback`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` + group: number + value: number + } + const nodes = createControlledCollection( + `prepared-facade-cleanup`, + [ + { id: 1, kind: `parent`, group: 1, value: 1 }, + { id: 10, kind: `child`, group: 1, value: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const rootSnapshots: Array<{ + status: string + rows: Array<{ id: number; value: number }> + }> = [] + const facadeSnapshots: Array<{ + status: string + rows: Array<{ id: number; value: number }> + }> = [] + let cleanupPromise: Promise | undefined + const rootSubscription = live.subscribeChanges( + () => { + rootSnapshots.push({ + status: facade.status, + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + }) + cleanupPromise = facade.cleanup() + }, + { includeInitialState: false }, + ) + const facadeSubscription = facade.subscribeChanges( + () => { + facadeSnapshots.push({ + status: facade.status, + rows: facade.toArray.map(({ id, value }) => ({ id, value })), + }) + }, + { includeInitialState: false }, + ) + + try { + nodes.writeBatch([ + { + type: `update`, + value: { id: 1, kind: `parent`, group: 1, value: 2 }, + }, + { + type: `update`, + value: { id: 10, kind: `child`, group: 1, value: 2 }, + }, + ]) + await cleanupPromise + + expect(rootSnapshots).toEqual([ + { + status: `ready`, + rows: [{ id: 10, value: 2 }], + }, + ]) + expect(facadeSnapshots).toEqual([]) + expect(facade.status).toBe(`cleaned-up`) + expect(facade.toArray).toEqual([]) + } finally { + rootSubscription.unsubscribe() + facadeSubscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + + fcTest(`cleanup cancels every handle in a prepared publication`, async () => { + const rows = createControlledCollection(`prepared-publication-cleanup`, [ + { id: 1, value: 1 }, + ]) + await rows.collection.preload() + const callbackValues: Array> = [] + const subscription = rows.collection.subscribeChanges( + (batch) => { + callbackValues.push(batch.map((change) => change.value.value)) + }, + { includeInitialState: false }, + ) + + try { + const firstPublication = rows.collection._deferPublication() + rows.write(`update`, { id: 1, value: 2 }) + const secondPublication = rows.collection._deferPublication() + rows.write(`update`, { id: 1, value: 3 }) + firstPublication.prepare() + secondPublication.prepare() + + expect(rows.collection.get(1)!.value).toBe(3) + expect(rows.collection.status).toBe(`ready`) + + await rows.collection.cleanup() + firstPublication.publish() + secondPublication.publish() + + expect(callbackValues).toEqual([]) + expect(rows.collection.status).toBe(`cleaned-up`) + expect(rows.collection.toArray).toEqual([]) + } finally { + subscription.unsubscribe() + await rows.collection.cleanup() + } + }) + + fcTest( + `coherent nested publication advances every revision before callbacks`, + async () => { + type NodeRow = { + id: number + kind: `parent` | `child` | `grandchild` + parentGroup: number + group: number + value: number + } + const initialRows: Array = [ + { + id: 1, + kind: `parent`, + parentGroup: 0, + group: 1, + value: 1, + }, + { + id: 10, + kind: `child`, + parentGroup: 1, + group: 10, + value: 1, + }, + { + id: 20, + kind: `child`, + parentGroup: 1, + group: 20, + value: 2, + }, + { + id: 100, + kind: `grandchild`, + parentGroup: 10, + group: 100, + value: 1, + }, + { + id: 200, + kind: `grandchild`, + parentGroup: 10, + group: 200, + value: 2, + }, + ] + const nodes = createControlledCollection( + `nested-publication-revisions`, + initialRows, + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: nodes.collection }) + .where(({ parent }) => eq(parent.kind, `parent`)) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: nodes.collection }) + .where(({ child }) => eq(child.kind, `child`)) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.value) + .select(({ child }) => ({ + id: child.id, + value: child.value, + grandchildren: q + .from({ grandchild: nodes.collection }) + .where(({ grandchild }) => eq(grandchild.kind, `grandchild`)) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ) + .orderBy(({ grandchild }) => grandchild.value), + })), + })), + ) + + await live.preload() + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(10)!.grandchildren + const childStateRevision = childFacade._stateRevision + const childLayoutRevision = childFacade._layoutRevision + const grandchildStateRevision = grandchildFacade._stateRevision + const grandchildLayoutRevision = grandchildFacade._layoutRevision + const callbackSnapshots: Array<{ + childRows: Array<{ id: number; value: number }> + childStateRevision: number + childLayoutRevision: number + grandchildRows: Array<{ id: number; value: number }> + grandchildStateRevision: number + grandchildLayoutRevision: number + }> = [] + const subscription = live.subscribeChanges( + () => { + callbackSnapshots.push({ + childRows: childFacade.toArray.map(({ id, value }) => ({ + id, + value, + })), + childStateRevision: childFacade._stateRevision, + childLayoutRevision: childFacade._layoutRevision, + grandchildRows: grandchildFacade.toArray.map(({ id, value }) => ({ + id, + value, + })), + grandchildStateRevision: grandchildFacade._stateRevision, + grandchildLayoutRevision: grandchildFacade._layoutRevision, + }) + }, + { includeInitialState: false }, + ) + + try { + nodes.writeBatch([ + { type: `update`, value: { ...initialRows[0]!, value: 3 } }, + { type: `update`, value: { ...initialRows[1]!, value: 4 } }, + { type: `update`, value: { ...initialRows[2]!, value: 3 } }, + { type: `update`, value: { ...initialRows[3]!, value: 4 } }, + { type: `update`, value: { ...initialRows[4]!, value: 3 } }, + ]) + + expect(callbackSnapshots).toEqual([ + { + childRows: [ + { id: 20, value: 3 }, + { id: 10, value: 4 }, + ], + childStateRevision: childStateRevision + 1, + childLayoutRevision: childLayoutRevision + 1, + grandchildRows: [ + { id: 200, value: 3 }, + { id: 100, value: 4 }, + ], + grandchildStateRevision: grandchildStateRevision + 1, + grandchildLayoutRevision: grandchildLayoutRevision + 1, + }, + ]) + } finally { + subscription.unsubscribe() + await Promise.all([live.cleanup(), nodes.collection.cleanup()]) + } + }, + ) + + fcTest( + `window callback-created source work follows the current facade publication`, + async () => { + const parents = createControlledCollection(`reentrant-window-parents`, [ + { id: 1, rank: 1, group: 1 }, + { id: 2, rank: 2, group: 2 }, + ]) + const children = createControlledCollection(`reentrant-window-children`, [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const observations: Array<{ + eventValues: Array + visibleValue: number + revision: number + }> = [] + let childSubscription: { unsubscribe: () => void } | undefined + let preparedRevision = -1 + let reentered = false + const rootSubscription = live.subscribeChanges( + () => { + if (reentered) return + reentered = true + const facade = live.get(2)!.children + preparedRevision = facade._stateRevision + childSubscription = facade.subscribeChanges( + (batch) => { + observations.push({ + eventValues: batch.map((change) => change.value.value), + visibleValue: facade.get(20)!.value, + revision: facade._stateRevision, + }) + }, + { includeInitialState: false }, + ) + children.write(`update`, { id: 20, group: 2, value: 3 }) + }, + { includeInitialState: false }, + ) + + try { + const result = live.utils.setWindow({ offset: 0, limit: 2 }) + if (result instanceof Promise) await result + await flushPromises() + + expect(children.collection.get(20)!.value).toBe(3) + expect(live.get(2)!.children.get(20)!.value).toBe(3) + expect(observations).toEqual([ + { + eventValues: [1], + visibleValue: 1, + revision: preparedRevision, + }, + { + eventValues: [3], + visibleValue: 3, + revision: preparedRevision + 1, + }, + ]) + } finally { + rootSubscription.unsubscribe() + childSubscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + for (const turnOrigin of [`source`, `window`] as const) { + for (const callbackAction of [`source-write`, `set-window`] as const) { + fcTest( + `${turnOrigin} graph turns serialize callback ${callbackAction} work`, + async () => { + const parents = createControlledCollection( + `callback-origin-parents`, + [ + { id: 1, rank: 1, group: 1, value: 1 }, + { id: 2, rank: 2, group: 2, value: 1 }, + ], + ) + const children = createControlledCollection( + `callback-origin-children`, + [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + value: parent.value, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const rootLayouts: Array> = [] + const rootWindows: Array< + { offset: number; limit: number } | undefined + > = [] + const childBatches: Array> = [] + let childSubscription: { unsubscribe: () => void } | undefined + let childRevision = -1 + let actionResult: true | Promise | undefined + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + rootWindows.push(live.utils.getWindow()) + if (acted) return + acted = true + if (callbackAction === `source-write`) { + const parentId = turnOrigin === `window` ? 2 : 1 + const childId = parentId === 1 ? 10 : 20 + const facade = live.get(parentId)!.children + childRevision = facade._stateRevision + childSubscription = facade.subscribeChanges( + (batch) => { + childBatches.push(batch.map((change) => change.value.value)) + }, + { includeInitialState: false }, + ) + children.write(`update`, { + id: childId, + group: parentId, + value: 3, + }) + } else { + actionResult = live.utils.setWindow( + turnOrigin === `window` + ? { offset: 1, limit: 1 } + : { offset: 0, limit: 2 }, + ) + } + }, + { includeInitialState: false }, + ) + + try { + if (turnOrigin === `source`) { + parents.write(`update`, { + id: 1, + rank: 1, + group: 1, + value: 2, + }) + } else { + const result = live.utils.setWindow({ offset: 0, limit: 2 }) + if (result instanceof Promise) await result + } + if (actionResult instanceof Promise) await actionResult + await flushPromises() + + if (callbackAction === `source-write`) { + const parentId = turnOrigin === `window` ? 2 : 1 + const childId = parentId === 1 ? 10 : 20 + const facade = live.get(parentId)!.children + expect(facade.get(childId)!.value).toBe(3) + expect(childBatches.at(-1)).toEqual([3]) + expect(facade._stateRevision).toBe(childRevision + 1) + expect(rootWindows).toEqual([ + turnOrigin === `window` + ? { offset: 0, limit: 2 } + : { offset: 0, limit: 1 }, + ]) + } else if (turnOrigin === `source`) { + expect(rootLayouts).toEqual([[1], [1, 2]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 1 }, + { offset: 0, limit: 2 }, + ]) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) + } else { + expect(rootLayouts).toEqual([[1, 2], [2]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 2 }, + { offset: 1, limit: 1 }, + ]) + expect(live.toArray.map(({ id }) => id)).toEqual([2]) + expect(live.utils.getWindow()).toEqual({ offset: 1, limit: 1 }) + } + } finally { + rootSubscription.unsubscribe() + childSubscription?.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + + fcTest( + `a rejected nested window restores its parent operation's window`, + async () => { + const parents = createControlledCollection(`nested-window-parents`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const nestedFailure = new Error(`nested window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNextGraph = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (failNextGraph) { + failNextGraph = false + builder.recordSubsetError(nestedFailure) + } + }) + + const rootLayouts: Array> = [] + let nestedError: unknown + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + if (acted) return + acted = true + failNextGraph = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + }, + { includeInitialState: false }, + ) + + try { + live.utils.setWindow({ offset: 0, limit: 2 }) + + expect(nestedError).toBe(nestedFailure) + expect(rootLayouts[0]).toEqual([1, 2]) + expect(rootLayouts.at(-1)).toEqual([1, 2]) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) + } finally { + rootSubscription.unsubscribe() + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `a rejected nested window preserves its parent operation outcome`, + async () => { + const parents = createControlledCollection(`parent-window-outcome`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + offset?: number + limit?: number + }) => void + const parentOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit === 2) { + builder.trackSubsetLoadOperationPromise(parentOutcome.promise, `root`) + } + }) + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + }, + { includeInitialState: false }, + ) + + try { + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(parentReady).toBeInstanceOf(Promise) + expect(nestedError).toBe(nestedFailure) + parentOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( + [expect.objectContaining({ demand: { limit: 2 } })], + ) + } finally { + subscription.unsubscribe() + parentOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `a rejected nested window restores its parent operation for follow-up work`, + async () => { + const parents = createControlledCollection(`parent-window-follow-up`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const rollbackOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const afterCatchOutcome = createDeferred<{ + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + }>() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + limit?: number + }) => void + let parentWindowCalls = 0 + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit === 2 && ++parentWindowCalls === 2) { + builder.trackSubsetLoadOperationPromise( + rollbackOutcome.promise, + `rollback`, + ) + } + }) + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + builder.trackSubsetLoadOperationPromise( + afterCatchOutcome.promise, + `after-catch`, + ) + }, + { includeInitialState: false }, + ) + + try { + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(nestedError).toBe(nestedFailure) + expect(parentReady).toBeInstanceOf(Promise) + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect(live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + sourceId: `rollback`, + demand: { limit: 2 }, + }), + expect.objectContaining({ + sourceId: `after-catch`, + demand: { limit: 2 }, + }), + ]), + ) + } finally { + subscription.unsubscribe() + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `a rejected nested window restores a waiting parent operation`, + async () => { + const parents = createControlledCollection(`waiting-window-parent`, [ + { id: 1, rank: 1, value: 1 }, + { id: 2, rank: 2, value: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id, value: parent.value })), + ) + + await live.preload() + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + type Outcome = { + collectionId: string + demand: { limit: number } + generation: number + extent: `exhausted` + } + const initialOutcome = createDeferred() + const beforeNestedOutcome = createDeferred() + const rollbackOutcome = createDeferred() + const afterCatchOutcome = createDeferred() + const originalWindowFn = Reflect.get(builder, `windowFn`) as (options: { + limit?: number + }) => void + let parentWindowCalls = 0 + Reflect.set(builder, `windowFn`, (options: { limit?: number }) => { + originalWindowFn(options) + if (options.limit !== 2) return + parentWindowCalls++ + if (parentWindowCalls === 1) { + builder.trackSubsetLoadOperationPromise( + initialOutcome.promise, + `initial`, + ) + } else if (parentWindowCalls === 2) { + builder.trackSubsetLoadOperationPromise( + rollbackOutcome.promise, + `rollback`, + ) + } + }) + + const parentReady = live.utils.setWindow({ offset: 0, limit: 2 }) + expect(parentReady).toBeInstanceOf(Promise) + + const nestedFailure = new Error(`nested failed`) + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failNested = false + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (!failNested) return + failNested = false + builder.recordSubsetError(nestedFailure) + }) + let nestedError: unknown + let acted = false + const subscription = live.subscribeChanges( + () => { + if (acted) return + acted = true + builder.trackSubsetLoadOperationPromise( + beforeNestedOutcome.promise, + `before-nested`, + ) + failNested = true + try { + live.utils.setWindow({ offset: 1, limit: 1 }) + } catch (error) { + nestedError = error + } + builder.trackSubsetLoadOperationPromise( + afterCatchOutcome.promise, + `after-catch`, + ) + }, + { includeInitialState: false }, + ) + + try { + parents.write(`update`, { id: 1, rank: 1, value: 3 }) + expect(nestedError).toBe(nestedFailure) + expect(parentWindowCalls).toBe(2) + expect(live.toArray.map(({ id, value }) => ({ id, value }))).toEqual([ + { id: 1, value: 3 }, + { id: 2, value: 2 }, + ]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 2 }) + + let parentSettled = false + void Promise.resolve(parentReady).then(() => { + parentSettled = true + }) + initialOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await flushPromises() + expect(parentSettled).toBe(false) + + beforeNestedOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await flushPromises() + expect(parentSettled).toBe(false) + + rollbackOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await flushPromises() + expect(parentSettled).toBe(false) + + afterCatchOutcome.resolve({ + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted`, + }) + await parentReady + expect( + live.utils[LIVE_QUERY_INTERNAL].getLastWindowOutcomes().map( + ({ sourceId }) => sourceId, + ), + ).toEqual([`initial`, `before-nested`, `rollback`, `after-catch`]) + } finally { + subscription.unsubscribe() + const outcome = { + collectionId: `parents`, + demand: { limit: 2 }, + generation: 1, + extent: `exhausted` as const, + } + initialOutcome.resolve(outcome) + beforeNestedOutcome.resolve(outcome) + rollbackOutcome.resolve(outcome) + afterCatchOutcome.resolve(outcome) + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `an older failed window cannot restore over a newer nested window`, + async () => { + const parents = createControlledCollection(`stale-window-parents`, [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ id: parent.id })), + ) + + await live.preload() + const outerFailure = new Error(`outer window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const rootLayouts: Array> = [] + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + rootLayouts.push(live.toArray.map(({ id }) => id)) + if (acted) return + acted = true + live.utils.setWindow({ offset: 1, limit: 1 }) + builder.recordSubsetError(outerFailure) + }, + { includeInitialState: false }, + ) + + try { + expect(() => live.utils.setWindow({ offset: 0, limit: 2 })).toThrow( + outerFailure, + ) + + expect(rootLayouts).toEqual([[1, 2], [2]]) + expect(live.toArray.map(({ id }) => id)).toEqual([2]) + expect(live.utils.getWindow()).toEqual({ offset: 1, limit: 1 }) + } finally { + rootSubscription.unsubscribe() + await Promise.all([live.cleanup(), parents.collection.cleanup()]) + } + }, + ) + + fcTest( + `failed window restoration serializes callback-created source work`, + async () => { + const parents = createControlledCollection(`rollback-window-parents`, [ + { id: 1, rank: 1, group: 1 }, + { id: 2, rank: 2, group: 2 }, + ]) + const children = createControlledCollection(`rollback-window-children`, [ + { id: 10, group: 1, value: 1 }, + { id: 20, group: 2, value: 1 }, + ]) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .orderBy(({ parent }) => parent.rank) + .limit(1) + .select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.group, parent.group)), + })), + ) + + await live.preload() + const failure = new Error(`requested window failed`) + const builder = live.utils[LIVE_QUERY_INTERNAL].getBuilder() + const runGraph = Reflect.get(builder, `maybeRunGraphFn`) as () => void + let failRequestedWindow = true + Reflect.set(builder, `maybeRunGraphFn`, () => { + runGraph() + if (failRequestedWindow) { + failRequestedWindow = false + builder.recordSubsetError(failure) + } + }) + + const facade = live.get(1)!.children + const rootLayouts: Array> = [] + const rootWindows: Array<{ offset: number; limit: number } | undefined> = + [] + const childBatches: Array> = [] + let sawRequestedWindow = false + let acted = false + const rootSubscription = live.subscribeChanges( + () => { + const layout = live.toArray.map(({ id }) => id) + rootLayouts.push(layout) + rootWindows.push(live.utils.getWindow()) + if (layout.length === 2) sawRequestedWindow = true + if (!sawRequestedWindow || acted || layout.length !== 1) return + acted = true + children.write(`update`, { id: 10, group: 1, value: 3 }) + }, + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + childBatches.push(batch.map((change) => change.value.value)) + }, + { includeInitialState: false }, + ) + + try { + expect(() => live.utils.setWindow({ offset: 0, limit: 2 })).toThrow( + failure, + ) + + expect(rootLayouts).toEqual([[1, 2], [1]]) + expect(rootWindows).toEqual([ + { offset: 0, limit: 2 }, + { offset: 0, limit: 1 }, + ]) + expect(live.toArray.map(({ id }) => id)).toEqual([1]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) + expect(live.utils.lastSubsetError).toBe(failure) + expect(facade.get(10)!.value).toBe(3) + expect(childBatches.at(-1)).toEqual([3]) + } finally { + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + for (const sourceOperation of pendingFacadeSourceOperations) { + for (const keyRelation of pendingFacadeKeyRelations) { + if (sourceOperation === `insert` && keyRelation === `same-key`) { + continue + } + for (const shape of pendingFacadeShapes) { + fcTest( + `publishes an ${shape} ${keyRelation} source ${sourceOperation} while a facade ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection( + `pending-facade-parents`, + [{ id: 1, group: 1 }], + ) + const children = createControlledCollection( + `pending-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q + .from({ parent: parents.collection }) + .select(({ parent }) => { + const childRows = q + .from({ child: children.collection }) + .where(({ child }) => + eq(child.parentGroup, parent.group), + ) + return { + id: parent.id, + children: + shape === `ordered` + ? childRows.orderBy(({ child }) => child.value) + : childRows, + } + }), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const childRows = () => + projectPendingFacadeRows(facade.toArray, shape) + const rootRows = () => + live.toArray.map( + ({ id: parentId, children: rootFacade }) => ({ + id: parentId, + children: projectPendingFacadeRows( + rootFacade.toArray, + shape, + ), + }), + ) + const rootPublications: Array = [] + const childPublications: Array> = [] + const childCallbackSnapshots: Array<{ + facade: Array + root: ReturnType + }> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + childPublications.push(batch.map(projectChildChange)) + childCallbackSnapshots.push({ + facade: childRows(), + root: rootRows(), + }) + }, + { includeInitialState: false }, + ) + const optimisticRow = + pendingOptimisticFacadeRow(optimisticOperation) + const sourceRow = pendingSourceFacadeRow( + sourceOperation, + keyRelation, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + facade.update(optimisticRow.id, (draft) => { + draft.value = optimisticRow.value + }) + } else { + facade.delete(optimisticRow.id) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map( + pendingFacadeInitialRows.map( + (row) => [row.id, { ...row }] as const, + ), + ) + const afterOptimistic = new Map(initialRows) + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + optimisticRow, + ) + const afterSource = new Map(initialRows) + applyPendingFacadeOperation( + afterSource, + sourceOperation, + sourceRow, + ) + const whilePending = new Map(afterSource) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + optimisticRow, + ) + const expectedOptimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + optimisticRow.id, + ) + const expectedSourceChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + sourceRow.id, + ) + const expectedSettlementChange = expectedPendingFacadeChange( + whilePending, + afterSource, + optimisticRow.id, + ) + const optimisticRows = expectedPendingFacadeRows( + afterOptimistic, + shape, + initialRows, + ) + const pendingRows = expectedPendingFacadeRows( + whilePending, + shape, + afterSource, + ) + const settledRows = expectedPendingFacadeRows( + afterSource, + shape, + afterSource, + ) + const sourceLayoutChanged = + shape === `ordered` && + (optimisticRows.length !== pendingRows.length || + optimisticRows.some( + (row, index) => row.id !== pendingRows[index]?.id, + )) + const expectedSourcePublication = expectedSourceChange + ? [expectedSourceChange] + : sourceLayoutChanged + ? [] + : undefined + const expectedSourcePublications = [ + [expectedOptimisticChange], + ...(expectedSourcePublication + ? [expectedSourcePublication] + : []), + ] + const expectedSourceSnapshots = [ + { + facade: optimisticRows, + root: [{ id: 1, children: optimisticRows }], + }, + ...(expectedSourcePublication + ? [ + { + facade: pendingRows, + root: [{ id: 1, children: pendingRows }], + }, + ] + : []), + ] + const expectedSettledPublications = [ + ...expectedSourcePublications, + ...(expectedSettlementChange + ? [[expectedSettlementChange]] + : []), + ] + const expectedSettledSnapshots = [ + ...expectedSourceSnapshots, + ...(expectedSettlementChange + ? [ + { + facade: settledRows, + root: [{ id: 1, children: settledRows }], + }, + ] + : []), + ] + + try { + expect(transaction.state).toBe(`persisting`) + expect(childRows()).toEqual(optimisticRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + ]) + expect(childCallbackSnapshots).toEqual( + expectedSourceSnapshots.slice(0, 1), + ) + + children.write(sourceOperation, sourceRow) + + expect(live.get(1)!.children).toBe(facade) + expect(childRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([]) + expect(childCallbackSnapshots).toEqual( + expectedSourceSnapshots, + ) + expect(childPublications).toEqual(expectedSourcePublications) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(childRows()).toEqual(settledRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual(expectedSettledPublications) + expect(childCallbackSnapshots).toEqual( + expectedSettledSnapshots, + ) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + } + } + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `publishes a non-projected same-key order move while its facade update ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`hidden-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `hidden-order-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const ids = () => facade.toArray.map(({ id }) => id) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackIds: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectChildChange)) + callbackIds.push(ids()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const revisionBeforeSource = facade._layoutRevision + + try { + expect(ids()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + ]) + + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 10, + position: 2, + }) + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`hidden order mutation rejected`)) + await persisted + await flushPromises() + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 10 }, + previousValue: { id: 10, parentGroup: 1, value: 11 }, + }, + ], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + [20, 10], + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `publishes an independent joined order move while a facade update ${settlement}s`, + async () => { + type SortRow = { id: number; childId: number; position: number } + const parents = createControlledCollection(`joined-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`joined-order-children`, [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + const sorts = createControlledCollection( + `joined-order-sorts`, + [ + { id: 100, childId: 10, position: 0 }, + { id: 200, childId: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .join({ sort: sorts.collection }, ({ child, sort }) => + eq(child.id, sort.childId), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ sort }) => sort.position) + .select(({ child }) => child), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const ids = () => facade.toArray.map(({ id }) => id) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackIds: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectChildChange)) + callbackIds.push(ids()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const revisionBeforeSource = facade._layoutRevision + + try { + expect(ids()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + + sorts.write(`update`, { id: 100, childId: 10, position: 2 }) + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`joined order mutation rejected`)) + await persisted + await flushPromises() + + expect(ids()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + expect(publications).toEqual([ + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 11 }, + previousValue: { id: 10, parentGroup: 1, value: 10 }, + }, + ], + [], + [ + { + type: `update`, + key: 10, + value: { id: 10, parentGroup: 1, value: 10 }, + previousValue: { id: 10, parentGroup: 1, value: 11 }, + }, + ], + ]) + expect(callbackIds).toEqual([ + [10, 20], + [20, 10], + [20, 10], + ]) + expect(callbackValues).toEqual([ + [11, 20], + [20, 11], + [20, 10], + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + sorts.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `keeps a projected optimistic value visible through a same-key base reinsert that ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`reinsert-order-parents`, [ + { id: 1, group: 1 }, + ]) + const sourceRows: ReadonlyArray = [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ] + const children = createControlledCollection( + `reinsert-order-children`, + sourceRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const callbackValues: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + callbackValues.push(values()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(transaction.state).toBe(`persisting`) + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + publications.length = 0 + callbackKeys.length = 0 + callbackValues.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`delete`, sourceRows[0]!) + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(publications).toEqual([[]]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(callbackValues).toEqual([[20, 11]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 1) + + children.write(`insert`, sourceRows[0]!) + + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([11, 20]) + expect(publications).toEqual([[], []]) + expect(callbackKeys).toEqual([ + [20, 10], + [10, 20], + ]) + expect(callbackValues).toEqual([ + [20, 11], + [11, 20], + ]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 2) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`reinsert mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([10, 20]) + expect(values()).toEqual([10, 20]) + expect(publications).toEqual([ + [], + [], + [ + { + type: `update`, + key: 10, + value: 10, + previousValue: 11, + }, + ], + ]) + expect(callbackKeys).toEqual([ + [20, 10], + [10, 20], + [10, 20], + ]) + expect(callbackValues).toEqual([ + [20, 11], + [11, 20], + [10, 20], + ]) + expect(facade._layoutRevision).toBe(revisionBeforeSource + 2) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + for (const targetPosition of [`first`, `last`] as const) { + fcTest( + `publishes a base-to-optimistic-suffix move only when a ${targetPosition} row changes layout and ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`suffix-order-parents`, [ + { id: 1, group: 1 }, + ]) + const target: OrderedSourceChild = { + id: 10, + parentGroup: 1, + value: 10, + position: targetPosition === `first` ? 0 : 1, + } + const peer: OrderedSourceChild = { + id: 20, + parentGroup: 1, + value: 20, + position: targetPosition === `first` ? 1 : 0, + } + const children = createControlledCollection(`suffix-order-children`, [ + target, + peer, + ]) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + facade.update(10, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual( + targetPosition === `first` ? [10, 20] : [20, 10], + ) + expect(values()).toEqual( + targetPosition === `first` ? [11, 20] : [20, 11], + ) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`delete`, target) + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 11]) + expect(publications).toEqual(targetPosition === `first` ? [[]] : []) + expect(callbackKeys).toEqual( + targetPosition === `first` ? [[20, 10]] : [], + ) + expect(facade._layoutRevision).toBe( + revisionBeforeSource + (targetPosition === `first` ? 1 : 0), + ) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`suffix mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications.at(-1)).toEqual([ + { + type: `delete`, + key: 10, + value: 11, + }, + ]) + expect(callbackKeys.at(-1)).toEqual([20]) + expect(facade._layoutRevision).toBe( + revisionBeforeSource + (targetPosition === `first` ? 1 : 0), + ) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `does not publish a same-source order move across an optimistically deleted peer that ${settlement}s`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`hidden-peer-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `hidden-peer-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => facade.delete(10), + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + children.write(`update`, { + id: 20, + parentGroup: 1, + value: 20, + position: -1, + }) + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications).toEqual([]) + expect(callbackKeys).toEqual([]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`hidden peer mutation rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(publications).toEqual([ + [{ type: `insert`, key: 10, value: 10 }], + ]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `does not publish a joined order move across an optimistically deleted peer that ${settlement}s`, + async () => { + type SortRow = { id: number; childId: number; position: number } + const parents = createControlledCollection(`joined-hidden-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`joined-hidden-children`, [ + { id: 10, parentGroup: 1, value: 10 }, + { id: 20, parentGroup: 1, value: 20 }, + ]) + const sorts = createControlledCollection( + `joined-hidden-sorts`, + [ + { id: 100, childId: 10, position: 0 }, + { id: 200, childId: 20, position: 1 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .join({ sort: sorts.collection }, ({ child, sort }) => + eq(child.id, sort.childId), + ) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ sort }) => sort.position) + .select(({ child }) => ({ value: child.value })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const keys = () => [...facade.keys()].map(Number) + const values = () => facade.toArray.map(({ value }) => value) + const publications: Array> = [] + const callbackKeys: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => { + publications.push(batch.map(projectValueChange)) + callbackKeys.push(keys()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => facade.delete(10), + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + publications.length = 0 + callbackKeys.length = 0 + const revisionBeforeSource = facade._layoutRevision + + sorts.write(`update`, { id: 200, childId: 20, position: -1 }) + + expect(keys()).toEqual([20]) + expect(values()).toEqual([20]) + expect(publications).toEqual([]) + expect(callbackKeys).toEqual([]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`joined hidden peer rejected`)) + await persisted + await flushPromises() + + expect(keys()).toEqual([20, 10]) + expect(values()).toEqual([20, 10]) + expect(publications).toEqual([ + [{ type: `insert`, key: 10, value: 10 }], + ]) + expect(callbackKeys).toEqual([[20, 10]]) + expect(facade._layoutRevision).toBe(revisionBeforeSource) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + sorts.collection.cleanup(), + ]) + } + }, + ) + } + + fcTest( + `does not publish an order token change that preserves facade layout`, + async () => { + type OrderedSourceChild = ChildRow & { position: number } + const parents = createControlledCollection(`stable-order-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `stable-order-children`, + [ + { id: 10, parentGroup: 1, value: 10, position: 0 }, + { id: 20, parentGroup: 1, value: 20, position: 2 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .orderBy(({ child }) => child.position) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + })), + ) + + await live.preload() + const facade = live.get(1)!.children + const publications: Array> = [] + const subscription = facade.subscribeChanges( + (batch) => publications.push(batch.map(projectChildChange)), + { includeInitialState: false }, + ) + const revision = facade._layoutRevision + + try { + children.write(`update`, { + id: 10, + parentGroup: 1, + value: 10, + position: 1, + }) + + expect(facade.toArray.map(({ id }) => id)).toEqual([10, 20]) + expect(facade._layoutRevision).toBe(revision) + expect(publications).toEqual([]) } finally { - rootSubscription.unsubscribe() - childSubscription.unsubscribe() + subscription.unsubscribe() await Promise.all([ live.cleanup(), parents.collection.cleanup(), @@ -762,6 +3921,517 @@ describe(`Collection-valued includes oracle`, () => { }, ) + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + fcTest( + `retires unrelated facade rows while a facade ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection( + `retiring-facade-parents`, + [{ id: 1, group: 1 }], + ) + const children = createControlledCollection( + `retiring-facade-children`, + pendingFacadeInitialRows, + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)), + })), + ) + const persistence = createDeferred() + + await live.preload() + const facade = live.get(1)!.children + const facadeRows = () => + facade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array< + Array<{ type: `insert` | `update` | `delete`; key: number }> + > = [] + const rootCallbackFacades: Array> = [] + const childPublications: Array> = [] + const childCallbackFacades: Array> = [] + const publicationTimeline: Array<`root` | `facade`> = [] + const rootSubscription = live.subscribeChanges( + (batch) => { + publicationTimeline.push(`root`) + rootPublications.push( + batch.map(({ type, key }) => ({ type, key: Number(key) })), + ) + rootCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const childSubscription = facade.subscribeChanges( + (batch) => { + publicationTimeline.push(`facade`) + childPublications.push(batch.map(projectChildChange)) + childCallbackFacades.push(facadeRows()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + facade.update(10, (draft) => { + draft.value = 11 + }) + } else { + facade.delete(10) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map( + pendingFacadeInitialRows.map( + (row) => [row.id, { ...row }] as const, + ), + ) + const optimisticRow = pendingOptimisticFacadeRow(optimisticOperation) + const afterOptimistic = new Map(initialRows) + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + optimisticRow, + ) + const emptyBase = new Map() + const whilePending = new Map(emptyBase) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + optimisticRow, + ) + const optimisticRows = expectedPendingFacadeRows(afterOptimistic) + const pendingRows = expectedPendingFacadeRows(whilePending) + const expectedOptimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + optimisticRow.id, + )! + const expectedRetirementChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + 20, + )! + const expectedSettlementChange = expectedPendingFacadeChange( + whilePending, + emptyBase, + optimisticRow.id, + ) + + try { + expect(facadeRows()).toEqual(optimisticRows) + expect(childPublications).toEqual([[expectedOptimisticChange]]) + expect(childCallbackFacades).toEqual([optimisticRows]) + expect(publicationTimeline).toEqual([`facade`]) + publicationTimeline.length = 0 + + parents.write(`delete`, { id: 1, group: 1 }) + + expect(live.has(1)).toBe(false) + expect(facadeRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(rootCallbackFacades).toEqual([pendingRows]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedRetirementChange], + ]) + expect(childCallbackFacades).toEqual([optimisticRows, pendingRows]) + expect(publicationTimeline).toEqual([`root`, `facade`]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`facade mutation rejected`)) + await persisted + await flushPromises() + + expect(facadeRows()).toEqual([]) + expect(rootPublications).toEqual([[{ type: `delete`, key: 1 }]]) + expect(childPublications).toEqual([ + [expectedOptimisticChange], + [expectedRetirementChange], + ...(expectedSettlementChange ? [[expectedSettlementChange]] : []), + ]) + expect(childCallbackFacades.at(-1)).toEqual([]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + ]) + } + }, + ) + } + } + + for (const settlement of pendingFacadeSettlements) { + fcTest( + `publishes a nested facade source update while a same-key delete ${settlement}s`, + async () => { + const parents = createControlledCollection(`nested-facade-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection(`nested-facade-children`, [ + { id: 100, parentGroup: 1, group: 7 }, + ]) + const grandchildren = createControlledCollection( + `nested-facade-grandchildren`, + [ + { id: 10, parentGroup: 7, value: 10 }, + { id: 20, parentGroup: 7, value: 20 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + id: child.id, + group: child.group, + grandchildren: q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ), + })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(100)!.grandchildren + const grandchildRows = () => + grandchildFacade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array = [] + const childPublications: Array = [] + const grandchildPublications: Array> = [] + const callbackRows: Array> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = childFacade.subscribeChanges( + (batch) => childPublications.push(batch), + { includeInitialState: false }, + ) + const grandchildSubscription = grandchildFacade.subscribeChanges( + (batch) => { + grandchildPublications.push(batch.map(projectChildChange)) + callbackRows.push(grandchildRows()) + }, + { includeInitialState: false }, + ) + const mutate = createOptimisticAction({ + onMutate: () => grandchildFacade.delete(10), + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(grandchildRows()).toEqual([ + { id: 20, parentGroup: 7, value: 20 }, + ]) + expect(grandchildPublications).toEqual([ + [ + { + type: `delete`, + key: 10, + value: { id: 10, parentGroup: 7, value: 10 }, + }, + ], + ]) + + grandchildren.write(`update`, { + id: 10, + parentGroup: 7, + value: 21, + }) + + expect(grandchildRows()).toEqual([ + { id: 20, parentGroup: 7, value: 20 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(grandchildPublications).toHaveLength(1) + expect(callbackRows).toEqual([ + [{ id: 20, parentGroup: 7, value: 20 }], + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`nested facade mutation rejected`)) + await persisted + await flushPromises() + + expect(grandchildRows()).toEqual([ + { id: 10, parentGroup: 7, value: 21 }, + { id: 20, parentGroup: 7, value: 20 }, + ]) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([]) + expect(grandchildPublications).toEqual([ + [ + { + type: `delete`, + key: 10, + value: { id: 10, parentGroup: 7, value: 10 }, + }, + ], + [ + { + type: `insert`, + key: 10, + value: { id: 10, parentGroup: 7, value: 21 }, + }, + ], + ]) + expect(callbackRows.at(-1)).toEqual([ + { id: 10, parentGroup: 7, value: 21 }, + { id: 20, parentGroup: 7, value: 20 }, + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + grandchildSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + grandchildren.collection.cleanup(), + ]) + } + }, + ) + } + + for (const settlement of pendingFacadeSettlements) { + for (const optimisticOperation of pendingFacadeOptimisticOperations) { + fcTest( + `retires a nested facade while its ${optimisticOperation} ${settlement}s`, + async () => { + const parents = createControlledCollection(`nested-retire-parents`, [ + { id: 1, group: 1 }, + ]) + const children = createControlledCollection( + `nested-retire-children`, + [{ id: 100, parentGroup: 1, group: 7 }], + ) + const grandchildren = createControlledCollection( + `nested-retire-grandchildren`, + [ + { id: 10, parentGroup: 7, value: 10 }, + { id: 20, parentGroup: 7, value: 20 }, + ], + ) + const live = createLiveQueryCollection((q) => + q.from({ parent: parents.collection }).select(({ parent }) => ({ + id: parent.id, + children: q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, parent.group)) + .select(({ child }) => ({ + id: child.id, + group: child.group, + grandchildren: q + .from({ grandchild: grandchildren.collection }) + .where(({ grandchild }) => + eq(grandchild.parentGroup, child.group), + ), + })), + })), + ) + const persistence = createDeferred() + + await live.preload() + const childFacade = live.get(1)!.children + const grandchildFacade = childFacade.get(100)!.grandchildren + const grandchildRows = () => + grandchildFacade.toArray + .map(({ id, parentGroup, value }) => ({ id, parentGroup, value })) + .sort((left, right) => left.id - right.id) + const rootPublications: Array = [] + const childPublications: Array<{ + type: `insert` | `update` | `delete` + key: number + id: number + group: number + grandchildren: boolean + }> = [] + const childCallbackSnapshots: Array<{ + childIds: Array + grandchildRows: Array + }> = [] + const grandchildPublications: Array> = [] + const grandchildCallbackRows: Array> = [] + const rootSubscription = live.subscribeChanges( + (batch) => rootPublications.push(batch), + { includeInitialState: false }, + ) + const childSubscription = childFacade.subscribeChanges( + (batch) => { + childPublications.push( + ...batch.map(({ type, key, value }) => ({ + type, + key: Number(key), + id: value.id, + group: value.group, + grandchildren: value.grandchildren === grandchildFacade, + })), + ) + childCallbackSnapshots.push({ + childIds: childFacade.toArray.map(({ id }) => id), + grandchildRows: grandchildRows(), + }) + }, + { includeInitialState: false }, + ) + const grandchildSubscription = grandchildFacade.subscribeChanges( + (batch) => { + grandchildPublications.push(batch.map(projectChildChange)) + grandchildCallbackRows.push(grandchildRows()) + }, + { includeInitialState: false }, + ) + const optimisticRow = pendingOptimisticFacadeRow(optimisticOperation) + const mutate = createOptimisticAction({ + onMutate: () => { + if (optimisticOperation === `update`) { + grandchildFacade.update(10, (draft) => { + draft.value = optimisticRow.value + }) + } else { + grandchildFacade.delete(10) + } + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + const initialRows = new Map([ + [10, { id: 10, parentGroup: 7, value: 10 }], + [20, { id: 20, parentGroup: 7, value: 20 }], + ]) + const afterOptimistic = new Map(initialRows) + const nestedOptimisticRow = { ...optimisticRow, parentGroup: 7 } + applyPendingFacadeOperation( + afterOptimistic, + optimisticOperation, + nestedOptimisticRow, + ) + const emptyBase = new Map() + const whilePending = new Map(emptyBase) + applyPendingFacadeOperation( + whilePending, + optimisticOperation, + nestedOptimisticRow, + ) + const optimisticRows = expectedPendingFacadeRows(afterOptimistic) + const pendingRows = expectedPendingFacadeRows(whilePending) + const optimisticChange = expectedPendingFacadeChange( + initialRows, + afterOptimistic, + 10, + )! + const retirementChange = expectedPendingFacadeChange( + afterOptimistic, + whilePending, + 20, + )! + const settlementChange = expectedPendingFacadeChange( + whilePending, + emptyBase, + 10, + ) + + try { + expect(grandchildRows()).toEqual(optimisticRows) + + children.write(`delete`, { + id: 100, + parentGroup: 1, + group: 7, + }) + + expect(live.has(1)).toBe(true) + expect(childFacade.toArray).toEqual([]) + expect(grandchildRows()).toEqual(pendingRows) + expect(rootPublications).toEqual([]) + expect(childPublications).toEqual([ + { + type: `delete`, + key: 100, + id: 100, + group: 7, + grandchildren: true, + }, + ]) + expect(childCallbackSnapshots).toEqual([ + { childIds: [], grandchildRows: pendingRows }, + ]) + expect(grandchildPublications).toEqual([ + [optimisticChange], + [retirementChange], + ]) + expect(grandchildCallbackRows).toEqual([ + optimisticRows, + pendingRows, + ]) + + const persisted = transaction.isPersisted.promise.catch( + () => undefined, + ) + if (settlement === `resolve`) persistence.resolve() + else persistence.reject(new Error(`nested retirement rejected`)) + await persisted + await flushPromises() + + expect(childFacade.toArray).toEqual([]) + expect(grandchildRows()).toEqual([]) + expect(rootPublications).toEqual([]) + expect(grandchildPublications).toEqual([ + [optimisticChange], + [retirementChange], + ...(settlementChange ? [[settlementChange]] : []), + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + rootSubscription.unsubscribe() + childSubscription.unsubscribe() + grandchildSubscription.unsubscribe() + await Promise.all([ + live.cleanup(), + parents.collection.cleanup(), + children.collection.cleanup(), + grandchildren.collection.cleanup(), + ]) + } + }, + ) + } + } + fcTest( `outer fn.select recomputes nested values after a union branch include changes`, async () => { @@ -1107,7 +4777,7 @@ describe(`Collection-valued includes oracle`, () => { wideId: fc.integer({ min: 10, max: 19 }), }), ], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `includes-collection.public-key-order`), )( `uses one raw public-key order across Collection and inline materializations`, async ({ smallId, wideId }) => { @@ -1261,6 +4931,52 @@ describe(`Collection-valued includes oracle`, () => { }, ) + fcTest( + `publishes every bounded internal order-only swap through root and facade producers`, + async () => { + const observedCells: Array = [] + for (let length = 4; length <= 12; length++) { + for (let swapIndex = 1; swapIndex <= length - 3; swapIndex++) { + await expectRootAndFacadeLayoutSwap({ length, swapIndex }) + observedCells.push(`${length}:${swapIndex}`) + } + } + + expect(observedCells).toEqual( + exhaustiveLayoutSwapScenarios.map( + ({ length, swapIndex }) => `${length}:${swapIndex}`, + ), + ) + }, + ) + + fcTest.prop([layoutSwapScenarioArbitrary], { + ...oraclePropertyOptions(20, `includes-collection.layout-swap`), + })( + `publishes replayable random internal order-only swaps through root and facade producers`, + expectRootAndFacadeLayoutSwap, + ) + + fcTest( + `scans every changed facade key before deciding whether layout may differ`, + async () => { + const observedScenarios: Array = [] + for (const candidatePosition of [`first`, `last`] as const) { + for (const finalLayout of [`moved`, `restored`] as const) { + await expectFacadeCandidateScan({ candidatePosition, finalLayout }) + observedScenarios.push(`${candidatePosition}:${finalLayout}`) + } + } + + expect(observedScenarios).toEqual( + facadeCandidateScanScenarios.map( + ({ candidatePosition, finalLayout }) => + `${candidatePosition}:${finalLayout}`, + ), + ) + }, + ) + fcTest( `reconstructs nested conditional includes through guard transitions`, async () => { @@ -1770,7 +5486,7 @@ describe(`Collection-valued includes oracle`, () => { value: fc.integer({ min: -10, max: 10 }), }), ], - oraclePropertyOptions(20), + oraclePropertyOptions(20, `includes-collection.optimistic-child-history`), )( `matches recomputation through optimistic child insert and delete confirmation and rollback`, async ({ group, insertedId, confirmedId, value }) => { diff --git a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts index 850aeb02c2..7770956fa7 100644 --- a/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts +++ b/packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts @@ -497,12 +497,18 @@ describe(`includes cross-formulation oracle`, () => { }), ) - fcTest.prop([scenarioArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(8, `includes-cross-formulation.equivalence`), + )( `agrees across nested includes, flat joins, per-parent queries, and TLP partitions`, expectFormulationsEquivalent, ) - fcTest.prop([windowedScenarioArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [windowedScenarioArbitrary], + oraclePropertyOptions(12, `includes-cross-formulation.ordered-window`), + )( `matches recomputation for ordered offset and limit child windows`, ({ scenario, offset, limit }) => expectWindowedIncludeMatches(scenario, offset, limit), diff --git a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts index 0e125cdfc8..23b4bf762d 100644 --- a/packages/db/tests/query/includes-optimistic-oracle.property.test.ts +++ b/packages/db/tests/query/includes-optimistic-oracle.property.test.ts @@ -499,7 +499,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.rekey-detach`), + )( `an optimistic rekey detaches its old descendants immediately`, async (routes) => { await expectHistoryMatches(routes, [ @@ -514,7 +517,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.rekey-rollback`), + )( `restores the authoritative relationship after an optimistic rekey rolls back`, async (routes) => { await expectHistoryMatches(routes, [ @@ -528,7 +534,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.descendant-rollback`), + )( `rolls back a descendant update made while its ancestor is reparented`, async (routes) => { await expectHistoryMatches(routes, [ @@ -552,7 +561,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.ancestor-rollback`), + )( `rolls back a reparented ancestor while its descendant update remains pending`, async (routes) => { await expectHistoryMatches(routes, [ @@ -576,7 +588,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.confirm-same-route`), + )( `settles a confirmed optimistic reparent on the same authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -614,7 +629,10 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.confirm-different-route`), + )( `settles a confirmed optimistic reparent on a different authoritative route`, async (routes) => { await expectHistoryMatches(routes, [ @@ -654,100 +672,100 @@ describe(`optimistic relationship-transition oracle`, () => { }, ) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( - `restores a rekey after a sibling enters its old route`, - async (routes) => { - await expectHistoryMatches(routes, [ - { - type: `optimisticRollback`, + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.sibling-route-rollback`), + )(`restores a rekey after a sibling enters its old route`, async (routes) => { + await expectHistoryMatches(routes, [ + { + type: `optimisticRollback`, + level: 1, + id: 11, + patch: { group: routes.optimistic }, + beforeRollback: { level: 1, - id: 11, - patch: { group: routes.optimistic }, - beforeRollback: { - level: 1, - changes: [ - { - type: `insert`, - value: { - id: 12, - parentGroup: routes.rootA, - group: routes.original, - value: 120, - position: 1, - }, - }, - ], - }, - }, - { - type: `sync`, - level: 2, changes: [ { - type: `update`, + type: `insert`, value: { - id: 21, - parentGroup: routes.original, - group: routes.original + 1000, - value: 211, - position: 0, + id: 12, + parentGroup: routes.rootA, + group: routes.original, + value: 120, + position: 1, }, }, ], }, - ]) - }, - ) + }, + { + type: `sync`, + level: 2, + changes: [ + { + type: `update`, + value: { + id: 21, + parentGroup: routes.original, + group: routes.original + 1000, + value: 211, + position: 0, + }, + }, + ], + }, + ]) + }) - fcTest.prop([routeValuesArbitrary], oraclePropertyOptions(12))( - `supports repeated rollback and confirmation histories`, - async (routes) => { - await expectHistoryMatches(routes, [ - { - type: `optimistic`, - handle: `first`, - level: 1, - id: 11, - patch: { parentGroup: routes.rootB }, - }, - { type: `rollback`, handle: `first` }, - { - type: `optimistic`, - handle: `second`, - level: 1, - id: 11, - patch: { parentGroup: routes.rootB }, - }, - { - type: `confirm`, - handle: `second`, - authoritative: firstChild(routes, { - parentGroup: routes.rootB, - }), - }, - { - type: `optimisticRollback`, - level: 1, - id: 11, - patch: { group: routes.optimistic }, - }, - { - type: `sync`, - level: 2, - changes: [ - { - type: `update`, - value: { - id: 21, - parentGroup: routes.original, - group: routes.original + 1000, - value: 212, - position: 0, - }, + fcTest.prop( + [routeValuesArbitrary], + oraclePropertyOptions(12, `includes-optimistic.repeated-history`), + )(`supports repeated rollback and confirmation histories`, async (routes) => { + await expectHistoryMatches(routes, [ + { + type: `optimistic`, + handle: `first`, + level: 1, + id: 11, + patch: { parentGroup: routes.rootB }, + }, + { type: `rollback`, handle: `first` }, + { + type: `optimistic`, + handle: `second`, + level: 1, + id: 11, + patch: { parentGroup: routes.rootB }, + }, + { + type: `confirm`, + handle: `second`, + authoritative: firstChild(routes, { + parentGroup: routes.rootB, + }), + }, + { + type: `optimisticRollback`, + level: 1, + id: 11, + patch: { group: routes.optimistic }, + }, + { + type: `sync`, + level: 2, + changes: [ + { + type: `update`, + value: { + id: 21, + parentGroup: routes.original, + group: routes.original + 1000, + value: 212, + position: 0, }, - ], - }, - ]) - }, - ) + }, + ], + }, + ]) + }) }) diff --git a/packages/db/tests/query/includes-oracle.property.test.ts b/packages/db/tests/query/includes-oracle.property.test.ts index 8fb7625876..a1e4ec2018 100644 --- a/packages/db/tests/query/includes-oracle.property.test.ts +++ b/packages/db/tests/query/includes-oracle.property.test.ts @@ -281,7 +281,7 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { fc.statistics( scenarioArbitrary, classifyScenarioCoverage, - oraclePropertyOptions(1_000), + oraclePropertyOptions(1_000, `includes.scenario-statistics`), ) } @@ -4295,7 +4295,10 @@ describe(`includes recompute oracle`, () => { }) }) - fcTest.prop([scenarioArbitrary], oraclePropertyOptions(40))( + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(40, `includes.incremental-history`), + )( `matches naive recomputation after every incremental change`, expectScenarioMatches, ) @@ -4306,7 +4309,7 @@ describe(`includes recompute oracle`, () => { ({ sharedIntermediate }) => !sharedIntermediate, ), ], - oraclePropertyOptions(30), + oraclePropertyOptions(30, `includes.nested-scalar-materialization`), )( `matches recomputation for nested scalar materialization`, expectMaterializeScenarioMatches, @@ -4334,7 +4337,7 @@ describe(`includes recompute oracle`, () => { { selector: (row) => row.id, maxLength: 7 }, ), ], - oraclePropertyOptions(25), + oraclePropertyOptions(25, `includes.alpha-renaming`), )( `is unchanged by alpha-renaming, sibling declaration order, or an unrelated sibling`, async (rootRows, childRows) => { @@ -4446,7 +4449,7 @@ describe(`includes recompute oracle`, () => { fcTest.prop( [fc.integer({ min: -5, max: 5 }).filter((value) => value !== 0)], - oraclePropertyOptions(15), + oraclePropertyOptions(15, `includes.optimistic-convergence`), )( `optimistic updates converge to confirmed-only state`, async (confirmedValue) => { diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts index e78656d70e..d95fdd1c90 100644 --- a/packages/db/tests/query/includes-publication-oracle.test.ts +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -1,6 +1,9 @@ import { fc, test as fcTest } from '@fast-check/vitest' -import { describe, expect } from 'vitest' +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' import { BasicIndex } from '../../src/indexes/basic-index.js' +import { createOptimisticAction } from '../../src/optimistic-action.js' import { createLiveQueryCollection, eq, @@ -10,7 +13,10 @@ import { runTrace } from '../trace-runner.js' import { oraclePropertyOptions } from '../oracle-config.js' import { flushPromises, withExpectedRejection } from '../utils.js' import { createControlledCollection } from './includes-oracle-helpers.js' +import type { StandardSchemaV1 } from '@standard-schema/spec' +import type { Collection } from '../../src/collection/index.js' import type { TraceDriver, TraceProjection } from '../trace-runner.js' +import type { ChangeMessage, SyncConfig, UtilsRecord } from '../../src/types.js' type ParentRow = { id: number @@ -37,6 +43,49 @@ type PublishedRow = { type Q2Shape = `passThrough` | `where` | `orderBy` | `select` type Q1Shape = `direct` | `joined` +type PendingPublicationOperation = `insert` | `update` | `delete` +type PendingPublicationDepth = `direct` | `layered` +type PendingPublicationShape = `passThrough` | `orderBy` | `select` +type PendingPublicationSettlement = `succeeds` | `rejects` +type SourceConfirmationOperation = `insert` | `update` | `delete` +type SourceConfirmationInterleaving = + | `handlerEcho` + | `replacementWhilePending` + | `replacementAfterSuccess` +type SourceConfirmationSettlement = `succeeds` | `rejects` + +type PendingPublicationRow = { + id: number + value: number +} + +type PendingPublicationEvent = + | { + type: `insert` | `delete` + key: number + value: PendingPublicationRow + } + | { + type: `update` + key: number + value: PendingPublicationRow + previousValue: PendingPublicationRow + } + +type PendingPublicationSourceChange = { + operation: PendingPublicationOperation + row: PendingPublicationRow +} + +type PendingPublicationScenario = { + optimisticOperation: PendingPublicationOperation + sourceChanges: ReadonlyArray + sameKey: boolean +} + +type OffDiagonalSameKeyHistory = PendingPublicationScenario & { + name: string +} const initialParent: ParentRow = { id: 1, group: 10, value: 0 } const initialChild: ChildRow = { id: 100, parentGroup: 10, value: 1 } @@ -401,6 +450,409 @@ async function expectPublicationMatches( const q2Shapes = [`passThrough`, `where`, `orderBy`, `select`] as const const q1Shapes = [`direct`, `joined`] as const +const pendingPublicationOperations = [`insert`, `update`, `delete`] as const +const pendingPublicationDepths = [`direct`, `layered`] as const +const pendingPublicationShapes = [`passThrough`, `orderBy`, `select`] as const +const pendingPublicationSettlements = [`succeeds`, `rejects`] as const + +const optimisticExistingRow: PendingPublicationRow = { id: 1, value: 10 } +const sourceExistingRow: PendingPublicationRow = { id: 2, value: 20 } +const optimisticInsertedRow: PendingPublicationRow = { id: 3, value: 30 } +const sourceInsertedRow: PendingPublicationRow = { id: 4, value: 15 } + +const offDiagonalSameKeyHistories = [ + { + name: `source inserts then updates the optimistic insert key`, + optimisticOperation: `insert`, + sourceChanges: [ + { operation: `insert`, row: { id: 3, value: 20 } }, + { operation: `update`, row: { id: 3, value: 15 } }, + ], + sameKey: true, + }, + { + name: `source inserts then deletes the optimistic insert key`, + optimisticOperation: `insert`, + sourceChanges: [ + { operation: `insert`, row: { id: 3, value: 20 } }, + { operation: `delete`, row: { id: 3, value: 20 } }, + ], + sameKey: true, + }, + { + name: `source deletes the optimistic update key`, + optimisticOperation: `update`, + sourceChanges: [{ operation: `delete`, row: { ...optimisticExistingRow } }], + sameKey: true, + }, + { + name: `source updates the optimistic delete key`, + optimisticOperation: `delete`, + sourceChanges: [{ operation: `update`, row: { id: 1, value: 5 } }], + sameKey: true, + }, +] as const satisfies ReadonlyArray + +function pendingOperationRow( + operation: PendingPublicationOperation, + owner: `optimistic` | `source`, +): PendingPublicationRow { + if (owner === `optimistic`) { + if (operation === `insert`) return { ...optimisticInsertedRow } + if (operation === `update`) return { ...optimisticExistingRow, value: 11 } + return { ...optimisticExistingRow } + } + + if (operation === `insert`) return { ...sourceInsertedRow } + if (operation === `update`) return { ...sourceExistingRow, value: 5 } + return { ...sourceExistingRow } +} + +function applyPendingOperation( + rows: Map, + operation: PendingPublicationOperation, + row: PendingPublicationRow, +): void { + if (operation === `delete`) rows.delete(row.id) + else rows.set(row.id, { ...row }) +} + +function expectedPendingRows( + rows: ReadonlyMap, + shape: PendingPublicationShape, + orderedBase: ReadonlyMap = rows, +): Array { + if (shape !== `orderBy`) { + return [...rows.values()] + .map((row) => ({ ...row })) + .sort((left, right) => left.id - right.id) + } + + const baseKeys = [...orderedBase.values()] + .sort((left, right) => left.value - right.value || left.id - right.id) + .map((row) => row.id) + const optimisticOnlyKeys = [...rows.keys()] + .filter((key) => !orderedBase.has(key)) + .sort((left, right) => left - right) + return [...baseKeys, ...optimisticOnlyKeys] + .filter((key) => rows.has(key)) + .map((key) => ({ ...rows.get(key)! })) +} + +function expectedPendingEvent( + type: PendingPublicationOperation, + key: number, + before: ReadonlyMap, + after: ReadonlyMap, +): PendingPublicationEvent { + if (type === `insert`) { + return { type, key, value: { ...after.get(key)! } } + } + if (type === `delete`) { + return { type, key, value: { ...before.get(key)! } } + } + return { + type, + key, + value: { ...after.get(key)! }, + previousValue: { ...before.get(key)! }, + } +} + +function pendingPublicationRowsEqual( + left: PendingPublicationRow | undefined, + right: PendingPublicationRow | undefined, +): boolean { + return left?.id === right?.id && left?.value === right?.value +} + +function expectedPendingTransition( + key: number, + before: ReadonlyMap, + after: ReadonlyMap, + includeLogicalNoopUpdate = false, +): PendingPublicationEvent | undefined { + const previousValue = before.get(key) + const value = after.get(key) + if (!previousValue && !value) return undefined + if (!previousValue) return { type: `insert`, key, value: { ...value! } } + if (!value) return { type: `delete`, key, value: { ...previousValue } } + if ( + !includeLogicalNoopUpdate && + pendingPublicationRowsEqual(previousValue, value) + ) { + return undefined + } + return { + type: `update`, + key, + value: { ...value }, + previousValue: { ...previousValue }, + } +} + +function pendingPublicationEvent< + TRow extends PendingPublicationRow, + TKey extends string | number, +>(change: ChangeMessage): PendingPublicationEvent { + const value = { id: change.value.id, value: change.value.value } + if (change.type !== `update`) { + return { type: change.type, key: Number(change.key), value } + } + return { + type: `update`, + key: Number(change.key), + value, + previousValue: { + id: change.previousValue!.id, + value: change.previousValue!.value, + }, + } +} + +function createPendingPublicationQuery< + TRow extends PendingPublicationRow, + TKey extends string | number, + TUtils extends UtilsRecord, + TSchema extends StandardSchemaV1, + TInput extends object, +>( + source: Collection, + shape: PendingPublicationShape, +) { + return createLiveQueryCollection({ + id: `pending-publication-${shape}-${nextCollectionId++}`, + query: (query) => { + const rows = query.from({ + row: source as unknown as Collection< + PendingPublicationRow, + string | number + >, + }) + if (shape === `orderBy`) { + return rows.orderBy(({ row }) => row.value) + } + if (shape === `select`) { + return rows.select(({ row }) => ({ id: row.id, value: row.value })) + } + return rows + }, + getKey: (row) => row.id, + }) +} + +function observePendingPublication< + TRow extends PendingPublicationRow, + TKey extends string | number, + TUtils extends UtilsRecord, + TSchema extends StandardSchemaV1, + TInput extends object, +>( + collection: Collection, + shape: PendingPublicationShape, +) { + const batches: Array> = [] + const callbackSnapshots: Array> = [] + const currentRows = () => { + const rows = collection.toArray.map((row) => ({ + id: row.id, + value: row.value, + })) + return shape === `orderBy` + ? rows + : rows.sort((left, right) => left.id - right.id) + } + const subscription = collection.subscribeChanges( + (changes) => { + batches.push(changes.map((change) => pendingPublicationEvent(change))) + callbackSnapshots.push(currentRows()) + }, + { includeInitialState: false }, + ) + + return { batches, callbackSnapshots, currentRows, subscription } +} + +async function expectSourcePublicationDuringPendingMutation( + scenario: PendingPublicationScenario, + depth: PendingPublicationDepth, + shape: PendingPublicationShape, + settlement: PendingPublicationSettlement, +): Promise { + const { optimisticOperation, sourceChanges, sameKey } = scenario + const initialRows = [optimisticExistingRow, sourceExistingRow] + const initialState = new Map( + initialRows.map((row) => [row.id, { ...row }] as const), + ) + const source = createControlledCollection( + `pending-publication-source`, + initialRows, + ) + const q1 = createPendingPublicationQuery(source.collection, shape) + const q2 = createPendingPublicationQuery(q1, shape) + const target = depth === `direct` ? q1 : q2 + const persistence = createDeferred() + const settlementError = new Error(`pending publication rollback`) + + await target.preload() + const terminal = observePendingPublication(target, shape) + const intermediate = + depth === `layered` ? observePendingPublication(q1, shape) : undefined + + const optimisticRow = pendingOperationRow(optimisticOperation, `optimistic`) + const insertTarget = target.insert.bind(target) as unknown as ( + row: PendingPublicationRow, + ) => unknown + const mutate = createOptimisticAction({ + onMutate: (operation) => { + if (operation === `insert`) { + insertTarget(optimisticRow) + } else if (operation === `update`) { + target.update(optimisticRow.id, (draft) => { + draft.value = optimisticRow.value + }) + } else { + target.delete(optimisticRow.id) + } + }, + mutationFn: () => persistence.promise, + }) + + const transaction = mutate(optimisticOperation) + const afterOptimistic = new Map(initialState) + applyPendingOperation(afterOptimistic, optimisticOperation, optimisticRow) + const optimisticEvent = expectedPendingEvent( + optimisticOperation, + optimisticRow.id, + initialState, + afterOptimistic, + ) + + try { + expect(terminal.batches).toEqual([[optimisticEvent]]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape, initialState), + ]) + expect(terminal.currentRows()).toEqual( + expectedPendingRows(afterOptimistic, shape, initialState), + ) + if (intermediate) { + expect(intermediate.batches).toEqual([]) + expect(intermediate.callbackSnapshots).toEqual([]) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows(initialState, shape), + ) + } + + const afterSource = new Map(initialState) + let whilePending = new Map(afterOptimistic) + const intermediateSourceBatches: Array> = [] + const intermediateSourceSnapshots: Array> = [] + const terminalSourceBatches: Array> = [] + const terminalSourceSnapshots: Array> = [] + + for (const { operation, row } of sourceChanges) { + const beforeSource = new Map(afterSource) + const beforeTerminal = new Map(whilePending) + source.write(operation, row) + applyPendingOperation(afterSource, operation, row) + + intermediateSourceBatches.push([ + expectedPendingEvent(operation, row.id, beforeSource, afterSource), + ]) + intermediateSourceSnapshots.push(expectedPendingRows(afterSource, shape)) + + const nextTerminal = new Map(afterSource) + applyPendingOperation(nextTerminal, optimisticOperation, optimisticRow) + const terminalSourceEvent = expectedPendingTransition( + row.id, + beforeTerminal, + nextTerminal, + ) + if (terminalSourceEvent) { + terminalSourceBatches.push([terminalSourceEvent]) + terminalSourceSnapshots.push( + expectedPendingRows(nextTerminal, shape, afterSource), + ) + } + whilePending = nextTerminal + } + + if (intermediate) { + expect(intermediate.batches).toEqual(intermediateSourceBatches) + expect(intermediate.callbackSnapshots).toEqual( + intermediateSourceSnapshots, + ) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows(afterSource, shape), + ) + } + + expect(terminal.batches).toEqual([ + [optimisticEvent], + ...terminalSourceBatches, + ]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape, initialState), + ...terminalSourceSnapshots, + ]) + expect(terminal.currentRows()).toEqual( + expectedPendingRows(whilePending, shape, afterSource), + ) + + if (settlement === `succeeds`) { + persistence.resolve() + await transaction.isPersisted.promise + } else { + persistence.reject(settlementError) + await expect(transaction.isPersisted.promise).rejects.toBe( + settlementError, + ) + } + await flushPromises() + + const settlementEvent = expectedPendingTransition( + optimisticRow.id, + whilePending, + afterSource, + sameKey, + ) + const settlementBatches = settlementEvent ? [[settlementEvent]] : [] + expect(terminal.batches).toEqual([ + [optimisticEvent], + ...terminalSourceBatches, + ...settlementBatches, + ]) + expect(terminal.callbackSnapshots).toEqual([ + expectedPendingRows(afterOptimistic, shape, initialState), + ...terminalSourceSnapshots, + ...settlementBatches.map(() => + expectedPendingRows(afterSource, shape, afterSource), + ), + ]) + expect(terminal.currentRows()).toEqual( + expectedPendingRows(afterSource, shape, afterSource), + ) + if (intermediate) { + expect(intermediate.batches).toEqual(intermediateSourceBatches) + expect(intermediate.callbackSnapshots).toEqual( + intermediateSourceSnapshots, + ) + expect(intermediate.currentRows()).toEqual( + expectedPendingRows(afterSource, shape), + ) + } + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + intermediate?.subscription.unsubscribe() + terminal.subscription.unsubscribe() + await q2.cleanup() + await q1.cleanup() + await source.collection.cleanup() + } +} + describe(`layered-query publication oracle`, () => { const changedValueArbitrary = fc.oneof( fc.integer({ min: -100, max: -1 }), @@ -413,8 +865,14 @@ describe(`layered-query publication oracle`, () => { for (const q1Shape of q1Shapes) { for (const q2Shape of q2Shapes) { - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(12))( - `publishes #1713 updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 12, + `includes-publication.parent-scalar.${q1Shape}.${q2Shape}`, + ), + )( + `publishes scalar parent updates through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( { type: `parentScalar`, value }, @@ -427,7 +885,10 @@ describe(`layered-query publication oracle`, () => { fcTest.prop( [changedValueArbitrary, changedChildValueArbitrary], - oraclePropertyOptions(12), + oraclePropertyOptions( + 12, + `includes-publication.parent-then-child.${q1Shape}.${q2Shape}`, + ), )( `recovers a ${q1Shape} Q1 and ${q2Shape} Q2 after a child update`, async (parentValue, childValue) => { @@ -440,7 +901,13 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 8, + `includes-publication.optimistic-before-confirm.${q1Shape}.${q2Shape}`, + ), + )( `publishes optimistic state before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -452,7 +919,13 @@ describe(`layered-query publication oracle`, () => { }, ) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(8))( + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions( + 8, + `includes-publication.optimistic-after-confirm.${q1Shape}.${q2Shape}`, + ), + )( `publishes state after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, async (value) => { await expectPublicationMatches( @@ -466,19 +939,22 @@ describe(`layered-query publication oracle`, () => { } } - fcTest.prop([changedChildValueArbitrary], oraclePropertyOptions(100))( + fcTest.prop( + [changedChildValueArbitrary], + oraclePropertyOptions(100, `includes-publication.child-scalar`), + )( `publishes child-only scalar updates through both layers`, async (value) => { await expectPublicationMatches({ type: `childScalar`, value }) }, ) - fcTest.prop([fc.constantFrom(20, 30)], oraclePropertyOptions(100))( - `compares route transitions at both query layers`, - async (group) => { - await expectPublicationMatches({ type: `parentRoute`, group }) - }, - ) + fcTest.prop( + [fc.constantFrom(20, 30)], + oraclePropertyOptions(100, `includes-publication.parent-route`), + )(`compares route transitions at both query layers`, async (group) => { + await expectPublicationMatches({ type: `parentRoute`, group }) + }) fcTest.prop( [ @@ -487,18 +963,479 @@ describe(`layered-query publication oracle`, () => { value: changedValueArbitrary, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions( + 100, + `includes-publication.atomic-parent-replacement`, + ), )(`compares atomic parent replacements at both query layers`, async (row) => { await expectPublicationMatches({ type: `atomicReplace`, ...row }) }) - fcTest.prop([changedValueArbitrary], oraclePropertyOptions(100))( - `publishes restored state after optimistic rollback`, - async (value) => { - await expectPublicationMatches({ - type: `optimisticRollback`, - value, + fcTest.prop( + [changedValueArbitrary], + oraclePropertyOptions(100, `includes-publication.optimistic-rollback`), + )(`publishes restored state after optimistic rollback`, async (value) => { + await expectPublicationMatches({ + type: `optimisticRollback`, + value, + }) + }) +}) + +describe(`source publication across pending derived mutations`, () => { + for (const settlement of pendingPublicationSettlements) { + it(`keeps ordinary source sync parked while layered graph publication ${settlement}`, async () => { + let sync!: Parameters< + SyncConfig[`sync`] + >[0] + const source = createCollection({ + id: `ordinary-source-prefix-${nextCollectionId++}`, + getKey: (row) => row.id, + sync: { + sync: (methods) => { + sync = methods + methods.markReady() + }, + }, }) - }, - ) + await source.preload() + sync.begin() + sync.write({ type: `insert`, value: { ...optimisticExistingRow } }) + sync.write({ type: `insert`, value: { ...sourceExistingRow } }) + const initialReceipt = sync.commit() + if (initialReceipt !== true) await initialReceipt + + const q1 = createPendingPublicationQuery(source, `passThrough`) + const q2 = createPendingPublicationQuery(q1, `select`) + await q2.preload() + const observed = observePendingPublication(q2, `select`) + const persistence = createDeferred() + const settlementError = new Error(`ordinary source prefix rollback`) + const mutate = createOptimisticAction({ + onMutate: () => { + source.update(1, (draft) => { + draft.value = 11 + }) + }, + mutationFn: () => persistence.promise, + }) + const transaction = mutate() + + try { + expect(q2.get(1)?.value).toBe(11) + observed.batches.length = 0 + observed.callbackSnapshots.length = 0 + + sync.begin() + sync.write({ type: `update`, value: { id: 2, value: 5 } }) + const parkedReceipt = sync.commit() + expect(parkedReceipt).not.toBe(true) + if (parkedReceipt === true) { + throw new Error(`ordinary source sync did not park`) + } + let parkedReceiptSettled = false + void parkedReceipt.then(() => { + parkedReceiptSettled = true + }) + await flushPromises() + + expect(parkedReceiptSettled).toBe(false) + expect(source.get(2)?.value).toBe(20) + expect(q2.get(2)?.value).toBe(20) + expect( + observed.batches.flat().filter((event) => event.key === 2), + ).toEqual([]) + + if (settlement === `succeeds`) { + persistence.resolve() + await transaction.isPersisted.promise + } else { + persistence.reject(settlementError) + await expect(transaction.isPersisted.promise).rejects.toBe( + settlementError, + ) + } + await parkedReceipt + await flushPromises() + + expect(parkedReceiptSettled).toBe(true) + expect(source.get(2)?.value).toBe(5) + expect(q2.get(2)?.value).toBe(5) + expect( + observed.batches.flat().filter((event) => event.key === 2), + ).toEqual([ + { + type: `update`, + key: 2, + value: { id: 2, value: 5 }, + previousValue: { id: 2, value: 20 }, + }, + ]) + } finally { + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + observed.subscription.unsubscribe() + await q2.cleanup() + await q1.cleanup() + await source.cleanup() + } + }) + } + + async function expectSourceConfirmationPreservesGraphIntegrity( + operation: SourceConfirmationOperation, + depth: PendingPublicationDepth, + interleaving: SourceConfirmationInterleaving, + settlement: SourceConfirmationSettlement, + ) { + type Row = { id: number; value: number } + let sync!: Parameters[`sync`]>[0] + const handlerCanFinish = createDeferred() + let echoFromHandler = interleaving === `handlerEcho` + let handlerFailure = + settlement === `rejects` + ? new Error(`source confirmation handler rejection`) + : undefined + + const commitSync = async () => { + const receipt = sync.commit() + if (receipt !== true) await receipt + } + + const source = createCollection({ + id: `same-key-source-confirmation-${nextCollectionId++}`, + getKey: (row) => row.id, + sync: { + sync: (config) => { + sync = config + config.markReady() + }, + }, + onInsert: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } + sync.begin() + sync.write({ + type: `insert`, + value: transaction.mutations[0].modified, + }) + await commitSync() + if (handlerFailure) throw handlerFailure + }, + onUpdate: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } + sync.begin() + sync.write({ + type: `update`, + value: transaction.mutations[0].modified, + }) + await commitSync() + if (handlerFailure) throw handlerFailure + }, + onDelete: async ({ transaction }) => { + if (!echoFromHandler) { + if (interleaving === `replacementWhilePending`) { + await handlerCanFinish.promise + } + if (handlerFailure) throw handlerFailure + return + } + sync.begin() + sync.write({ + type: `delete`, + key: transaction.mutations[0].key, + }) + await commitSync() + if (handlerFailure) throw handlerFailure + }, + }) + await source.preload() + + if (operation !== `insert`) { + sync.begin() + sync.write({ type: `insert`, value: { id: 1, value: 0 } }) + await commitSync() + } + + const q1 = createLiveQueryCollection({ + id: `same-key-source-confirmation-query-${nextCollectionId++}`, + query: (q) => + q.from({ row: source }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + const q2 = + depth === `layered` + ? createLiveQueryCollection({ + id: `same-key-source-confirmation-layer-${nextCollectionId++}`, + query: (q) => + q.from({ row: q1 }).select(({ row }) => ({ + id: row.id, + value: row.value, + })), + getKey: (row) => row.id, + }) + : undefined + const query = q2 ?? q1 + const sourceEvents: Array<{ + type: string + key: string | number + value?: number + }> = [] + const queryEvents: Array<{ + type: string + key: string | number + value?: number + }> = [] + const sourceSubscription = source.subscribeChanges((changes) => { + sourceEvents.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + value: change.value.value, + })), + ) + }) + const subscription = query.subscribeChanges((changes) => { + queryEvents.push( + ...changes.map((change) => ({ + type: change.type, + key: change.key, + value: change.value.value, + })), + ) + }) + + try { + await query.preload() + + const firstTransaction = (() => { + switch (operation) { + case `insert`: + return source.insert({ id: 1, value: 1 }) + case `update`: + return source.update(1, (draft) => { + draft.value = 1 + }) + case `delete`: + return source.delete(1) + } + })() + sourceEvents.length = 0 + queryEvents.length = 0 + + if (interleaving === `replacementWhilePending`) { + sync.begin() + sync.truncate() + sync.write({ type: `insert`, value: { id: 1, value: 99 } }) + await commitSync() + + if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(false) + expect(query.get(1)?.value).toBe(1) + } + expect(sourceEvents).toEqual( + operation === `delete` + ? [] + : [ + { type: `delete`, key: 1, value: 1 }, + { type: `insert`, key: 1, value: 1 }, + ], + ) + expect(queryEvents).toEqual([]) + + handlerCanFinish.resolve() + } + if (handlerFailure) { + await expect(firstTransaction.isPersisted.promise).rejects.toBe( + handlerFailure, + ) + handlerFailure = undefined + } else { + await firstTransaction.isPersisted.promise + } + + if (interleaving === `replacementAfterSuccess`) { + sync.begin() + sync.truncate() + sync.write({ type: `insert`, value: { id: 1, value: 99 } }) + await commitSync() + } + + if (interleaving !== `handlerEcho` && settlement === `succeeds`) { + if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(false) + expect(query.get(1)?.value).toBe(1) + } + + echoFromHandler = true + await source.insert({ id: 2, value: 2 }).isPersisted.promise + + // A replacement is not confirmation, so the optimistic value survives + // it. Once persistence has succeeded, however, the next ordinary sync + // drain retires an unconfirmed direct overlay and reveals the base. + expect(source.get(1)?.value).toBe(99) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(99) + + sync.begin() + if (operation === `delete`) { + sync.write({ type: `delete`, key: 1 }) + } else { + sync.write({ type: `update`, value: { id: 1, value: 1 } }) + } + await commitSync() + } + + if ( + interleaving === `replacementWhilePending` && + settlement === `rejects` + ) { + expect(source.get(1)?.value).toBe(99) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(99) + echoFromHandler = true + } else if (operation === `delete`) { + expect(source.get(1)).toBeUndefined() + expect(query.get(1)).toBeUndefined() + } else { + expect(source.get(1)?.value).toBe(1) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(1) + } + + const probeTransaction = + operation === `delete` && + !( + interleaving === `replacementWhilePending` && settlement === `rejects` + ) + ? source.insert({ id: 1, value: 2 }) + : source.update(1, (draft) => { + draft.value = 2 + }) + await probeTransaction.isPersisted.promise + + expect(source.get(1)?.value).toBe(2) + expect(source.get(1)?.$synced).toBe(true) + expect(query.get(1)?.value).toBe(2) + } finally { + subscription.unsubscribe() + sourceSubscription.unsubscribe() + if (q2) await q2.cleanup() + await q1.cleanup() + await source.cleanup() + } + } + + for (const depth of pendingPublicationDepths) { + for (const interleaving of [ + `handlerEcho`, + `replacementWhilePending`, + `replacementAfterSuccess`, + ] as const satisfies ReadonlyArray) { + for (const settlement of [ + `succeeds`, + `rejects`, + ] as const satisfies ReadonlyArray) { + if ( + interleaving === `replacementAfterSuccess` && + settlement === `rejects` + ) { + continue + } + for (const operation of [ + `insert`, + `update`, + `delete`, + ] as const satisfies ReadonlyArray) { + it(`preserves ${depth} graph integrity after a same-key optimistic ${operation} with ${interleaving} that ${settlement}`, async () => { + await expectSourceConfirmationPreservesGraphIntegrity( + operation, + depth, + interleaving, + settlement, + ) + }) + } + } + } + } + + for (const depth of pendingPublicationDepths) { + for (const shape of pendingPublicationShapes) { + for (const settlement of pendingPublicationSettlements) { + for (const optimisticOperation of pendingPublicationOperations) { + for (const sourceOperation of pendingPublicationOperations) { + it(`publishes a disjoint source ${sourceOperation} through a ${depth} ${shape} query while an optimistic ${optimisticOperation} ${settlement}`, async () => { + await expectSourcePublicationDuringPendingMutation( + { + optimisticOperation, + sourceChanges: [ + { + operation: sourceOperation, + row: pendingOperationRow(sourceOperation, `source`), + }, + ], + sameKey: false, + }, + depth, + shape, + settlement, + ) + }) + } + + it(`retains a same-key source ${optimisticOperation} through a ${depth} ${shape} query while its optimistic mutation ${settlement}`, async () => { + await expectSourcePublicationDuringPendingMutation( + { + optimisticOperation, + sourceChanges: [ + { + operation: optimisticOperation, + row: pendingOperationRow(optimisticOperation, `optimistic`), + }, + ], + sameKey: true, + }, + depth, + shape, + settlement, + ) + }) + } + + for (const history of offDiagonalSameKeyHistories) { + it(`retains the synced base when the ${history.name} through a ${depth} ${shape} query and the optimistic mutation ${settlement}`, async () => { + await expectSourcePublicationDuringPendingMutation( + history, + depth, + shape, + settlement, + ) + }) + } + } + } + } }) diff --git a/packages/db/tests/query/includes-temporal-oracle.test.ts b/packages/db/tests/query/includes-temporal-oracle.test.ts index 649dccd49a..f7612bf984 100644 --- a/packages/db/tests/query/includes-temporal-oracle.test.ts +++ b/packages/db/tests/query/includes-temporal-oracle.test.ts @@ -839,6 +839,115 @@ async function expectFailedDemandRetriesSameCoverage(): Promise { } } +async function expectDemandReactivationRetriesAfterReleaseFailure( + keys: ReadonlyArray, +): Promise { + let loadCount = 0 + let allowUnload = false + const releaseError = new Error(`child release failed`) + const comments = createCollection({ + id: nextCollectionId(`temporal-release-retry-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + markReady() + return true + }, + unloadSubset: () => { + if (!allowUnload) throw releaseError + }, + }), + }, + }) + comments.createIndex((comment) => comment.postId) + const subscription = comments.subscribeChanges(() => {}, { + includeInitialState: false, + }) + const controller = new SubsetDemandController() + const plan: LazyDemandPlan = { + id: `release-failure-retry`, + path: [`postId`], + collectionId: comments.id, + initialKeys: new Set(), + } + + try { + expect( + controller.setDemand(subscription, plan, new Set(keys)), + ).toMatchObject({ changed: true, empty: false }) + expect(loadCount).toBe(1) + + const retired = controller.setDemand(subscription, plan, new Set()) + expect(retired).toMatchObject({ changed: true, empty: true }) + expect(retired.releaseFailure?.error).toBe(releaseError) + + const reactivated = controller.setDemand(subscription, plan, new Set(keys)) + expect(reactivated).toMatchObject({ changed: true, empty: false }) + expect(loadCount).toBe(2) + } finally { + allowUnload = true + controller.clear() + subscription.unsubscribe() + await comments.cleanup() + } +} + +async function expectRetiredDemandStaysNonfatalAfterReleaseFailure(): Promise { + const post = { id: 1, authorId: `selected`, title: `one` } + const posts = createMutablePosts([post]) + let loadCount = 0 + let allowUnload = false + const releaseError = new Error(`child release failed`) + const comments = createCollection({ + id: nextCollectionId(`temporal-retired-release-comments`), + getKey: (comment) => comment.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BasicIndex, + sync: { + sync: ({ markReady }) => ({ + loadSubset: () => { + loadCount += 1 + markReady() + return true + }, + unloadSubset: () => { + if (!allowUnload) throw releaseError + }, + }), + }, + }) + const live = createPostsWithCommentsLive(posts.collection, comments) + const consoleError = vi.spyOn(console, `error`).mockImplementation(() => {}) + + try { + await live.preload() + expect(loadCount).toBe(1) + expect(live.status).toBe(`ready`) + + posts.write(`delete`, post) + await flushPromises() + expect(live.size).toBe(0) + expect(live.status).toBe(`ready`) + expect(live.utils.lastSubsetError).toBe(releaseError) + + posts.write(`insert`, post) + await flushPromises() + expect(loadCount).toBe(2) + expect(live.status).toBe(`ready`) + } finally { + allowUnload = true + await live.cleanup() + await Promise.all([posts.collection.cleanup(), comments.cleanup()]) + consoleError.mockRestore() + } +} + async function expectSynchronousEmptyDemandIsReady(): Promise { const posts = createMutablePosts([ { id: 1, authorId: `selected`, title: `one` }, @@ -1196,7 +1305,10 @@ describe(`includes temporal oracle`, () => { expectObsoleteDemandCannotPublishAfterReactivation, ) - fcTest.prop([fc.scheduler()], oraclePropertyOptions(20))( + fcTest.prop( + [fc.scheduler()], + oraclePropertyOptions(20, `includes-temporal.demand-scheduling`), + )( `obsolete and current demand completions are generation-safe in either order`, expectScheduledDemandCompletionsStayGenerationSafe, ) @@ -1218,6 +1330,27 @@ describe(`includes temporal oracle`, () => { expectFailedDemandRetriesSameCoverage, ) + it(`reactivated demand retries after its prior release fails`, () => + expectDemandReactivationRetriesAfterReleaseFailure([1])) + + fcTest.prop( + [ + fc.uniqueArray(fc.integer({ min: -3, max: 3 }), { + minLength: 1, + maxLength: 5, + }), + ], + oraclePropertyOptions(20, `includes-temporal.release-reentry`), + )( + `failed release never suppresses a later demand incarnation`, + expectDemandReactivationRetriesAfterReleaseFailure, + ) + + it( + `failed release retires an empty live-query demand without poisoning reentry`, + expectRetiredDemandStaysNonfatalAfterReleaseFailure, + ) + it( `a synchronous empty demand can establish ready coverage`, expectSynchronousEmptyDemandIsReady, diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index c3968b0f5a..78d584f4b7 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -47,6 +47,7 @@ import { } from '../../src/query/ir.js' import { compileExpression, + compileSingleRowExpression, toBooleanPredicate, } from '../../src/query/compiler/evaluators.js' import { isLoadSubsetRequestSubsumedBy } from '../../src/query/predicate-utils.js' @@ -58,6 +59,8 @@ import { cloneLoadSubsetOptions, snapshotLoadSubsetDemand, } from '../../src/query/load-subset-options.js' +import { areValuesEqual, normalizeValue } from '../../src/utils/comparison.js' +import { createCrossRealmUint8Array } from '../utils.js' import type { BasicExpression, QueryIR } from '../../src/query/ir.js' import type { LoadSubsetOptions } from '../../src/types.js' @@ -508,20 +511,6 @@ describe(`loadSubset demand identity`, () => { const createDemands = (value: unknown): Array => [ { where: new Func(`eq`, [field, new Value(value)]) }, { where: new Func(`in`, [field, new Value([value])]) }, - { - orderBy: [ - { - expression: new Func(`coalesce`, [field, new Value(value)]), - compareOptions: { direction: `asc`, nulls: `first` }, - }, - ], - }, - { - cursor: { - whereFrom: new Func(`gt`, [field, new Value(value)]), - whereCurrent: new Func(`eq`, [field, new Value(value)]), - }, - }, ] for (const [firstValue, secondValue] of [ @@ -552,6 +541,466 @@ describe(`loadSubset demand identity`, () => { ).toThrow(/function value/) }) + it(`snapshots structural function operands without changing demand identity`, () => { + const bytes = Buffer.from([65]) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(bytes)]), + new Value(`A`), + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ( + ((snapshot.where as Func).args[0] as Func).args[0] as Value + ).value + + expect(snapshotBytes).not.toBe(bytes) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + + bytes[0] = 66 + expect(compileExpression(demand.where!)({})).toBe(false) + expect(compileExpression(snapshot.where!)({})).toBe(true) + }) + + it(`snapshots large binary equality values without changing demand identity`, () => { + const bytes = new Uint8Array(129).fill(7) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(bytes) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + + bytes.fill(8) + expect( + compileSingleRowExpression(demand.where!)({ + id: new Uint8Array(129).fill(7), + }), + ).toBe(false) + expect( + compileSingleRowExpression(snapshot.where!)({ + id: new Uint8Array(129).fill(7), + }), + ).toBe(true) + }) + + it(`copies binary equality values without calling an overridden slice`, () => { + const bytes = new Uint8Array([1, 2, 3]) + Object.defineProperty(bytes, `slice`, { + value: () => bytes, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(new Uint8Array([1, 2, 3])) + + bytes.fill(9) + expect( + compileSingleRowExpression(snapshot.where!)({ + id: new Uint8Array([1, 2, 3]), + }), + ).toBe(true) + }) + + it(`derives binary equality identity from intrinsic bytes`, () => { + const bytes = new Uint8Array([2]) + Object.defineProperty(bytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + const predicate = new Func(`eq`, [ + new PropRef([`id`]), + new Value(bytes), + ]) + + expect(getLoadSubsetDemandKey({ where: predicate })).toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [ + new PropRef([`id`]), + new Value(new Uint8Array([2])), + ]), + }), + ) + expect(getLoadSubsetDemandKey({ where: predicate })).not.toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [ + new PropRef([`id`]), + new Value(new Uint8Array([1])), + ]), + }), + ) + }) + + it(`rejects binary values without intrinsic typed-array slots`, () => { + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot binary equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot binary equality value/, + ) + }) + + it(`snapshots intrinsic Uint8Array values across realms`, () => { + const bytes = createCrossRealmUint8Array([1, 2, 3]) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`id`]), new Value(bytes)]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotBytes = ((snapshot.where as Func).args[1] as Value).value + + expect(areValuesEqual(bytes, new Uint8Array([1, 2, 3]))).toBe(true) + expect(normalizeValue(bytes)).toBe( + normalizeValue(new Uint8Array([1, 2, 3])), + ) + + bytes[0] = 9 + + expect(snapshotBytes).not.toBe(bytes) + expect(snapshotBytes).toEqual(new Uint8Array([1, 2, 3])) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + }) + + it.each([`coalesce`, `caseWhen`] as const)( + `snapshots equality candidates returned by %s`, + (wrapper) => { + const candidates = [new Uint8Array([1])] + const candidateExpression = + wrapper === `coalesce` + ? new Func(`coalesce`, [new Value(candidates)]) + : new Func(`caseWhen`, [ + new Value(true), + new Value(candidates), + new Value([]), + ]) + const demand: LoadSubsetOptions = { + where: new Func(`in`, [ + new PropRef([`token`]), + candidateExpression, + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + + candidates[0]![0] = 2 + candidates.push(new Uint8Array([3])) + + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + expect( + compileSingleRowExpression(snapshot.where!)({ + token: new Uint8Array([1]), + }), + ).toBe(true) + expect( + compileSingleRowExpression(snapshot.where!)({ + token: new Uint8Array([2]), + }), + ).toBe(false) + }, + ) + + it(`rejects membership arrays with custom observation hooks`, () => { + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, + }) + const demand: LoadSubsetOptions = { + where: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot membership candidates/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot membership candidates/, + ) + }) + + it(`rejects mutable Temporal-branded equality lookalikes`, () => { + let callerDate = `2024-01-15` + const callerValue = { + [Symbol.toStringTag]: `Temporal.PlainDate`, + toString: () => callerDate, + } + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new PropRef([`date`]), + new Value(callerValue), + ]), + } + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + callerDate = `2024-01-16` + }) + + it(`rejects constructor-shaped Temporal equality lookalikes`, () => { + class TemporalLookalike { + static shared = `2024-01-15` + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return TemporalLookalike.shared + } + } + const value = new TemporalLookalike() + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`date`]), new Value(value)]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot Temporal.PlainDate equality value/, + ) + }) + + it(`reads Date equality values through the intrinsic getTime`, () => { + const date = new Date(2) + Object.defineProperty(date, `getTime`, { + value: () => 1, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [new PropRef([`date`]), new Value(date)]), + } + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotDate = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotDate.getTime()).toBe(2) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey({ + where: new Func(`eq`, [new PropRef([`date`]), new Value(new Date(2))]), + }), + ) + }) + + it.each([ + [`Duration`, Temporal.Duration.from(`P1DT2H`)], + [`Instant`, Temporal.Instant.from(`2024-01-15T12:00:00Z`)], + [`PlainDate`, Temporal.PlainDate.from(`2024-01-15`)], + [`PlainDateTime`, Temporal.PlainDateTime.from(`2024-01-15T12:00:00`)], + [`PlainMonthDay`, Temporal.PlainMonthDay.from(`01-15`)], + [`PlainTime`, Temporal.PlainTime.from(`12:00:00`)], + [`PlainYearMonth`, Temporal.PlainYearMonth.from(`2024-01`)], + [`ZonedDateTime`, Temporal.ZonedDateTime.from(`2024-01-15T12:00:00Z[UTC]`)], + ])( + `clones genuine Temporal.%s equality values without changing type or identity`, + (_name, value) => { + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new PropRef([`value`]), + new Value(value), + ]), + } + const demandKey = getLoadSubsetDemandKey(demand) + const snapshot = cloneLoadSubsetOptions(demand) + const snapshotValue = ((snapshot.where as Func).args[1] as Value).value + + expect(snapshotValue).not.toBe(value) + expect(Object.getPrototypeOf(snapshotValue)).toBe( + Object.getPrototypeOf(value), + ) + expect(String(snapshotValue)).toBe(String(value)) + expect(getLoadSubsetDemandKey(snapshot)).toBe(demandKey) + expect(compileSingleRowExpression(snapshot.where!)({ value })).toBe(true) + }, + ) + + it.each([ + [`function`, () => () => 1], + [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => 1 })], + [ + `indexed accessor`, + () => { + const value: Array = [] + Object.defineProperty(value, `0`, { + enumerable: true, + get: () => 1, + }) + return value + }, + ], + [ + `cycle`, + () => { + const value: Array = [] + value.push(value) + return value + }, + ], + ])(`rejects %s in ordering operands`, (_name, createValue) => { + const demand: LoadSubsetOptions = { + where: new Func(`gt`, [ + new PropRef([`value`]), + new Value(createValue()), + ]), + } + + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /Cannot snapshot structural expression value/, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /Cannot snapshot structural expression value/, + ) + }) + + it.each([ + [`symbol coercion`, () => ({ [Symbol.toPrimitive]: () => `A` }), `A`], + [ + `non-enumerable coercion`, + () => { + const value = {} + Object.defineProperty(value, `toString`, { + value: () => `A`, + }) + return value + }, + `A`, + ], + [ + `opaque mutable coercion`, + () => + new (class { + value = `A`; + [Symbol.toPrimitive]() { + return this.value + } + })(), + `A`, + ], + [ + `indexed accessor coercion`, + () => { + const value: Array = [] + Object.defineProperty(value, `0`, { + enumerable: true, + get: () => `A`, + }) + return value + }, + `A`, + ], + [ + `built-in subclass coercion`, + () => + new (class extends Array { + [Symbol.toPrimitive]() { + return `A` + } + })(), + `A`, + ], + [ + `cyclic structure`, + () => { + const value: { self?: unknown } = {} + value.self = value + return value + }, + `[object Object]`, + ], + ] as const)( + `rejects unsupported %s before retaining structural demand state`, + (_label, createValue, expected) => { + const value = createValue() + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(expected), + ]), + } + + expect(compileExpression(demand.where!)({})).toBe(true) + expect(() => cloneLoadSubsetOptions(demand)).toThrow( + /snapshot structural expression value/i, + ) + expect(() => getLoadSubsetDemandKey(demand)).toThrow( + /snapshot structural expression value/i, + ) + }, + ) + + it.each([ + [`nested invalid Date`, [new Date(Number.NaN)]], + [`nested symbol`, [Symbol(`immutable`)]], + [`sparse array`, new Array(1)], + ] as const)( + `preserves structural demand identity while cloning %s`, + (_label, value) => { + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value( + compileExpression(new Func(`concat`, [new Value(value)]))({}), + ), + ]), + } + const snapshot = cloneLoadSubsetOptions(demand) + + expect(compileExpression(snapshot.where!)({})).toBe(true) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + }, + ) + + it(`preserves an enumerable __proto__ data property while cloning`, () => { + const value: Record = {} + Object.defineProperty(value, `__proto__`, { + enumerable: true, + value: null, + }) + const demand: LoadSubsetOptions = { + where: new Func(`eq`, [ + new Func(`concat`, [new Value(value)]), + new Value(`[object Object]`), + ]), + } + const snapshot = cloneLoadSubsetOptions(demand) + + expect(compileExpression(demand.where!)({})).toBe(true) + expect(compileExpression(snapshot.where!)({})).toBe(true) + expect(getLoadSubsetDemandKey(snapshot)).toBe( + getLoadSubsetDemandKey(demand), + ) + }) + it.each([ [`signed zero`, -0, 0], [`invalid Date`, new Date(Number.NaN), new Date(Number.NaN)], diff --git a/packages/db/tests/query/join-subquery.test.ts b/packages/db/tests/query/join-subquery.test.ts index 3468dedd3c..ed3261cc57 100644 --- a/packages/db/tests/query/join-subquery.test.ts +++ b/packages/db/tests/query/join-subquery.test.ts @@ -954,7 +954,68 @@ describe(`Lazy join: subquery whose join key resolves to an indexed collection`, }) }) -describe(`Lazy join without a usable index`, () => { +describe(`Lazy join index availability`, () => { + test(`uses an auto-index with omitted locale options`, async () => { + type Team = { id: string } + type Member = { id: string; teamId: string } + const teams = createCollection( + mockSyncCollectionOptions({ + id: `lazy-default-collation-teams`, + getKey: (team) => team.id, + initialData: [{ id: `t1` }], + }), + ) + const members = createCollection( + mockSyncCollectionOptions({ + id: `lazy-default-collation-members`, + getKey: (member) => member.id, + initialData: [{ id: `m1`, teamId: `t1` }], + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + defaultStringCollation: { + stringSort: `locale`, + localeOptions: { sensitivity: undefined }, + }, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `m1`, teamId: `t1` } }) + commit() + markReady() + return { loadSubset: () => true } + }, + }, + }), + ) + const warnSpy = vi.spyOn(console, `warn`).mockImplementation(() => {}) + const live = createLiveQueryCollection((q) => + q + .from({ team: teams }) + .leftJoin({ member: members }, ({ team, member }) => + eq(team.id, member.teamId), + ) + .select(({ team, member }) => ({ + id: team.id, + memberId: member.id, + })), + ) + + try { + await live.preload() + expect(live.toArray.map(stripVirtualProps)).toEqual([ + { id: `t1`, memberId: `m1` }, + ]) + expect(members.indexes.size).toBe(1) + expect(warnSpy).not.toHaveBeenCalledWith( + expect.stringContaining(`Join requires an index`), + ) + } finally { + warnSpy.mockRestore() + await Promise.all([live.cleanup(), teams.cleanup(), members.cleanup()]) + } + }) + test(`warns when demand falls back to a full local scan`, async () => { type Team = { id: string } type Member = { id: string; teamId: string } diff --git a/packages/db/tests/query/live-query-collection.test.ts b/packages/db/tests/query/live-query-collection.test.ts index fc4d7f06bd..af9c061836 100644 --- a/packages/db/tests/query/live-query-collection.test.ts +++ b/packages/db/tests/query/live-query-collection.test.ts @@ -2836,7 +2836,7 @@ describe(`createLiveQueryCollection`, () => { } }) - it(`passes single orderBy clause to loadSubset when using limit`, async () => { + it(`loads an ordered source without a range index unbounded`, async () => { const capturedOptions: Array = [] let resolveLoadSubset: () => void const loadSubsetPromise = new Promise((resolve) => { @@ -2886,7 +2886,7 @@ describe(`createLiveQueryCollection`, () => { expect(callWithOrderBy).toBeDefined() expect(callWithOrderBy?.orderBy).toHaveLength(1) expect(callWithOrderBy?.orderBy?.[0]?.expression.type).toBe(`ref`) - expect(callWithOrderBy?.limit).toBe(10) + expect(callWithOrderBy?.limit).toBeUndefined() // Resolve the loadSubset promise so preload can complete resolveLoadSubset!() @@ -2894,7 +2894,7 @@ describe(`createLiveQueryCollection`, () => { await preloadPromise }) - it(`passes multiple orderBy columns to loadSubset when using limit`, async () => { + it(`loads a multi-column ordered source without an index unbounded`, async () => { const capturedOptions: Array = [] let resolveLoadSubset: () => void const loadSubsetPromise = new Promise((resolve) => { @@ -2950,7 +2950,7 @@ describe(`createLiveQueryCollection`, () => { expect(callWithMultiOrderBy?.orderBy).toHaveLength(2) expect(callWithMultiOrderBy?.orderBy?.[0]?.expression.type).toBe(`ref`) expect(callWithMultiOrderBy?.orderBy?.[1]?.expression.type).toBe(`ref`) - expect(callWithMultiOrderBy?.limit).toBe(10) + expect(callWithMultiOrderBy?.limit).toBeUndefined() // Resolve the loadSubset promise so preload can complete resolveLoadSubset!() diff --git a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts index c1116eca77..5a79036b16 100644 --- a/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-full-flow-oracle.property.test.ts @@ -2,11 +2,15 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { expect, it, vi } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' +import { SyncTransactionAbortedError } from '../../src/errors.js' import { BTreeIndex, ReverseIndex } from '../../src/index.js' +import { localOnlyCollectionOptions } from '../../src/local-only.js' import { Func, PropRef, Value } from '../../src/query/ir.js' import { createEffect } from '../../src/query/effect.js' -import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createLiveQueryCollection, eq, gte } from '../../src/query/index.js' +import { getLoadSubsetDemandKey } from '../../src/query/ir-stable-identity.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' +import { normalizeValue } from '../../src/utils/comparison.js' import { LIVE_QUERY_INTERNAL } from '../../src/query/live/internal.js' import { computeOrderedLoadCursor } from '../../src/query/live/utils.js' import { WindowState } from '../../src/query/live/window-state.js' @@ -18,18 +22,32 @@ import { projectAuthorizedContinuationStarts, projectOrderedContinuationEvidence, projectOrderedPublicationBoundary, + projectOrderedSourceProgress, projectRetainedRowKeys, + projectRetainedSourceRows, projectReusableDemands, + projectReusableSourceDemands, projectTransportLoads, } from '../load-subset-full-flow-model.js' -import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import { + createCrossRealmUint8Array, + flushPromises, + mockSyncCollectionOptions, +} from '../utils.js' import { oracleRandomParameters, readOracleRunConfig, } from '../oracle-config.js' import type { InitialQueryBuilder } from '../../src/query/builder/index.js' -import type { LoadSubsetOptions, WritableDeep } from '../../src/types.js' -import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { + LoadSubsetOptions, + LoadSubsetResult, + WritableDeep, +} from '../../src/types.js' +import type { + LoadSubsetFullFlowEvent, + OrderedSourceStep, +} from '../load-subset-full-flow-model.js' type AdapterLifecycleEvent = | { type: `start`; options: LoadSubsetOptions } @@ -47,300 +65,4048 @@ function visibleRows( return Array.from(values, ({ id, value }) => ({ id, value })) } -type TruncateCoverageScenario = { - oldRequest: `none` | `settles-late` - freshResult: `authoritative` | `unknown` | `reject` - settlementOrder: `old-first` | `fresh-first` +it(`loads each side of a filtered inner join once`, async () => { + type Order = { + id: number + scheduledAt: string + status: string + addressId: number + } + type Charge = { id: number; addressId: number } + + const orderLoads: Array = [] + const chargeLoads: Array = [] + const orders = createCollection({ + id: `full-flow-filtered-join-orders`, + getKey: (order) => order.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { + id: 1, + scheduledAt: `2024-01-15`, + status: `queued`, + addressId: 1, + }, + }) + write({ + type: `insert`, + value: { + id: 2, + scheduledAt: `2024-01-10`, + status: `queued`, + addressId: 2, + }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + orderLoads.push(options) + return true + }, + } + }, + }, + }) + const charges = createCollection({ + id: `full-flow-filtered-join-charges`, + getKey: (charge) => charge.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: 10, addressId: 1 } }) + write({ type: `insert`, value: { id: 20, addressId: 2 } }) + commit() + markReady() + return { + loadSubset: (options) => { + chargeLoads.push(options) + return true + }, + } + }, + }, + }) + const query = createLiveQueryCollection((q) => + q + .from({ order: orders }) + .where(({ order }) => gte(order.scheduledAt, `2024-01-12`)) + .where(({ order }) => eq(order.status, `queued`)) + .innerJoin({ charge: charges }, ({ order, charge }) => + eq(order.addressId, charge.addressId), + ), + ) + + try { + await query.preload() + + expect( + [...query.values()].map(({ order, charge }) => [order.id, charge.id]), + ).toEqual([[1, 10]]) + expect(orderLoads).toHaveLength(1) + expect(chargeLoads).toHaveLength(1) + } finally { + await Promise.all([query.cleanup(), orders.cleanup(), charges.cleanup()]) + } +}) + +const { multiplier: fullFlowMultiplier, ...fullFlowReplay } = + readOracleRunConfig() + +type MultiSourceOrderedScenario = { + primaryRows: ReadonlyArray<{ + id: string + rank: number + joinKey: string + }> + secondaryRows: ReadonlyArray<{ id: string; joinKey: string }> + offset: number + limit: number + direction: `asc` | `desc` + primaryAutoIndex: `eager` | `off` + secondaryPublication: + | `preloaded` + | `preloaded-delayed-receipt` + | `after-primary-continuation` + | `after-primary-exhaustion` + secondaryPageSize: 1 | 2 + secondaryCommitOrder: `insertion` | `reverse` } -const truncateCoverageScenarioArbitrary: fc.Arbitrary = - fc.record({ - oldRequest: fc.constantFrom(`none` as const, `settles-late` as const), - freshResult: fc.constantFrom( - `authoritative` as const, - `unknown` as const, - `reject` as const, - ), - settlementOrder: fc.constantFrom( - `old-first` as const, - `fresh-first` as const, +const multiSourceJoinKeyArbitrary = fc.constantFrom(`x`, `y`, `z`) +const secondaryJoinKeyOrders = [ + [`x`, `y`, `z`], + [`x`, `z`, `y`], + [`y`, `x`, `z`], + [`y`, `z`, `x`], + [`z`, `x`, `y`], + [`z`, `y`, `x`], +] as const +const multiSourceOrderedScenarioArbitrary: fc.Arbitrary = + fc + .record({ + ranks: fc.tuple( + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + ), + joinKeys: fc.tuple( + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + multiSourceJoinKeyArbitrary, + ), + secondaryMatchCounts: fc.tuple( + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 2 }), + ), + secondaryJoinKeyOrder: fc.constantFrom(...secondaryJoinKeyOrders), + reverseSecondaryMatches: fc.boolean(), + offset: fc.integer({ min: 0, max: 2 }), + limit: fc.integer({ min: 0, max: 2 }), + direction: fc.constantFrom(`asc` as const, `desc` as const), + primaryAutoIndex: fc.constantFrom(`eager` as const, `off` as const), + secondaryPublication: fc.constantFrom( + `preloaded` as const, + `preloaded-delayed-receipt` as const, + `after-primary-continuation` as const, + `after-primary-exhaustion` as const, + ), + secondaryPageSize: fc.constantFrom(1 as const, 2 as const), + secondaryCommitOrder: fc.constantFrom( + `insertion` as const, + `reverse` as const, + ), + }) + .map( + ({ + ranks, + joinKeys, + secondaryMatchCounts, + secondaryJoinKeyOrder, + reverseSecondaryMatches, + ...scenario + }) => ({ + ...scenario, + primaryRows: [`a`, `b`, `c`, `d`].map((id, index) => ({ + id, + rank: ranks[index]!, + joinKey: joinKeys[index]!, + })), + secondaryRows: secondaryJoinKeyOrder.flatMap((joinKey) => { + const count = secondaryMatchCounts[[`x`, `y`, `z`].indexOf(joinKey)]! + const rows = Array.from({ length: count }, (_, matchIndex) => ({ + id: `${joinKey}-${matchIndex}`, + joinKey, + })) + return reverseSecondaryMatches ? rows.reverse() : rows + }), + }), + ) + +if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + fc.statistics( + multiSourceOrderedScenarioArbitrary, + ({ + primaryRows, + secondaryRows, + offset, + limit, + direction, + primaryAutoIndex, + secondaryPublication, + secondaryPageSize, + secondaryCommitOrder, + }) => [ + `direction=${direction}`, + `primary-auto-index=${primaryAutoIndex}`, + `offset=${offset}`, + `limit=${limit}`, + `secondary=${secondaryPublication}`, + `secondary-page-size=${secondaryPageSize}`, + `secondary-commit-order=${secondaryCommitOrder}`, + `secondary-insertion-order=${secondaryRows + .map(({ id }) => id) + .join(`,`)}`, + `exhaustion=${ + primaryRows.reduce( + (count, { joinKey }) => + count + + secondaryRows.filter((row) => row.joinKey === joinKey).length, + 0, + ) < + offset + limit + }`, + `leading-exclusion=${!secondaryRows.some( + ({ joinKey }) => + joinKey === + orderedPrimaryRows({ + primaryRows, + secondaryRows, + offset, + limit, + direction, + primaryAutoIndex, + secondaryPublication, + secondaryPageSize, + secondaryCommitOrder, + })[0]!.joinKey, + )}`, + `multiplicity=${new Set(secondaryRows.map(({ joinKey }) => joinKey)).size < secondaryRows.length}`, + `tied=${new Set(primaryRows.map(({ rank }) => rank)).size < primaryRows.length}`, + ], + oracleRandomParameters( + 1_000, + fullFlowReplay, + `load-subset-full-flow.multi-source-statistics`, ), + ) +} + +function orderedPrimaryRows( + scenario: MultiSourceOrderedScenario, +): Array { + return [...scenario.primaryRows].sort((left, right) => { + const rankOrder = + scenario.direction === `asc` + ? left.rank - right.rank + : right.rank - left.rank + return rankOrder || left.id.localeCompare(right.id) }) +} -const exhaustiveTruncateCoverageScenarios: Array = [ - `none` as const, - `settles-late` as const, -].flatMap((oldRequest) => - ([`authoritative`, `unknown`, `reject`] as const).flatMap((freshResult) => - ([`old-first`, `fresh-first`] as const).map((settlementOrder) => ({ - oldRequest, - freshResult, - settlementOrder, - })), - ), -) +let multiSourceOrderedControlId = 0 -const { multiplier: fullFlowMultiplier, replaySeed: fullFlowReplaySeed } = - readOracleRunConfig() +async function observeOrderedSourceSteps( + scenario: MultiSourceOrderedScenario, +): Promise> { + const controlId = multiSourceOrderedControlId++ + const primary = createCollection( + localOnlyCollectionOptions({ + id: `multi-source-control-primary-${controlId}`, + getKey: (row) => row.id, + initialData: [...scenario.primaryRows], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + const secondary = createCollection( + localOnlyCollectionOptions({ + id: `multi-source-control-secondary-${controlId}`, + getKey: (row) => row.id, + initialData: [...scenario.secondaryRows], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + const result = createLiveQueryCollection({ + id: `multi-source-control-result-${controlId}`, + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank, scenario.direction), + startSync: true, + }) -let truncateCoverageHarnessId = 0 + try { + await result.preload() + const resultKeysBySource = new Map>() + for (const { primaryRow, secondaryRow } of result.toArray) { + const keys = resultKeysBySource.get(primaryRow.id) ?? [] + keys.push(`${primaryRow.id}:${secondaryRow.id}`) + resultKeysBySource.set(primaryRow.id, keys) + } + return orderedPrimaryRows(scenario).map((row) => ({ + sourceKey: row.id, + resultKeys: resultKeysBySource.get(row.id) ?? [], + demandKeys: [row.joinKey], + })) + } finally { + await Promise.all([ + result.cleanup(), + primary.cleanup(), + secondary.cleanup(), + ]) + } +} -async function runTruncateCoverageScenario( - scenario: TruncateCoverageScenario, +function hasPreloadedSecondary(scenario: MultiSourceOrderedScenario): boolean { + return ( + scenario.secondaryPublication === `preloaded` || + scenario.secondaryPublication === `preloaded-delayed-receipt` + ) +} + +function collectStringLiterals( + expression: Func | PropRef | Value, +): Array { + if (expression instanceof Func) { + return expression.args.flatMap((argument) => + collectStringLiterals(argument), + ) + } + if (!(expression instanceof Value)) return [] + if (typeof expression.value === `string`) return [expression.value] + if (!Array.isArray(expression.value)) return [] + return expression.value.filter( + (value): value is string => typeof value === `string`, + ) +} + +let multiSourceOrderedHarnessId = 0 + +async function expectMultiSourceStepToSettle( + scenario: MultiSourceOrderedScenario, + step: string, + result: T, +): Promise> { + let timeout: ReturnType | undefined + try { + return await Promise.race([ + Promise.resolve(result), + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error(`${step} did not settle for ${JSON.stringify(scenario)}`), + ) + }, 5_000) + }), + ]) + } finally { + if (timeout !== undefined) clearTimeout(timeout) + } +} + +async function runMultiSourceOrderedScenario( + scenario: MultiSourceOrderedScenario, ): Promise { - type Row = { id: string; value: number } - type AdapterResult = { - hasMore: boolean | undefined + type PrimaryRow = MultiSourceOrderedScenario[`primaryRows`][number] + type SecondaryRow = { id: string; joinKey: string } + + const primaryOrder = orderedPrimaryRows(scenario) + const sourceSteps = await expectMultiSourceStepToSettle( + scenario, + `control projection`, + observeOrderedSourceSteps(scenario), + ) + expect( + sourceSteps.map(({ sourceKey, demandKeys }) => ({ sourceKey, demandKeys })), + ).toEqual( + primaryOrder.map(({ id, joinKey }) => ({ + sourceKey: id, + demandKeys: [joinKey], + })), + ) + const projection = projectOrderedSourceProgress({ + sourceSteps, + offset: scenario.offset, + limit: scenario.limit, + }) + const primaryCalls: Array = [] + const primaryCallProgress: Array<{ + demandKey: string + establishedPrimaryCount: number + establishedSecondaryCount: number + }> = [] + const primaryReceipts: Array<{ + demandKey: string + expectedRowKeys: ReadonlyArray + appliedRowKeys: ReadonlyArray + }> = [] + const primaryOrderedVisitedKeys: Array = [] + const secondaryCalls: Array = [] + const secondaryReceipts: Array<{ + demandKey: string + expectedRowKeys: ReadonlyArray appliedRowKeys: ReadonlyArray + }> = [] + const secondaryLoadCommitSizes: Array = [] + const delayedSecondaryReceiptWaiters: Array<{ + index: number + gate: ReturnType> + }> = [] + const delayedSecondaryReceiptCompletionOrder: Array = [] + let releaseDelayedSecondaryReceipts = false + const secondaryPublicationGate = createDeferred() + const establishedPrimaryKeys = new Set() + const committedPrimaryKeys = new Set() + const establishedSecondaryKeys = new Set() + let primaryOrderedCallCount = 0 + let primaryOrderedCallCountAtSecondaryRelease: number | undefined + let primaryKeysAtSecondaryRelease: ReadonlyArray | undefined + let primaryCommittedKeysAtSecondaryRelease: ReadonlyArray | undefined + let primaryKeysBeforeSecondaryPublication: ReadonlyArray | undefined + let primaryBegin!: () => void + let primaryWrite!: (message: { type: `insert`; value: PrimaryRow }) => void + let primaryCommit!: (signal?: AbortSignal) => true | Promise + + const applyPrimaryRows = async ( + rows: ReadonlyArray, + signal: AbortSignal | undefined, + ): Promise> => { + if (rows.length === 0) return [] + primaryBegin() + for (const row of rows) { + establishedPrimaryKeys.add(row.id) + primaryWrite({ type: `insert`, value: row }) + } + const applied = primaryCommit(signal) + if (applied !== true) await applied + for (const row of rows) committedPrimaryKeys.add(row.id) + return rows.map(({ id }) => id) } - let begin!: () => void - let write!: (message: { type: `insert`; value: Row }) => void - let commit!: () => true | Promise - let truncate!: () => void - const pending = new Map< - LoadSubsetOptions, - ReturnType> - >() - const unloadSubset = vi.fn() - const source = createCollection({ - id: `truncate-coverage-oracle-${truncateCoverageHarnessId++}`, + + const releaseSecondaryPublication = (): void => { + primaryOrderedCallCountAtSecondaryRelease ??= primaryOrderedCallCount + primaryKeysAtSecondaryRelease ??= [...new Set(primaryOrderedVisitedKeys)] + primaryCommittedKeysAtSecondaryRelease ??= [...committedPrimaryKeys] + secondaryPublicationGate.resolve() + } + if (scenario.limit === 0) secondaryPublicationGate.resolve() + + const recordPrimaryCall = (options: LoadSubsetOptions): void => { + primaryCalls.push(options) + // Four source rows, one initial window, and one positive refinement cannot + // require an unbounded number of physical acquisitions. Keep a generous + // ceiling so a microtask refill loop becomes a shrinkable oracle failure. + if (primaryCalls.length > 32) { + throw new Error( + `primary loadSubset exceeded the bounded source grammar at call ${primaryCalls.length}: ${JSON.stringify( + { limit: options.limit, cursor: options.cursor }, + )}`, + ) + } + primaryCallProgress.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + establishedPrimaryCount: establishedPrimaryKeys.size, + establishedSecondaryCount: establishedSecondaryKeys.size, + }) + } + + const primary = createCollection({ + id: `multi-source-ordered-primary-${multiSourceOrderedHarnessId}`, getKey: (row) => row.id, syncMode: `on-demand`, startSync: true, + autoIndex: scenario.primaryAutoIndex, + defaultIndexType: BTreeIndex, sync: { sync: (params) => { - begin = params.begin - write = params.write - commit = params.commit - truncate = params.truncate + primaryBegin = params.begin + primaryWrite = params.write + primaryCommit = params.commit params.markReady() return { - loadSubset: (options) => { - const request = createDeferred() - pending.set(options, request) - return request.promise + loadSubset: async (options) => { + recordPrimaryCall(options) + if (!options.orderBy) { + const rows = primaryOrder.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + const appliedRowKeys = await applyPrimaryRows( + rows, + options.signal, + ) + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: rows.map(({ id }) => id), + appliedRowKeys, + }) + return { + hasMore: false, + appliedRowKeys, + } + } + + primaryOrderedCallCount++ + if (options.limit === undefined) { + primaryOrderedVisitedKeys.push( + ...primaryOrder.map(({ id }) => id), + ) + const appliedRowKeys = await applyPrimaryRows( + primaryOrder, + options.signal, + ) + if ( + scenario.secondaryPublication === + `after-primary-continuation` || + scenario.secondaryPublication === `after-primary-exhaustion` + ) { + releaseSecondaryPublication() + } + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: primaryOrder.map(({ id }) => id), + appliedRowKeys, + }) + return { + hasMore: false, + appliedRowKeys, + } + } + const lastKey = options.cursor?.lastKey + const previousIndex = + lastKey === undefined + ? -1 + : primaryOrder.findIndex(({ id }) => id === lastKey) + if (lastKey !== undefined && previousIndex < 0) { + throw new Error(`Unknown primary cursor ${String(lastKey)}`) + } + const row = primaryOrder[previousIndex + 1] + let appliedRowKeys: Array = [] + if (row) { + primaryOrderedVisitedKeys.push(row.id) + appliedRowKeys = await applyPrimaryRows([row], options.signal) + } + const hasMore = previousIndex + 1 < primaryOrder.length - 1 + if ( + scenario.secondaryPublication === `after-primary-continuation` && + primaryOrderedCallCount >= 2 + ) { + releaseSecondaryPublication() + } + if ( + scenario.secondaryPublication === `after-primary-exhaustion` && + !hasMore + ) { + releaseSecondaryPublication() + } + primaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: row ? [row.id] : [], + appliedRowKeys, + }) + return { + hasMore, + appliedRowKeys, + } }, - unloadSubset, + unloadSubset: () => {}, } }, }, }) - const initialOptions = { limit: 1 } - const oldOptions = { limit: 2 } - const freshOptions = { limit: 3 } - const histories: Array = [] - const activeOptions: Array = [] - - const request = (ownerId: string, options: LoadSubsetOptions) => { - histories.push({ - type: `requestDemand`, - ownerId, - sessionId: `session`, - demandId: `prefix-${options.limit}`, - alreadyAborted: false, - }) - activeOptions.push(options) - const result = source._sync.loadSubset(options) - if (result === true) throw new Error(`Expected a controlled async request`) - return result - } - const apply = async ( - ownerId: string, - options: LoadSubsetOptions, - rows: ReadonlyArray, - hasMore: boolean | undefined, - ) => { - begin() - for (const row of rows) write({ type: `insert`, value: row }) - const applied = commit() + let secondaryBegin!: () => void + let secondaryWrite!: (message: { + type: `insert` + value: SecondaryRow + }) => void + let secondaryCommit!: (signal?: AbortSignal) => true | Promise + const secondaryRows = scenario.secondaryRows + const applySecondaryRows = async ( + rows: ReadonlyArray, + signal: AbortSignal | undefined, + ): Promise> => { + if (rows.length === 0) return [] + secondaryLoadCommitSizes.push(rows.length) + secondaryBegin() + for (const row of rows) { + establishedSecondaryKeys.add(row.id) + secondaryWrite({ type: `insert`, value: row }) + } + const applied = secondaryCommit(signal) if (applied !== true) await applied - pending.get(options)!.resolve({ - hasMore, - appliedRowKeys: rows.map(({ id }) => id), - }) - histories.push({ - type: - hasMore === undefined ? `applyUnprovenRows` : `applyAuthoritativeRows`, - ownerId, - demandId: `prefix-${options.limit}`, - rowKeys: rows.map(({ id }) => id), - }) - } - - const reject = (ownerId: string, options: LoadSubsetOptions) => { - pending.get(options)!.reject(new Error(`fresh replay failed`)) - histories.push({ - type: `rejectDemand`, - ownerId, - demandId: `prefix-${options.limit}`, - }) + return rows.map(({ id }) => id) } + const secondary = createCollection({ + id: `multi-source-ordered-secondary-${multiSourceOrderedHarnessId}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: hasPreloadedSecondary(scenario), + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + secondaryBegin = params.begin + secondaryWrite = params.write + secondaryCommit = params.commit + if (hasPreloadedSecondary(scenario) && secondaryRows.length > 0) { + secondaryBegin() + for (const row of secondaryRows) { + establishedSecondaryKeys.add(row.id) + secondaryWrite({ type: `insert`, value: row }) + } + const applied = secondaryCommit() + if (applied !== true) { + throw new Error(`Expected synchronous initial secondary rows`) + } + } + params.markReady() + return { + loadSubset: async (options) => { + secondaryCalls.push(options) + if (secondaryCalls.length > 32) { + throw new Error( + `secondary loadSubset exceeded the bounded source grammar`, + ) + } + if (!hasPreloadedSecondary(scenario)) { + await secondaryPublicationGate.promise + primaryKeysBeforeSecondaryPublication ??= [ + ...new Set(primaryOrderedVisitedKeys), + ] + } + if ( + scenario.secondaryPublication === `preloaded-delayed-receipt` && + !releaseDelayedSecondaryReceipts + ) { + const waiter = { + index: delayedSecondaryReceiptWaiters.length, + gate: createDeferred(), + } + delayedSecondaryReceiptWaiters.push(waiter) + await waiter.gate.promise + delayedSecondaryReceiptCompletionOrder.push(waiter.index) + } + const matchingRows = secondaryRows.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + const rowsInCommitOrder = + scenario.secondaryCommitOrder === `reverse` + ? [...matchingRows].reverse() + : matchingRows + const appliedRowKeys: Array = [] + for ( + let index = 0; + index < rowsInCommitOrder.length; + index += scenario.secondaryPageSize + ) { + appliedRowKeys.push( + ...(await applySecondaryRows( + rowsInCommitOrder.slice( + index, + index + scenario.secondaryPageSize, + ), + options.signal, + )), + ) + } + secondaryReceipts.push({ + demandKey: getLoadSubsetDemandKey(options) ?? `unfiltered`, + expectedRowKeys: rowsInCommitOrder.map(({ id }) => id), + appliedRowKeys, + }) + return { + hasMore: false, + appliedRowKeys, + } + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `multi-source-ordered-live-${multiSourceOrderedHarnessId++}`, + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank, scenario.direction) + .offset(scenario.offset) + .limit(scenario.limit), + startSync: true, + }) - const expectModel = () => { - const actualReusable = activeOptions - .filter( - (options) => source._sync.getLoadSubsetOutcome(options) !== undefined, - ) - .map((options) => `prefix-${options.limit}`) - .sort() - expect(actualReusable).toEqual(projectReusableDemands(histories)) - expect(Array.from(source.keys()).sort()).toEqual( - projectRetainedRowKeys(histories), + try { + const preload = live.preload() + let preloadSettled = false + void preload.then( + () => { + preloadSettled = true + }, + () => { + preloadSettled = true + }, ) - } + if (scenario.secondaryPublication === `preloaded-delayed-receipt`) { + await flushPromises() + if (scenario.secondaryRows.length > 0 && scenario.limit > 0) { + expect(delayedSecondaryReceiptWaiters.length).toBeGreaterThan(0) + expect(preloadSettled).toBe(false) + expect(live.isReady()).toBe(false) + } + releaseDelayedSecondaryReceipts = true + for (const waiter of [...delayedSecondaryReceiptWaiters].reverse()) { + waiter.gate.resolve() + await flushPromises() + } + } + await expectMultiSourceStepToSettle(scenario, `preload`, preload) + await flushPromises() + expect(preloadSettled).toBe(true) - try { - const initialLoad = request(`initial`, initialOptions) - await apply(`initial`, initialOptions, [{ id: `initial`, value: 1 }], false) - await initialLoad - expectModel() + expect( + live.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual(projection.visibleResultKeys) - const oldLoad = - scenario.oldRequest === `settles-late` - ? request(`old`, oldOptions) - : undefined + const initialPrimaryCallCount = primaryCalls.length + if (scenario.limit === 0) { + expect( + primaryCalls + .slice(0, initialPrimaryCallCount) + .filter(({ orderBy }) => orderBy !== undefined), + ).toEqual([]) + } - begin() - truncate() - const truncated = commit() - if (truncated !== true) await truncated - histories.push({ type: `truncateSource`, sessionId: `session` }) - expectModel() + const refinedOffset = scenario.offset === 0 ? 1 : 0 + const refinedLimit = scenario.limit === 0 ? 1 : scenario.limit + 1 + const refinedProjection = projectOrderedSourceProgress({ + sourceSteps, + offset: refinedOffset, + limit: refinedLimit, + }) + await expectMultiSourceStepToSettle( + scenario, + `positive window refinement`, + live.utils.setWindow({ + offset: refinedOffset, + limit: refinedLimit, + }), + ) + await flushPromises() + expect( + live.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual(refinedProjection.visibleResultKeys) + if (scenario.limit === 0) { + const refinementCalls = primaryCalls + .slice(initialPrimaryCallCount) + .filter(({ orderBy }) => orderBy !== undefined) + expect(refinementCalls.length).toBeGreaterThan(0) + if (scenario.primaryAutoIndex === `off`) { + expect(refinementCalls).toHaveLength(1) + expect(refinementCalls[0]?.limit).toBeUndefined() + } + } - const freshLoad = request(`fresh`, freshOptions) - const settleOld = async () => { - if (!oldLoad) return - await apply(`old`, oldOptions, [{ id: `old`, value: 2 }], false) - await oldLoad - expectModel() + const primaryCallsBeforeZeroShrink = primaryCalls.length + await expectMultiSourceStepToSettle( + scenario, + `zero window refinement`, + live.utils.setWindow({ offset: 2, limit: 0 }), + ) + await flushPromises() + expect(live.toArray).toEqual([]) + expect( + primaryCalls + .slice(primaryCallsBeforeZeroShrink) + .filter(({ orderBy }) => orderBy !== undefined), + ).toEqual([]) + + if (scenario.limit > 0) { + expect(primaryCalls.some(({ orderBy }) => orderBy !== undefined)).toBe( + true, + ) + expect(secondaryCalls.length).toBeGreaterThan(0) } - const settleFresh = async () => { - if (scenario.freshResult === `reject`) { - reject(`fresh`, freshOptions) - await expect(freshLoad).rejects.toThrow(`fresh replay failed`) - } else { - await apply( - `fresh`, - freshOptions, - [{ id: `fresh`, value: 3 }], - scenario.freshResult === `authoritative` ? false : undefined, - ) - await freshLoad + + expect(primaryCallProgress).toHaveLength(primaryCalls.length) + const previousProgressByDemand = new Map< + string, + (typeof primaryCallProgress)[number] + >() + for (const progress of primaryCallProgress) { + const previous = previousProgressByDemand.get(progress.demandKey) + if (previous) { + expect( + progress.establishedPrimaryCount > previous.establishedPrimaryCount || + progress.establishedSecondaryCount > + previous.establishedSecondaryCount, + ).toBe(true) } - expectModel() + previousProgressByDemand.set(progress.demandKey, progress) + } + expect(primaryReceipts).toHaveLength(primaryCalls.length) + for (const receipt of primaryReceipts) { + expect(new Set(receipt.appliedRowKeys).size).toBe( + receipt.appliedRowKeys.length, + ) + expect([...receipt.appliedRowKeys].sort(), receipt.demandKey).toEqual( + [...receipt.expectedRowKeys].sort(), + ) } - if (scenario.settlementOrder === `fresh-first`) { - await settleFresh() - await settleOld() - } else { - await settleOld() - await settleFresh() + expect(secondaryReceipts).toHaveLength(secondaryCalls.length) + for (const receipt of secondaryReceipts) { + expect(new Set(receipt.appliedRowKeys).size).toBe( + receipt.appliedRowKeys.length, + ) + expect([...receipt.appliedRowKeys].sort(), receipt.demandKey).toEqual( + [...receipt.expectedRowKeys].sort(), + ) } + expect( + secondaryLoadCommitSizes.every( + (commitSize) => commitSize <= scenario.secondaryPageSize, + ), + ).toBe(true) - for (const options of activeOptions) { - source._sync.unloadSubset(options) - histories.push({ - type: `releaseDemand`, - ownerId: - options === initialOptions - ? `initial` - : options === oldOptions - ? `old` - : `fresh`, - demandId: `prefix-${options.limit}`, - rowKeys: - options === initialOptions - ? [`initial`] - : options === oldOptions - ? [`old`] - : scenario.freshResult === `reject` - ? [] - : [`fresh`], - finalRowOwner: true, - invalidatesAdapterEvidence: true, - }) + const primaryJoinKeys = new Set( + scenario.primaryRows.map(({ joinKey }) => joinKey), + ) + const joinCalls = secondaryCalls.filter(({ where }) => where !== undefined) + if ( + hasPreloadedSecondary(scenario) && + scenario.secondaryRows.length > 0 && + scenario.limit > 0 + ) { + expect(joinCalls.length).toBeGreaterThan(0) } - expect(unloadSubset.mock.calls.map(([options]) => options)).toEqual( - activeOptions, + if (scenario.secondaryPublication === `preloaded-delayed-receipt`) { + expect(delayedSecondaryReceiptCompletionOrder).toEqual( + delayedSecondaryReceiptWaiters.map(({ index }) => index).reverse(), + ) + } + const requestedJoinKeys = new Set( + joinCalls.flatMap(({ where }) => + [...primaryJoinKeys].filter((joinKey) => + evaluateReferenceExpression(where!, { + id: `probe-${joinKey}`, + joinKey, + }), + ), + ), ) - expectModel() - } finally { - for (const pendingRequest of pending.values()) { - pendingRequest.reject(new Error(`test cleanup`)) + const literalJoinKeys = joinCalls.flatMap(({ where }) => + collectStringLiterals(where!), + ) + expect( + literalJoinKeys.every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) + expect( + [...requestedJoinKeys].every((joinKey) => primaryJoinKeys.has(joinKey)), + ).toBe(true) + const requiredJoinKeys = new Set([ + ...projection.demandedKeys, + ...refinedProjection.demandedKeys, + ]) + if (joinCalls.length > 0) { + for (const joinKey of requiredJoinKeys) { + expect(requestedJoinKeys.has(joinKey)).toBe(true) + } } - await source.cleanup() + + if (scenario.secondaryPublication === `after-primary-continuation`) { + if (scenario.limit > 0) { + if (scenario.primaryAutoIndex === `eager`) { + expect(primaryOrderedCallCountAtSecondaryRelease).toBe(2) + expect(primaryCommittedKeysAtSecondaryRelease).toEqual( + expect.arrayContaining(primaryOrderedVisitedKeys.slice(0, 2)), + ) + expect(primaryKeysBeforeSecondaryPublication?.length).toBe(2) + } else { + expect(primaryOrderedCallCountAtSecondaryRelease).toBe(1) + expect(primaryCommittedKeysAtSecondaryRelease).toEqual( + expect.arrayContaining(primaryOrder.map(({ id }) => id)), + ) + expect(primaryKeysBeforeSecondaryPublication).toEqual( + primaryOrder.map(({ id }) => id), + ) + } + } + } + if (scenario.secondaryPublication === `after-primary-exhaustion`) { + if (scenario.limit > 0) { + expect(primaryKeysAtSecondaryRelease).toEqual( + primaryOrder.map(({ id }) => id), + ) + expect(primaryKeysBeforeSecondaryPublication).toEqual( + primaryOrder.map(({ id }) => id), + ) + } + } + } finally { + secondaryPublicationGate.resolve() + for (const waiter of delayedSecondaryReceiptWaiters) waiter.gate.resolve() + await expectMultiSourceStepToSettle( + scenario, + `cleanup`, + Promise.all([live.cleanup(), primary.cleanup(), secondary.cleanup()]), + ) } } -it(`does not release physical work when an already-aborted demand skips adapter start`, async () => { - const ownerId = `aborted-owner` - const requestEvent: LoadSubsetFullFlowEvent = { - type: `requestDemand`, - ownerId, - sessionId: `session-1`, - demandId: `all-rows`, - alreadyAborted: true, - } - const history: ReadonlyArray = [ - requestEvent, - { - type: `releaseDemand`, - ownerId, - demandId: `all-rows`, - rowKeys: [], - finalRowOwner: false, - invalidatesAdapterEvidence: false, +const orderedPrimaryFixture = [ + { id: `a`, rank: 1, joinKey: `a` }, + { id: `b`, rank: 2, joinKey: `b` }, + { id: `c`, rank: 3, joinKey: `c` }, + { id: `d`, rank: 4, joinKey: `d` }, +] + +it.each([ + { + name: `preloaded rejection continuation`, + secondaryRows: [ + { id: `c-0`, joinKey: `c` }, + { id: `d-0`, joinKey: `d` }, + ], + offset: 0, + limit: 2, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `late secondary after continuation`, + secondaryRows: [ + { id: `b-0`, joinKey: `b` }, + { id: `a-0`, joinKey: `a` }, + ], + offset: 0, + limit: 2, + secondaryPublication: `after-primary-continuation` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `reverse` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `delayed filtered secondary receipt`, + secondaryRows: [ + { id: `c-0`, joinKey: `c` }, + { id: `d-0`, joinKey: `d` }, + ], + offset: 0, + limit: 2, + secondaryPublication: `preloaded-delayed-receipt` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `reverse` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `late secondary after primary exhaustion`, + secondaryRows: [{ id: `d-0`, joinKey: `d` }], + offset: 0, + limit: 2, + secondaryPublication: `after-primary-exhaustion` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `joined multiplicity before offset`, + secondaryRows: [ + { id: `a-1`, joinKey: `a` }, + { id: `a-0`, joinKey: `a` }, + ], + offset: 1, + limit: 1, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 2 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `indexed zero-limit window`, + secondaryRows: [{ id: `a-0`, joinKey: `a` }], + offset: 2, + limit: 0, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `eager` as const, + }, + { + name: `unindexed zero-limit window`, + secondaryRows: [{ id: `a-0`, joinKey: `a` }], + offset: 2, + limit: 0, + secondaryPublication: `preloaded` as const, + secondaryPageSize: 1 as const, + secondaryCommitOrder: `insertion` as const, + primaryAutoIndex: `off` as const, + }, +] satisfies ReadonlyArray< + Pick< + MultiSourceOrderedScenario, + | `secondaryRows` + | `offset` + | `limit` + | `secondaryPublication` + | `secondaryPageSize` + | `secondaryCommitOrder` + | `primaryAutoIndex` + > & { name: string } +>)(`$name`, async ({ name: _name, ...scenario }) => { + await runMultiSourceOrderedScenario({ + ...scenario, + primaryRows: orderedPrimaryFixture, + direction: `asc`, + }) +}) + +it(`settles a late secondary load after tied primary continuations`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 1, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-continuation`, + secondaryPageSize: 1, + secondaryCommitOrder: `reverse`, + primaryRows: [ + { id: `a`, rank: 2, joinKey: `x` }, + { id: `b`, rank: 0, joinKey: `z` }, + { id: `c`, rank: 0, joinKey: `y` }, + { id: `d`, rank: 2, joinKey: `y` }, + ], + secondaryRows: [ + { id: `x-0`, joinKey: `x` }, + { id: `z-0`, joinKey: `z` }, + ], + }) +}) + +it(`settles an empty join after exhausting tied primary rows`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 1, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-exhaustion`, + secondaryPageSize: 1, + secondaryCommitOrder: `insertion`, + primaryRows: [ + { id: `a`, rank: 0, joinKey: `x` }, + { id: `b`, rank: 0, joinKey: `x` }, + { id: `c`, rank: 0, joinKey: `x` }, + { id: `d`, rank: 0, joinKey: `x` }, + ], + secondaryRows: [], + }) +}) + +it(`does not start duplicate ordered work from an applying receipt`, async () => { + await runMultiSourceOrderedScenario({ + offset: 0, + limit: 2, + direction: `asc`, + primaryAutoIndex: `eager`, + secondaryPublication: `after-primary-continuation`, + secondaryPageSize: 1, + secondaryCommitOrder: `insertion`, + primaryRows: [ + { id: `a`, rank: 0, joinKey: `y` }, + { id: `b`, rank: 1, joinKey: `x` }, + { id: `c`, rank: 1, joinKey: `y` }, + { id: `d`, rank: 0, joinKey: `z` }, + ], + secondaryRows: [ + { id: `z-0`, joinKey: `z` }, + { id: `z-1`, joinKey: `z` }, + { id: `y-0`, joinKey: `y` }, + ], + }) +}) + +it(`preserves a synchronous unindexed load error after reentrant cleanup`, async () => { + type Row = { id: string; rank: number } + const failure = new Error(`unindexed load failed after cleanup`) + let cleanupLive: () => Promise = () => Promise.resolve() + const source = createCollection({ + id: `unindexed-reentrant-cleanup-error`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + void cleanupLive() + throw failure + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-reentrant-cleanup-error-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + cleanupLive = () => live.cleanup() + + try { + await live.preload() + + let thrown: unknown + try { + live.utils.setWindow({ offset: 0, limit: 1 }) + } catch (error) { + thrown = error + } + expect(thrown).toBe(failure) + } finally { + await Promise.all([live.cleanup(), source.cleanup()]) + } +}) + +it.each([ + { + name: `indexed sync throw without cleanup`, + autoIndex: `eager` as const, + failureMode: `sync throw` as const, + reentrantCleanup: false, + }, + { + name: `indexed async reject without cleanup`, + autoIndex: `eager` as const, + failureMode: `async reject` as const, + reentrantCleanup: false, + }, + { + name: `unindexed sync throw without cleanup`, + autoIndex: `off` as const, + failureMode: `sync throw` as const, + reentrantCleanup: false, + }, + { + name: `unindexed async reject without cleanup`, + autoIndex: `off` as const, + failureMode: `async reject` as const, + reentrantCleanup: false, + }, + { + name: `indexed sync throw with cleanup`, + autoIndex: `eager` as const, + failureMode: `sync throw` as const, + reentrantCleanup: true, + }, + { + name: `indexed async reject with cleanup`, + autoIndex: `eager` as const, + failureMode: `async reject` as const, + reentrantCleanup: true, + }, + { + name: `unindexed sync throw with cleanup`, + autoIndex: `off` as const, + failureMode: `sync throw` as const, + reentrantCleanup: true, + }, + { + name: `unindexed async reject with cleanup`, + autoIndex: `off` as const, + failureMode: `async reject` as const, + reentrantCleanup: true, + }, +])( + `preserves refinement failure and retry state for $name`, + async ({ autoIndex, failureMode, reentrantCleanup }) => { + type Row = { id: string; rank: number } + const failure = new Error(`fallback failed`) + let attempts = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let cleanupLive: () => Promise = () => Promise.resolve() + let staleFailure: ReturnType> | undefined + const signals: Array = [] + let unloads = 0 + const source = createCollection({ + id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: ({ signal }) => { + attempts++ + signals.push(signal) + if (attempts === 1) { + if (reentrantCleanup) void cleanupLive() + if (failureMode === `sync throw`) throw failure + if (reentrantCleanup) { + staleFailure = createDeferred() + return staleFailure.promise + } + return Promise.reject(failure) + } + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + const outcome = { hasMore: false, appliedRowKeys: [`a`] } + return applied === true + ? Promise.resolve(outcome) + : applied.then(() => outcome) + }, + unloadSubset: () => { + unloads++ + }, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `zero-refinement-${autoIndex}-${failureMode}-${reentrantCleanup}-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + cleanupLive = () => live.cleanup() + + try { + await live.preload() + expect(attempts).toBe(0) + + if (failureMode === `sync throw`) { + let thrown: unknown + try { + live.utils.setWindow({ offset: 0, limit: 1 }) + } catch (error) { + thrown = error + } + expect(thrown).toBe(failure) + } else if (reentrantCleanup) { + expect(live.utils.setWindow({ offset: 0, limit: 1 })).toBe(true) + } else { + await expect( + live.utils.setWindow({ offset: 0, limit: 1 }), + ).rejects.toBe(failure) + } + await flushPromises() + + if (reentrantCleanup) { + expect(attempts).toBe(1) + expect(live.status).toBe(`cleaned-up`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray).toEqual([]) + expect(signals).toHaveLength(1) + expect(signals[0]).toBeInstanceOf(AbortSignal) + expect(signals[0]?.aborted).toBe(true) + expect(unloads).toBe(1) + expect(live.utils.getWindow()).toEqual({ + offset: 0, + limit: failureMode === `sync throw` ? 0 : 1, + }) + + await live.preload() + if (failureMode === `sync throw`) { + expect(attempts).toBe(1) + await live.utils.setWindow({ offset: 0, limit: 1 }) + } + await flushPromises() + + expect(attempts).toBe(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) + + staleFailure?.reject(failure) + await flushPromises() + + expect(attempts).toBe(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 1 }) + return + } + + expect(attempts).toBe(1) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBe(failure) + expect(live.toArray).toEqual([]) + expect(live.utils.getWindow()).toEqual({ + offset: 0, + limit: failureMode === `sync throw` ? 0 : 1, + }) + + if (failureMode === `sync throw`) { + begin() + write({ type: `insert`, value: { id: `b`, rank: 2 } }) + await commit() + await flushPromises() + expect(live.toArray).toEqual([]) + expect(live.utils.getWindow()).toEqual({ offset: 0, limit: 0 }) + } + + await live.utils.setWindow({ offset: 0, limit: 1 }) + await flushPromises() + + expect(attempts).toBe(2) + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + staleFailure?.reject(failure) + await Promise.all([live.cleanup(), source.cleanup()]) + } + }, +) + +it(`publishes once after a loader fills an indexed window across graph turns`, async () => { + type Row = { id: string; rank: number } + type ObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: Row + } + const remoteRows: ReadonlyArray = [ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ] + const batches: Array> = [] + const callbackReads: Array> = [] + let loads = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `indexed-loader-quiescent-publication`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + const row = remoteRows[loads++] + if (!row) return true + begin() + write({ type: `insert`, value: row }) + const applied = commit() + if (applied !== true) { + throw new Error(`Expected synchronous source application`) + } + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `indexed-loader-quiescent-publication-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + const readRows = () => live.toArray.map(({ id, rank }) => ({ id, rank })) + let subscription: ReturnType | undefined + + try { + await live.preload() + subscription = live.subscribeChanges( + (changes) => { + batches.push( + changes + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), + ) + callbackReads.push(readRows()) + }, + { includeInitialState: false }, + ) + await live.utils.setWindow({ offset: 0, limit: 2 }) + await flushPromises() + + expect(loads).toBe(2) + expect(readRows()).toEqual([ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ]) + expect(batches).toEqual([ + [ + { type: `insert`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(callbackReads).toEqual([ + [ + { id: `a`, rank: 1 }, + { id: `b`, rank: 2 }, + ], + ]) + } finally { + subscription?.unsubscribe() + await Promise.all([live.cleanup(), source.cleanup()]) + } +}) + +it(`fences an unindexed fallback settlement from a cleaned query session`, async () => { + type Row = { id: string; rank: number } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `unindexed-fallback-session-fence`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-fallback-session-fence-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(0), + startSync: true, + }) + let failedWindow: true | Promise | undefined + let firstWindow: true | Promise | undefined + let secondWindow: true | Promise | undefined + let repeatedWindow: true | Promise | undefined + + try { + await live.preload() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + + const visibleFailure = new Error(`visible fallback failed`) + failedWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + expect(pending).toHaveLength(1) + expect(live.isLoadingSubset).toBe(true) + pending[0]!.reject(visibleFailure) + await expect(Promise.resolve(failedWindow)).rejects.toBe(visibleFailure) + await flushPromises() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.hasSubsetError).toBe(true) + expect(live.utils.lastSubsetError).toBe(visibleFailure) + + firstWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + void Promise.resolve(firstWindow).catch(() => {}) + expect(pending).toHaveLength(2) + expect(live.isLoadingSubset).toBe(true) + + await live.cleanup() + await live.preload() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.hasSubsetError).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + secondWindow = live.utils.setWindow({ offset: 0, limit: 1 }) + expect(pending).toHaveLength(3) + expect(live.isLoadingSubset).toBe(true) + + pending[1]!.reject(new Error(`stale fallback failed`)) + await flushPromises() + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(true) + repeatedWindow = live.utils.setWindow({ offset: 0, limit: 2 }) + void Promise.resolve(repeatedWindow).catch(() => {}) + expect(pending).toHaveLength(3) + + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + if (applied !== true) await applied + pending[2]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await Promise.all([secondWindow, repeatedWindow]) + await flushPromises() + + expect(pending).toHaveLength(3) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await Promise.all([ + Promise.resolve(failedWindow).catch(() => undefined), + Promise.resolve(firstWindow).catch(() => undefined), + Promise.resolve(secondWindow).catch(() => undefined), + Promise.resolve(repeatedWindow).catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) + } +}) + +it(`keeps an initial unindexed load scoped to its query session`, async () => { + type Row = { id: string; rank: number } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `unindexed-initial-session-fence`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-initial-session-fence-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + let firstPreload: Promise | undefined + let secondPreload: Promise | undefined + + try { + firstPreload = live.preload() + void firstPreload.catch(() => {}) + expect(pending).toHaveLength(1) + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + + await live.cleanup() + secondPreload = live.preload() + expect(pending).toHaveLength(2) + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() + + pending[0]!.reject(new Error(`stale initial fallback failed`)) + await flushPromises() + expect(live.status).toBe(`loading`) + expect(live.isLoadingSubset).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() + + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const applied = commit() + if (applied !== true) await applied + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await secondPreload + await flushPromises() + + expect(pending).toHaveLength(2) + expect(live.status).toBe(`ready`) + expect(live.isLoadingSubset).toBe(false) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(live.toArray.map(({ id }) => id)).toEqual([`a`]) + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await Promise.all([ + firstPreload?.catch(() => undefined), + secondPreload?.catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) + } +}) + +it(`replays one unindexed fallback and publishes one replacement after truncate`, async () => { + type Row = { id: string; rank: number } + type ObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: Row + } + type Result = { + hasMore: boolean + appliedRowKeys: ReadonlyArray + } + const pending: Array>> = [] + const batches: Array> = [] + const callbackReads: Array> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `unindexed-fallback-truncate-replay`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `unindexed-fallback-truncate-replay-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + const readRows = () => live.toArray.map(({ id, rank }) => ({ id, rank })) + const subscription = live.subscribeChanges( + (changes) => { + batches.push( + changes + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), + ) + callbackReads.push(readRows()) + }, + { includeInitialState: false }, + ) + const preload = live.preload() + + try { + expect(pending).toHaveLength(1) + begin() + write({ type: `insert`, value: { id: `a`, rank: 1 } }) + const initialApplied = commit() + if (initialApplied !== true) await initialApplied + pending[0]!.resolve({ hasMore: false, appliedRowKeys: [`a`] }) + await preload + await flushPromises() + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + + batches.length = 0 + callbackReads.length = 0 + begin() + truncate() + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + const replacement = commit() + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + await flushPromises() + expect(pending).toHaveLength(2) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + + begin() + write({ type: `insert`, value: { id: `b`, rank: 2 } }) + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + const replacementApplied = commit() + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + if (replacementApplied !== true) await replacementApplied + expect(readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(batches).toHaveLength(0) + expect(callbackReads).toHaveLength(0) + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [`b`] }) + if (replacement !== true) await replacement + await flushPromises() + + expect(pending).toHaveLength(2) + expect(readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + subscription.unsubscribe() + await Promise.all([ + preload.catch(() => undefined), + live.cleanup(), + source.cleanup(), + ]) + } +}) + +type UnindexedReplayRow = { id: string; rank: number } +type UnindexedReplayResult = { + hasMore: boolean + appliedRowKeys: ReadonlyArray +} +type UnindexedReplayObservedChange = { + type: `insert` | `update` | `delete` + key: string + value: UnindexedReplayRow +} + +function createUnindexedReplayHarness(id: string) { + const pending: Array<{ + options: LoadSubsetOptions + request?: ReturnType> + }> = [] + const loadResults: Array> = [] + const unloads: Array = [] + const synchronousLoads = new Map>() + const batches: Array> = [] + const callbackReads: Array> = [] + let begin!: () => void + let write!: (message: { type: `insert`; value: UnindexedReplayRow }) => void + let commit!: (signal?: AbortSignal) => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `${id}-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `off`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: (options) => { + const loadIndex = pending.length + const synchronousRows = synchronousLoads.get(loadIndex) + if (synchronousRows) { + pending.push({ options }) + begin() + for (const row of synchronousRows) { + write({ type: `insert`, value: row }) + } + commit() + const result = true as const + loadResults.push(result) + return result + } + const request = createDeferred() + pending.push({ options, request }) + const result = request.promise + loadResults.push(result) + return result + }, + unloadSubset: (options) => { + unloads.push(options) + }, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `${id}-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(1), + startSync: true, + }) + const readRows = () => + live.toArray.map(({ id: rowId, rank }) => ({ id: rowId, rank })) + let observer: ReturnType | undefined + const startObserving = () => { + observer = live.subscribeChanges( + (changes) => { + batches.push( + changes + .map(({ type, key, value }) => ({ + type, + key: String(key), + value: { id: value.id, rank: value.rank }, + })) + .sort((left, right) => left.key.localeCompare(right.key)), + ) + callbackReads.push(readRows()) + }, + { includeInitialState: false }, + ) + } + const stopObserving = () => { + observer?.unsubscribe() + observer = undefined + } + const clearObservations = () => { + batches.length = 0 + callbackReads.length = 0 + } + const applyRows = async (rows: ReadonlyArray) => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + const receipt = commit() + if (receipt !== true) await receipt + } + const applyRowsForRequest = ( + requestIndex: number, + rows: ReadonlyArray, + ): Promise => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + return Promise.resolve(commit(pending[requestIndex]!.options.signal)) + } + const startTruncate = () => { + begin() + truncate() + return commit() + } + const cleanup = async () => { + for (const { request } of pending) { + request?.reject(new Error(`test cleanup`)) + } + stopObserving() + await Promise.all([live.cleanup(), source.cleanup()]) + } + + startObserving() + return { + source, + live, + pending, + loadResults, + unloads, + synchronousLoads, + batches, + callbackReads, + readRows, + startObserving, + stopObserving, + clearObservations, + applyRows, + applyRowsForRequest, + startTruncate, + cleanup, + } +} + +function expectUnindexedFullSnapshotRequest(options: LoadSubsetOptions): void { + expect(Object.keys(options).sort()).toEqual([ + `cursor`, + `limit`, + `orderBy`, + `signal`, + `subscription`, + `where`, + ]) + expect(options.where).toBeUndefined() + expect(options.limit).toBeUndefined() + expect(options.offset).toBeUndefined() + expect(options.cursor).toBeUndefined() + expect(options.orderBy).toHaveLength(1) + const ordering = options.orderBy![0]! + expect(Object.keys(ordering).sort()).toEqual([`compareOptions`, `expression`]) + expect(Object.keys(ordering.expression).sort()).toEqual([`path`, `type`]) + expect(ordering.expression).toEqual({ type: `ref`, path: [`rank`] }) + expect(ordering.compareOptions).toStrictEqual({ + direction: `asc`, + nulls: `first`, + stringSort: `locale`, + }) + expect(options.signal).toBeInstanceOf(AbortSignal) + expect(options.subscription).toBeDefined() +} + +function acquisitionIndices( + acquisitions: ReadonlyArray<{ options: LoadSubsetOptions }>, + releases: ReadonlyArray, +): ReadonlyArray { + return releases.map((options) => + acquisitions.findIndex((acquisition) => acquisition.options === options), + ) +} + +it.each([`async`, `sync`] as const)( + `retries one unindexed fallback after a rejected truncate replay with %s success`, + async (successMode) => { + const harness = createUnindexedReplayHarness( + `unindexed-rejected-truncate-retry-${successMode}`, + ) + const preload = harness.live.preload() + let cleaned = false + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await preload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + const replayFailure = new Error(`truncate replay failed`) + const failedReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + harness.pending[1]!.request!.reject(replayFailure) + await Promise.resolve(failedReplacement).catch(() => undefined) + await flushPromises() + + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + if (successMode === `sync`) { + harness.synchronousLoads.set(2, [{ id: `b`, rank: 2 }]) + } + const successfulReplacement = harness.startTruncate() + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.isLoadingSubset).toBe(successMode === `async`) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + if (successMode === `async`) { + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + await harness.applyRows([{ id: `b`, rank: 2 }]) + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`b`], + }) + } else { + expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + } + if (successfulReplacement !== true) await successfulReplacement + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([{ id: `b`, rank: 2 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `b`, value: { id: `b`, rank: 2 } }, + ], + ]) + expect(harness.callbackReads).toEqual([[{ id: `b`, rank: 2 }]]) + + for (const { options } of harness.pending) { + expectUnindexedFullSnapshotRequest(options) + } + expect( + harness.loadResults.map((result) => + result === true ? `sync` : `async`, + ), + ).toEqual([`async`, `async`, successMode === `sync` ? `sync` : `async`]) + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect( + new Set(harness.pending.map(({ options }) => options.subscription)), + ).toHaveLength(1) + expect(harness.unloads).toHaveLength(2) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, + ]) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBe(replayFailure) + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(harness.unloads).toHaveLength(3) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, 2, + ]) + } finally { + await Promise.all([ + preload.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) + +it.each([`resolve`, `reject`] as const)( + `fences a %s settlement from a truncate replay cleaned before completion`, + async (lateSettlement) => { + const harness = createUnindexedReplayHarness( + `unindexed-pending-replay-cleanup-${lateSettlement}`, + ) + const firstPreload = harness.live.preload() + let restartPreload: Promise | undefined + let replacement: true | Promise | undefined + let cleaned = false + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await firstPreload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + replacement = harness.startTruncate() + void Promise.resolve(replacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(2) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const firstSessionSubscription = harness.pending[0]!.options.subscription + harness.stopObserving() + await harness.live.cleanup() + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + expect(harness.unloads).toHaveLength(2) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, + ]) + + restartPreload = harness.live.preload() + harness.startObserving() + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`loading`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + + const staleError = new Error(`stale replay failed`) + await expect( + harness.applyRowsForRequest(1, [{ id: `stale`, rank: -1 }]), + ).rejects.toBeInstanceOf(SyncTransactionAbortedError) + if (lateSettlement === `resolve`) { + harness.pending[1]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`stale`], + }) + } else { + harness.pending[1]!.request!.reject(staleError) + } + await Promise.resolve(replacement).catch(() => undefined) + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`loading`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + await harness.applyRows([{ id: `c`, rank: 3 }]) + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [{ type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }], + ]) + expect(harness.callbackReads).toEqual([[{ id: `c`, rank: 3 }]]) + const appliedStateRevision = harness.live._stateRevision + const appliedLayoutRevision = harness.live._layoutRevision + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`c`], + }) + await restartPreload + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [{ type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }], + [], + ]) + expect(harness.callbackReads).toEqual([ + [{ id: `c`, rank: 3 }], + [{ id: `c`, rank: 3 }], + ]) + expect(harness.live._stateRevision).toBe(appliedStateRevision) + expect(harness.live._layoutRevision).toBe(appliedLayoutRevision) + + for (const { options } of harness.pending) { + expectUnindexedFullSnapshotRequest(options) + } + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect(harness.pending[1]!.options.subscription).toBe( + firstSessionSubscription, + ) + expect(harness.pending[2]!.options.subscription).not.toBe( + firstSessionSubscription, + ) + const loadResults = await Promise.allSettled( + harness.loadResults.map((result) => Promise.resolve(result)), + ) + expect(loadResults[0]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`a`] }, + }) + if (lateSettlement === `resolve`) { + expect(loadResults[1]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`stale`] }, + }) + } else { + expect(loadResults[1]!.status).toBe(`rejected`) + if (loadResults[1]!.status === `rejected`) { + expect(loadResults[1]!.reason).toBe(staleError) + } + } + expect(loadResults[2]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`c`] }, + }) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(harness.unloads).toHaveLength(3) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual([ + 1, 0, 2, + ]) + } finally { + await Promise.all([ + firstPreload.catch(() => undefined), + restartPreload?.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) + +it.each( + ([`resolve`, `reject`] as const).flatMap((supersededSettlement) => + ([`superseded-first`, `current-first`] as const).map((settlementOrder) => ({ + supersededSettlement, + settlementOrder, + })), + ), +)( + `publishes only the current replay when an overlapping replay settles $settlementOrder with $supersededSettlement`, + async ({ supersededSettlement, settlementOrder }) => { + const harness = createUnindexedReplayHarness( + `unindexed-overlapping-replays-${supersededSettlement}-${settlementOrder}`, + ) + const preload = harness.live.preload() + let firstReplacement: true | Promise | undefined + let currentReplacement: true | Promise | undefined + let cleaned = false + + try { + expect(harness.pending).toHaveLength(1) + await harness.applyRows([{ id: `a`, rank: 1 }]) + harness.pending[0]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`a`], + }) + await preload + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + + harness.clearObservations() + firstReplacement = harness.startTruncate() + void Promise.resolve(firstReplacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(2) + + currentReplacement = harness.startTruncate() + void Promise.resolve(currentReplacement).catch(() => {}) + await flushPromises() + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const signals = harness.pending.map(({ options }) => options.signal!) + expect(new Set(signals)).toHaveLength(3) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, false]) + expect( + new Set(harness.pending.map(({ options }) => options.subscription)), + ).toHaveLength(1) + expect(harness.unloads).toEqual([]) + + await expect( + harness.applyRowsForRequest(1, [{ id: `stale`, rank: -1 }]), + ).rejects.toBeInstanceOf(SyncTransactionAbortedError) + await harness.applyRowsForRequest(2, [{ id: `c`, rank: 3 }]) + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + + const supersededError = new Error(`superseded replay failed`) + const settleSuperseded = () => { + if (supersededSettlement === `resolve`) { + harness.pending[1]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`stale`], + }) + } else { + harness.pending[1]!.request!.reject(supersededError) + } + } + const settleCurrent = () => { + harness.pending[2]!.request!.resolve({ + hasMore: false, + appliedRowKeys: [`c`], + }) + } + const settleFirst = + settlementOrder === `superseded-first` + ? settleSuperseded + : settleCurrent + const settleLast = + settlementOrder === `superseded-first` + ? settleCurrent + : settleSuperseded + + settleFirst() + await flushPromises() + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(true) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `a`, rank: 1 }]) + expect(harness.batches).toEqual([]) + expect(harness.callbackReads).toEqual([]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1] : [0], + ) + + settleLast() + await Promise.all([ + Promise.resolve(firstReplacement), + Promise.resolve(currentReplacement), + ]) + await flushPromises() + + expect(harness.pending).toHaveLength(3) + expect(harness.live.status).toBe(`ready`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([{ id: `c`, rank: 3 }]) + expect(harness.batches).toEqual([ + [ + { type: `delete`, key: `a`, value: { id: `a`, rank: 1 } }, + { type: `insert`, key: `c`, value: { id: `c`, rank: 3 } }, + ], + ]) + expect(harness.callbackReads).toEqual([[{ id: `c`, rank: 3 }]]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1, 0] : [0, 1], + ) + + for (const { options } of harness.pending) { + expectUnindexedFullSnapshotRequest(options) + } + const loadResults = await Promise.allSettled( + harness.loadResults.map((result) => Promise.resolve(result)), + ) + expect(loadResults[0]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`a`] }, + }) + if (supersededSettlement === `resolve`) { + expect(loadResults[1]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`stale`] }, + }) + } else { + expect(loadResults[1]!.status).toBe(`rejected`) + if (loadResults[1]!.status === `rejected`) { + expect(loadResults[1]!.reason).toBe(supersededError) + } + } + expect(loadResults[2]).toStrictEqual({ + status: `fulfilled`, + value: { hasMore: false, appliedRowKeys: [`c`] }, + }) + + await harness.cleanup() + cleaned = true + expect(harness.live.status).toBe(`cleaned-up`) + expect(harness.live.isLoadingSubset).toBe(false) + expect(harness.live.utils.lastSubsetError).toBeUndefined() + expect(harness.readRows()).toEqual([]) + expect(signals.map(({ aborted }) => aborted)).toEqual([true, true, true]) + expect(acquisitionIndices(harness.pending, harness.unloads)).toEqual( + settlementOrder === `superseded-first` ? [1, 0, 2] : [0, 1, 2], + ) + } finally { + await Promise.all([ + preload.catch(() => undefined), + cleaned ? Promise.resolve() : harness.cleanup(), + ]) + } + }, +) + +it.each([`eager`, `off`] as const)( + `keeps an Effect zero-limit join free of ordered transport work with autoIndex %s`, + async (autoIndex) => { + type PrimaryRow = { id: string; rank: number; joinKey: string } + type SecondaryRow = { id: string; joinKey: string } + const primaryLoads: Array = [] + const primary = createCollection({ + id: `multi-source-zero-limit-effect-primary-${autoIndex}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + primaryLoads.push(options) + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const secondary = createCollection({ + id: `multi-source-zero-limit-effect-secondary-${autoIndex}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => {}, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => + q + .from({ primaryRow: primary }) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank) + .offset(2) + .limit(0), + onBatch: () => {}, + }) + + try { + await flushPromises() + expect(primaryLoads).toEqual([]) + } finally { + await effect.dispose() + await Promise.all([primary.cleanup(), secondary.cleanup()]) + } + }, +) + +it(`settles concurrent secondary loads out of order across paged commits`, async () => { + type PrimaryRow = { id: string; rank: number; joinKey: string } + type SecondaryRow = { id: string; joinKey: string } + type PendingSecondaryLoad = { + requestIndex: number + options: LoadSubsetOptions + gate: ReturnType> + joinKeys: ReadonlyArray + } + + const primaryOptions = mockSyncCollectionOptions({ + id: `multi-source-filtered-primary`, + initialData: [ + { id: `a`, rank: 1, joinKey: `a` }, + { id: `b`, rank: 2, joinKey: `b` }, + ], + getKey: (row) => row.id, + syncMode: `eager`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }) + const primary = createCollection(primaryOptions) + const secondaryRows = [ + { id: `a-1`, joinKey: `a` }, + { id: `a-0`, joinKey: `a` }, + { id: `b-1`, joinKey: `b` }, + { id: `b-0`, joinKey: `b` }, + { id: `c-0`, joinKey: `c` }, + ] + const pendingSecondaryLoads: Array = [] + const secondaryCompletionOrder: Array = [] + const secondaryReceipts: Array<{ + requestIndex: number + appliedRowKeys: ReadonlyArray + }> = [] + const secondaryLoadCommitSizes: Array = [] + let secondaryBegin!: () => void + let secondaryWrite!: (message: { + type: `insert` + value: SecondaryRow + }) => void + let secondaryCommit!: () => true | Promise + const establishedSecondaryKeys = new Set() + const secondary = createCollection({ + id: `multi-source-filtered-secondary`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + secondaryBegin = params.begin + secondaryWrite = params.write + secondaryCommit = params.commit + secondaryBegin() + secondaryWrite({ + type: `insert`, + value: { id: `unrelated`, joinKey: `unrelated` }, + }) + establishedSecondaryKeys.add(`unrelated`) + const seeded = secondaryCommit() + if (seeded !== true) { + throw new Error(`Expected synchronous secondary seed`) + } + params.markReady() + return { + loadSubset: async (options) => { + const matchingRows = secondaryRows.filter( + (row) => + options.where === undefined || + evaluateReferenceExpression(options.where, row), + ) + const joinKeys = [ + ...new Set(matchingRows.map(({ joinKey }) => joinKey)), + ] + const pending = { + requestIndex: pendingSecondaryLoads.length, + options, + gate: createDeferred(), + joinKeys, + } + pendingSecondaryLoads.push(pending) + await pending.gate.promise + + const appliedRowKeys: Array = [] + for (const row of [...matchingRows].reverse()) { + if (establishedSecondaryKeys.has(row.id)) continue + establishedSecondaryKeys.add(row.id) + secondaryLoadCommitSizes.push(1) + secondaryBegin() + secondaryWrite({ type: `insert`, value: row }) + const applied = secondaryCommit() + if (applied !== true) await applied + appliedRowKeys.push(row.id) + } + secondaryCompletionOrder.push(pending.requestIndex) + secondaryReceipts.push({ + requestIndex: pending.requestIndex, + appliedRowKeys, + }) + return { hasMore: false, appliedRowKeys } + }, + unloadSubset: () => {}, + } + }, + }, + }) + const createFilteredLive = (id: string, primaryId: string) => + createLiveQueryCollection({ + id, + query: (q) => + q + .from({ primaryRow: primary }) + .where(({ primaryRow }) => eq(primaryRow.id, primaryId)) + .innerJoin( + { secondaryRow: secondary }, + ({ primaryRow, secondaryRow }) => + eq(primaryRow.joinKey, secondaryRow.joinKey), + ) + .orderBy(({ primaryRow }) => primaryRow.rank) + .limit(2), + startSync: true, + }) + const liveA = createFilteredLive(`multi-source-filtered-live-a`, `a`) + const liveB = createFilteredLive(`multi-source-filtered-live-b`, `b`) + + try { + const preload = Promise.all([liveA.preload(), liveB.preload()]) + await flushPromises() + expect(pendingSecondaryLoads).toHaveLength(2) + expect(pendingSecondaryLoads.every(({ options }) => !options.where)).toBe( + true, + ) + expect(pendingSecondaryLoads.map(({ joinKeys }) => joinKeys)).toEqual([ + [`a`, `b`, `c`], + [`a`, `b`, `c`], + ]) + + pendingSecondaryLoads[1]!.gate.resolve() + await flushPromises() + pendingSecondaryLoads[0]!.gate.resolve() + await preload + await flushPromises() + + expect(secondaryCompletionOrder).toEqual([1, 0]) + expect(secondaryLoadCommitSizes).toEqual([1, 1, 1, 1, 1]) + expect(secondaryReceipts.map(({ requestIndex }) => requestIndex)).toEqual([ + 1, 0, + ]) + const claimedSecondaryKeys = secondaryReceipts.flatMap( + ({ appliedRowKeys }) => appliedRowKeys, + ) + expect(new Set(claimedSecondaryKeys).size).toBe(claimedSecondaryKeys.length) + expect(new Set(claimedSecondaryKeys)).toEqual( + new Set(secondaryRows.map(({ id }) => id)), + ) + expect(secondaryReceipts[0]?.appliedRowKeys).toEqual( + [...secondaryRows].reverse().map(({ id }) => id), + ) + expect(secondaryReceipts[1]?.appliedRowKeys).toEqual([]) + expect( + liveA.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual([`a:a-0`, `a:a-1`]) + expect( + liveB.toArray.map( + ({ primaryRow, secondaryRow }) => `${primaryRow.id}:${secondaryRow.id}`, + ), + ).toEqual([`b:b-0`, `b:b-1`]) + } finally { + for (const pending of pendingSecondaryLoads) pending.gate.resolve() + await Promise.all([ + liveA.cleanup(), + liveB.cleanup(), + primary.cleanup(), + secondary.cleanup(), + ]) + } +}) + +it(`projects the minimal source prefix needed by evaluated result contributions`, () => { + const projection = projectOrderedSourceProgress({ + sourceSteps: [ + { sourceKey: `a`, resultKeys: [`a:x-0`], demandKeys: [`x`] }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`y`] }, + { sourceKey: `c`, resultKeys: [`c:z-0`], demandKeys: [`z`] }, + { sourceKey: `d`, resultKeys: [`d:x-0`], demandKeys: [`x`] }, + ], + offset: 0, + limit: 2, + }) + + expect(projection).toEqual({ + visibleResultKeys: [`a:x-0`, `c:z-0`], + scannedSourceKeys: [`a`, `b`, `c`], + sourceCursorKeys: [undefined, `a`, `b`], + demandedKeys: [`x`, `y`, `z`], + rowsNeeded: 0, + sourceExhausted: false, + }) +}) + +it(`erases demand-key spelling without changing source progress`, () => { + const original = projectOrderedSourceProgress({ + sourceSteps: [ + { sourceKey: `a`, resultKeys: [`a:match-0`], demandKeys: [`x`] }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`y`] }, + { sourceKey: `c`, resultKeys: [`c:match-0`], demandKeys: [`x`] }, + ], + offset: 0, + limit: 2, + }) + const renamed = projectOrderedSourceProgress({ + sourceSteps: [ + { + sourceKey: `a`, + resultKeys: [`a:match-0`], + demandKeys: [`renamed-x`], + }, + { sourceKey: `b`, resultKeys: [], demandKeys: [`renamed-y`] }, + { + sourceKey: `c`, + resultKeys: [`c:match-0`], + demandKeys: [`renamed-x`], + }, + ], + offset: 0, + limit: 2, + }) + + expect({ + visibleResultKeys: original.visibleResultKeys, + scannedSourceKeys: original.scannedSourceKeys, + sourceCursorKeys: original.sourceCursorKeys, + rowsNeeded: original.rowsNeeded, + sourceExhausted: original.sourceExhausted, + }).toEqual({ + visibleResultKeys: renamed.visibleResultKeys, + scannedSourceKeys: renamed.scannedSourceKeys, + sourceCursorKeys: renamed.sourceCursorKeys, + rowsNeeded: renamed.rowsNeeded, + sourceExhausted: renamed.sourceExhausted, + }) +}) + +it(`exhausts the bounded multi-source ordered-window model`, () => { + const rows = [ + { key: `a`, joinKey: `x` }, + { key: `b`, joinKey: `y` }, + { key: `c`, joinKey: `z` }, + ] + for (const xCount of [0, 1, 2]) { + for (const yCount of [0, 1, 2]) { + for (const zCount of [0, 1, 2]) { + const counts = [xCount, yCount, zCount] + const sourceSteps = rows.map((row, index) => ({ + sourceKey: row.key, + resultKeys: Array.from( + { length: counts[index]! }, + (_, matchIndex) => `${row.key}:${row.joinKey}-${matchIndex}`, + ), + demandKeys: [row.joinKey], + })) + for (const offset of [0, 1, 2]) { + for (const limit of [0, 1, 2]) { + const projection = projectOrderedSourceProgress({ + sourceSteps, + offset, + limit, + }) + const direct = sourceSteps + .flatMap(({ resultKeys }) => resultKeys) + .slice(offset, offset + limit) + + expect(projection.visibleResultKeys).toEqual(direct) + expect(projection.rowsNeeded).toBe( + Math.max(0, limit - direct.length), + ) + if (limit === 0) { + expect(projection.scannedSourceKeys).toEqual([]) + continue + } + if (projection.scannedSourceKeys.length < sourceSteps.length) { + const shorterPrefix = sourceSteps.slice( + 0, + projection.scannedSourceKeys.length - 1, + ) + const shorterPairCount = shorterPrefix.reduce( + (count, step) => count + step.resultKeys.length, + 0, + ) + expect(shorterPairCount).toBeLessThan(offset + limit) + } else { + expect(projection.sourceExhausted).toBe(true) + } + } + } + } + } + } +}) + +fcTest.prop([multiSourceOrderedScenarioArbitrary], { + numRuns: 12 * fullFlowMultiplier, + seed: 17802, +})( + `fills joined ordered windows for a fixed seed`, + runMultiSourceOrderedScenario, +) + +fcTest.prop( + [multiSourceOrderedScenarioArbitrary], + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.multi-source-ordered`, + ), +)( + `fills joined ordered windows for a random or replayed seed`, + runMultiSourceOrderedScenario, +) + +type TruncateCoverageScenario = { + oldRequest: `none` | `settles-late` + freshResult: `authoritative` | `unknown` | `reject` + settlementOrder: `old-first` | `fresh-first` +} + +const truncateCoverageScenarioArbitrary: fc.Arbitrary = + fc.record({ + oldRequest: fc.constantFrom(`none` as const, `settles-late` as const), + freshResult: fc.constantFrom( + `authoritative` as const, + `unknown` as const, + `reject` as const, + ), + settlementOrder: fc.constantFrom( + `old-first` as const, + `fresh-first` as const, + ), + }) + +const exhaustiveTruncateCoverageScenarios: Array = [ + `none` as const, + `settles-late` as const, +].flatMap((oldRequest) => + ([`authoritative`, `unknown`, `reject`] as const).flatMap((freshResult) => + ([`old-first`, `fresh-first`] as const).map((settlementOrder) => ({ + oldRequest, + freshResult, + settlementOrder, + })), + ), +) + +let truncateCoverageHarnessId = 0 + +async function runTruncateCoverageScenario( + scenario: TruncateCoverageScenario, +): Promise { + type Row = { id: string; value: number } + type AdapterResult = { + hasMore: boolean | undefined + appliedRowKeys: ReadonlyArray + } + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const pending = new Map< + LoadSubsetOptions, + ReturnType> + >() + const unloadSubset = vi.fn() + const source = createCollection({ + id: `truncate-coverage-oracle-${truncateCoverageHarnessId++}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: (options) => { + const request = createDeferred() + pending.set(options, request) + return request.promise + }, + unloadSubset, + } + }, + }, + }) + const initialOptions = { limit: 1 } + const oldOptions = { limit: 2 } + const freshOptions = { limit: 3 } + const histories: Array = [] + const activeOptions: Array = [] + + const request = (ownerId: string, options: LoadSubsetOptions) => { + histories.push({ + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session`, + demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, + alreadyAborted: false, + }) + activeOptions.push(options) + const result = source._sync.loadSubset(options) + if (result === true) throw new Error(`Expected a controlled async request`) + return result + } + + const apply = async ( + ownerId: string, + options: LoadSubsetOptions, + rows: ReadonlyArray, + hasMore: boolean | undefined, + ) => { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + const applied = commit() + if (applied !== true) await applied + pending.get(options)!.resolve({ + hasMore, + appliedRowKeys: rows.map(({ id }) => id), + }) + histories.push({ + type: + hasMore === undefined ? `applyUnprovenRows` : `applyAuthoritativeRows`, + sourceId: `source`, + ownerId, + demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, + rowKeys: rows.map(({ id }) => id), + }) + } + + const reject = (ownerId: string, options: LoadSubsetOptions) => { + pending.get(options)!.reject(new Error(`fresh replay failed`)) + histories.push({ + type: `rejectDemand`, + sourceId: `source`, + ownerId, + demandId: `prefix-${options.limit}`, + attemptId: `${ownerId}-attempt`, + }) + } + + const expectModel = () => { + const actualReusable = activeOptions + .filter( + (options) => source._sync.getLoadSubsetOutcome(options) !== undefined, + ) + .map((options) => `prefix-${options.limit}`) + .sort() + expect(actualReusable).toEqual(projectReusableDemands(histories)) + expect(Array.from(source.keys()).sort()).toEqual( + projectRetainedRowKeys(histories), + ) + } + + try { + const initialLoad = request(`initial`, initialOptions) + await apply(`initial`, initialOptions, [{ id: `initial`, value: 1 }], false) + await initialLoad + expectModel() + + const oldLoad = + scenario.oldRequest === `settles-late` + ? request(`old`, oldOptions) + : undefined + + begin() + truncate() + const truncated = commit() + if (truncated !== true) await truncated + histories.push({ + type: `truncateSource`, + sessionId: `session`, + sourceId: `source`, + }) + expectModel() + + const freshLoad = request(`fresh`, freshOptions) + const settleOld = async () => { + if (!oldLoad) return + await apply(`old`, oldOptions, [{ id: `old`, value: 2 }], false) + await oldLoad + expectModel() + } + const settleFresh = async () => { + if (scenario.freshResult === `reject`) { + reject(`fresh`, freshOptions) + await expect(freshLoad).rejects.toThrow(`fresh replay failed`) + } else { + await apply( + `fresh`, + freshOptions, + [{ id: `fresh`, value: 3 }], + scenario.freshResult === `authoritative` ? false : undefined, + ) + await freshLoad + } + expectModel() + } + + if (scenario.settlementOrder === `fresh-first`) { + await settleFresh() + await settleOld() + } else { + await settleOld() + await settleFresh() + } + + for (const options of activeOptions) { + source._sync.unloadSubset(options) + histories.push({ + type: `releaseDemand`, + sourceId: `source`, + ownerId: + options === initialOptions + ? `initial` + : options === oldOptions + ? `old` + : `fresh`, + demandId: `prefix-${options.limit}`, + attemptId: `${ + options === initialOptions + ? `initial` + : options === oldOptions + ? `old` + : `fresh` + }-attempt`, + }) + } + expect(unloadSubset.mock.calls.map(([options]) => options)).toEqual( + activeOptions, + ) + expectModel() + } finally { + for (const pendingRequest of pending.values()) { + pendingRequest.reject(new Error(`test cleanup`)) + } + await source.cleanup() + } +} + +it.each([ + { oldOutcome: `authoritative`, freshSettlesFirst: false }, + { oldOutcome: `unproven`, freshSettlesFirst: false }, + { oldOutcome: `rejected`, freshSettlesFirst: false }, + { oldOutcome: `evidence-free`, freshSettlesFirst: false }, + { oldOutcome: `released`, freshSettlesFirst: false }, + { oldOutcome: `released`, freshSettlesFirst: true }, +] as const)( + `keeps fresh exact-demand work shared after a pre-truncate $oldOutcome request (freshSettlesFirst=$freshSettlesFirst)`, + async ({ oldOutcome, freshSettlesFirst }) => { + type Row = { id: string; value: number } + type AdapterResult = + | { + hasMore: boolean | undefined + appliedRowKeys: ReadonlyArray + } + | undefined + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const pending: Array>> = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: () => { + const request = createDeferred() + pending.push(request) + return request.promise + }, + }) + const source = createCollection({ + id: `same-demand-truncate-${oldOutcome}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: deduplicated.loadSubset, + unloadSubset: deduplicated.unloadSubset, + } + }, + }, + }) + const oldOptions = { limit: 2 } + const freshOptions = { limit: 2 } + const peerOptions = { limit: 2 } + const applyRows = async (rows: ReadonlyArray) => { + begin() + rows.forEach((row) => write({ type: `insert`, value: row })) + const applied = commit() + if (applied !== true) await applied + } + + try { + const oldLoad = source._sync.loadSubset(oldOptions) + if (oldLoad === true) throw new Error(`Expected an async old request`) + expect(pending).toHaveLength(1) + + begin() + truncate() + const truncated = commit() + if (truncated !== true) await truncated + deduplicated.reset() + + const freshLoad = source._sync.loadSubset(freshOptions) + if (freshLoad === true) throw new Error(`Expected an async fresh request`) + expect(pending).toHaveLength(2) + + if (freshSettlesFirst) { + await applyRows([{ id: `fresh-row`, value: 2 }]) + pending[1]!.resolve({ + hasMore: false, + appliedRowKeys: [`fresh-row`], + }) + await freshLoad + + source._sync.unloadSubset(oldOptions) + expect(source._sync.loadSubset(peerOptions)).toBe(true) + expect(pending).toHaveLength(2) + expect(source._sync.getLoadSubsetOutcome(peerOptions)).toBeDefined() + + pending[0]!.resolve(undefined) + await oldLoad + return + } + + if (oldOutcome === `released`) { + source._sync.unloadSubset(oldOptions) + } else if (oldOutcome === `rejected`) { + const rejection = expect(oldLoad).rejects.toThrow(`old request failed`) + pending[0]!.reject(new Error(`old request failed`)) + await rejection + } else if (oldOutcome === `evidence-free`) { + pending[0]!.resolve(undefined) + await oldLoad + } else { + await applyRows([{ id: `old-row`, value: 1 }]) + pending[0]!.resolve({ + hasMore: oldOutcome === `authoritative` ? false : undefined, + appliedRowKeys: [`old-row`], + }) + await oldLoad + } + + expect(source._sync.getLoadSubsetOutcome(freshOptions)).toBeUndefined() + const peerLoad = source._sync.loadSubset(peerOptions) + if (peerLoad === true) throw new Error(`Expected a shared peer request`) + expect(pending).toHaveLength(2) + + await applyRows([{ id: `fresh-row`, value: 2 }]) + pending[1]!.resolve({ + hasMore: false, + appliedRowKeys: [`fresh-row`], + }) + await Promise.all([freshLoad, peerLoad]) + expect(source._sync.getLoadSubsetOutcome(peerOptions)).toBeDefined() + if (oldOutcome === `released`) { + pending[0]!.resolve(undefined) + await oldLoad + } + } finally { + for (const request of pending) { + request.reject(new Error(`test cleanup`)) + } + await source.cleanup() + } + }, +) + +it(`keeps adapter release obligations distinct across attempts by one owner`, () => { + const history: ReadonlyArray = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt-1`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt-2`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt-1`, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt-2`, + }, + ] + + expect(projectAdapterLifecycle(history)).toEqual([ + { + type: `invoke`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-1`, + }, + { + type: `invoke`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-2`, + }, + { + type: `release`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-1`, + }, + { + type: `release`, + ownerId: `owner`, + sourceId: `source`, + attemptId: `attempt-2`, + }, + ]) +}) + +let sourceIdentityHarnessId = 0 + +it(`keeps identical demand and row identities local to each source`, async () => { + type Row = { id: string } + type Result = { hasMore: false; appliedRowKeys: ReadonlyArray } + const createSource = (sourceId: string) => { + const result = createDeferred() + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const collection = createCollection({ + id: `source-identity-${sourceIdentityHarnessId++}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { loadSubset: () => result.promise } + }, + }, + }) + const options = { limit: 1 } + const load = collection._sync.loadSubset(options) + if (load === true) throw new Error(`Expected a controlled async load`) + return { + sourceId, + collection, + options, + load, + settle: async () => { + begin() + write({ type: `insert`, value: { id: `shared-row` } }) + const applied = commit() + if (applied !== true) await applied + result.resolve({ hasMore: false, appliedRowKeys: [`shared-row`] }) + await load + }, + } + } + const sourceA = createSource(`source-a`) + const sourceB = createSource(`source-b`) + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + rowKeys: [`shared-row`], + }) + const history: Array = [ + request(sourceA.sourceId), + request(sourceB.sourceId), + ] + const actualRows = () => + [sourceA, sourceB].flatMap(({ sourceId, collection }) => + Array.from(collection.keys(), (rowKey) => ({ sourceId, rowKey })), + ) + + try { + await sourceA.settle() + history.push(settle(sourceA.sourceId)) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectReusableSourceDemands(history)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + ]) + + await sourceB.settle() + history.push(settle(sourceB.sourceId)) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectTransportLoads(history)).toBe(2) + + sourceA.collection._sync.unloadSubset(sourceA.options) + history.push({ + type: `releaseDemand`, + sourceId: sourceA.sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }) + expect(actualRows()).toEqual(projectRetainedSourceRows(history)) + expect(projectReusableSourceDemands(history)).toEqual([ + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) + } finally { + await Promise.all([ + sourceA.collection.cleanup(), + sourceB.collection.cleanup(), + ]) + } +}) + +it(`derives shared row and evidence lifetime from active attempts`, () => { + const sharedHistory: ReadonlyArray = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `shared`, + attemptId: `attempt-a`, + rowKeys: [`x`], + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `shared`, + attemptId: `attempt-a`, + }, + ] + + expect(projectRetainedRowKeys(sharedHistory)).toEqual([`x`]) + expect( + projectTransportLoads([ + ...sharedHistory, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-c`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-c`, + alreadyAborted: false, + }, + ]), + ).toBe(1) + + expect( + projectRetainedRowKeys([ + ...sharedHistory, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-b`, + demandId: `shared`, + attemptId: `attempt-b`, + }, + ]), + ).toEqual([]) +}) + +it(`keeps an additional demand active until its final attempt releases`, () => { + const history: ReadonlyArray = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `other`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `other`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `next`, + sourceId: `source`, + demandId: `ordered`, + rows: [{ key: `o`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `next`, + sourceId: `source`, + demandId: `other`, + rows: [{ key: `x`, orderValue: 1 }], + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `other`, + attemptId: `attempt-a`, + }, + { type: `commitPublication`, publicationId: `next` }, + ] + + expect( + projectAtomicOrderedPublicationState(history, { + sourceId: `source`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`o`, `x`]) +}) + +it(`does not release physical work when an already-aborted demand skips adapter start`, async () => { + const ownerId = `aborted-owner` + const requestEvent: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session-1`, + demandId: `all-rows`, + attemptId: `aborted-attempt`, + alreadyAborted: true, + } + const history: ReadonlyArray = [ + requestEvent, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId, + demandId: `all-rows`, + attemptId: `aborted-attempt`, + }, + ] + const adapterEvents: Array = [] + const collection = createCollection<{ id: string }>({ + id: `full-flow-aborted-before-start`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: (options) => { + adapterEvents.push({ type: `start`, options }) + return true + }, + unloadSubset: (options) => { + adapterEvents.push({ type: `release`, options }) + }, + } + }, + }, + }) + const subscription = collection.subscribeChanges(() => {}, { + includeInitialState: false, + }) + const request = new AbortController() + request.abort() + + try { + subscription.requestSnapshot({ + signal: request.signal, + optimizedOnly: false, + }) + expect(eventTypes(adapterEvents)).toEqual( + projectAdapterLifecycle([requestEvent]).map(({ type }) => + type === `invoke` ? `start` : `release`, + ), + ) + + subscription.unsubscribe() + + // A skipped adapter call creates no physical resource to release. + expect(eventTypes(adapterEvents)).toEqual( + projectAdapterLifecycle(history).map(({ type }) => + type === `invoke` ? `start` : `release`, + ), + ) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it.each([127, 128, 129])( + `freezes a %i-byte equality constant across local filtering and adapter acquisition`, + async (byteLength) => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const originalToken = new Uint8Array(byteLength).fill(1) + const changedToken = new Uint8Array(byteLength).fill(2) + const callerToken = new Uint8Array(originalToken) + Object.defineProperty(callerToken, `slice`, { + value: () => callerToken, + }) + const rows: ReadonlyArray = [ + { id: `original`, token: originalToken }, + { id: `changed`, token: changedToken }, + ] + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-binary-equality-${byteLength}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + rows.forEach((value) => write({ type: `insert`, value })) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const visible = new Set() + const where = new Func(`eq`, [ + new PropRef([`token`]), + new Value(callerToken), + ]) + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { whereExpression: where }, + ) + + try { + callerToken.fill(2) + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredValue = ( + (acquired?.where as Func | undefined)?.args[1] as + | Value + | undefined + )?.value + expect(acquiredValue).toEqual(originalToken) + expect(acquiredValue).not.toBe(callerToken) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } + }, +) + +it(`rejects binary values without intrinsic typed-array slots before adapter acquisition`, async () => { + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + let adapterCalls = 0 + const collection = createCollection<{ id: string; token: Uint8Array }>({ + id: `reject-binary-proxy`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(bytes), + ]), + }), + ).toThrow(/Cannot snapshot binary equality value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + +it(`freezes cross-realm binary equality across filtering and acquisition`, async () => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const rows: ReadonlyArray = [ + { id: `original`, token: new Uint8Array([1]) }, + { id: `changed`, token: new Uint8Array([2]) }, + ] + const callerToken = createCrossRealmUint8Array([1]) + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-cross-realm-binary-equality`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + rows.forEach((value) => write({ type: `insert`, value })) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + unloadSubset: () => {}, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(callerToken), + ]), + }, + ) + + try { + callerToken[0] = 2 + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredValue = ( + (acquired?.where as Func | undefined)?.args[1] as + | Value + | undefined + )?.value + expect(acquiredValue).toEqual(new Uint8Array([1])) + expect(acquiredValue).not.toBe(callerToken) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`keeps binary equality distinct from a sentinel-looking string`, async () => { + type Row = { id: `binary` | `string`; token: Uint8Array | string } + const binary = new Uint8Array([1, 2, 3]) + const sentinel = normalizeValue(binary) as string + const collection = createCollection({ + id: `binary-string-normalization-domains`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `binary`, token: binary } }) + write({ type: `insert`, value: { id: `string`, token: sentinel } }) + commit() + markReady() + return { loadSubset: () => true } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`token`]), + new Value(binary), + ]), + }, + ) + + try { + subscription.requestSnapshot({ optimizedOnly: false }) + expect([...visible]).toEqual([`binary`]) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`freezes computed membership candidates across local filtering and adapter acquisition`, async () => { + type Row = { id: `original` | `changed`; token: Uint8Array } + const candidates = [new Uint8Array([1])] + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `frozen-computed-membership-candidates`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `original`, token: new Uint8Array([1]) }, + }) + write({ + type: `insert`, + value: { id: `changed`, token: new Uint8Array([2]) }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + }, + ) + + try { + candidates[0]![0] = 2 + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`original`]) + const acquiredCandidates = ( + ((acquired?.where as Func).args[1] as Func).args[0] as Value< + Array + > + ).value + expect(acquiredCandidates).toEqual([new Uint8Array([1])]) + expect(acquiredCandidates).not.toBe(candidates) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`rejects custom membership observation before adapter acquisition`, async () => { + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, + }) + let adapterCalls = 0 + const collection = createCollection<{ id: string; token: Uint8Array }>({ + id: `reject-custom-membership-observation`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, + }, + }) + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`in`, [ + new PropRef([`token`]), + new Func(`coalesce`, [new Value(candidates)]), + ]), + }), + ).toThrow(/Cannot snapshot membership candidates/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + +it(`uses intrinsic Date state for local filtering and adapter acquisition`, async () => { + type Row = { id: `instance-hook` | `intrinsic`; date: Date } + const callerDate = new Date(2) + Object.defineProperty(callerDate, `getTime`, { value: () => 1 }) + let acquired: LoadSubsetOptions | undefined + const collection = createCollection({ + id: `intrinsic-date-equality`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ + type: `insert`, + value: { id: `instance-hook`, date: new Date(1) }, + }) + write({ + type: `insert`, + value: { id: `intrinsic`, date: new Date(2) }, + }) + commit() + markReady() + return { + loadSubset: (options) => { + acquired = options + return true + }, + } + }, + }, + }) + const visible = new Set() + const subscription = collection.subscribeChanges( + (changes) => { + for (const change of changes) { + if (change.type === `delete`) visible.delete(change.key as Row[`id`]) + else visible.add(change.key as Row[`id`]) + } + }, + { + whereExpression: new Func(`eq`, [ + new PropRef([`date`]), + new Value(callerDate), + ]), + }, + ) + + try { + subscription.requestSnapshot({ optimizedOnly: false }) + + expect([...visible]).toEqual([`intrinsic`]) + const acquiredDate = ((acquired?.where as Func).args[1] as Value) + .value + expect(acquiredDate.getTime()).toBe(2) + } finally { + subscription.unsubscribe() + await collection.cleanup() + } +}) + +it(`rejects constructor-shaped Temporal lookalikes before adapter acquisition`, async () => { + class TemporalLookalike { + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return `2024-01-15` + } + } + let adapterCalls = 0 + const collection = createCollection<{ id: string; date: TemporalLookalike }>({ + id: `reject-constructor-shaped-temporal`, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => { + adapterCalls += 1 + return true + }, + } + }, }, - ] - const adapterEvents: Array = [] - const collection = createCollection<{ id: string }>({ - id: `full-flow-aborted-before-start`, + }) + + try { + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`eq`, [ + new PropRef([`date`]), + new Value(new TemporalLookalike()), + ]), + }), + ).toThrow(/Cannot snapshot Temporal.PlainDate equality value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) + } finally { + await collection.cleanup() + } +}) + +it(`rejects unsupported relational coercion before adapter entry`, async () => { + let adapterCalls = 0 + const collection = createCollection<{ id: string; value: number }>({ + id: `unsupported-relational-coercion`, getKey: (row) => row.id, syncMode: `on-demand`, sync: { sync: ({ markReady }) => { markReady() return { - loadSubset: (options) => { - adapterEvents.push({ type: `start`, options }) + loadSubset: () => { + adapterCalls += 1 return true }, - unloadSubset: (options) => { - adapterEvents.push({ type: `release`, options }) - }, } }, }, }) - const subscription = collection.subscribeChanges(() => {}, { - includeInitialState: false, - }) - const request = new AbortController() - request.abort() + const coercion = { [Symbol.toPrimitive]: () => 1 } try { - subscription.requestSnapshot({ - signal: request.signal, - optimizedOnly: false, - }) - expect(eventTypes(adapterEvents)).toEqual( - projectAdapterLifecycle([requestEvent]).map(({ type }) => - type === `invoke` ? `start` : `release`, - ), - ) - - subscription.unsubscribe() - - // A skipped adapter call creates no physical resource to release. - expect(eventTypes(adapterEvents)).toEqual( - projectAdapterLifecycle(history).map(({ type }) => - type === `invoke` ? `start` : `release`, - ), - ) + expect(() => + collection.subscribeChanges(() => {}, { + whereExpression: new Func(`gt`, [ + new PropRef([`value`]), + new Value(coercion), + ]), + }), + ).toThrow(/Cannot snapshot structural expression value/) + expect(adapterCalls).toBe(0) + expect(collection.subscriberCount).toBe(0) } finally { - subscription.unsubscribe() await collection.cleanup() } }) @@ -351,24 +4117,27 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-1`, sessionId: `session-1`, demandId: `all-rows`, + attemptId: `attempt-1`, alreadyAborted: false, }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-1`, demandId: `all-rows`, + attemptId: `attempt-1`, rowKeys: [row.id], }, { type: `releaseDemand`, + sourceId: `source`, ownerId: `owner-1`, demandId: `all-rows`, - rowKeys: [row.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, + attemptId: `attempt-1`, }, { type: `restartSession`, @@ -377,15 +4146,19 @@ it(`reloads authoritative rows after final-owner cleanup invalidates retained ad }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-2`, sessionId: `session-2`, demandId: `all-rows`, + attemptId: `attempt-2`, alreadyAborted: false, }, { type: `applyAuthoritativeRows`, + sourceId: `source`, ownerId: `owner-2`, demandId: `all-rows`, + attemptId: `attempt-2`, rowKeys: [row.id], }, ] @@ -460,9 +4233,11 @@ it(`does not let an ordered continuation from a cleaned session start new work a const history: ReadonlyArray = [ { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-1`, sessionId: `session-1`, demandId: `top-1`, + attemptId: `attempt-1`, alreadyAborted: false, }, { @@ -479,9 +4254,11 @@ it(`does not let an ordered continuation from a cleaned session start new work a }, { type: `requestDemand`, + sourceId: `source`, ownerId: `owner-2`, sessionId: `session-2`, demandId: `top-1`, + attemptId: `attempt-2`, alreadyAborted: false, }, { type: `runContinuation`, taskId: `load-1-settlement` }, @@ -619,6 +4396,40 @@ it.each([`sync`, `async`] as const)( expect(demands).toHaveLength(2) expect(demands[1]).toMatchObject({ limit: 2, offset: 0 }) expect(demands[1]?.cursor).toBeUndefined() + + await live.utils.setWindow({ offset: 0, limit: 4 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(4) + expect(demands[2]).toMatchObject({ limit: 4, offset: 0 }) + expect(demands[2]?.cursor).toBeUndefined() + expect(demands[3]).toMatchObject({ limit: 4, offset: 0 }) + expect(demands[3]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + await live.utils.setWindow({ offset: 0, limit: 5 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(5) + expect(demands[4]).toMatchObject({ limit: 5, offset: 0 }) + expect(demands[4]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + await live.utils.setWindow({ offset: 0, limit: 2 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(demands).toHaveLength(5) + + await live.utils.setWindow({ offset: 0, limit: 5 }) + + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2, 3]) + expect(live.status).toBe(`ready`) + expect(demands).toHaveLength(6) + expect(demands[5]).toMatchObject({ limit: 5, offset: 0 }) + expect(demands[5]?.cursor).toBeUndefined() + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) } finally { await live.cleanup() await source.cleanup() @@ -626,6 +4437,166 @@ it.each([`sync`, `async`] as const)( }, ) +it(`does not treat explicit continuation as outcome-free satisfaction`, async () => { + type Row = { id: number; rank: number } + const pending: Array>> = [] + const calls: Array = [] + const source = createCollection({ + id: `full-flow-explicit-continuation-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options) => { + calls.push(options) + if (calls.length === 1) { + begin() + write({ type: `insert`, value: { id: 1, rank: 1 } }) + commit() + } + const deferred = createDeferred() + pending.push(deferred) + return deferred.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `full-flow-explicit-continuation-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(2), + startSync: true, + }) + const preload = live.preload() + + try { + expect(pending).toHaveLength(1) + pending[0]!.resolve({ hasMore: true }) + await flushPromises() + + expect(pending).toHaveLength(2) + expect(calls[1]?.limit).toBeUndefined() + const [subscription] = Object.values( + live.utils[LIVE_QUERY_INTERNAL].getBuilder().subscriptions, + ) + expect(subscription?.hasOrderedResultForActiveWindow).toBe(false) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + pending[1]!.resolve({ hasMore: false, appliedRowKeys: [] }) + await preload + expect(live.toArray.map(({ id }) => id)).toEqual([1]) + } finally { + for (const request of pending) { + request.resolve({ hasMore: false, appliedRowKeys: [] }) + } + await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await source.cleanup() + } +}) + +it(`keeps the prior ordered publication until truncate replay gains authoritative coverage`, async () => { + type Row = { id: number; rank: number } + const oldRows: ReadonlyArray = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + ] + const replacementRows: ReadonlyArray = [ + { id: 3, rank: 3 }, + { id: 4, rank: 4 }, + ] + const authoritative = createDeferred() + let calls = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + let truncate!: () => void + const source = createCollection({ + id: `full-flow-outcome-free-truncate-source`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: () => { + calls++ + const rows = calls === 1 ? oldRows : replacementRows + if (calls <= 2) { + begin() + for (const row of rows) write({ type: `insert`, value: row }) + commit() + } + if (calls === 1) { + return Promise.resolve({ + hasMore: false, + appliedRowKeys: oldRows.map(({ id }) => id), + }) + } + if (calls === 2) return Promise.resolve() + if (calls === 3) return authoritative.promise + throw new Error(`Unexpected fourth replay request`) + }, + unloadSubset: () => {}, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `full-flow-outcome-free-truncate-live`, + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(2), + startSync: true, + }) + const preload = live.preload() + + try { + await preload + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + + begin() + truncate() + const replacement = commit() + await flushPromises() + + expect(calls).toBe(3) + expect(live.toArray.map(({ id }) => id)).toEqual([1, 2]) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + + authoritative.resolve({ + hasMore: false, + appliedRowKeys: replacementRows.map(({ id }) => id), + }) + if (replacement !== true) await replacement + await flushPromises() + + expect(live.toArray.map(({ id }) => id)).toEqual([3, 4]) + } finally { + authoritative.resolve({ hasMore: false, appliedRowKeys: [] }) + await Promise.all([preload.catch(() => undefined), live.cleanup()]) + await source.cleanup() + } +}) + it.each([ { name: `continues past an excluded source row`, @@ -1001,7 +4972,11 @@ fcTest.prop([orderedConsumerParityScenarioArbitrary], { fcTest.prop( [orderedConsumerParityScenarioArbitrary], - oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.consumer-parity`, + ), )( `keeps ordered collection consumers equal for a random or replayed seed`, assertOrderedConsumerParity, @@ -1211,7 +5186,7 @@ it(`retries an evidence-free ordered Effect after truncate`, async () => { } }) -it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { +it(`rechecks an ordered Effect until truncate replay proves replacement coverage`, async () => { type Row = { id: number; rank: number } type Result = { hasMore: boolean @@ -1255,6 +5230,10 @@ it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { begin() write({ type: `insert`, value: finalRow }) commit() + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [finalRow.id], + }) } return true }, @@ -1301,6 +5280,81 @@ it(`rechecks an ordered Effect after synchronous truncate replay`, async () => { } }) +it(`settles an outcome-free ordered Effect when its boundary stops advancing`, async () => { + type Row = { id: number; rank: number } + const rows: ReadonlyArray = [ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ] + const visible = new Map() + let calls = 0 + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const source = createCollection({ + id: `full-flow-effect-outcome-free-no-progress`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: () => { + calls++ + if (calls <= rows.length) { + begin() + write({ type: `insert`, value: rows[calls - 1]! }) + commit() + } + + // Bound the old loop. A correct implementation stops when the + // fourth request completes without moving the local boundary. + if (calls === 5) { + return Promise.resolve({ + hasMore: false, + appliedRowKeys: [], + }) + } + return Promise.resolve() + }, + unloadSubset: () => {}, + } + }, + }, + }) + const effect = createEffect({ + query: (q) => + q + .from({ row: source }) + .orderBy(({ row }) => row.rank) + .limit(4), + onBatch: (events) => { + for (const event of events) { + if (event.type === `exit`) visible.delete(event.key) + else visible.set(event.key, event.value) + } + }, + }) + + try { + await flushPromises() + + expect([...visible.keys()]).toEqual([1, 2, 3]) + expect(calls).toBe(4) + expect(source._sync.getLoadSubsetCoverage()).toEqual([]) + } finally { + await effect.dispose() + await source.cleanup() + } +}) + it(`replaces an ordered Effect only after a rejected continuation disposes it`, async () => { type Row = { id: number; rank: number } const firstRow: Row = { id: 1, rank: 1 } @@ -1657,7 +5711,11 @@ if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { )}`, `exhaustion=${pages.some((page) => page.extent === `exhausted`)}`, ], - oracleRandomParameters(1_000, fullFlowReplaySeed), + oracleRandomParameters( + 1_000, + fullFlowReplay, + `load-subset-full-flow.continuation-statistics`, + ), ) } @@ -1928,7 +5986,11 @@ fcTest.prop( maxLength: 8, }), ], - oracleRandomParameters(128 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 128 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.automatic-progress`, + ), )( `starts automatic continuation only for new semantic progress with a random or replayed seed`, assertAutomaticOrderedProgress, @@ -1944,7 +6006,11 @@ fcTest.prop([orderedContinuationEvidenceScenarioArbitrary], { fcTest.prop( [orderedContinuationEvidenceScenarioArbitrary], - oracleRandomParameters(64 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 64 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.continuation-evidence`, + ), )( `derives ordered progress from applied eligible evidence for a random or replayed seed`, runOrderedContinuationEvidenceScenario, @@ -2041,6 +6107,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `initial-publication`, + sourceId: `source`, demandId: `ordered-window`, rows: orderedForDirection.slice(0, prefixSize).map((row) => ({ key: row.id, @@ -2056,6 +6123,7 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `additional-publication`, + sourceId: `source`, demandId: `ordered-window`, rows: expectedOrderedPrefix.map((row) => ({ key: row.id, @@ -2067,14 +6135,16 @@ async function runOrderedBoundaryProvenanceScenario( { type: `stagePublicationRows`, publicationId: `additional-publication`, + sourceId: `source`, demandId: `unordered-retention`, rows: [{ key: addedRow.id, orderValue: addedRow.rank }], }, { type: `commitPublication`, publicationId: `additional-publication` }, - { type: `truncateSource`, sessionId: `session` }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, { type: `stagePublicationRows`, publicationId: `failed-replacement`, + sourceId: `source`, demandId: `ordered-window`, rows: [ { @@ -2087,11 +6157,14 @@ async function runOrderedBoundaryProvenanceScenario( }, { type: `rejectDemand`, + sourceId: `source`, ownerId: `ordered-owner`, demandId: `ordered-window`, + attemptId: `ordered-attempt`, }, ] const expectedBoundary = projectOrderedPublicationBoundary(history, { + sourceId: `source`, demandId: `ordered-window`, direction: scenario.direction, prefixSize, @@ -2274,7 +6347,11 @@ fcTest.prop([orderedBoundaryProvenanceArbitrary], { fcTest.prop( [orderedBoundaryProvenanceArbitrary], - oracleRandomParameters(32 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 32 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.boundary-provenance`, + ), )( `keeps ordered boundary provenance for a random or replayed seed`, runOrderedBoundaryProvenanceScenario, @@ -2457,6 +6534,7 @@ async function runAtomicOrderedReplayScenario( { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `ordered`, rows: toModelRows(initialRows), }, @@ -2555,12 +6633,14 @@ async function runAtomicOrderedReplayScenario( const expectedPublicationProjection = () => projectAtomicOrderedPublicationState(history, { + sourceId: `source`, demandId: `ordered`, direction: scenario.direction, initialWindowSize, }) const expectedPublications = () => projectAtomicOrderedPublications(history, { + sourceId: `source`, demandId: `ordered`, direction: scenario.direction, initialWindowSize, @@ -2597,9 +6677,10 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `beginReplacement`, publicationId, - demandIds: acquisitions.map((acquisition) => - acquisition === ordered ? `ordered` : `other`, - ), + demands: acquisitions.map((acquisition) => ({ + sourceId: `source`, + demandId: acquisition === ordered ? `ordered` : `other`, + })), }) expectPublicationHistory() return { publicationId, acquisitions, ordered } satisfies PendingAttempt @@ -2624,6 +6705,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: replay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows(rows), }) @@ -2663,6 +6745,7 @@ async function runAtomicOrderedReplayScenario( ? { type: `settleReplacement`, publicationId: replay.publicationId, + sourceId: `source`, demandId, outcome: settledOutcome, extent: isOrdered ? extent : `exhausted`, @@ -2670,6 +6753,7 @@ async function runAtomicOrderedReplayScenario( : { type: `settleReplacement`, publicationId: replay.publicationId, + sourceId: `source`, demandId, outcome: settledOutcome, }, @@ -2685,11 +6769,10 @@ async function runAtomicOrderedReplayScenario( expect(released?.options.signal?.aborted).toBe(true) history.push({ type: `releaseDemand`, + sourceId: `source`, ownerId: `other-owner`, demandId: `other`, - rowKeys: [replacementOtherRow.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, + attemptId: `other-attempt`, }) expectPublicationHistory() } @@ -2708,9 +6791,11 @@ async function runAtomicOrderedReplayScenario( if (scenario.otherDemand !== `none`) { history.push({ type: `requestDemand`, + sourceId: `source`, ownerId: `other-owner`, sessionId: `atomic-session`, demandId: `other`, + attemptId: `other-attempt`, alreadyAborted: false, }) subscription.requestSnapshot({ where: otherWhere }) @@ -2719,6 +6804,7 @@ async function runAtomicOrderedReplayScenario( { type: `stagePublicationRows`, publicationId: `initial`, + sourceId: `source`, demandId: `other`, rows: toModelRows(initialOtherRows), }, @@ -2733,6 +6819,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: firstReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([obsoleteRow]), }) @@ -2754,7 +6841,12 @@ async function runAtomicOrderedReplayScenario( ? ([2, 0] as const) : ([0, 2] as const) for (const size of resizeSizes) { - history.push({ type: `resizeOrderedWindow`, size }) + history.push({ + type: `resizeOrderedWindow`, + sourceId: `source`, + demandId: `ordered`, + size, + }) subscription.ensureOrderedWindowSize(size) expectPublicationHistory() } @@ -2764,6 +6856,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `other`, rows: toModelRows([replacementOtherRow]), }) @@ -2772,11 +6865,10 @@ async function runAtomicOrderedReplayScenario( subscription.releaseSnapshot(otherWhere) history.push({ type: `releaseDemand`, + sourceId: `source`, ownerId: `other-owner`, demandId: `other`, - rowKeys: [replacementOtherRow.id], - finalRowOwner: true, - invalidatesAdapterEvidence: true, + attemptId: `other-attempt`, }) expectPublicationHistory() } @@ -2787,6 +6879,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([sourceDelta]), }) @@ -2798,6 +6891,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([partialRow]), }) @@ -2815,6 +6909,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([partialRow, continuationRow]), }) @@ -2880,6 +6975,7 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `stagePublicationRows`, publicationId: currentReplay.publicationId, + sourceId: `source`, demandId: `ordered`, rows: toModelRows([...finalRows, continuationRow]), }) @@ -2946,6 +7042,8 @@ async function runAtomicOrderedReplayScenario( history.push({ type: `establishReplacementCoverage`, publicationId: currentReplay.publicationId, + sourceId: `source`, + demandId: `ordered`, }) await flushPromises() expectPublicationHistory() @@ -3062,21 +7160,23 @@ const mixedDemandSettlementScenarios: ReadonlyArray }, ], ), - { - direction, - resizeOrder: `grow-shrink` as const, - overlap: false, - currentOutcome: `resolve` as const, - currentExtent: `exhausted` as const, - settleCurrentFirst: false, - sourceDelta: false, - otherDemand: `active` as const, - otherOutcome: `reject` as const, - demandSettlementOrder: `ordered-first` as const, - releaseAfterOrdered: true, - }, ]) +const releaseDuringPrivateReplayScenarios: ReadonlyArray = + ([`asc`, `desc`] as const).map((direction) => ({ + direction, + resizeOrder: `grow-shrink`, + overlap: false, + currentOutcome: `resolve`, + currentExtent: `exhausted`, + settleCurrentFirst: false, + sourceDelta: false, + otherDemand: `active`, + otherOutcome: `reject`, + demandSettlementOrder: `ordered-first`, + releaseAfterOrdered: true, + })) + it(`does not reuse caller or public continuation state when an active replacement has no progress`, async () => { for (const direction of [`asc`, `desc`] as const) { for (const callerContinuation of [ @@ -3153,6 +7253,12 @@ it(`keeps mixed demand settlements inside one replacement epoch`, async () => { } }) +it(`removes a released peer from the public baseline while replay remains private`, async () => { + for (const scenario of releaseDuringPrivateReplayScenarios) { + await runAtomicOrderedReplayScenario(scenario) + } +}) + it(`discards pending replacement epochs on teardown`, async () => { for (const direction of [`asc`, `desc`] as const) { for (const overlap of [false, true]) { @@ -3187,7 +7293,11 @@ fcTest.prop([atomicOrderedReplayArbitrary], { fcTest.prop( [atomicOrderedReplayArbitrary], - oracleRandomParameters(32 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 32 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.atomic-replacement`, + ), )( `keeps ordered replacement publication atomic for a random or replayed seed`, runAtomicOrderedReplayScenario, @@ -3206,7 +7316,11 @@ fcTest.prop([truncateCoverageScenarioArbitrary], { fcTest.prop( [truncateCoverageScenarioArbitrary], - oracleRandomParameters(12 * fullFlowMultiplier, fullFlowReplaySeed), + oracleRandomParameters( + 12 * fullFlowMultiplier, + fullFlowReplay, + `load-subset-full-flow.truncate-evidence`, + ), )( `fences pre-truncate evidence for a random or replayed seed`, runTruncateCoverageScenario, diff --git a/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts b/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts index 433881dde7..8dc4e634b6 100644 --- a/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-lifecycle-oracle.property.test.ts @@ -348,7 +348,7 @@ fcTest.prop( maxCommands: 20, }), ], - oraclePropertyOptions(100), + oraclePropertyOptions(100, `load-subset-lifecycle.state-machine`), )( `matches the scheduled acquisition, coverage, release, teardown, and stale-settlement lifecycle`, (commands) => { diff --git a/packages/db/tests/query/load-subset-oracle.property.test.ts b/packages/db/tests/query/load-subset-oracle.property.test.ts index fed44b28be..21c1b27e44 100644 --- a/packages/db/tests/query/load-subset-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-oracle.property.test.ts @@ -2,7 +2,6 @@ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' -import { createOptimisticAction } from '../../src/optimistic-action.js' import { createLiveQueryCollection, eq } from '../../src/query/index.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { Func, PropRef, Value } from '../../src/query/ir.js' @@ -73,11 +72,6 @@ type PersistedLoadRow = { projectId: string } -type OptimisticDerivedRow = { - id: string - value: string -} - type CoverageSubject = { loadSubset: LoadSubsetFn reset?: () => void @@ -1119,12 +1113,10 @@ async function runAsyncScenarioWithKnownFailures( } } -const { multiplier, replaySeed } = readOracleRunConfig() +const { multiplier, ...replay } = readOracleRunConfig() const coverageScenarioRuns = 40 * multiplier -const coverageRandomParameters = oracleRandomParameters( - coverageScenarioRuns, - replaySeed, -) +const coverageRandomParameters = (property: string) => + oracleRandomParameters(coverageScenarioRuns, replay, property) let collectionSequence = 0 @@ -1789,6 +1781,8 @@ async function expectCanceledReceiptReleasesOnlyItsSuppression() { expect(source._state.recentlySyncedKeys).toEqual(new Set([`second`])) expect(source._state.preSyncVisibleState.has(`first`)).toBe(false) expect(source._state.preSyncVisibleState.has(`second`)).toBe(true) + expect(source._state.preSyncVirtualState.has(`first`)).toBe(false) + expect(source._state.preSyncVirtualState.has(`second`)).toBe(true) if (canceled !== true) { await expect(canceled).rejects.toMatchObject({ name: `AbortError` }) } @@ -1879,75 +1873,23 @@ async function expectCleanupRejectsReceiptOnce() { await source.cleanup() } -async function expectDerivedSyncDuringOptimisticMutation(): Promise { - let begin!: () => void - let write!: (message: { type: `insert`; value: OptimisticDerivedRow }) => void - let commit!: () => void - const source = createCollection({ - id: `optimistic-derived-source-${collectionSequence++}`, - getKey: (row) => row.id, - sync: { - sync: (params) => { - begin = params.begin - write = params.write - commit = params.commit - params.markReady() - }, - }, - }) - const derived = createLiveQueryCollection({ - query: (query) => - query - .from({ row: source }) - .select(({ row }) => ({ id: row.id, value: row.value })), - getKey: (row) => row.id, - startSync: true, - }) - const persistence = createDeferred() - // Query collections currently expose read-side virtual properties in their - // insert input type even though the runtime accepts the plain selected row. - const insertDerived = derived.insert.bind(derived) as unknown as ( - row: OptimisticDerivedRow, - ) => ReturnType - const insertOptimistically = createOptimisticAction({ - onMutate: insertDerived, - mutationFn: () => persistence.promise, - }) - - await derived.preload() - const transaction = insertOptimistically({ - id: `optimistic`, - value: `optimistic`, - }) - try { - begin() - write({ type: `insert`, value: { id: `synced`, value: `synced` } }) - commit() - - try { - expect([...derived.keys()].sort()).toEqual([`optimistic`, `synced`]) - } catch (error) { - throw new TraceAssertionError(0, error) - } - } finally { - persistence.resolve() - await transaction.isPersisted.promise - await derived.cleanup() - await source.cleanup() - } -} - async function expectDeduplicatedWaiterHandlesRejection( scenario: RejectedWaiterScenario, ): Promise { - const detachedBranches: Array> = [] + let sourceRejectionObservers = 0 class LocallyTrackedPromise extends Promise { - catch( - onRejected?: ((reason: unknown) => TResult | PromiseLike) | null, - ): Promise { - const branch = super.catch(onRejected) - detachedBranches.push(branch) - return branch + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: ((value: T) => TResult1 | PromiseLike) | null, + onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + if (onrejected) sourceRejectionObservers += 1 + return super.then(onfulfilled, onrejected) } } @@ -1970,7 +1912,6 @@ async function expectDeduplicatedWaiterHandlesRejection( } const callerOutcomes = Promise.allSettled([first, second]) - const detachedOutcomes = Promise.allSettled(detachedBranches) rejectSource(new Error(`transport failed`)) expect((await callerOutcomes).map(({ status }) => status)).toEqual([ `rejected`, @@ -1978,10 +1919,7 @@ async function expectDeduplicatedWaiterHandlesRejection( ]) try { - expect({ - branchCount: detachedBranches.length, - statuses: (await detachedOutcomes).map(({ status }) => status), - }).toEqual({ branchCount: 1, statuses: [`fulfilled`] }) + expect(sourceRejectionObservers).toBe(1) } catch (error) { throw new TraceAssertionError(0, error) } @@ -2164,14 +2102,37 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: an empty ordered window issues no transport work`, - expectExactCountFailure( - () => countWindowLoads([{ direction: `asc`, offset: 0, limit: 0 }]), - 1, + it(`an empty ordered window issues no transport work`, () => { + expect(countWindowLoads([{ direction: `asc`, offset: 0, limit: 0 }])).toBe( 0, - ), - ) + ) + }) + + it(`releases a reused zero-window owner without invalidating later coverage`, () => { + let loads = 0 + const dedupe = new DeduplicatedLoadSubset({ + loadSubset: () => { + loads++ + return true + }, + }) + const reusedOptions = toWindowOptions({ + direction: `asc`, + offset: 0, + limit: 0, + }) + + expect(dedupe.loadSubset(reusedOptions)).toBe(true) + reusedOptions.limit = 1 + expect(dedupe.loadSubset(reusedOptions)).toBe(true) + dedupe.unloadSubset(reusedOptions) + expect( + dedupe.loadSubset( + toWindowOptions({ direction: `asc`, offset: 0, limit: 1 }), + ), + ).toBe(true) + expect(loads).toBe(1) + }) it( `discovered trace: an empty filtered window issues no transport work`, @@ -2405,19 +2366,15 @@ describe(`loadSubset coverage oracle`, () => { ), ) - it( - `discovered trace: a composed predicate state forgets one loaded region`, - expectExactCountFailure( - () => - countLoads([ - { kind: `in`, values: [0] }, - { kind: `in`, values: [2] }, - { kind: `eq`, value: 2 }, - ]), - 3, - 2, - ), - ) + it(`retains exact coverage when predicate regions compose`, () => { + expect( + countLoads([ + { kind: `in`, values: [0] }, + { kind: `in`, values: [2] }, + { kind: `eq`, value: 2 }, + ]), + ).toBe(2) + }) it(`rejects repeated transport work for one identical compound predicate`, () => { const predicate: PredicateSpec = { @@ -2666,11 +2623,8 @@ describe(`loadSubset coverage oracle`, () => { }, ) - it(`discovered trace: settled predicate regions cover their union`, async () => { - await expectAssertionFailure(runAsyncScenario, { - checkpoint: 2, - classify: ({ actual, expected }) => actual === true && expected === false, - })({ + it(`settled predicate regions cover their union`, async () => { + await runAsyncScenario({ first: [0], second: [1], firstOutcome: `resolve`, @@ -2688,7 +2642,10 @@ describe(`loadSubset coverage oracle`, () => { runCoverageTraceWithKnownFailures, ) - fcTest.prop([requestTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [requestTraceArbitrary], + coverageRandomParameters(`load-subset.coverage`), + )( `matches finite-domain coverage for a random or replayed seed`, runCoverageTraceWithKnownFailures, ) @@ -2701,7 +2658,10 @@ describe(`loadSubset coverage oracle`, () => { runAsyncScenarioWithKnownFailures, ) - fcTest.prop([asyncScenarioArbitrary], coverageRandomParameters)( + fcTest.prop( + [asyncScenarioArbitrary], + coverageRandomParameters(`load-subset.async-settlement`), + )( `settles, retries, and resets in-flight set requests for a random or replayed seed`, runAsyncScenarioWithKnownFailures, ) @@ -2716,7 +2676,7 @@ describe(`loadSubset coverage oracle`, () => { fcTest.prop( [concurrentAsyncScenarioArbitrary, resultWrapperModeArbitrary], - coverageRandomParameters, + coverageRandomParameters(`load-subset.concurrent-dedupe`), )( `deduplicates three or more concurrent requests for a random or replayed seed`, runConcurrentAsyncScenario, @@ -2730,7 +2690,10 @@ describe(`loadSubset coverage oracle`, () => { expectDeduplicatedWaiterHandlesRejection, ) - fcTest.prop([rejectedWaiterScenarioArbitrary], coverageRandomParameters)( + fcTest.prop( + [rejectedWaiterScenarioArbitrary], + coverageRandomParameters(`load-subset.rejected-waiter`), + )( `checks rejected requests observed by an in-flight waiter for a random or replayed seed`, expectDeduplicatedWaiterHandlesRejection, ) @@ -2743,7 +2706,10 @@ describe(`loadSubset coverage oracle`, () => { runWindowCoverageTraceWithKnownFailures, ) - fcTest.prop([windowTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [windowTraceArbitrary], + coverageRandomParameters(`load-subset.ordered-window`), + )( `never treats uncovered ordered windows as loaded for a random or replayed seed`, runWindowCoverageTraceWithKnownFailures, ) @@ -2756,7 +2722,10 @@ describe(`loadSubset coverage oracle`, () => { runWindowCoverageTraceWithKnownFailures, ) - fcTest.prop([changingWhereWindowTraceArbitrary], coverageRandomParameters)( + fcTest.prop( + [changingWhereWindowTraceArbitrary], + coverageRandomParameters(`load-subset.changing-predicate`), + )( `keeps changing predicates distinct across window histories for a random or replayed seed`, runWindowCoverageTraceWithKnownFailures, ) @@ -2769,7 +2738,10 @@ describe(`loadSubset coverage oracle`, () => { expectDistinctWhereStartsDistinctLimitedWindowLoads, ) - fcTest.prop([distinctWindowWherePairArbitrary], coverageRandomParameters)( + fcTest.prop( + [distinctWindowWherePairArbitrary], + coverageRandomParameters(`load-subset.distinct-window-predicate`), + )( `keeps distinct limited-window predicates separate for a random or replayed seed`, expectDistinctWhereStartsDistinctLimitedWindowLoads, ) @@ -2840,17 +2812,6 @@ describe(`loadSubset coverage oracle`, () => { await expectCleanupRejectsReceiptOnce() }) - it(`publishes synced source rows while a derived mutation persists`, async () => { - await expectAssertionFailure(expectDerivedSyncDuringOptimisticMutation, { - checkpoint: 0, - classify: ({ actual, expected }) => - Array.isArray(actual) && - actual.join(`,`) === `optimistic` && - Array.isArray(expected) && - expected.join(`,`) === `optimistic,synced`, - })() - }) - it( `discovered trace: adjacent ordered windows do not cover their combined window`, expectAssertionFailure( diff --git a/packages/db/tests/query/load-subset-projection-oracle.property.test.ts b/packages/db/tests/query/load-subset-projection-oracle.property.test.ts index d17a396b40..ca721343c8 100644 --- a/packages/db/tests/query/load-subset-projection-oracle.property.test.ts +++ b/packages/db/tests/query/load-subset-projection-oracle.property.test.ts @@ -256,7 +256,10 @@ const projectionScenarioArbitrary = fc return { sourceSize, callerOffset, callerLimit } }) -fcTest.prop([projectionScenarioArbitrary], oraclePropertyOptions(50))( +fcTest.prop( + [projectionScenarioArbitrary], + oraclePropertyOptions(50, `load-subset-projection.state-equivalence`), +)( `projects covering exhaustion relative to a finite source world`, async ({ sourceSize, callerOffset, callerLimit }) => { const rows = Array.from({ length: sourceSize }, (_, id) => ({ id })) diff --git a/packages/db/tests/query/load-subset-refinement-model.property.test.ts b/packages/db/tests/query/load-subset-refinement-model.property.test.ts new file mode 100644 index 0000000000..9282c1e75e --- /dev/null +++ b/packages/db/tests/query/load-subset-refinement-model.property.test.ts @@ -0,0 +1,4022 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { createLiveQueryCollection } from '../../src/query/index.js' +import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' +import { + projectAcquisitionSettlement, + projectAdapterLifecycle, + projectAtomicOrderedPublicationState, + projectAuthorizedContinuationStarts, + projectOrderedPublicationBoundary, + projectReplayPublication, + projectRetainedRowKeys, + projectRetainedSourceRows, + projectReusableDemands, + projectReusableSourceDemands, + projectSourceReadiness, + projectSyncTransactions, + projectTransportLoads, +} from '../load-subset-full-flow-model.js' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' +import { flushPromises } from '../utils.js' +import type { + FullFlowVersionedRow, + LoadSubsetFullFlowEvent, +} from '../load-subset-full-flow-model.js' +import type { LoadSubsetResult } from '../../src/types.js' + +function refinementCampaigns(fixedSeed: number) { + return [ + { + label: `fixed seed ${fixedSeed}`, + options: { numRuns: oracleRuns(50), seed: fixedSeed }, + }, + { + label: `random or replayed seed`, + options: oraclePropertyOptions(50, `load-subset-refinement.${fixedSeed}`), + }, + ] as const +} + +function successfulTransaction( + transactionId: string, + sourceId: string, + rowKey: string, +): Array { + return [ + { + type: `stageSyncTransaction`, + transactionId, + sourceId, + rowKeys: [rowKey], + }, + { + type: `commitSyncTransaction`, + transactionId, + parked: false, + signalAborted: false, + }, + { type: `enterSyncApplication`, transactionId }, + { type: `publishSyncTransaction`, transactionId }, + { type: `settleSyncReceipt`, transactionId }, + ] +} + +for (const campaign of refinementCampaigns(1_779_001)) { + fcTest.prop( + [ + fc.string({ minLength: 1, maxLength: 4 }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( + `commuting independent transactions preserves final public state and receipts (${campaign.label})`, + (leftKey, rightKey) => { + const left = successfulTransaction(`left-tx`, `left-source`, leftKey) + const right = successfulTransaction(`right-tx`, `right-source`, rightKey) + const leftThenRight = projectSyncTransactions([...left, ...right]) + const rightThenLeft = projectSyncTransactions([...right, ...left]) + + // Event-batch order is intentionally observable and may differ. The + // metamorphic law concerns the final independent state and receipts. + expect(leftThenRight.visibleRows).toEqual(rightThenLeft.visibleRows) + expect(leftThenRight.receipts).toEqual(rightThenLeft.receipts) + }, + ) +} + +type DemandLifecycleCase = { + history: Array + expected: Array<{ + type: `invoke` | `release` + ownerId: string + sourceId: string + attemptId: string + }> +} + +function enumerateDemandLifecycles(): Array { + const cases: Array = [] + const visit = ( + history: Array, + expected: DemandLifecycleCase[`expected`], + unseenOwners: ReadonlyArray, + activeOwners: ReadonlyArray, + ) => { + cases.push({ history, expected }) + if (history.length === 4) return + + for (const ownerId of unseenOwners) { + for (const alreadyAborted of [false, true]) { + visit( + [ + ...history, + { + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session`, + demandId: `demand`, + attemptId: `${ownerId}-attempt`, + alreadyAborted, + }, + ], + alreadyAborted + ? expected + : [ + ...expected, + { + type: `invoke`, + ownerId, + sourceId: `source`, + attemptId: `${ownerId}-attempt`, + }, + ], + unseenOwners.filter((owner) => owner !== ownerId), + alreadyAborted ? activeOwners : [...activeOwners, ownerId], + ) + } + } + for (const ownerId of activeOwners) { + visit( + [ + ...history, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId, + demandId: `demand`, + attemptId: `${ownerId}-attempt`, + }, + ], + [ + ...expected, + { + type: `release`, + ownerId, + sourceId: `source`, + attemptId: `${ownerId}-attempt`, + }, + ], + unseenOwners, + activeOwners.filter((owner) => owner !== ownerId), + ) + } + } + + visit([], [], [`owner-a`, `owner-b`], []) + return cases +} + +it(`exhaustively projects exact adapter starts and releases for two owners`, () => { + for (const { history, expected } of enumerateDemandLifecycles()) { + const lifecycle = projectAdapterLifecycle(history) + expect(lifecycle, JSON.stringify(history)).toEqual(expected) + const activeAttempts = new Set() + + for (const event of lifecycle) { + if (event.type === `invoke`) { + expect( + activeAttempts.has(event.attemptId), + JSON.stringify(history), + ).toBe(false) + activeAttempts.add(event.attemptId) + } else { + expect( + activeAttempts.delete(event.attemptId), + JSON.stringify(history), + ).toBe(true) + } + } + } +}) + +it(`shares concurrent exact demand and retries after evidence-free settlement`, () => { + const request = ( + ownerId: string, + attemptId = `${ownerId}-attempt`, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session`, + demandId: `exact-demand`, + attemptId, + alreadyAborted: false, + }) + const concurrent = [request(`owner-a`), request(`owner-b`)] + + expect(projectTransportLoads(concurrent)).toBe( + projectAcquisitionSettlement(acquisitionHistory(`shared`, [`row`])) + .physicalStarts.length, + ) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `exact-demand`, + attemptId: `owner-a-attempt`, + }, + request(`owner-c`), + ]), + ).toBe(1) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `exact-demand`, + attemptId: `owner-a-attempt`, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-b`, + demandId: `exact-demand`, + attemptId: `owner-b-attempt`, + }, + request(`owner-c`), + ]), + ).toBe(2) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `settleDemandWithoutEvidence`, + sourceId: `source`, + demandId: `exact-demand`, + attemptId: `owner-a-attempt`, + }, + request(`owner-c`), + ]), + ).toBe(2) + expect( + projectTransportLoads([ + ...concurrent, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `exact-demand`, + attemptId: `owner-a-attempt`, + rowKeys: [`row`], + }, + request(`owner-c`), + ]), + ).toBe(1) +}) + +it(`scopes identical demand attempts, rows, and evidence to their source`, () => { + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + rowKeys: [`shared-row`], + }) + const sourceASettled = [ + request(`source-a`), + request(`source-b`), + settle(`source-a`), + ] + + expect(projectTransportLoads(sourceASettled)).toBe(2) + expect(projectRetainedSourceRows(sourceASettled)).toEqual([ + { sourceId: `source-a`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(sourceASettled)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + ]) + + const bothSettled = [...sourceASettled, settle(`source-b`)] + expect(projectRetainedSourceRows(bothSettled)).toEqual([ + { sourceId: `source-a`, rowKey: `shared-row` }, + { sourceId: `source-b`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(bothSettled)).toEqual([ + { sourceId: `source-a`, demandId: `shared-demand` }, + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) + + const sourceATruncated = [ + ...bothSettled, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: `source-a`, + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(sourceATruncated)).toEqual([ + { sourceId: `source-b`, rowKey: `shared-row` }, + ]) + expect(projectReusableSourceDemands(sourceATruncated)).toEqual([ + { sourceId: `source-b`, demandId: `shared-demand` }, + ]) +}) + +it(`fences stale same-source settlement from a fresh demand generation`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + ...oldRequest, + attemptId: `fresh-attempt`, + } + const beforeFreshSettlement: ReadonlyArray = [ + oldRequest, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, + freshRequest, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + ] + + expect(projectTransportLoads(beforeFreshSettlement)).toBe(2) + expect(projectRetainedSourceRows(beforeFreshSettlement)).toEqual([ + { sourceId: `source`, rowKey: `stale-row` }, + ]) + expect(projectReusableSourceDemands(beforeFreshSettlement)).toEqual([]) + + const oldReleased = [ + ...beforeFreshSettlement, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `old-attempt`, + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(oldReleased)).toEqual([]) + expect(projectReusableSourceDemands(oldReleased)).toEqual([]) + + const freshSettled = [ + ...oldReleased, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `fresh-attempt`, + rowKeys: [`fresh-row`], + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectRetainedSourceRows(freshSettled)).toEqual([ + { sourceId: `source`, rowKey: `fresh-row` }, + ]) + expect(projectReusableSourceDemands(freshSettled)).toEqual([ + { sourceId: `source`, demandId: `demand` }, + ]) +}) + +for (const campaign of refinementCampaigns(1_779_010)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ maxLength: 6 }), { + minLength: 2, + maxLength: 2, + }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + ], + campaign.options, + )( + `source identity scopes equal demand histories (${campaign.label})`, + (sourceIds, demandId, attemptId, rowKey) => { + const [sourceA, sourceB] = sourceIds as [string, string] + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const settle = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId, + rowKeys: [rowKey], + }) + const settled = [ + request(sourceA), + request(sourceB), + settle(sourceA), + settle(sourceB), + ] + const surviving = [ + ...settled, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: sourceA, + } satisfies LoadSubsetFullFlowEvent, + ] + + expect(projectTransportLoads(settled)).toBe(2) + expect(projectRetainedSourceRows(settled)).toEqual( + [sourceA, sourceB] + .sort((left, right) => left.localeCompare(right)) + .map((sourceId) => ({ sourceId, rowKey })), + ) + expect(projectRetainedSourceRows(surviving)).toEqual([ + { sourceId: sourceB, rowKey }, + ]) + expect(projectReusableSourceDemands(surviving)).toEqual([ + { sourceId: sourceB, demandId }, + ]) + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_011)) { + fcTest.prop( + [ + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + fc.uniqueArray(fc.string({ maxLength: 6 }), { + minLength: 2, + maxLength: 2, + }), + fc.string({ maxLength: 6 }), + fc.string({ maxLength: 6 }), + ], + campaign.options, + )( + `truncate fences stale settlement from the next demand generation (${campaign.label})`, + (sourceId, demandId, attemptIds, staleRowKey, freshRowKey) => { + const [oldAttemptId, freshAttemptId] = attemptIds as [string, string] + const request = (attemptId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const oldSettlesThenReleases: ReadonlyArray = [ + request(oldAttemptId), + { type: `truncateSource`, sessionId: `session`, sourceId }, + request(freshAttemptId), + { + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: oldAttemptId, + rowKeys: [staleRowKey], + }, + { + type: `releaseDemand`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: oldAttemptId, + }, + ] + const freshSettles = [ + ...oldSettlesThenReleases, + { + type: `applyAuthoritativeRows`, + sourceId, + ownerId: `owner`, + demandId, + attemptId: freshAttemptId, + rowKeys: [freshRowKey], + } satisfies LoadSubsetFullFlowEvent, + ] + + expect(projectTransportLoads(oldSettlesThenReleases)).toBe(2) + expect(projectRetainedSourceRows(oldSettlesThenReleases)).toEqual([]) + expect(projectReusableSourceDemands(oldSettlesThenReleases)).toEqual([]) + expect(projectRetainedSourceRows(freshSettles)).toEqual([ + { sourceId, rowKey: freshRowKey }, + ]) + expect(projectReusableSourceDemands(freshSettles)).toEqual([ + { sourceId, demandId }, + ]) + }, + ) +} + +type LegalOrderAction = + | `release-old` + | `release-peer` + | `settle-old` + | `settle-fresh` + +function interleaveLegalOrderChains( + left: ReadonlyArray, + right: ReadonlyArray, +): Array> { + if (left.length === 0) return [[...right]] + if (right.length === 0) return [[...left]] + + return [ + ...interleaveLegalOrderChains(left.slice(1), right).map((suffix) => [ + left[0]!, + ...suffix, + ]), + ...interleaveLegalOrderChains(left, right.slice(1)).map((suffix) => [ + right[0]!, + ...suffix, + ]), + ] +} + +function legalOrderBaseHistory(): Array { + return [ + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-old`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-peer`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-peer`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-b`, + ownerId: `owner-peer`, + demandId: `shared`, + attemptId: `attempt-peer`, + rowKeys: [`peer-row`], + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + outcome: `resolve`, + }, + { + type: `stagePublicationRows`, + publicationId: `initial-publication`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `old-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `initial-publication`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `peer-row`, orderValue: 2 }], + }, + { type: `commitPublication`, publicationId: `initial-publication` }, + { + type: `stagePublicationRows`, + publicationId: `old-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `obsolete-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `stale-row`, orderValue: 1 }], + }, + { + type: `beginReplacement`, + publicationId: `old-replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source-a` }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-fresh`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-fresh`, + alreadyAborted: false, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-fresh`, + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `fresh-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `fresh-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `fresh-replacement`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `peer-row`, orderValue: 2 }], + }, + { + type: `beginReplacement`, + publicationId: `fresh-replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + { sourceId: `source-b`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `source-b`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] +} + +function legalOrderEvents( + action: LegalOrderAction, + oldOutcome: `resolve` | `reject`, +): Array { + switch (action) { + case `release-old`: + return [ + { + type: `releaseDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + ] + case `release-peer`: + return [ + { + type: `releaseDemand`, + sourceId: `source-b`, + ownerId: `owner-peer`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared`, + attemptId: `attempt-peer`, + }, + ] + case `settle-old`: + return [ + oldOutcome === `resolve` + ? { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + rowKeys: [`stale-row`], + } + : { + type: `rejectDemand`, + sourceId: `source-a`, + ownerId: `owner-old`, + demandId: `shared`, + attemptId: `attempt-old`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-old`, + outcome: oldOutcome, + }, + oldOutcome === `resolve` + ? { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + } + : { + type: `settleReplacement`, + publicationId: `old-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `failure`, + }, + ] + case `settle-fresh`: + return [ + { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner-fresh`, + demandId: `shared`, + attemptId: `attempt-fresh`, + rowKeys: [`fresh-row`], + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared`, + attemptId: `attempt-fresh`, + outcome: `resolve`, + }, + { + type: `settleReplacement`, + publicationId: `fresh-replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] + } +} + +it(`enumerates legal release and settlement orders across every refinement projection`, () => { + const publicationOptions = { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc` as const, + initialWindowSize: 1, + } + + for (const releaseOrder of [ + [`release-old`, `release-peer`], + [`release-peer`, `release-old`], + ] as const) { + for (const settlementOrder of [ + [`settle-old`, `settle-fresh`], + [`settle-fresh`, `settle-old`], + ] as const) { + for (const actions of interleaveLegalOrderChains( + releaseOrder, + settlementOrder, + )) { + for (const oldOutcome of [`resolve`, `reject`] as const) { + const concreteSteps = actions.flatMap((action) => + legalOrderEvents(action, oldOutcome).map((event) => ({ + action, + event, + })), + ) + for ( + let prefixLength = 0; + prefixLength <= concreteSteps.length; + prefixLength++ + ) { + const prefix = concreteSteps.slice(0, prefixLength) + const prefixEvents = prefix.map(({ event }) => event) + const history = [...legalOrderBaseHistory(), ...prefixEvents] + const diagnostic = JSON.stringify({ + releaseOrder, + settlementOrder, + actions, + oldOutcome, + prefixLength, + prefix: prefix.map(({ action, event }) => ({ + action, + event: event.type, + })), + }) + const eventIndex = ( + predicate: (event: LoadSubsetFullFlowEvent) => boolean, + ) => prefixEvents.findIndex(predicate) + const oldReleased = eventIndex( + (event) => + event.type === `releaseDemand` && + event.attemptId === `attempt-old`, + ) + const peerReleased = eventIndex( + (event) => + event.type === `releaseDemand` && + event.attemptId === `attempt-peer`, + ) + const oldRowsApplied = eventIndex( + (event) => + event.type === `applyAuthoritativeRows` && + event.attemptId === `attempt-old`, + ) + const freshRowsApplied = eventIndex( + (event) => + event.type === `applyAuthoritativeRows` && + event.attemptId === `attempt-fresh`, + ) + const freshSourceSettled = eventIndex( + (event) => + event.type === `settleSourceDemand` && + event.attemptId === `attempt-fresh`, + ) + const oldReplacementSettled = eventIndex( + (event) => + event.type === `settleReplacement` && + event.publicationId === `old-replacement` && + event.sourceId === `source-a`, + ) + const freshReplacementSettled = eventIndex( + (event) => + event.type === `settleReplacement` && + event.publicationId === `fresh-replacement` && + event.sourceId === `source-a`, + ) + const replacementComplete = + oldReplacementSettled >= 0 && freshReplacementSettled >= 0 + const replacementCompletionIndex = Math.max( + oldReplacementSettled, + freshReplacementSettled, + ) + const expectedRows = [ + ...(freshRowsApplied >= 0 + ? [{ sourceId: `source-a`, rowKey: `fresh-row` }] + : []), + ...(oldRowsApplied >= 0 && oldReleased < 0 + ? [{ sourceId: `source-a`, rowKey: `stale-row` }] + : []), + ...(peerReleased < 0 + ? [{ sourceId: `source-b`, rowKey: `peer-row` }] + : []), + ] + const expectedEvidence = [ + ...(freshRowsApplied >= 0 + ? [{ sourceId: `source-a`, demandId: `shared` }] + : []), + ...(peerReleased < 0 + ? [{ sourceId: `source-b`, demandId: `shared` }] + : []), + ] + const oldOrderedRow = { + key: `old-ordered-row`, + orderValue: 0, + } + const freshOrderedRow = { + key: `fresh-ordered-row`, + orderValue: 0, + } + const freshRow = { key: `fresh-row`, orderValue: 1 } + const peerRow = { key: `peer-row`, orderValue: 2 } + const initialPublication = [oldOrderedRow, peerRow] + const publicationTransitions: Array<{ + index: number + rows: Array<{ key: string; orderValue: number }> + }> = [] + if (replacementComplete) { + publicationTransitions.push({ + index: replacementCompletionIndex, + rows: [ + freshOrderedRow, + freshRow, + ...(peerReleased < 0 || + peerReleased > replacementCompletionIndex + ? [peerRow] + : []), + ], + }) + } + if (peerReleased >= 0) { + publicationTransitions.push({ + index: peerReleased, + rows: + replacementComplete && + replacementCompletionIndex < peerReleased + ? [freshOrderedRow, freshRow] + : [oldOrderedRow], + }) + } + publicationTransitions.sort( + (left, right) => left.index - right.index, + ) + const expectedPublications = [ + initialPublication, + ...publicationTransitions.map(({ rows }) => rows), + ] + const expectedCurrentRows = expectedPublications.at(-1)! + const expectedOrderedBoundary = replacementComplete + ? freshOrderedRow + : oldOrderedRow + + expect(projectTransportLoads(history), diagnostic).toBe(3) + expect(projectRetainedSourceRows(history), diagnostic).toEqual( + expectedRows, + ) + expect(projectReusableSourceDemands(history), diagnostic).toEqual( + expectedEvidence, + ) + expect(projectSourceReadiness(history), diagnostic).toEqual({ + status: freshSourceSettled >= 0 ? `ready` : `loading`, + pendingSources: freshSourceSettled >= 0 ? [] : [`source-a`], + failedSources: [], + }) + const publication = projectAtomicOrderedPublicationState( + history, + publicationOptions, + ) + expect(publication, diagnostic).toEqual({ + publications: expectedPublications, + currentPublication: { + rows: expectedCurrentRows, + orderedPrefixSize: 1, + orderedBoundary: expectedOrderedBoundary, + }, + retainsPreviousPublication: !replacementComplete, + }) + } + } + } + } + } +}) + +it(`retains a row until its last independent demand claim releases`, () => { + const request = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId: attemptId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const apply = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `applyUnprovenRows`, + sourceId: `source`, + ownerId: attemptId, + demandId, + attemptId, + rowKeys: [`x`], + }) + const release = ( + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + sourceId: `source`, + ownerId: attemptId, + demandId, + attemptId, + }) + const sharedClaims = [ + request(`left`, `left-attempt`), + request(`right`, `right-attempt`), + apply(`left`, `left-attempt`), + apply(`right`, `right-attempt`), + ] + + expect( + projectRetainedRowKeys([...sharedClaims, release(`left`, `left-attempt`)]), + ).toEqual([`x`]) + expect( + projectRetainedRowKeys([ + ...sharedClaims, + release(`left`, `left-attempt`), + release(`right`, `right-attempt`), + ]), + ).toEqual([]) +}) + +it(`attaches late rows only to attempts that shared the settling acquisition`, () => { + const request = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session`, + demandId: `shared`, + attemptId, + alreadyAborted: false, + }) + const release = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + sourceId: `source`, + ownerId, + demandId: `shared`, + attemptId, + }) + const lateSettlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `shared`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + } + const oldRequest = request(`old-owner`, `old-attempt`) + const oldRelease = release(`old-owner`, `old-attempt`) + + const freshCohort = [ + oldRequest, + oldRelease, + request(`fresh-owner`, `fresh-attempt`), + lateSettlement, + ] + expect(projectRetainedRowKeys(freshCohort)).toEqual([]) + expect(projectReusableDemands(freshCohort)).toEqual([]) + + const attachedPeer = [ + oldRequest, + request(`peer-owner`, `peer-attempt`), + oldRelease, + lateSettlement, + ] + expect(projectRetainedRowKeys(attachedPeer)).toEqual([`stale-row`]) + expect(projectReusableDemands(attachedPeer)).toEqual([`shared`]) +}) + +it(`retires an ownerless acquisition without disturbing another cohort for the same demand`, () => { + const demandId = `shared` + const request = (attemptId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId: attemptId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }) + const history: ReadonlyArray = [ + request(`attempt-a`), + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, + request(`attempt-b`), + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `attempt-b`, + demandId, + attemptId: `attempt-b`, + }, + request(`attempt-c`), + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `attempt-b`, + demandId, + attemptId: `attempt-b`, + rowKeys: [`stale-b`], + }, + ] + + expect(projectTransportLoads(history)).toBe(3) + expect(projectRetainedRowKeys(history)).toEqual([]) + expect(projectReusableDemands(history)).toEqual([]) + + const survivingAcquisitionSettles = [ + ...history, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `attempt-a`, + demandId, + attemptId: `attempt-a`, + rowKeys: [`live-a`], + } satisfies LoadSubsetFullFlowEvent, + ] + expect(projectTransportLoads(survivingAcquisitionSettles)).toBe(3) + expect(projectRetainedRowKeys(survivingAcquisitionSettles)).toEqual([ + `live-a`, + ]) + expect(projectReusableDemands(survivingAcquisitionSettles)).toEqual([]) +}) + +it.each([ + { + name: `one owner releases the first attempt first`, + owners: [`owner`, `owner`] as const, + releaseOrder: [0, 1] as const, + }, + { + name: `one owner releases the second attempt first`, + owners: [`owner`, `owner`] as const, + releaseOrder: [1, 0] as const, + }, + { + name: `two owners release the first attempt first`, + owners: [`owner-a`, `owner-b`] as const, + releaseOrder: [0, 1] as const, + }, + { + name: `two owners release the second attempt first`, + owners: [`owner-a`, `owner-b`] as const, + releaseOrder: [1, 0] as const, + }, +])(`derives shared ownership for $name`, ({ owners, releaseOrder }) => { + const demandId = `shared` + const attempts = owners.map((ownerId, index) => ({ + ownerId, + attemptId: `attempt-${index}`, + })) + const requests = attempts.map( + ({ ownerId, attemptId }) => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session`, + demandId, + attemptId, + alreadyAborted: false, + }), + ) + const settlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: attempts[0]!.ownerId, + demandId, + attemptId: attempts[0]!.attemptId, + rowKeys: [`x`], + } + const releases = releaseOrder.map((index) => ({ + type: `releaseDemand`, + sourceId: `source`, + ownerId: attempts[index]!.ownerId, + demandId, + attemptId: attempts[index]!.attemptId, + })) + + for (let released = 0; released <= releases.length; released++) { + const active = released < releases.length + const history = [...requests, settlement, ...releases.slice(0, released)] + const lifecycle = projectAdapterLifecycle(history) + + expect(lifecycle.filter(({ type }) => type === `invoke`)).toHaveLength(2) + expect(lifecycle.filter(({ type }) => type === `release`)).toHaveLength( + released, + ) + expect(projectRetainedRowKeys(history)).toEqual(active ? [`x`] : []) + expect(projectReusableDemands(history)).toEqual(active ? [demandId] : []) + const peerRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `peer`, + sessionId: `session`, + demandId, + attemptId: `peer-after-${released}`, + alreadyAborted: false, + } + expect(projectRetainedRowKeys([...history, peerRequest])).toEqual( + active ? [`x`] : [], + ) + expect(projectTransportLoads([...history, peerRequest])).toBe( + active ? 1 : 2, + ) + + const publication = projectAtomicOrderedPublicationState( + [ + ...history, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source`, + demandId: `ordered`, + rows: [{ key: `o`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source`, + demandId, + rows: [{ key: `x`, orderValue: 1 }], + }, + { type: `commitPublication`, publicationId: `publication` }, + ], + { + sourceId: `source`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }, + ) + expect(publication.currentPublication?.rows.map(({ key }) => key)).toEqual( + active ? [`o`, `x`] : [`o`], + ) + } + + const lateSharedSettlement = [ + requests[0]!, + requests[1]!, + releases[0]!, + settlement, + ] + expect(projectRetainedRowKeys(lateSharedSettlement)).toEqual([`x`]) + expect(projectReusableDemands(lateSharedSettlement)).toEqual([demandId]) + + const fullyReleasedBeforeSettlement = [ + ...requests, + ...releases, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId, + attemptId: `fresh-attempt`, + alreadyAborted: false, + } satisfies LoadSubsetFullFlowEvent, + settlement, + ] + expect(projectRetainedRowKeys(fullyReleasedBeforeSettlement)).toEqual([]) + expect(projectReusableDemands(fullyReleasedBeforeSettlement)).toEqual([]) + expect(projectTransportLoads(fullyReleasedBeforeSettlement)).toBe(2) +}) + +it(`keeps a same-name publication demand active on its surviving source`, () => { + const request = (sourceId: string): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId, + ownerId: `owner`, + sessionId: `session`, + demandId: `shared`, + attemptId: `same-attempt`, + alreadyAborted: false, + }) + const projection = projectAtomicOrderedPublicationState( + [ + request(`source-a`), + request(`source-b`), + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `other-ordered-source`, + demandId: `ordered`, + rows: [{ key: `wrong-ordered-row`, orderValue: -1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `source-b-row`, orderValue: 2 }], + }, + { + type: `releaseDemand`, + sourceId: `source-a`, + ownerId: `owner`, + demandId: `shared`, + attemptId: `same-attempt`, + }, + { type: `commitPublication`, publicationId: `publication` }, + ], + { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }, + ) + + expect(projection.currentPublication?.rows.map(({ key }) => key)).toEqual([ + `ordered-row`, + `source-b-row`, + ]) +}) + +it(`treats a same-name demand from another source as additional`, () => { + const history: Array = [ + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `ordered`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `ordered`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `ordered`, + rows: [{ key: `source-b-row`, orderValue: 2 }], + }, + { type: `commitPublication`, publicationId: `publication` }, + ] + + expect( + projectAtomicOrderedPublicationState(history, { + sourceId: `source-a`, + demandId: `ordered`, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`source-a-row`, `source-b-row`]) +}) + +it(`settles same-name replacement demands independently by source`, () => { + const history: Array = [ + { + type: `stagePublicationRows`, + publicationId: `initial`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `old-ordered-row`, orderValue: 0 }], + }, + { type: `commitPublication`, publicationId: `initial` }, + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner-a`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-a`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner-b`, + sessionId: `session`, + demandId: `shared`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + rows: [{ key: `new-ordered-row`, orderValue: 0 }], + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `source-a`, + demandId: `shared`, + rows: [{ key: `source-a-row`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + sourceId: `source-b`, + demandId: `shared`, + rows: [{ key: `source-b-row`, orderValue: 2 }], + }, + { + type: `beginReplacement`, + publicationId: `replacement`, + demands: [ + { sourceId: `ordered-source`, demandId: `ordered` }, + { sourceId: `source-a`, demandId: `shared` }, + { sourceId: `source-b`, demandId: `shared` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `ordered-source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `source-a`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }, + ] + const options = { + sourceId: `ordered-source`, + demandId: `ordered`, + direction: `asc` as const, + initialWindowSize: 1, + } + + expect( + projectAtomicOrderedPublicationState( + history, + options, + ).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`old-ordered-row`]) + + history.push({ + type: `settleReplacement`, + publicationId: `replacement`, + sourceId: `source-b`, + demandId: `shared`, + outcome: `success`, + extent: `exhausted`, + }) + expect( + projectAtomicOrderedPublicationState( + history, + options, + ).currentPublication?.rows.map(({ key }) => key), + ).toEqual([`new-ordered-row`, `source-a-row`, `source-b-row`]) +}) + +it.each([`a-first`, `b-first`] as const)( + `keeps ordered boundaries source-qualified when staged %s`, + (stageOrder) => { + const stages: Array = [ + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-a`, + demandId: `ordered`, + rows: [{ key: `row-a`, orderValue: 1 }], + }, + { + type: `stagePublicationRows`, + publicationId: `publication`, + sourceId: `source-b`, + demandId: `ordered`, + rows: [{ key: `row-b`, orderValue: 2 }], + }, + ] + if (stageOrder === `b-first`) stages.reverse() + const history = [ + ...stages, + { type: `commitPublication`, publicationId: `publication` } as const, + ] + const boundary = (sourceId: string) => + projectOrderedPublicationBoundary(history, { + sourceId, + demandId: `ordered`, + direction: `asc`, + prefixSize: 1, + })?.key + + expect(boundary(`source-a`)).toBe(`row-a`) + expect(boundary(`source-b`)).toBe(`row-b`) + }, +) + +it(`applies target events only to their named source and demand`, () => { + const target = { sourceId: `source-a`, demandId: `ordered` } as const + const base: Array = [ + { + type: `stagePublicationRows`, + publicationId: `initial`, + ...target, + rows: [{ key: `old-row`, orderValue: 0 }], + }, + { type: `commitPublication`, publicationId: `initial` }, + { + type: `stagePublicationRows`, + publicationId: `replacement`, + ...target, + rows: [ + { key: `new-row-a`, orderValue: 1 }, + { key: `new-row-b`, orderValue: 2 }, + ], + }, + { + type: `beginReplacement`, + publicationId: `replacement`, + demands: [target], + }, + ] + const settle: LoadSubsetFullFlowEvent = { + type: `settleReplacement`, + publicationId: `replacement`, + ...target, + outcome: `success`, + extent: `continues`, + } + const establish = ( + sourceId: string, + demandId = `ordered`, + publicationId = `replacement`, + ): LoadSubsetFullFlowEvent => ({ + type: `establishReplacementCoverage`, + publicationId, + sourceId, + demandId, + }) + const resize = ( + sourceId: string, + demandId = `ordered`, + ): LoadSubsetFullFlowEvent => ({ + type: `resizeOrderedWindow`, + sourceId, + demandId, + size: 2, + }) + const rows = (history: ReadonlyArray) => + projectAtomicOrderedPublicationState(history, { + ...target, + direction: `asc`, + initialWindowSize: 1, + }).currentPublication?.rows.map(({ key }) => key) + + expect(rows([...base, settle, establish(`source-b`)])).toEqual([`old-row`]) + expect(rows([...base, settle, establish(`source-a`, `other`)])).toEqual([ + `old-row`, + ]) + expect( + rows([...base, settle, establish(`source-a`, `ordered`, `obsolete`)]), + ).toEqual([`old-row`]) + expect(rows([...base, settle, establish(`source-a`)])).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-b`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-a`, `other`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`]) + expect( + rows([...base, resize(`source-a`), settle, establish(`source-a`)]), + ).toEqual([`new-row-a`, `new-row-b`]) +}) + +it.each([ + { + name: `authoritative`, + event: { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + }, + { + name: `unproven`, + event: { + type: `applyUnprovenRows`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + }, + { + name: `rejected`, + event: { + type: `rejectDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + }, + { + name: `evidence-free`, + event: { + type: `settleDemandWithoutEvidence`, + sourceId: `source`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + }, + { + name: `released`, + event: { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + }, +] satisfies ReadonlyArray<{ + name: string + event: LoadSubsetFullFlowEvent +}>)( + `keeps fresh same-demand work shared when an old attempt is $name after truncate`, + ({ event }) => { + expect( + projectTransportLoads([ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + }, + { type: `truncateSource`, sessionId: `session`, sourceId: `source` }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + }, + event, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `peer-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `peer-attempt`, + alreadyAborted: false, + }, + ]), + ).toBe(2) + }, +) + +it(`scopes reusable evidence to the physical attempt when an owner is reused`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `stable-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + ...oldRequest, + attemptId: `fresh-attempt`, + } + const oldSettlement: LoadSubsetFullFlowEvent = { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `stable-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + } + const freshSettlement: LoadSubsetFullFlowEvent = { + ...oldSettlement, + attemptId: `fresh-attempt`, + rowKeys: [`fresh-row`], + } + const staleRelease: LoadSubsetFullFlowEvent = { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `stable-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + } + const beforeFreshSettlement = [ + oldRequest, + { + type: `truncateSource`, + sessionId: `session`, + sourceId: `source`, + } as const, + freshRequest, + oldSettlement, + ] + + expect(projectReusableDemands(beforeFreshSettlement)).toEqual([]) + expect( + projectReusableDemands([...beforeFreshSettlement, freshSettlement]), + ).toEqual([`exact-demand`]) + expect( + projectReusableDemands([ + ...beforeFreshSettlement, + freshSettlement, + staleRelease, + ]), + ).toEqual([`exact-demand`]) + expect( + projectTransportLoads([ + ...beforeFreshSettlement, + freshSettlement, + staleRelease, + { + ...freshRequest, + ownerId: `peer-owner`, + attemptId: `peer-attempt`, + }, + ]), + ).toBe(2) +}) + +it(`does not rebuild coverage when a released attempt settles after its replacement starts`, () => { + expect( + projectReusableDemands([ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + rowKeys: [`stale-row`], + }, + ]), + ).toEqual([]) +}) + +it(`keeps fresh same-epoch work shared after an older rejected attempt releases`, () => { + const oldRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + alreadyAborted: false, + } + const freshRequest: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `fresh-attempt`, + alreadyAborted: false, + } + + expect( + projectTransportLoads([ + oldRequest, + { + type: `rejectDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + freshRequest, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `old-attempt`, + }, + { + ...freshRequest, + ownerId: `peer-owner`, + attemptId: `peer-attempt`, + }, + ]), + ).toBe(2) +}) + +it(`rejects histories that reuse one demand attempt identity`, () => { + const history: Array = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `old-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + alreadyAborted: false, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `fresh-owner`, + sessionId: `session`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `old-owner`, + demandId: `exact-demand`, + attemptId: `reused-attempt`, + rowKeys: [`stale-row`], + }, + ] + + expect(() => projectTransportLoads(history)).toThrow( + `Demand attempt "reused-attempt" was requested more than once`, + ) + expect(() => projectReusableDemands(history)).toThrow( + `Demand attempt "reused-attempt" was requested more than once`, + ) +}) + +it(`rejects histories that settle one demand attempt twice`, () => { + const history: Array = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `settleDemandWithoutEvidence`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt`, + }, + { + type: `rejectDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + }, + ] + + expect(() => projectTransportLoads(history)).toThrow( + `Demand attempt "attempt" settled more than once`, + ) + expect(() => projectReusableDemands(history)).toThrow( + `Demand attempt "attempt" settled more than once`, + ) +}) + +function renameHistoryIds( + history: ReadonlyArray, + suffix: string, +): Array { + return history.map((event) => { + switch (event.type) { + case `requestDemand`: + return { + ...event, + ownerId: `${event.ownerId}-${suffix}`, + sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, + } + case `applyAuthoritativeRows`: + case `applyUnprovenRows`: + case `rejectDemand`: + case `releaseDemand`: + return { + ...event, + ownerId: `${event.ownerId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, + } + case `truncateSource`: + return { + ...event, + sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + } + case `settleDemandWithoutEvidence`: + return { + ...event, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, + } + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + return { + ...event, + sessionId: `${event.sessionId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + attemptId: `${event.attemptId}-${suffix}`, + } + case `cleanupSession`: + return { ...event, sessionId: `${event.sessionId}-${suffix}` } + case `restartSession`: + return { + ...event, + previousSessionId: `${event.previousSessionId}-${suffix}`, + nextSessionId: `${event.nextSessionId}-${suffix}`, + } + case `advanceWindowRevision`: + return { ...event, sessionId: `${event.sessionId}-${suffix}` } + case `scheduleContinuation`: + return { + ...event, + taskId: `${event.taskId}-${suffix}`, + sessionId: `${event.sessionId}-${suffix}`, + } + case `runContinuation`: + return { ...event, taskId: `${event.taskId}-${suffix}` } + case `stageSyncTransaction`: + return { + ...event, + transactionId: `${event.transactionId}-${suffix}`, + } + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + return { + ...event, + transactionId: `${event.transactionId}-${suffix}`, + } + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + return { + ...event, + attemptId: `${event.attemptId}-${suffix}`, + } + case `startAcquisition`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `attachAcquisitionOwner`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + ownerId: `${event.ownerId}-${suffix}`, + } + case `settleAcquisition`: + return { + ...event, + acquisitionId: `${event.acquisitionId}-${suffix}`, + } + case `stagePublicationRows`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `commitPublication`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + } + case `establishReplacementCoverage`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `beginReplacement`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + demands: event.demands.map(({ sourceId, demandId }) => ({ + sourceId: `${sourceId}-${suffix}`, + demandId: `${demandId}-${suffix}`, + })), + } + case `settleReplacement`: + return { + ...event, + publicationId: `${event.publicationId}-${suffix}`, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + case `resizeOrderedWindow`: + return { + ...event, + sourceId: `${event.sourceId}-${suffix}`, + demandId: `${event.demandId}-${suffix}`, + } + default: + return event + } + }) +} + +function expectObservationPreservedAfterEveryPrefix( + history: ReadonlyArray, + suffix: string, + project: ( + prefix: ReadonlyArray, + suffix: string, + ) => T, + normalize: (observation: T, suffix: string) => unknown = (observation) => + observation, +): void { + for (let prefixLength = 0; prefixLength <= history.length; prefixLength++) { + const prefix = history.slice(0, prefixLength) + expect( + normalize(project(renameHistoryIds(prefix, suffix), suffix), suffix), + JSON.stringify({ prefixLength, prefix }), + ).toEqual(normalize(project(prefix, ``), ``)) + } +} + +function removeRenamingSuffix(value: string, suffix: string): string { + const marker = `-${suffix}` + return suffix !== `` && value.endsWith(marker) + ? value.slice(0, -marker.length) + : value +} + +function normalizeSourceReadiness( + observation: ReturnType, + suffix: string, +) { + return { + ...observation, + pendingSources: observation.pendingSources.map((sourceId) => + removeRenamingSuffix(sourceId, suffix), + ), + failedSources: observation.failedSources.map((sourceId) => + removeRenamingSuffix(sourceId, suffix), + ), + } +} + +it(`settles source readiness by exact demand attempt`, () => { + const pendingReplacement: ReadonlyArray = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-a`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-b`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-a`, + outcome: `resolve`, + }, + ] + expect(projectSourceReadiness(pendingReplacement)).toEqual({ + status: `loading`, + pendingSources: [`source`], + failedSources: [], + }) + expect( + projectSourceReadiness([ + ...pendingReplacement, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source`, + demandId: `demand`, + attemptId: `attempt-b`, + outcome: `resolve`, + }, + ]), + ).toEqual({ status: `ready`, pendingSources: [], failedSources: [] }) +}) + +it(`retires source demand attempts without crossing source identity`, () => { + const survivingSource: ReadonlyArray = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `retireSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + outcome: `reject`, + }, + ] + expect(projectSourceReadiness(survivingSource)).toEqual({ + status: `loading`, + pendingSources: [`source-b`], + failedSources: [], + }) + expect( + projectSourceReadiness([ + ...survivingSource, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `shared-demand`, + attemptId: `shared-attempt`, + outcome: `resolve`, + }, + ]), + ).toEqual({ status: `ready`, pendingSources: [], failedSources: [] }) +}) + +for (const campaign of refinementCampaigns(1_779_002)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `source demand names are observationally erased (${campaign.label})`, + (suffix) => { + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + { + type: `registerSourceDemand`, + sessionId: `session`, + sourceId: `source-b`, + demandId: `demand-b`, + attemptId: `attempt-b`, + }, + { + type: `settleSourceDemand`, + sessionId: `session`, + sourceId: `source-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + outcome: `resolve`, + }, + ] + + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSourceReadiness, + normalizeSourceReadiness, + ) + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_003)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `demand, attempt, owner, session, and task names preserve projected laws (${campaign.label})`, + (suffix) => { + const demandHistory: Array = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + rowKeys: [`row`], + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + }, + ] + const continuationHistory: Array = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner`, + sessionId: `session`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `scheduleContinuation`, + taskId: `task`, + sessionId: `session`, + windowRevision: 0, + }, + { type: `runContinuation`, taskId: `task` }, + ] + + const evidenceFreeHistory: Array = [ + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `evidence-free-owner`, + sessionId: `session`, + demandId: `evidence-free-demand`, + attemptId: `evidence-free-attempt`, + alreadyAborted: false, + }, + { + type: `settleDemandWithoutEvidence`, + sourceId: `source`, + demandId: `evidence-free-demand`, + attemptId: `evidence-free-attempt`, + }, + ] + + const renamedDemand = renameHistoryIds(demandHistory, suffix) + expect( + renamedDemand.flatMap((event) => + `attemptId` in event ? [event.attemptId] : [], + ), + ).toEqual( + demandHistory.flatMap((event) => + `attemptId` in event ? [`${event.attemptId}-${suffix}`] : [], + ), + ) + expect( + renameHistoryIds(evidenceFreeHistory, suffix).flatMap((event) => + `attemptId` in event ? [event.attemptId] : [], + ), + ).toEqual([ + `evidence-free-attempt-${suffix}`, + `evidence-free-attempt-${suffix}`, + ]) + expect(projectTransportLoads(renamedDemand)).toBe( + projectTransportLoads(demandHistory), + ) + expect(projectRetainedRowKeys(renamedDemand)).toEqual( + projectRetainedRowKeys(demandHistory), + ) + expect( + projectAdapterLifecycle(renamedDemand).map(({ type }) => type), + ).toEqual(projectAdapterLifecycle(demandHistory).map(({ type }) => type)) + expect( + projectAuthorizedContinuationStarts( + renameHistoryIds(continuationHistory, suffix), + ), + ).toBe(projectAuthorizedContinuationStarts(continuationHistory)) + + for (const history of [ + demandHistory, + evidenceFreeHistory, + continuationHistory, + ]) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectTransportLoads, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedRowKeys, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableDemands, + (demandIds, renamingSuffix) => + demandIds.map((demandId) => + removeRenamingSuffix(demandId, renamingSuffix), + ), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAdapterLifecycle, + (events, renamingSuffix) => + events.map(({ type, ownerId, attemptId }) => ({ + type, + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + attemptId: removeRenamingSuffix(attemptId, renamingSuffix), + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAuthorizedContinuationStarts, + ) + } + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_004)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `transaction names do not change publication semantics (${campaign.label})`, + (suffix) => { + const history = successfulTransaction(`transaction`, `source`, `row`) + const original = projectSyncTransactions(history) + const renamed = projectSyncTransactions(renameHistoryIds(history, suffix)) + + expect({ + visibleRows: renamed.visibleRows, + publishedBatches: renamed.publishedBatches, + callbackReads: renamed.callbackReads, + receiptStates: renamed.receipts.map(({ state }) => state), + }).toEqual({ + visibleRows: original.visibleRows, + publishedBatches: original.publishedBatches, + callbackReads: original.callbackReads, + receiptStates: original.receipts.map(({ state }) => state), + }) + + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSyncTransactions, + (observation, renamingSuffix) => ({ + ...observation, + receipts: observation.receipts.map(({ transactionId, state }) => ({ + transactionId: removeRenamingSuffix(transactionId, renamingSuffix), + state, + })), + }), + ) + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_005)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( + `acquisition and owner names are semantically erased (${campaign.label})`, + (rowKeys, suffix) => { + const history = acquisitionHistory(`shared`, rowKeys) + const renamed = renameHistoryIds(history, suffix) + + const normalizeOwners = ( + observation: ReturnType, + ) => ({ + owners: observation.owners.map(({ state, rowKeys: keys }) => ({ + state, + rowKeys: keys, + })), + visibleRowKeys: observation.visibleRowKeys, + }) + + expect(normalizeOwners(projectAcquisitionSettlement(renamed))).toEqual( + normalizeOwners(projectAcquisitionSettlement(history)), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAcquisitionSettlement, + (observation, renamingSuffix) => ({ + physicalStarts: observation.physicalStarts.map((acquisitionId) => + removeRenamingSuffix(acquisitionId, renamingSuffix), + ), + owners: observation.owners.map( + ({ ownerId, state, rowKeys: keys }) => ({ + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + state, + rowKeys: keys, + }), + ), + visibleRowKeys: observation.visibleRowKeys, + }), + ) + }, + ) +} + +function overlappingReplayHistory( + baseline: FullFlowVersionedRow, + replacement: FullFlowVersionedRow, + oldAttemptId: string, + newAttemptId: string, + settlementOrder: `old-first` | `new-first`, +): Array { + const settlements: Array = + settlementOrder === `old-first` + ? [ + { + type: `settleReplay`, + attemptId: oldAttemptId, + outcome: `reject`, + }, + { + type: `settleReplay`, + attemptId: newAttemptId, + outcome: `resolve`, + }, + ] + : [ + { + type: `settleReplay`, + attemptId: newAttemptId, + outcome: `resolve`, + }, + { + type: `settleReplay`, + attemptId: oldAttemptId, + outcome: `reject`, + }, + ] + return [ + { + type: `establishPublication`, + sourceId: baseline.sourceId, + rows: [baseline], + }, + { + type: `startReplay`, + attemptId: oldAttemptId, + sourceId: baseline.sourceId, + }, + { + type: `startReplay`, + attemptId: newAttemptId, + sourceId: baseline.sourceId, + }, + { + type: `writeReplayRows`, + attemptId: newAttemptId, + rows: [replacement], + acceptedByCore: true, + }, + ...settlements, + ] +} + +for (const campaign of refinementCampaigns(1_779_006)) { + fcTest.prop( + [fc.integer({ min: -10, max: 10 }), fc.integer({ min: -10, max: 10 })], + campaign.options, + )( + `overlapping replay settlement order does not change the newest complete replacement (${campaign.label})`, + (baselineVersion, replacementVersion) => { + const baseline = { + sourceId: `source`, + rowKey: `row`, + version: baselineVersion, + } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } + + expect( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `old-first`, + ), + ), + ).toEqual( + projectReplayPublication( + overlappingReplayHistory( + baseline, + replacement, + `old`, + `new`, + `new-first`, + ), + ), + ) + }, + ) +} + +for (const campaign of refinementCampaigns(1_779_007)) { + fcTest.prop( + [ + fc.integer({ min: -10, max: 10 }), + fc.string({ minLength: 1, maxLength: 4 }), + ], + campaign.options, + )( + `replay attempt names are observationally erased (${campaign.label})`, + (replacementVersion, suffix) => { + const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } + const replacement = { + sourceId: `source`, + rowKey: `row`, + version: replacementVersion, + } + + const history = overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReplayPublication, + ) + }, + ) +} + +type AcquisitionTopology = `shared` | `separate` + +function acquisitionHistory( + topology: AcquisitionTopology, + rowKeys: ReadonlyArray, +): Array { + const start = (acquisitionId: string): LoadSubsetFullFlowEvent => ({ + type: `startAcquisition`, + acquisitionId, + sourceId: `source`, + demandId: `exact-demand`, + }) + const attach = ( + acquisitionId: string, + ownerId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `attachAcquisitionOwner`, + acquisitionId, + ownerId, + }) + const settle = (acquisitionId: string): LoadSubsetFullFlowEvent => ({ + type: `settleAcquisition`, + acquisitionId, + outcome: `resolve`, + rowKeys, + }) + + return topology === `shared` + ? [ + start(`shared-acquisition`), + attach(`shared-acquisition`, `owner-a`), + attach(`shared-acquisition`, `owner-b`), + settle(`shared-acquisition`), + ] + : [ + start(`acquisition-a`), + attach(`acquisition-a`, `owner-a`), + settle(`acquisition-a`), + start(`acquisition-b`), + attach(`acquisition-b`, `owner-b`), + settle(`acquisition-b`), + ] +} + +function sourceErasureHistories(): Array> { + const register = ( + sessionId: string, + sourceId: string, + demandId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `registerSourceDemand`, + sessionId, + sourceId, + demandId, + attemptId, + }) + const settle = ( + sessionId: string, + sourceId: string, + demandId: string, + attemptId: string, + outcome: `resolve` | `reject`, + ): LoadSubsetFullFlowEvent => ({ + type: `settleSourceDemand`, + sessionId, + sourceId, + demandId, + attemptId, + outcome, + }) + + return [ + [ + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), + register(`session-a`, `source-b`, `demand-b`, `attempt-b`), + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `resolve`), + settle(`session-a`, `source-b`, `demand-b`, `attempt-b`, `reject`), + ], + [ + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), + { type: `cleanupSession`, sessionId: `session-a` }, + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `resolve`), + ], + [ + register(`session-a`, `source-a`, `demand-a`, `attempt-a`), + { + type: `restartSession`, + previousSessionId: `session-a`, + nextSessionId: `session-b`, + }, + settle(`session-a`, `source-a`, `demand-a`, `attempt-a`, `reject`), + register(`session-b`, `source-b`, `demand-b`, `attempt-b`), + settle(`session-b`, `source-b`, `demand-b`, `attempt-b`, `resolve`), + ], + ] +} + +function demandErasureHistories(): Array> { + const request = ( + ownerId: string, + attemptId: string, + alreadyAborted = false, + ): LoadSubsetFullFlowEvent => ({ + type: `requestDemand`, + sourceId: `source`, + ownerId, + sessionId: `session-a`, + demandId: `demand-a`, + attemptId, + alreadyAborted, + }) + const release = ( + ownerId: string, + attemptId: string, + ): LoadSubsetFullFlowEvent => ({ + type: `releaseDemand`, + sourceId: `source`, + ownerId, + demandId: `demand-a`, + attemptId, + }) + + return [ + [ + request(`owner-a`, `attempt-a`), + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`row-a`], + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `applyUnprovenRows`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`row-a`], + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `rejectDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + release(`owner-a`, `attempt-a`), + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `settleDemandWithoutEvidence`, + sourceId: `source`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + release(`owner-a`, `attempt-a`), + ], + [request(`owner-a`, `attempt-a`, true), release(`owner-a`, `attempt-a`)], + [ + request(`owner-a`, `attempt-a`), + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + }, + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `truncateSource`, + sessionId: `session-a`, + sourceId: `source`, + }, + request(`owner-b`, `attempt-b`), + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner-a`, + demandId: `demand-a`, + attemptId: `attempt-a`, + rowKeys: [`stale-row`], + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source`, + ownerId: `owner-b`, + demandId: `demand-a`, + attemptId: `attempt-b`, + rowKeys: [`row-a`], + }, + ], + [ + request(`owner-a`, `attempt-a`), + { + type: `scheduleContinuation`, + taskId: `task-a`, + sessionId: `session-a`, + windowRevision: 0, + }, + { + type: `advanceWindowRevision`, + sessionId: `session-a`, + revision: 1, + }, + { type: `runContinuation`, taskId: `task-a` }, + { type: `cleanupSession`, sessionId: `session-a` }, + { + type: `restartSession`, + previousSessionId: `session-a`, + nextSessionId: `session-b`, + }, + { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-b`, + sessionId: `session-b`, + demandId: `demand-b`, + attemptId: `attempt-b`, + alreadyAborted: false, + }, + { + type: `scheduleContinuation`, + taskId: `task-b`, + sessionId: `session-b`, + windowRevision: 0, + }, + { type: `runContinuation`, taskId: `task-b` }, + ], + [ + { + type: `requestDemand`, + sourceId: `source-a`, + ownerId: `owner`, + sessionId: `session-a`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `requestDemand`, + sourceId: `source-b`, + ownerId: `owner`, + sessionId: `session-a`, + demandId: `demand`, + attemptId: `attempt`, + alreadyAborted: false, + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-a`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + rowKeys: [`row`], + }, + { + type: `applyAuthoritativeRows`, + sourceId: `source-b`, + ownerId: `owner`, + demandId: `demand`, + attemptId: `attempt`, + rowKeys: [`row`], + }, + { + type: `truncateSource`, + sessionId: `session-a`, + sourceId: `source-a`, + }, + ], + ] +} + +function transactionErasureHistories(): Array> { + const stage: LoadSubsetFullFlowEvent = { + type: `stageSyncTransaction`, + transactionId: `transaction`, + sourceId: `source`, + rowKeys: [`row`], + } + const settle: LoadSubsetFullFlowEvent = { + type: `settleSyncReceipt`, + transactionId: `transaction`, + } + + return [ + successfulTransaction(`transaction`, `source`, `row`), + [ + ...successfulTransaction(`transaction-a`, `source-a`, `row-a`), + ...successfulTransaction(`transaction-b`, `source-b`, `row-b`), + ], + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: false, + signalAborted: true, + }, + settle, + ], + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: true, + signalAborted: false, + }, + { type: `abortSyncTransaction`, transactionId: `transaction` }, + settle, + ], + [ + stage, + { + type: `commitSyncTransaction`, + transactionId: `transaction`, + parked: true, + signalAborted: false, + }, + { type: `enterSyncApplication`, transactionId: `transaction` }, + { type: `publishSyncTransaction`, transactionId: `transaction` }, + settle, + ], + ] +} + +function replayErasureHistories(): Array> { + const baseline = { sourceId: `source`, rowKey: `row`, version: 0 } + const replacement = { sourceId: `source`, rowKey: `row`, version: 1 } + + return [ + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `old-first`, + ), + overlappingReplayHistory( + baseline, + replacement, + `attempt-a`, + `attempt-b`, + `new-first`, + ), + [ + { type: `establishPublication`, sourceId: `source`, rows: [baseline] }, + { type: `startReplay`, attemptId: `attempt-a`, sourceId: `source` }, + { + type: `writeReplayRows`, + attemptId: `attempt-a`, + rows: [replacement], + acceptedByCore: false, + }, + { type: `settleReplay`, attemptId: `attempt-a`, outcome: `resolve` }, + ], + [ + { type: `establishPublication`, sourceId: `source`, rows: [baseline] }, + { type: `startReplay`, attemptId: `attempt-a`, sourceId: `source` }, + { type: `settleReplay`, attemptId: `attempt-a`, outcome: `reject` }, + ], + ] +} + +function erasedIdentityReferences( + history: ReadonlyArray, +): Array<{ path: string; field: string; value: string }> { + const references: Array<{ path: string; field: string; value: string }> = [] + const add = ( + eventIndex: number, + field: string, + value: string, + fieldPath = field, + ) => { + references.push({ path: `${eventIndex}.${fieldPath}`, field, value }) + } + + for (const [eventIndex, event] of history.entries()) { + switch (event.type) { + case `requestDemand`: + add(eventIndex, `ownerId`, event.ownerId) + add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) + break + case `applyAuthoritativeRows`: + case `applyUnprovenRows`: + case `rejectDemand`: + case `releaseDemand`: + add(eventIndex, `ownerId`, event.ownerId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) + break + case `settleDemandWithoutEvidence`: + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) + break + case `truncateSource`: + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `sessionId`, event.sessionId) + break + case `cleanupSession`: + case `advanceWindowRevision`: + add(eventIndex, `sessionId`, event.sessionId) + break + case `restartSession`: + add(eventIndex, `previousSessionId`, event.previousSessionId) + add(eventIndex, `nextSessionId`, event.nextSessionId) + break + case `scheduleContinuation`: + add(eventIndex, `taskId`, event.taskId) + add(eventIndex, `sessionId`, event.sessionId) + break + case `runContinuation`: + add(eventIndex, `taskId`, event.taskId) + break + case `stageSyncTransaction`: + case `commitSyncTransaction`: + case `enterSyncApplication`: + case `abortSyncTransaction`: + case `publishSyncTransaction`: + case `settleSyncReceipt`: + add(eventIndex, `transactionId`, event.transactionId) + break + case `startReplay`: + case `writeReplayRows`: + case `settleReplay`: + add(eventIndex, `attemptId`, event.attemptId) + break + case `registerSourceDemand`: + case `settleSourceDemand`: + case `retireSourceDemand`: + add(eventIndex, `sessionId`, event.sessionId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + add(eventIndex, `attemptId`, event.attemptId) + break + case `startAcquisition`: + add(eventIndex, `acquisitionId`, event.acquisitionId) + add(eventIndex, `demandId`, event.demandId) + break + case `attachAcquisitionOwner`: + add(eventIndex, `acquisitionId`, event.acquisitionId) + add(eventIndex, `ownerId`, event.ownerId) + break + case `settleAcquisition`: + add(eventIndex, `acquisitionId`, event.acquisitionId) + break + case `stagePublicationRows`: + add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + break + case `commitPublication`: + add(eventIndex, `publicationId`, event.publicationId) + break + case `establishReplacementCoverage`: + add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + break + case `beginReplacement`: + add(eventIndex, `publicationId`, event.publicationId) + event.demands.forEach(({ sourceId, demandId }, demandIndex) => { + add( + eventIndex, + `sourceId`, + sourceId, + `demands.${demandIndex}.sourceId`, + ) + add( + eventIndex, + `demandId`, + demandId, + `demands.${demandIndex}.demandId`, + ) + }) + break + case `settleReplacement`: + add(eventIndex, `publicationId`, event.publicationId) + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + break + case `establishPublication`: + break + case `resizeOrderedWindow`: + add(eventIndex, `sourceId`, event.sourceId) + add(eventIndex, `demandId`, event.demandId) + break + } + } + + return references +} + +function changedLeafPaths( + left: unknown, + right: unknown, + path = ``, +): Array { + if (Object.is(left, right)) return [] + if (Array.isArray(left) && Array.isArray(right)) { + if (left.length !== right.length) return [path] + return left.flatMap((value, index) => + changedLeafPaths( + value, + right[index], + path === `` ? `${index}` : `${path}.${index}`, + ), + ) + } + if ( + typeof left === `object` && + left !== null && + typeof right === `object` && + right !== null + ) { + const leftRecord = left as Record + const rightRecord = right as Record + const keys = [ + ...new Set([...Object.keys(leftRecord), ...Object.keys(rightRecord)]), + ].sort() + return keys.flatMap((key) => + changedLeafPaths( + leftRecord[key], + rightRecord[key], + path === `` ? key : `${path}.${key}`, + ), + ) + } + return [path] +} + +function expectEveryErasedIdentityRenamed( + history: ReadonlyArray, + suffix: string, +): void { + const renamed = renameHistoryIds(history, suffix) + const references = erasedIdentityReferences(history) + expect(erasedIdentityReferences(renamed), JSON.stringify(history)).toEqual( + references.map(({ path, field, value }) => ({ + path, + field, + value: `${value}-${suffix}`, + })), + ) + expect( + changedLeafPaths(history, renamed).sort(), + JSON.stringify(history), + ).toEqual(references.map(({ path }) => path).sort()) +} + +function publicationErasureHistories(): Array> { + const orderedRows = [ + { key: `row-a`, orderValue: 1 }, + { key: `row-b`, orderValue: 2 }, + ] + const relatedRows = [{ key: `related`, orderValue: 3 }] + const requestRelated: LoadSubsetFullFlowEvent = { + type: `requestDemand`, + sourceId: `source`, + ownerId: `owner-related`, + sessionId: `session`, + demandId: `related`, + attemptId: `attempt-related`, + alreadyAborted: false, + } + + return [ + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + sourceId: `source`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + { + type: `resizeOrderedWindow`, + sourceId: `source`, + demandId: `ordered`, + size: 2, + }, + ], + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + sourceId: `source`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + requestRelated, + { + type: `stagePublicationRows`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, + rows: orderedRows.slice(1), + }, + { + type: `stagePublicationRows`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `related`, + rows: relatedRows, + }, + { + type: `beginReplacement`, + publicationId: `publication-b`, + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `related` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `related`, + outcome: `success`, + extent: `exhausted`, + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, + outcome: `success`, + extent: `continues`, + }, + { + type: `establishReplacementCoverage`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, + }, + { + type: `releaseDemand`, + sourceId: `source`, + ownerId: `owner-related`, + demandId: `related`, + attemptId: `attempt-related`, + }, + ], + [ + { + type: `stagePublicationRows`, + publicationId: `publication-a`, + sourceId: `source`, + demandId: `ordered`, + rows: orderedRows, + }, + { + type: `commitPublication`, + publicationId: `publication-a`, + }, + requestRelated, + { + type: `beginReplacement`, + publicationId: `publication-b`, + demands: [ + { sourceId: `source`, demandId: `ordered` }, + { sourceId: `source`, demandId: `related` }, + ], + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `related`, + outcome: `abort`, + }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, + outcome: `failure`, + }, + { type: `cleanupSession`, sessionId: `session` }, + { + type: `settleReplacement`, + publicationId: `publication-b`, + sourceId: `source`, + demandId: `ordered`, + outcome: `success`, + extent: `exhausted`, + }, + ], + ] +} + +for (const campaign of refinementCampaigns(1_779_009)) { + fcTest.prop([fc.string({ minLength: 1, maxLength: 4 })], campaign.options)( + `erased identities preserve every bounded next-command observation (${campaign.label})`, + (suffix) => { + for (const history of [ + ...sourceErasureHistories(), + ...demandErasureHistories(), + ...transactionErasureHistories(), + ...replayErasureHistories(), + ...publicationErasureHistories(), + acquisitionHistory(`shared`, [`row-a`, `row-b`]), + acquisitionHistory(`separate`, [`row-a`, `row-b`]), + [ + { + type: `startAcquisition`, + acquisitionId: `acquisition`, + sourceId: `source`, + demandId: `demand`, + }, + { + type: `attachAcquisitionOwner`, + acquisitionId: `acquisition`, + ownerId: `owner`, + }, + { + type: `settleAcquisition`, + acquisitionId: `acquisition`, + outcome: `reject`, + rowKeys: [`ghost-row`], + }, + ] satisfies Array, + ]) { + expectEveryErasedIdentityRenamed(history, suffix) + } + + for (const history of sourceErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSourceReadiness, + normalizeSourceReadiness, + ) + } + + for (const history of demandErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectTransportLoads, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedRowKeys, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectRetainedSourceRows, + (rows, renamingSuffix) => + rows.map(({ sourceId, rowKey }) => ({ + sourceId: removeRenamingSuffix(sourceId, renamingSuffix), + rowKey, + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableDemands, + (demandIds, renamingSuffix) => + demandIds.map((demandId) => + removeRenamingSuffix(demandId, renamingSuffix), + ), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReusableSourceDemands, + (demands, renamingSuffix) => + demands.map(({ sourceId, demandId }) => ({ + sourceId: removeRenamingSuffix(sourceId, renamingSuffix), + demandId: removeRenamingSuffix(demandId, renamingSuffix), + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAdapterLifecycle, + (events, renamingSuffix) => + events.map(({ type, ownerId, attemptId }) => ({ + type, + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + attemptId: removeRenamingSuffix(attemptId, renamingSuffix), + })), + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAuthorizedContinuationStarts, + ) + } + + for (const history of transactionErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectSyncTransactions, + (observation, renamingSuffix) => ({ + ...observation, + receipts: observation.receipts.map(({ transactionId, state }) => ({ + transactionId: removeRenamingSuffix( + transactionId, + renamingSuffix, + ), + state, + })), + }), + ) + } + + for (const history of replayErasureHistories()) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectReplayPublication, + ) + } + + for (const history of publicationErasureHistories()) { + const orderedProjection = ( + prefix: ReadonlyArray, + renamingSuffix: string, + ) => + projectAtomicOrderedPublicationState(prefix, { + sourceId: + renamingSuffix === `` ? `source` : `source-${renamingSuffix}`, + demandId: + renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, + direction: `asc`, + initialWindowSize: 1, + }) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + orderedProjection, + ) + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + (prefix, renamingSuffix) => + projectOrderedPublicationBoundary(prefix, { + sourceId: + renamingSuffix === `` ? `source` : `source-${renamingSuffix}`, + demandId: + renamingSuffix === `` ? `ordered` : `ordered-${renamingSuffix}`, + direction: `asc`, + prefixSize: 2, + }), + ) + } + + for (const history of [ + acquisitionHistory(`shared`, [`row-a`, `row-b`]), + acquisitionHistory(`separate`, [`row-a`, `row-b`]), + [ + { + type: `startAcquisition`, + acquisitionId: `acquisition`, + sourceId: `source`, + demandId: `demand`, + }, + { + type: `attachAcquisitionOwner`, + acquisitionId: `acquisition`, + ownerId: `owner`, + }, + { + type: `settleAcquisition`, + acquisitionId: `acquisition`, + outcome: `reject`, + rowKeys: [`ghost-row`], + }, + ] satisfies Array, + ]) { + expectObservationPreservedAfterEveryPrefix( + history, + suffix, + projectAcquisitionSettlement, + (observation, renamingSuffix) => ({ + physicalStarts: observation.physicalStarts.map((acquisitionId) => + removeRenamingSuffix(acquisitionId, renamingSuffix), + ), + owners: observation.owners.map(({ ownerId, state, rowKeys }) => ({ + ownerId: removeRenamingSuffix(ownerId, renamingSuffix), + state, + rowKeys, + })), + visibleRowKeys: observation.visibleRowKeys, + }), + ) + } + }, + ) +} + +function semanticAcquisitionResult( + history: ReadonlyArray, +) { + const { owners, visibleRowKeys } = projectAcquisitionSettlement(history) + return { owners, visibleRowKeys } +} + +async function runAcquisitionTopology( + topology: AcquisitionTopology, + rowKeys: ReadonlyArray, +) { + const runId = ++acquisitionRunId + let physicalStarts = 0 + let logicalStarts = 0 + let logicalReleases = 0 + let deduplications = 0 + const delivery = createDeferred() + const createSource = (suffix: string) => { + type Row = { id: string } + let begin!: () => void + let write!: (message: { type: `insert`; value: Row }) => void + let commit!: () => true | Promise + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: async () => { + physicalStarts++ + await delivery.promise + begin() + for (const id of rowKeys) write({ type: `insert`, value: { id } }) + const applied = commit() + if (applied !== true) await applied + return { + hasMore: false, + appliedRowKeys: rowKeys, + } satisfies LoadSubsetResult + }, + onDeduplicate: () => { + deduplications++ + }, + }) + return createCollection({ + id: `refinement-acquisition-${runId}-${suffix}`, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + params.markReady() + return { + loadSubset: (options) => { + logicalStarts++ + return deduplicated.loadSubset(options) + }, + unloadSubset: (options) => { + logicalReleases++ + deduplicated.unloadSubset(options) + }, + } + }, + }, + }) + } + const sharedSource = createSource(`shared`) + const ownerSources = + topology === `shared` + ? [sharedSource, sharedSource] + : [sharedSource, createSource(`separate`)] + const sources = [...new Set(ownerSources)] + const ownerIds = [`owner-a`, `owner-b`] as const + const liveQueries = ownerSources.map((source, index) => + createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-${ownerIds[index]}`, + query: (q) => q.from({ row: source }), + startSync: true, + }), + ) + const batches: Array>> = [[], []] + const callbackReads: Array>> = [[], []] + const subscriptions = liveQueries.map((live, index) => + live.subscribeChanges( + (changes) => { + batches[index]!.push(changes.map(({ key }) => String(key)).sort()) + callbackReads[index]!.push( + live.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ), + ) + const preloads = liveQueries.map((live) => live.preload()) + const expectedPhysicalStarts = topology === `shared` ? 1 : 2 + let owners: Array<{ + ownerId: (typeof ownerIds)[number] + state: `resolved` + rowKeys: Array + }> = [] + let settledBatches: Array>> = [[], []] + let settledCallbackReads: Array>> = [[], []] + let initialPhysicalStarts = 0 + let initialLogicalStarts = 0 + let initialDeduplications = 0 + let retainedOwnerRowKeys: Array = [] + let retainedOwnerReady = false + let coOwnerPhysicalStarts = 0 + let coOwnerLogicalStarts = 0 + let coOwnerDeduplications = 0 + let coOwnerRowKeys: Array = [] + let coOwnerBatches: Array> = [] + let coOwnerCallbackReads: Array> = [] + let coOwnerBatchesAfterUnsubscribe: Array> = [] + let coOwnerCallbackReadsAfterUnsubscribe: Array> = [] + let remountRowKeys: Array = [] + let remountBatches: Array> = [] + let remountCallbackReads: Array> = [] + let remountBatchesAfterUnsubscribe: Array> = [] + let remountCallbackReadsAfterUnsubscribe: Array> = [] + + try { + for ( + let attempt = 0; + attempt < 20 && physicalStarts < expectedPhysicalStarts; + attempt++ + ) { + await flushPromises() + } + expect(physicalStarts).toBe(expectedPhysicalStarts) + expect(logicalStarts).toBe(2) + expect(liveQueries.map((live) => live.isReady())).toEqual([false, false]) + expect(liveQueries.map((live) => live.isLoadingSubset)).toEqual([ + true, + true, + ]) + expect(liveQueries.map((live) => live.toArray)).toEqual([[], []]) + expect(batches).toEqual([[], []]) + expect(callbackReads).toEqual([[], []]) + delivery.resolve() + await Promise.all(preloads) + + owners = liveQueries.map((live, index) => ({ + ownerId: ownerIds[index]!, + state: `resolved` as const, + rowKeys: live.toArray.map(({ id }) => String(id)).sort(), + })) + expect(liveQueries.map((live) => live.isReady())).toEqual([true, true]) + expect(liveQueries.map((live) => live.isLoadingSubset)).toEqual([ + false, + false, + ]) + settledBatches = batches.map((ownerBatches) => + ownerBatches.map((batch) => [...batch]), + ) + settledCallbackReads = callbackReads.map((ownerReads) => + ownerReads.map((read) => [...read]), + ) + + initialPhysicalStarts = physicalStarts + initialLogicalStarts = logicalStarts + initialDeduplications = deduplications + subscriptions[0]!.unsubscribe() + await liveQueries[0]!.cleanup() + expect(logicalReleases).toBe(1) + + const coOwner = createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-co-owner`, + query: (q) => q.from({ row: sharedSource }), + startSync: false, + }) + const observedCoOwnerBatches: Array> = [] + const observedCoOwnerCallbackReads: Array> = [] + const coOwnerSubscription = coOwner.subscribeChanges( + (changes) => { + observedCoOwnerBatches.push( + changes.map(({ key }) => String(key)).sort(), + ) + observedCoOwnerCallbackReads.push( + coOwner.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ) + try { + await coOwner.preload() + retainedOwnerRowKeys = liveQueries[1]!.toArray + .map(({ id }) => String(id)) + .sort() + retainedOwnerReady = liveQueries[1]!.isReady() + coOwnerPhysicalStarts = physicalStarts + coOwnerLogicalStarts = logicalStarts + coOwnerDeduplications = deduplications + coOwnerRowKeys = coOwner.toArray.map(({ id }) => String(id)).sort() + coOwnerBatches = observedCoOwnerBatches.map((batch) => [...batch]) + coOwnerCallbackReads = observedCoOwnerCallbackReads.map((read) => [ + ...read, + ]) + + subscriptions[1]!.unsubscribe() + await liveQueries[1]!.cleanup() + } finally { + coOwnerSubscription.unsubscribe() + await coOwner.cleanup() + coOwnerBatchesAfterUnsubscribe = observedCoOwnerBatches.map((batch) => [ + ...batch, + ]) + coOwnerCallbackReadsAfterUnsubscribe = observedCoOwnerCallbackReads.map( + (read) => [...read], + ) + } + expect(logicalReleases).toBe(3) + + const remount = createLiveQueryCollection({ + id: `refinement-acquisition-${runId}-remount`, + query: (q) => q.from({ row: sharedSource }), + startSync: false, + }) + const observedRemountBatches: Array> = [] + const observedRemountCallbackReads: Array> = [] + const remountSubscription = remount.subscribeChanges( + (changes) => { + observedRemountBatches.push( + changes.map(({ key }) => String(key)).sort(), + ) + observedRemountCallbackReads.push( + remount.toArray.map(({ id }) => String(id)).sort(), + ) + }, + { includeInitialState: false }, + ) + try { + await remount.preload() + remountRowKeys = remount.toArray.map(({ id }) => String(id)).sort() + remountBatches = observedRemountBatches.map((batch) => [...batch]) + remountCallbackReads = observedRemountCallbackReads.map((read) => [ + ...read, + ]) + } finally { + remountSubscription.unsubscribe() + await remount.cleanup() + remountBatchesAfterUnsubscribe = observedRemountBatches.map((batch) => [ + ...batch, + ]) + remountCallbackReadsAfterUnsubscribe = observedRemountCallbackReads.map( + (read) => [...read], + ) + } + } finally { + delivery.resolve() + subscriptions.forEach((subscription) => subscription.unsubscribe()) + await Promise.all([ + ...liveQueries.map((live) => live.cleanup()), + ...sources.map((source) => source.cleanup()), + ]) + } + + return { + initialPhysicalStarts, + initialLogicalStarts, + initialDeduplications, + retainedOwnerRowKeys, + retainedOwnerReady, + coOwnerPhysicalStarts, + coOwnerLogicalStarts, + coOwnerDeduplications, + coOwnerRowKeys, + coOwnerBatches, + coOwnerCallbackReads, + coOwnerBatchesAfterUnsubscribe, + coOwnerCallbackReadsAfterUnsubscribe, + totalPhysicalStarts: physicalStarts, + totalLogicalStarts: logicalStarts, + logicalReleases, + totalDeduplications: deduplications, + owners, + visibleRowKeys: [ + ...new Set(owners.flatMap(({ rowKeys: keys }) => keys)), + ].sort(), + batches: settledBatches, + callbackReads: settledCallbackReads, + batchesAfterUnsubscribe: batches, + callbackReadsAfterUnsubscribe: callbackReads, + remountRowKeys, + remountBatches, + remountCallbackReads, + remountBatchesAfterUnsubscribe, + remountCallbackReadsAfterUnsubscribe, + } +} + +let acquisitionRunId = 0 + +for (const campaign of refinementCampaigns(1_779_008)) { + fcTest.prop( + [ + fc.uniqueArray(fc.string({ minLength: 1, maxLength: 4 }), { + minLength: 1, + maxLength: 3, + }), + ], + campaign.options, + )( + `sharing an exact physical acquisition changes work, not logical results (${campaign.label})`, + async (rowKeys) => { + const sharedHistory = acquisitionHistory(`shared`, rowKeys) + const separateHistory = acquisitionHistory(`separate`, rowKeys) + const sharedExpected = projectAcquisitionSettlement(sharedHistory) + const separateExpected = projectAcquisitionSettlement(separateHistory) + const sharedSemantic = semanticAcquisitionResult(sharedHistory) + const separateSemantic = semanticAcquisitionResult(separateHistory) + + expect(sharedSemantic).toEqual(separateSemantic) + expect(sharedExpected.physicalStarts).toHaveLength(1) + expect(separateExpected.physicalStarts).toHaveLength(2) + + const sharedActual = await runAcquisitionTopology(`shared`, rowKeys) + const separateActual = await runAcquisitionTopology(`separate`, rowKeys) + expect({ + owners: sharedActual.owners, + visibleRowKeys: sharedActual.visibleRowKeys, + }).toEqual(sharedSemantic) + expect({ + owners: separateActual.owners, + visibleRowKeys: separateActual.visibleRowKeys, + }).toEqual(separateSemantic) + expect(sharedActual.batches).toEqual(separateActual.batches) + expect(sharedActual.callbackReads).toEqual(separateActual.callbackReads) + const expectedKeys = [...rowKeys].sort() + const expectedBatches = [ + [expectedKeys, []], + [expectedKeys, []], + ] + const expectedCallbackReads = [ + [expectedKeys, expectedKeys], + [expectedKeys, expectedKeys], + ] + expect(sharedActual.batches).toEqual(expectedBatches) + expect(sharedActual.callbackReads).toEqual(expectedCallbackReads) + expect(sharedActual.batchesAfterUnsubscribe).toEqual(sharedActual.batches) + expect(sharedActual.callbackReadsAfterUnsubscribe).toEqual( + sharedActual.callbackReads, + ) + expect(separateActual.batchesAfterUnsubscribe).toEqual( + separateActual.batches, + ) + expect(separateActual.callbackReadsAfterUnsubscribe).toEqual( + separateActual.callbackReads, + ) + expect(sharedActual.initialLogicalStarts).toBe(2) + expect(separateActual.initialLogicalStarts).toBe(2) + expect(sharedActual.initialPhysicalStarts).toBe(1) + expect(separateActual.initialPhysicalStarts).toBe(2) + expect(sharedActual.initialDeduplications).toBe(1) + expect(separateActual.initialDeduplications).toBe(0) + expect(sharedActual.retainedOwnerRowKeys).toEqual(expectedKeys) + expect(separateActual.retainedOwnerRowKeys).toEqual(expectedKeys) + expect(sharedActual.retainedOwnerReady).toBe(true) + expect(separateActual.retainedOwnerReady).toBe(true) + expect(sharedActual.coOwnerRowKeys).toEqual(expectedKeys) + expect(separateActual.coOwnerRowKeys).toEqual(expectedKeys) + expect(sharedActual.coOwnerBatches).toEqual([]) + expect(separateActual.coOwnerBatches).toEqual([expectedKeys, []]) + expect(sharedActual.coOwnerCallbackReads).toEqual([]) + expect(separateActual.coOwnerCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(sharedActual.coOwnerBatchesAfterUnsubscribe).toEqual( + sharedActual.coOwnerBatches, + ) + expect(sharedActual.coOwnerCallbackReadsAfterUnsubscribe).toEqual( + sharedActual.coOwnerCallbackReads, + ) + expect(separateActual.coOwnerBatchesAfterUnsubscribe).toEqual( + separateActual.coOwnerBatches, + ) + expect(separateActual.coOwnerCallbackReadsAfterUnsubscribe).toEqual( + separateActual.coOwnerCallbackReads, + ) + expect(sharedActual.coOwnerLogicalStarts).toBe(3) + expect(separateActual.coOwnerLogicalStarts).toBe(3) + expect(sharedActual.coOwnerPhysicalStarts).toBe(1) + expect(separateActual.coOwnerPhysicalStarts).toBe(3) + expect(sharedActual.coOwnerDeduplications).toBe(2) + expect(separateActual.coOwnerDeduplications).toBe(0) + expect(sharedActual.remountRowKeys).toEqual(expectedKeys) + expect(separateActual.remountRowKeys).toEqual(expectedKeys) + expect(sharedActual.remountBatches).toEqual([expectedKeys, []]) + expect(separateActual.remountBatches).toEqual([expectedKeys, []]) + expect(sharedActual.remountCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(separateActual.remountCallbackReads).toEqual([ + expectedKeys, + expectedKeys, + ]) + expect(sharedActual.remountBatchesAfterUnsubscribe).toEqual( + sharedActual.remountBatches, + ) + expect(sharedActual.remountCallbackReadsAfterUnsubscribe).toEqual( + sharedActual.remountCallbackReads, + ) + expect(separateActual.remountBatchesAfterUnsubscribe).toEqual( + separateActual.remountBatches, + ) + expect(separateActual.remountCallbackReadsAfterUnsubscribe).toEqual( + separateActual.remountCallbackReads, + ) + expect(sharedActual.totalLogicalStarts).toBe(4) + expect(separateActual.totalLogicalStarts).toBe(4) + expect(sharedActual.logicalReleases).toBe(4) + expect(separateActual.logicalReleases).toBe(4) + expect(sharedActual.totalPhysicalStarts).toBe(2) + expect(separateActual.totalPhysicalStarts).toBe(4) + expect(sharedActual.totalDeduplications).toBe(2) + expect(separateActual.totalDeduplications).toBe(0) + expect( + sharedActual.totalPhysicalStarts + sharedActual.totalDeduplications, + ).toBe(sharedActual.totalLogicalStarts) + expect( + separateActual.totalPhysicalStarts + separateActual.totalDeduplications, + ).toBe(separateActual.totalLogicalStarts) + }, + ) +} diff --git a/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts new file mode 100644 index 0000000000..c81af33ab7 --- /dev/null +++ b/packages/db/tests/query/load-subset-replay-refinement-oracle.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { createLiveQueryCollection } from '../../src/query/index.js' +import { projectReplayPublication } from '../load-subset-full-flow-model.js' +import { flushPromises } from '../utils.js' +import type { + FullFlowVersionedRow, + LoadSubsetFullFlowEvent, +} from '../load-subset-full-flow-model.js' +import type { + ChangeMessageOrDeleteKeyMessage, + LoadSubsetOptions, +} from '../../src/types.js' + +type Row = { id: string; version: number } + +describe(`loadSubset replay refinement`, () => { + function createHarness(sourceId: string) { + let begin!: () => void + let write!: (message: ChangeMessageOrDeleteKeyMessage) => void + let commit!: () => void + let truncate!: () => void + let loadCount = 0 + const pending: Array<{ + options: LoadSubsetOptions + deferred: ReturnType> + }> = [] + const batches: Array< + Array<{ + type: `insert` | `update` | `delete` + row: { sourceId: string; rowKey: string; version: number } + previousVersion?: number + }> + > = [] + const source = createCollection({ + id: sourceId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: (params) => { + begin = params.begin + write = params.write + commit = params.commit + truncate = params.truncate + params.markReady() + return { + loadSubset: (options) => { + loadCount++ + if (loadCount === 1) { + begin() + write({ type: `insert`, value: { id: `row`, version: 1 } }) + commit() + return true + } + const deferred = createDeferred() + pending.push({ options, deferred }) + return deferred.promise + }, + unloadSubset: () => {}, + } + }, + }, + }) + const downstream = createLiveQueryCollection({ + id: `${sourceId}-downstream`, + query: (q) => + q.from({ row: source }).select(({ row }) => ({ + id: row.id, + version: row.version, + })), + startSync: true, + }) + const callbackReads: Array> = [] + const subscription = downstream.subscribeChanges( + (changes) => { + const batch = changes.map((change) => ({ + type: change.type, + row: { + sourceId, + rowKey: String(change.key), + version: change.value.version, + }, + ...(change.previousValue === undefined + ? {} + : { previousVersion: change.previousValue.version }), + })) + if (batch.length > 0) { + batches.push(batch) + callbackReads.push( + downstream.toArray.map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })), + ) + } + }, + { includeInitialState: true }, + ) + + const replaceCore = (version: number) => { + begin() + write({ type: `insert`, value: { id: `row`, version } }) + commit() + } + const startReplay = async () => { + begin() + truncate() + commit() + await flushPromises() + } + const coreRows = () => + source.toArray.map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })) + const visibleRows = () => + downstream.toArray.map(({ id, version }) => ({ + sourceId, + rowKey: id, + version, + })) + + return { + source, + downstream, + subscription, + pending, + batches, + callbackReads, + replaceCore, + startReplay, + coreRows, + visibleRows, + } + } + + it(`retains the last complete publication when replay fails after writing`, async () => { + const sourceId = `replay-refinement-failure` + const row = (version: number) => ({ + sourceId, + rowKey: `row`, + version, + }) + const history: Array = [ + { type: `establishPublication`, sourceId, rows: [row(1)] }, + ] + const harness = createHarness(sourceId) + + try { + await harness.downstream.preload() + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) + + harness.replaceCore(2) + history.push({ + type: `writeReplayRows`, + attemptId: `replay-1`, + rows: [row(2)], + acceptedByCore: true, + }) + harness.pending[0]?.deferred.reject(new Error(`replay failed`)) + history.push({ + type: `settleReplay`, + attemptId: `replay-1`, + outcome: `reject`, + }) + await flushPromises() + + const expected = projectReplayPublication(history) + expect(harness.coreRows()).toEqual(expected.coreRows) + expect(harness.visibleRows()).toEqual(expected.visibleRows) + expect(harness.batches).toEqual(expected.publishedBatches) + expect(harness.callbackReads).toEqual(expected.callbackReads) + } finally { + harness.subscription.unsubscribe() + await Promise.all([ + harness.downstream.cleanup(), + harness.source.cleanup(), + ]) + } + }) + + it(`waits for every overlapping replay before publishing the newest success`, async () => { + const sourceId = `replay-refinement-overlap` + const row = (version: number) => ({ + sourceId, + rowKey: `row`, + version, + }) + const history: Array = [ + { type: `establishPublication`, sourceId, rows: [row(1)] }, + ] + const harness = createHarness(sourceId) + + try { + await harness.downstream.preload() + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-1`, sourceId }) + await harness.startReplay() + history.push({ type: `startReplay`, attemptId: `replay-2`, sourceId }) + + expect(harness.pending[0]?.options.signal?.aborted).toBe(true) + harness.replaceCore(3) + history.push({ + type: `writeReplayRows`, + attemptId: `replay-2`, + rows: [row(3)], + acceptedByCore: true, + }) + harness.pending[1]?.deferred.resolve() + history.push({ + type: `settleReplay`, + attemptId: `replay-2`, + outcome: `resolve`, + }) + await flushPromises() + + const beforeObsoleteSettlement = projectReplayPublication(history) + expect(harness.visibleRows()).toEqual( + beforeObsoleteSettlement.visibleRows, + ) + expect(harness.batches).toEqual(beforeObsoleteSettlement.publishedBatches) + expect(harness.callbackReads).toEqual( + beforeObsoleteSettlement.callbackReads, + ) + + harness.pending[0]?.deferred.reject( + new DOMException(`obsolete`, `AbortError`), + ) + history.push({ + type: `settleReplay`, + attemptId: `replay-1`, + outcome: `reject`, + }) + await flushPromises() + + const expected = projectReplayPublication(history) + expect(harness.coreRows()).toEqual(expected.coreRows) + expect(harness.visibleRows()).toEqual(expected.visibleRows) + expect(harness.batches).toEqual(expected.publishedBatches) + expect(harness.callbackReads).toEqual(expected.callbackReads) + } finally { + for (const replay of harness.pending) replay.deferred.resolve() + harness.subscription.unsubscribe() + await Promise.all([ + harness.downstream.cleanup(), + harness.source.cleanup(), + ]) + } + }) +}) diff --git a/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts new file mode 100644 index 0000000000..7dde2e4e7e --- /dev/null +++ b/packages/db/tests/query/load-subset-source-readiness-refinement-oracle.test.ts @@ -0,0 +1,441 @@ +import { expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { BTreeIndex } from '../../src/index.js' +import { extractSimpleComparisons } from '../../src/query/expression-helpers.js' +import { + createLiveQueryCollection, + eq, + toArray, +} from '../../src/query/index.js' +import { projectSourceReadiness } from '../load-subset-full-flow-model.js' +import { flushPromises } from '../utils.js' +import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' +import type { LoadSubsetOptions } from '../../src/types.js' + +type Row = { id: string; group: string } + +it.each([ + { oldOutcome: `resolve`, settlementOrder: `old-first` }, + { oldOutcome: `reject`, settlementOrder: `old-first` }, + { oldOutcome: `resolve`, settlementOrder: `fresh-first` }, + { oldOutcome: `reject`, settlementOrder: `fresh-first` }, +] as const)( + `fences a retired source-demand attempt across $settlementOrder $oldOutcome settlement`, + async ({ oldOutcome, settlementOrder }) => { + type Parent = { id: string; group: string } + type Child = { id: string; group: string } + type PendingRequest = { + options: LoadSubsetOptions + rows: ReturnType>> + } + const sessionId = `session` + const caseId = `${oldOutcome}-${settlementOrder}` + const parentId = `readiness-generation-parent-${caseId}` + const childId = `readiness-generation-child-${caseId}` + const oldAttemptId = `old-attempt` + const freshAttemptId = `fresh-attempt` + let parentBegin!: () => void + let parentWrite!: (message: { + type: `update` + value: Parent + previousValue: Parent + }) => void + let parentCommit!: () => true | Promise + const oldParent: Parent = { id: `parent`, group: `old` } + const freshParent: Parent = { ...oldParent, group: `fresh` } + const parent = createCollection({ + id: parentId, + getKey: (row) => row.id, + sync: { + sync: ({ begin, write, commit, markReady }) => { + parentBegin = begin + parentWrite = write + parentCommit = commit + begin() + write({ type: `insert`, value: oldParent }) + commit() + markReady() + }, + }, + }) + let childBegin!: () => void + let childWrite!: (message: { type: `insert`; value: Child }) => void + let childCommit!: () => true | Promise + const pending: Array = [] + const unloads: Array<{ + options: LoadSubsetOptions + abortedAtUnload: boolean | undefined + }> = [] + const child = createCollection({ + id: childId, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + childBegin = begin + childWrite = write + childCommit = commit + markReady() + return { + loadSubset: (options) => { + const rows = createDeferred>() + pending.push({ options, rows }) + return rows.promise.then(async (acquiredRows) => { + if (acquiredRows.length > 0) { + childBegin() + for (const row of acquiredRows) { + childWrite({ type: `insert`, value: row }) + } + const applied = childCommit() + if (applied !== true) await applied + } + return { + hasMore: false, + appliedRowKeys: acquiredRows.map((row) => row.id), + } + }) + }, + unloadSubset: (options) => { + unloads.push({ + options, + abortedAtUnload: options.signal?.aborted, + }) + }, + } + }, + }, + }) + const live = createLiveQueryCollection({ + id: `readiness-generation-live-${caseId}`, + query: (q) => + q.from({ parent }).select(({ parent: parentRow }) => ({ + id: parentRow.id, + children: toArray( + q + .from({ child }) + .where(({ child: childRow }) => + eq(childRow.group, parentRow.group), + ), + ), + })), + startSync: true, + }) + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + }, + ] + let preloadState: `pending` | `resolved` | `rejected` = `pending` + const preload = live.preload() + void preload.then( + () => { + preloadState = `resolved` + }, + () => { + preloadState = `rejected` + }, + ) + const requestedGroups = (options: LoadSubsetOptions): Array => + extractSimpleComparisons(options.where).flatMap((comparison) => { + if (comparison.field.join(`.`) !== `group`) return [] + if (comparison.operator === `eq`) { + return typeof comparison.value === `string` ? [comparison.value] : [] + } + if (comparison.operator !== `in` || !Array.isArray(comparison.value)) { + return [] + } + return comparison.value.filter( + (value): value is string => typeof value === `string`, + ) + }) + const expectUnloads = ( + ...expectedOptions: ReadonlyArray + ): void => { + expect(unloads).toHaveLength(expectedOptions.length) + for (const [index, options] of expectedOptions.entries()) { + expect(unloads[index]!.options).toBe(options) + expect(unloads[index]!.abortedAtUnload).toBe(true) + } + } + let liveCleaned = false + + try { + await flushPromises() + expect(pending).toHaveLength(1) + expect(requestedGroups(pending[0]!.options)).toEqual([`old`]) + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`pending`) + + parentBegin() + parentWrite({ + type: `update`, + value: freshParent, + previousValue: oldParent, + }) + const parentApplied = parentCommit() + if (parentApplied !== true) await parentApplied + await flushPromises() + + expect(pending).toHaveLength(2) + expect(requestedGroups(pending[0]!.options)).toEqual([`old`]) + expect(requestedGroups(pending[1]!.options)).toEqual([`fresh`]) + expect(pending[0]!.options.signal?.aborted).toBe(true) + expect(pending[1]!.options.signal?.aborted).toBe(false) + expectUnloads(pending[0]!.options) + history.push( + { + type: `retireSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + }, + { + type: `registerSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: freshAttemptId, + }, + ) + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`pending`) + + const freshChild: Child = { id: `fresh-child`, group: `fresh` } + const settleOld = async () => { + if (oldOutcome === `resolve`) { + pending[0]!.rows.resolve([]) + } else { + pending[0]!.rows.reject(new Error(`retired source demand failed`)) + } + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: oldAttemptId, + outcome: oldOutcome, + }) + await flushPromises() + } + const settleFresh = async () => { + expect(child.get(freshChild.id)).toBeUndefined() + pending[1]!.rows.resolve([freshChild]) + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: childId, + demandId: `children`, + attemptId: freshAttemptId, + outcome: `resolve`, + }) + await flushPromises() + } + const settlements = + settlementOrder === `old-first` + ? [settleOld, settleFresh] + : [settleFresh, settleOld] + for (const settle of settlements) { + await settle() + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe( + projectSourceReadiness(history).status === `ready` + ? `resolved` + : `pending`, + ) + expect(live.utils.lastSubsetError).toBeUndefined() + } + + await preload + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(preloadState).toBe(`resolved`) + expect(live.utils.lastSubsetError).toBeUndefined() + expect(child.get(freshChild.id)).toEqual( + expect.objectContaining(freshChild), + ) + expect(live.toArray).toEqual([ + expect.objectContaining({ + id: `parent`, + children: [expect.objectContaining({ id: `fresh-child` })], + }), + ]) + expect(pending[1]!.options.signal?.aborted).toBe(false) + expectUnloads(pending[0]!.options) + + await live.cleanup() + liveCleaned = true + expect(pending[1]!.options.signal?.aborted).toBe(true) + expectUnloads(pending[0]!.options, pending[1]!.options) + } finally { + for (const request of pending) { + request.rows.resolve([]) + } + await Promise.all([ + preload.catch(() => undefined), + liveCleaned ? Promise.resolve() : live.cleanup(), + ]) + await Promise.all([parent.cleanup(), child.cleanup()]) + } + }, +) + +it.each([`resolve`, `reject`, `cleanup`] as const)( + `matches cross-source initial readiness through %s`, + async (secondOutcome) => { + const sessionId = `session-1` + const leftId = `readiness-left-${secondOutcome}` + const rightId = `readiness-right-${secondOutcome}` + const leftDelivery = createDeferred() + const rightDelivery = createDeferred() + const history: Array = [ + { + type: `registerSourceDemand`, + sessionId, + sourceId: leftId, + demandId: `all`, + attemptId: `left-attempt`, + }, + { + type: `registerSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + attemptId: `right-attempt`, + }, + ] + const createSource = ( + id: string, + row: Row, + delivery: ReturnType>, + ) => + createCollection({ + id, + getKey: (value) => value.id, + syncMode: `on-demand`, + startSync: true, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: () => + delivery.promise.then(async () => { + begin() + write({ type: `insert`, value: row }) + const applied = commit() + if (applied !== true) await applied + return { hasMore: false, appliedRowKeys: [row.id] } + }), + unloadSubset: () => {}, + } + }, + }, + }) + const left = createSource( + leftId, + { id: `left`, group: `shared` }, + leftDelivery, + ) + const right = createSource( + rightId, + { id: `right`, group: `shared` }, + rightDelivery, + ) + const live = createLiveQueryCollection({ + id: `readiness-live-${secondOutcome}`, + query: (q) => + q + .from({ left }) + .innerJoin({ right }, ({ left: leftRow, right: rightRow }) => + eq(leftRow.group, rightRow.group), + ) + .select(({ left: leftRow, right: rightRow }) => ({ + leftId: leftRow.id, + rightId: rightRow.id, + })), + startSync: true, + }) + const preload = live.preload() + void preload.catch(() => undefined) + + try { + expect(live.status).toBe(projectSourceReadiness(history).status) + + leftDelivery.resolve() + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: leftId, + demandId: `all`, + attemptId: `left-attempt`, + outcome: `resolve`, + }) + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(live.toArray).toEqual([]) + + if (secondOutcome === `cleanup`) { + await live.cleanup() + history.push({ type: `cleanupSession`, sessionId }) + expect(live.status).toBe(projectSourceReadiness(history).status) + + rightDelivery.resolve() + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + attemptId: `right-attempt`, + outcome: `resolve`, + }) + await flushPromises() + + expect(live.status).toBe(projectSourceReadiness(history).status) + expect(live.toArray).toEqual([]) + return + } else if (secondOutcome === `resolve`) { + rightDelivery.resolve() + } else { + rightDelivery.reject(new Error(`right source failed`)) + } + history.push({ + type: `settleSourceDemand`, + sessionId, + sourceId: rightId, + demandId: `all`, + attemptId: `right-attempt`, + outcome: secondOutcome, + }) + await flushPromises() + + const expected = projectSourceReadiness(history) + expect(live.status).toBe(expected.status) + if (secondOutcome === `resolve`) { + await expect(preload).resolves.toBeUndefined() + expect(live.toArray).toEqual([ + expect.objectContaining({ leftId: `left`, rightId: `right` }), + ]) + } else { + await expect(preload).rejects.toThrow(`right source failed`) + expect(expected.failedSources).toEqual([rightId]) + } + } finally { + leftDelivery.resolve() + rightDelivery.resolve() + await live.cleanup() + await Promise.all([left.cleanup(), right.cleanup()]) + } + }, +) diff --git a/packages/db/tests/query/load-subset-subquery.test.ts b/packages/db/tests/query/load-subset-subquery.test.ts index 2888753c39..002100dca7 100644 --- a/packages/db/tests/query/load-subset-subquery.test.ts +++ b/packages/db/tests/query/load-subset-subquery.test.ts @@ -328,11 +328,12 @@ describe(`loadSubset with subqueries`, () => { // Verify loadSubset was called expect(loadSubsetCalls.length).toBeGreaterThan(0) - // Verify the last call has the orderBy clause and limit + // Without a range index, core asks the adapter for the full ordered source + // and applies the query limit locally. const lastCall = loadSubsetCalls[loadSubsetCalls.length - 1] expect(lastCall).toBeDefined() expect(lastCall!.orderBy).toBeDefined() - expect(lastCall!.limit).toBe(2) + expect(lastCall!.limit).toBeUndefined() const expectedOrderBy: OrderBy = [ { @@ -373,11 +374,12 @@ describe(`loadSubset with subqueries`, () => { // Verify loadSubset was called for the orders collection expect(loadSubsetCalls.length).toBeGreaterThan(0) - // Verify the last call has the orderBy clause and limit + // Without a range index, core asks the adapter for the full ordered source + // and applies the subquery limit locally. const lastCall = loadSubsetCalls[loadSubsetCalls.length - 1] expect(lastCall).toBeDefined() expect(lastCall!.orderBy).toBeDefined() - expect(lastCall!.limit).toBe(2) + expect(lastCall!.limit).toBeUndefined() const expectedOrderBy: OrderBy = [ { diff --git a/packages/db/tests/query/load-subset-transaction-refinement-oracle.test.ts b/packages/db/tests/query/load-subset-transaction-refinement-oracle.test.ts new file mode 100644 index 0000000000..424c447b92 --- /dev/null +++ b/packages/db/tests/query/load-subset-transaction-refinement-oracle.test.ts @@ -0,0 +1,134 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createDeferred } from '../../src/deferred.js' +import { createTransaction } from '../../src/transactions.js' +import { projectSyncTransactions } from '../load-subset-full-flow-model.js' +import type { LoadSubsetFullFlowEvent } from '../load-subset-full-flow-model.js' + +type Row = { id: string; group: string } + +describe(`loadSubset transaction refinement`, () => { + it.each([`at-commit`, `while-parked`, `after-publication-starts`] as const)( + `matches the independent receipt and publication model when aborting %s`, + async (abortPhase) => { + const sourceId = `transaction-refinement-${abortPhase}` + const transactionId = `subset-transaction` + const remoteRow: Row = { id: `remote`, group: `requested` } + const history: Array = [ + { + type: `stageSyncTransaction`, + transactionId, + sourceId, + rowKeys: [remoteRow.id], + }, + { + type: `commitSyncTransaction`, + transactionId, + parked: true, + signalAborted: abortPhase === `at-commit`, + }, + ] + const controller = new AbortController() + const persistence = createDeferred() + const publishedBatches: Array> = [] + const callbackReads: Array> = [] + const source = createCollection({ + id: sourceId, + getKey: (row) => row.id, + syncMode: `on-demand`, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: ({ signal }) => { + begin() + write({ type: `insert`, value: remoteRow }) + if (abortPhase === `at-commit`) controller.abort() + return commit(signal) + }, + } + }, + }, + }) + source.startSyncImmediate() + const blocker = createTransaction({ + mutationFn: () => persistence.promise, + }) + blocker.mutate(() => + source.insert({ id: `local`, group: `outside-request` }), + ) + const subscription = source.subscribeChanges( + (changes) => { + const remoteKeys = changes + .filter((change) => change.key === remoteRow.id) + .map((change) => String(change.key)) + if (remoteKeys.length === 0) return + publishedBatches.push(remoteKeys) + callbackReads.push(source.has(remoteRow.id) ? [remoteRow.id] : []) + if (abortPhase === `after-publication-starts`) { + controller.abort() + } + }, + { includeInitialState: false }, + ) + const load = source._sync.loadSubset({ signal: controller.signal }) + expect(load).toBeInstanceOf(Promise) + + try { + if (abortPhase === `while-parked`) { + controller.abort() + history.push({ type: `abortSyncTransaction`, transactionId }) + } else if (abortPhase === `after-publication-starts`) { + history.push( + { type: `enterSyncApplication`, transactionId }, + { type: `publishSyncTransaction`, transactionId }, + { type: `abortSyncTransaction`, transactionId }, + { type: `settleSyncReceipt`, transactionId }, + ) + } + + persistence.resolve() + await blocker.isPersisted.promise + + if (abortPhase !== `after-publication-starts`) { + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } else { + await expect(load).resolves.toEqual( + expect.objectContaining({ collectionId: sourceId }), + ) + } + + const expected = projectSyncTransactions(history) + const visibleRows = source.has(remoteRow.id) + ? [{ sourceId, rowKey: remoteRow.id }] + : [] + + expect(visibleRows).toEqual(expected.visibleRows) + expect(publishedBatches).toEqual( + expected.publishedBatches.map((batch) => + batch.map(({ rowKey }) => rowKey), + ), + ) + expect(callbackReads).toEqual( + expected.callbackReads.map((rows) => + rows.map(({ rowKey }) => rowKey), + ), + ) + expect(expected.receipts).toEqual([ + { + transactionId, + state: + abortPhase === `after-publication-starts` + ? `resolved` + : `rejected`, + }, + ]) + } finally { + persistence.resolve() + await blocker.isPersisted.promise.catch(() => undefined) + subscription.unsubscribe() + await source.cleanup() + } + }, + ) +}) diff --git a/packages/db/tests/query/ordered-work-oracle.property.test.ts b/packages/db/tests/query/ordered-work-oracle.property.test.ts new file mode 100644 index 0000000000..71fa4a0daf --- /dev/null +++ b/packages/db/tests/query/ordered-work-oracle.property.test.ts @@ -0,0 +1,3522 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it, vi } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { CollectionSubscription } from '../../src/collection/subscription.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' +import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { ReverseIndex } from '../../src/indexes/reverse-index.js' +import { localOnlyCollectionOptions } from '../../src/local-only.js' +import { eq } from '../../src/query/builder/functions.js' +import { compileSingleRowExpression } from '../../src/query/compiler/evaluators.js' +import { PropRef } from '../../src/query/ir.js' +import { TotalOrder } from '../../src/query/total-order.js' +import { makeComparator } from '../../src/utils/comparison.js' +import { + WindowState, + diffPublications, +} from '../../src/query/live/window-state.js' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config.js' +import type * as DbIvm from '@tanstack/db-ivm' +import type { CollectionImpl } from '../../src/collection/index.js' +import type { CompareOptions } from '../../src/query/builder/types.js' +import type { + ChangeMessage, + CurrentStateAsChangesOptions, + StringCollationConfig, +} from '../../src/types.js' +import type { OrderBy, OrderByDirection } from '../../src/query/ir.js' + +const keyComparisonCounter = vi.hoisted(() => ({ count: 0 })) + +vi.mock(`@tanstack/db-ivm`, async (importOriginal) => { + const actual = await importOriginal() + return { + ...actual, + compareKeys: (left: string | number, right: string | number) => { + keyComparisonCounter.count++ + return actual.compareKeys(left, right) + }, + } +}) + +type RankedRow = { + id: string + rank: number + included: boolean +} + +class CountingReadonlyMap implements ReadonlyMap { + private readonly valuesByKey: Map + iterationReads = 0 + membershipReads = 0 + valueReads = 0 + + constructor( + entries: Iterable = [], + private readonly onIteration?: () => void, + private readonly onMembershipRead?: () => void, + ) { + this.valuesByKey = new Map(entries) + } + + get size(): number { + return this.valuesByKey.size + } + + private *countIterator( + iterator: Iterator, + ): Generator { + for (let next = iterator.next(); !next.done; next = iterator.next()) { + this.iterationReads++ + this.onIteration?.() + yield next.value + } + return undefined + } + + [Symbol.iterator](): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(this.valuesByKey[Symbol.iterator]()) + } + + entries(): Generator<[TKey, TValue], undefined, unknown> { + return this.countIterator(this.valuesByKey.entries()) + } + + keys(): Generator { + return this.countIterator(this.valuesByKey.keys()) + } + + values(): Generator { + return this.countIterator(this.valuesByKey.values()) + } + + forEach( + callback: ( + value: TValue, + key: TKey, + map: ReadonlyMap, + ) => void, + thisArg?: unknown, + ): void { + this.valuesByKey.forEach((value, key) => { + this.iterationReads++ + this.onIteration?.() + callback.call(thisArg, value, key, this) + }) + } + + get(key: TKey): TValue | undefined { + this.valueReads++ + return this.valuesByKey.get(key) + } + + has(key: TKey): boolean { + this.membershipReads++ + this.onMembershipRead?.() + return this.valuesByKey.has(key) + } +} + +type PublicKeyRankedRow = Omit & { + id: string | number +} + +type OrderedWork = { + keys: Array + sourceReads: Array + expectedValueReads: number + valueReads: number + expectedBucketReads: number + bucketReads: number + expectedCursorCalls: number + cursorCalls: number + expectedBucketYields: number + bucketYields: number + unexpectedTraversalCalls: number + expectedKeyComparisons: number + keyComparisons: number + totalOrderComparisons: number +} + +type OrderedReadProbe = { + getValueReads: () => number + getBucketReads: () => number + getCursorCalls: () => number + getUnexpectedTraversalCalls: () => number + restore: () => void +} + +function isArrayIndex(property: PropertyKey): boolean { + if (typeof property !== `string` || property.length === 0) return false + const index = Number(property) + return Number.isSafeInteger(index) && index >= 0 && String(index) === property +} + +const traversalMethods = new Set([ + Symbol.iterator, + `entries`, + `keys`, + `values`, + `forEach`, +]) + +function observeUnexpectedTraversals( + target: T, + onTraversal: () => void, +): T { + return new Proxy(target, { + get(inner, property) { + const member = Reflect.get(inner, property, inner) as unknown + if (typeof member !== `function`) return member + return (...args: Array) => { + if (traversalMethods.has(property)) onTraversal() + return Reflect.apply(member, inner, args) as unknown + } + }, + }) +} + +function observeOrderedIndexReads( + index: BasicIndex | BTreeIndex, + indexKind: `basic` | `btree`, + direction: OrderByDirection, +): OrderedReadProbe { + let valueReads = 0 + let bucketReads = 0 + let cursorCalls = 0 + let unexpectedTraversalCalls = 0 + + if (indexKind === `basic`) { + const internals = index as unknown as { + sortedValues: Array + valueMap: Map> + indexedKeys: Set + } + const sortedValues = internals.sortedValues + const valueMap = internals.valueMap + const indexedKeys = internals.indexedKeys + internals.sortedValues = new Proxy(sortedValues, { + get(target, property, receiver) { + if (isArrayIndex(property)) valueReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + internals.valueMap = new Proxy(valueMap, { + get(target, property) { + const member = Reflect.get(target, property, target) as unknown + if (property === `get`) { + return (value: unknown) => { + bucketReads++ + return target.get(value) + } + } + if (typeof member === `function`) { + return (...args: Array) => { + unexpectedTraversalCalls++ + return member.apply(target, args) + } + } + return member + }, + }) + internals.indexedKeys = observeUnexpectedTraversals(indexedKeys, () => { + unexpectedTraversalCalls++ + }) + return { + getValueReads: () => valueReads, + getBucketReads: () => bucketReads, + getCursorCalls: () => cursorCalls, + getUnexpectedTraversalCalls: () => unexpectedTraversalCalls, + restore: () => { + internals.sortedValues = sortedValues + internals.valueMap = valueMap + internals.indexedKeys = indexedKeys + }, + } + } + + const internals = index as unknown as { + orderedEntries: { + nextHigherPair: (key?: unknown) => readonly [unknown, unknown] | undefined + nextLowerPair: (key?: unknown) => readonly [unknown, unknown] | undefined + } + valueMap: Map> + indexedKeys: Set + } + const orderedEntries = internals.orderedEntries + const valueMap = internals.valueMap + const indexedKeys = internals.indexedKeys + const expectedMethod = + direction === `asc` ? `nextHigherPair` : `nextLowerPair` + internals.orderedEntries = new Proxy(orderedEntries, { + get(target, property) { + if (property !== expectedMethod) unexpectedTraversalCalls++ + const member = Reflect.get(target, property, target) as unknown + if (typeof member !== `function`) return member + return (...args: Array) => { + if (property === expectedMethod) cursorCalls++ + const result = member.apply(target, args) as + | readonly [unknown, unknown] + | undefined + if (property === `nextHigherPair` || property === `nextLowerPair`) { + if (result !== undefined) { + valueReads++ + bucketReads++ + } + } + return result + } + }, + }) + internals.valueMap = observeUnexpectedTraversals(valueMap, () => { + unexpectedTraversalCalls++ + }) + internals.indexedKeys = observeUnexpectedTraversals(indexedKeys, () => { + unexpectedTraversalCalls++ + }) + return { + getValueReads: () => valueReads, + getBucketReads: () => bucketReads, + getCursorCalls: () => cursorCalls, + getUnexpectedTraversalCalls: () => unexpectedTraversalCalls, + restore: () => { + internals.orderedEntries = orderedEntries + internals.valueMap = valueMap + internals.indexedKeys = indexedKeys + }, + } +} + +function orderedWorkCampaigns(property: string, fixedSeed: number) { + return [ + { + label: `fixed seed ${fixedSeed}`, + options: { numRuns: oracleRuns(40), seed: fixedSeed }, + }, + { + label: `random or replayed seed`, + options: oraclePropertyOptions(40, property), + }, + ] as const +} + +function orderBy( + direction: OrderByDirection, + nulls: `first` | `last` = `first`, +): OrderBy { + return [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction, nulls }, + }, + ] +} + +function orderByWithOptions(compareOptions: CompareOptions): OrderBy { + return [{ expression: new PropRef([`rank`]), compareOptions }] +} + +const publicKeyIndexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, +} satisfies CompareOptions + +function publicKeyOrderBy(direction: OrderByDirection): OrderBy { + return orderByWithOptions({ + ...publicKeyIndexCompareOptions, + direction, + nulls: direction === `asc` ? `last` : `first`, + }) +} + +function comparePublicKeys( + left: string | number, + right: string | number, +): number { + if (typeof left !== typeof right) { + return typeof left === `string` ? -1 : 1 + } + if (typeof left === `number` && typeof right === `number`) { + const leftIsNaN = Number.isNaN(left) + const rightIsNaN = Number.isNaN(right) + if (leftIsNaN || rightIsNaN) { + if (leftIsNaN && rightIsNaN) return 0 + return leftIsNaN ? 1 : -1 + } + } + return left < right ? -1 : left > right ? 1 : 0 +} + +const orderedIndexCompatibilityCases = ([`basic`, `btree`] as const).flatMap( + (indexKind) => + ([`asc`, `desc`] as const).flatMap((indexDirection) => + ([`first`, `last`] as const).flatMap((indexNulls) => + ([`asc`, `desc`] as const).flatMap((queryDirection) => + ([`first`, `last`] as const).map((queryNulls) => ({ + indexKind, + indexDirection, + indexNulls, + queryDirection, + queryNulls, + compatible: + indexDirection === queryDirection + ? indexNulls === queryNulls + : indexNulls !== queryNulls, + })), + ), + ), + ), +) + +const stringComparisonVariants = [ + { + name: `the same locale options`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: true, + }, + { + name: `lexical string order`, + collation: { stringSort: `lexical` }, + compatible: false, + }, + { + name: `another locale`, + collation: { + stringSort: `locale`, + locale: `de`, + localeOptions: { numeric: true, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another numeric option`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: false, sensitivity: `base` }, + }, + compatible: false, + }, + { + name: `another sensitivity option`, + collation: { + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `accent` }, + }, + compatible: false, + }, +] satisfies Array<{ + name: string + collation: StringCollationConfig + compatible: boolean +}> + +const orderedStringCompatibilityCases = ([`basic`, `btree`] as const).flatMap( + (indexKind) => + ([`asc`, `desc`] as const).flatMap((queryDirection) => + stringComparisonVariants.map(({ name, collation, compatible }) => ({ + name, + indexKind, + queryDirection, + compareOptions: { + ...collation, + direction: queryDirection, + nulls: + queryDirection === `asc` ? (`last` as const) : (`first` as const), + } satisfies CompareOptions, + compatible, + })), + ), +) + +async function observeOrderedPrefix( + rows: ReadonlyArray, + limit: number | undefined, + indexKind: `basic` | `btree` = `btree`, + direction: OrderByDirection = `desc`, +): Promise { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-${Math.random()}`, + getKey: (row) => row.id, + initialData: [...rows], + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + }, + }, + }) as BasicIndex | BTreeIndex + + let expectedKeyComparisons = 0 + let expectedMatches = 0 + let expectedBucketYields = 0 + if (limit === undefined || limit > 0) { + const keysByRank = new Map>() + const rowsInCollectionOrder = [...rows].sort((left, right) => + comparePublicKeys(left.id, right.id), + ) + for (const { id, rank } of rowsInCollectionOrder) { + const bucket = keysByRank.get(rank) + if (bucket === undefined) keysByRank.set(rank, [id]) + else bucket.push(id) + } + const expectedRanks = [...keysByRank.keys()].sort((left, right) => + direction === `asc` ? left - right : right - left, + ) + for (const rank of expectedRanks) { + expectedBucketYields++ + const orderedKeys = [...keysByRank.get(rank)!] + orderedKeys.sort((left, right) => { + expectedKeyComparisons++ + return comparePublicKeys(left, right) + }) + expectedMatches += orderedKeys.filter( + (key) => rows.find((row) => row.id === key)?.included === true, + ).length + if (limit !== undefined && expectedMatches >= limit) break + } + } + + // Observe private value traversal and bucket construction independently + // from public generator yields. A generator can materialize all private + // values or groups before yielding only the requested prefix. + const readProbe = observeOrderedIndexReads(index, indexKind, direction) + const distinctValueCount = new Set(rows.map(({ rank }) => rank)).size + const expectedValueReads = + indexKind === `btree` || limit === 0 + ? expectedBucketYields + : limit === undefined + ? distinctValueCount + : Math.min( + distinctValueCount, + expectedBucketYields + + (expectedBucketYields < distinctValueCount ? 1 : 0), + ) + + let bucketYields = 0 + const originalOrderedBuckets = index.orderedBuckets.bind(index) + const originalOrderedBucketsReversed = + index.orderedBucketsReversed.bind(index) + index.orderedBuckets = function* () { + for (const bucket of originalOrderedBuckets()) { + bucketYields++ + yield bucket + } + } + index.orderedBucketsReversed = function* () { + for (const bucket of originalOrderedBucketsReversed()) { + bucketYields++ + yield bucket + } + } + + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(String(key)) + return originalGet(key) + } + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + + try { + keyComparisonCounter.count = 0 + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: orderBy(direction, direction === `asc` ? `last` : `first`), + limit, + })! + + return { + keys: changes.map(({ key }) => String(key)), + sourceReads, + expectedValueReads, + valueReads: readProbe.getValueReads(), + expectedBucketReads: expectedBucketYields, + bucketReads: readProbe.getBucketReads(), + expectedCursorCalls: + indexKind === `btree` + ? expectedBucketYields + + Number(limit === undefined || expectedMatches < limit) + : 0, + cursorCalls: readProbe.getCursorCalls(), + expectedBucketYields, + bucketYields, + unexpectedTraversalCalls: readProbe.getUnexpectedTraversalCalls(), + expectedKeyComparisons, + keyComparisons: keyComparisonCounter.count, + totalOrderComparisons: compareEntries.mock.calls.length, + } + } finally { + compareEntries.mockRestore() + readProbe.restore() + } + } finally { + await collection.cleanup() + } +} + +function createOrderedPrefixRows( + options: { + leadingRejects: number + limit: number + extraBoundaryMatches: number + boundaryRejects: number + trailingRows: number + }, + direction: OrderByDirection = `desc`, +): { + rows: Array + expectedKeys: Array + expectedSourceReads: Array +} { + const rank = (descendingRank: number) => + direction === `desc` ? descendingRank : -descendingRank + const leading = Array.from( + { length: options.leadingRejects }, + (_, index): RankedRow => ({ + id: `leading-${index.toString().padStart(2, `0`)}`, + rank: rank(100 + index), + included: false, + }), + ) + const matchingBoundary = Array.from( + { length: options.limit + options.extraBoundaryMatches }, + (_, index): RankedRow => ({ + id: `boundary-match-${index.toString().padStart(2, `0`)}`, + rank: rank(50), + included: true, + }), + ).reverse() + const rejectedBoundary = Array.from( + { length: options.boundaryRejects }, + (_, index): RankedRow => ({ + id: `boundary-reject-${index.toString().padStart(2, `0`)}`, + rank: rank(50), + included: false, + }), + ) + const trailing = Array.from( + { length: options.trailingRows }, + (_, index): RankedRow => ({ + id: `trailing-${index.toString().padStart(3, `0`)}`, + rank: rank(10 - index), + included: true, + }), + ) + const expectedKeys = matchingBoundary + .map(({ id }) => id) + .sort() + .slice(0, options.limit) + const expectedCandidateReads = [ + ...leading, + ...matchingBoundary, + ...rejectedBoundary, + ] + .sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) + return { + rows: [...trailing, ...rejectedBoundary, ...matchingBoundary, ...leading], + expectedKeys, + // Every row through the boundary bucket is tested once. The selected rows + // are then read once more to materialize their change messages. + expectedSourceReads: + options.limit === 0 ? [] : [...expectedCandidateReads, ...expectedKeys], + } +} + +describe(`ordered source work oracle`, () => { + it.each([`off`, `eager`] as const)( + `does no setup work for an empty ordered window with auto-indexing %s`, + async (autoIndex) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-empty-${autoIndex}`, + getKey: (row) => row.id, + initialData: [ + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: false }, + { id: `three`, rank: 3, included: true }, + ], + autoIndex, + ...(autoIndex === `eager` && { defaultIndexType: BTreeIndex }), + }), + ) + + try { + await collection.preload() + let whereExpressionReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + whereExpressionReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const entries = vi.spyOn(collection, `entries`) + const get = vi.spyOn(collection, `get`) + const createIndex = vi.spyOn(collection, `createIndex`) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + const indexesBefore = collection.indexes.size + + const changes = collection.currentStateAsChanges({ + where, + orderBy: orderBy(`asc`, `last`), + limit: 0, + }) + + expect(changes).toEqual([]) + expect(whereExpressionReads).toBe(0) + expect(entries).not.toHaveBeenCalled() + expect(get).not.toHaveBeenCalled() + expect(createIndex).not.toHaveBeenCalled() + expect(collection.indexes.size).toBe(indexesBefore) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + await collection.cleanup() + } + }, + ) + + it(`defers live ordered setup until a zero window becomes positive`, async () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-live-zero-window`, + getKey: (row) => row.id, + initialData: [{ id: `one`, rank: 1, included: true }], + }), + ) + let subscription: CollectionSubscription | undefined + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + subscription = new CollectionSubscription(collection, () => {}, {}) + subscription.setOrderByIndex(index) + let orderCompilationReads = 0 + const order: OrderBy = [ + { + expression: new Proxy(new PropRef([`rank`]), { + get(target, property, receiver) { + if (property === `type`) orderCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + compareOptions: publicKeyIndexCompareOptions, + }, + ] + + const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) + try { + subscription.requestLimitedSnapshot({ + orderBy: order, + limit: 0, + trackLoadSubsetPromise: false, + }) + expect( + ( + subscription as unknown as { + orderedWindow: WindowState | undefined + } + ).orderedWindow, + ).toBeUndefined() + // Freezing the request reads the expression tag once. It must not also + // construct TotalOrder or compile the frozen expression for no rows. + expect(orderCompilationReads).toBe(1) + expect(readProbe.getValueReads()).toBe(0) + expect(readProbe.getBucketReads()).toBe(0) + expect(readProbe.getCursorCalls()).toBe(0) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + } finally { + readProbe.restore() + } + + subscription.requestLimitedSnapshot({ + orderBy: order, + limit: 1, + trackLoadSubsetPromise: false, + }) + expect( + ( + subscription as unknown as { + orderedWindow: WindowState | undefined + } + ).orderedWindow, + ).toBeDefined() + } finally { + subscription?.unsubscribe() + await collection.cleanup() + } + }) + + it.each([ + { + name: `ascending numbers`, + direction: `asc` as const, + left: 1, + right: 2, + expected: -1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 0, + locale: 0, + localeOptions: 0, + ownKeys: 0, + descriptors: 0, + prototype: 0, + }, + }, + { + name: `ascending strings`, + direction: `asc` as const, + left: `a`, + right: `b`, + expected: -1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 1, + locale: 1, + localeOptions: 1, + ownKeys: 0, + descriptors: 0, + prototype: 0, + }, + }, + { + name: `descending numbers`, + direction: `desc` as const, + left: 1, + right: 2, + expected: 1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 0, + locale: 0, + localeOptions: 0, + ownKeys: 0, + descriptors: 0, + prototype: 0, + }, + }, + { + name: `descending strings`, + direction: `desc` as const, + left: `a`, + right: `b`, + expected: 1, + expectedReads: { + direction: 1, + nulls: 1, + stringSort: 1, + locale: 1, + localeOptions: 1, + ownKeys: 0, + descriptors: 0, + prototype: 0, + }, + }, + ])( + `executes the inner comparator once for $name`, + ({ direction, left, right, expected, expectedReads }) => { + const reads = { + direction: 0, + nulls: 0, + stringSort: 0, + locale: 0, + localeOptions: 0, + ownKeys: 0, + descriptors: 0, + prototype: 0, + } + const options = new Proxy( + { + direction, + nulls: `last` as const, + stringSort: `locale` as const, + locale: `en`, + localeOptions: { sensitivity: `base` as const }, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (typeof property === `string` && property in reads) { + reads[property as keyof typeof reads]++ + } + return Reflect.get(target, property, receiver) as unknown + }, + ownKeys(target) { + reads.ownKeys++ + return Reflect.ownKeys(target) + }, + getOwnPropertyDescriptor(target, property) { + reads.descriptors++ + return Reflect.getOwnPropertyDescriptor(target, property) + }, + getPrototypeOf(target) { + reads.prototype++ + return Reflect.getPrototypeOf(target) + }, + }, + ) + + const descriptorCopies = vi.spyOn(Object, `getOwnPropertyDescriptors`) + const prototypeReads = vi.spyOn(Object, `getPrototypeOf`) + let actual: number + let descriptorCopyCount: number + let prototypeReadCount: number + try { + actual = makeComparator(options)(left, right) + descriptorCopyCount = descriptorCopies.mock.calls.length + prototypeReadCount = prototypeReads.mock.calls.length + } finally { + descriptorCopies.mockRestore() + prototypeReads.mockRestore() + } + expect(descriptorCopyCount).toBe(0) + expect(prototypeReadCount).toBe(0) + expect(actual).toBe(expected) + expect(reads).toEqual(expectedReads) + }, + ) + + it.each([`asc`, `desc`] as const)( + `executes the inner comparator's date work once in %s order`, + (direction) => { + const getTime = vi.spyOn(Date.prototype, `getTime`) + try { + expect( + makeComparator({ direction, nulls: `last` })( + new Date(0), + new Date(1), + ), + ).toBe(direction === `asc` ? -1 : 1) + // Each valid Date is read once while checking the unorderable case and + // once more for the comparison itself. + expect(getTime).toHaveBeenCalledTimes(4) + } finally { + getTime.mockRestore() + } + }, + ) + + it.each([`asc`, `desc`] as const)( + `executes the inner comparator's string work once in %s order`, + (direction) => { + const localeCompare = vi.spyOn(String.prototype, `localeCompare`) + try { + expect( + makeComparator({ + direction, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + })(`a`, `b`), + ).toBe(direction === `asc` ? -1 : 1) + expect(localeCompare).toHaveBeenCalledTimes(1) + } finally { + localeCompare.mockRestore() + } + }, + ) + + type ComparatorArrayValue = number | Array + type ComparatorArray = Array + type ArrayInputReads = { lengths: number; elements: number } + type ArrayComparisonModel = { + sign: number + nullReads: number + leftReads: ArrayInputReads + rightReads: ArrayInputReads + } + + const modelAscendingArrayComparison = ( + left: ComparatorArrayValue, + right: ComparatorArrayValue, + ): ArrayComparisonModel => { + const leftReads: ArrayInputReads = { lengths: 0, elements: 0 } + const rightReads: ArrayInputReads = { lengths: 0, elements: 0 } + + if (Array.isArray(left) && Array.isArray(right)) { + leftReads.lengths++ + rightReads.lengths++ + const commonLength = Math.min(left.length, right.length) + let nullReads = 1 + + for (let index = 0; index < commonLength; index++) { + leftReads.elements++ + rightReads.elements++ + const child = modelAscendingArrayComparison(left[index]!, right[index]!) + nullReads += child.nullReads + leftReads.lengths += child.leftReads.lengths + leftReads.elements += child.leftReads.elements + rightReads.lengths += child.rightReads.lengths + rightReads.elements += child.rightReads.elements + if (child.sign !== 0) { + return { sign: child.sign, nullReads, leftReads, rightReads } + } + } + + return { + sign: Math.sign(left.length - right.length), + nullReads, + leftReads, + rightReads, + } + } + + const sign = Array.isArray(left) + ? 1 + : Array.isArray(right) + ? -1 + : Math.sign(left - right) + return { sign, nullReads: 1, leftReads, rightReads } + } + + const modelArrayComparison = ( + left: ComparatorArray, + right: ComparatorArray, + direction: `asc` | `desc`, + ): ArrayComparisonModel => { + if (direction === `asc`) { + return modelAscendingArrayComparison(left, right) + } + + const reversed = modelAscendingArrayComparison(right, left) + return { + sign: reversed.sign, + nullReads: reversed.nullReads, + leftReads: reversed.rightReads, + rightReads: reversed.leftReads, + } + } + + const recursiveArrayComparisonScenarios = [ + { name: `empty equality`, left: [], right: [] }, + { name: `no common element`, left: [], right: [1] }, + { name: `primitive equality`, left: [1], right: [1] }, + { name: `primitive difference`, left: [1], right: [2] }, + { name: `equal prefix then difference`, left: [1, 2], right: [1, 3] }, + { name: `equal prefix then length`, left: [1], right: [1, 2] }, + { name: `array then primitive`, left: [[]], right: [1] }, + { name: `primitive then array`, left: [1], right: [[]] }, + { + name: `equal nested prefix then outer difference`, + left: [[1], 2], + right: [[1], 3], + }, + { + name: `equal nested prefix then outer length`, + left: [[1]], + right: [[1], 2], + }, + ].flatMap(({ name, left: initialLeft, right: initialRight }) => { + const scenarios: Array<{ + name: string + left: ComparatorArray + right: ComparatorArray + }> = [] + let left: ComparatorArray = initialLeft + let right: ComparatorArray = initialRight + + for (let depth = 1; depth <= 3; depth++) { + scenarios.push({ name: `${name} at depth ${depth}`, left, right }) + left = [left] + right = [right] + } + return scenarios + }) + + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + recursiveArrayComparisonScenarios.map((scenario) => ({ + direction, + ...scenario, + })), + ), + )( + `reads each visited array input once for $name in $direction order`, + ({ direction, left, right }) => { + let nullReads = 0 + const observeArrayReads = ( + value: ComparatorArray, + reads: { lengths: number; elements: number; structural: number }, + ): ComparatorArray => { + const nested = value.map((element) => + Array.isArray(element) ? observeArrayReads(element, reads) : element, + ) + return new Proxy(nested, { + get(target, property, receiver) { + if (property === `length`) { + reads.lengths++ + } else if ( + typeof property === `string` && + /^(0|[1-9]\d*)$/.test(property) + ) { + reads.elements++ + } else if (property !== Symbol.toStringTag) { + // Array/scalar comparisons perform constant-time brand checks. + // The work law counts input-dependent traversal, not those checks. + reads.structural++ + } + return Reflect.get( + target, + property, + receiver, + ) as ComparatorArrayValue + }, + ownKeys(target) { + reads.structural++ + return Reflect.ownKeys(target) + }, + getOwnPropertyDescriptor(target, property) { + reads.structural++ + return Reflect.getOwnPropertyDescriptor(target, property) + }, + has(target, property) { + reads.structural++ + return Reflect.has(target, property) + }, + }) + } + const leftReads = { lengths: 0, elements: 0, structural: 0 } + const rightReads = { lengths: 0, elements: 0, structural: 0 } + const observedLeft = observeArrayReads(left, leftReads) + const observedRight = observeArrayReads(right, rightReads) + const expected = modelArrayComparison(left, right, direction) + const options = new Proxy( + { + direction, + nulls: `last` as const, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `nulls`) nullReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) + + expect( + Math.sign(makeComparator(options)(observedLeft, observedRight)), + ).toBe(expected.sign) + expect(nullReads).toBe(expected.nullReads) + expect(leftReads).toEqual({ ...expected.leftReads, structural: 0 }) + expect(rightReads).toEqual({ ...expected.rightReads, structural: 0 }) + }, + ) + + it.each([ + { indexKind: `basic`, direction: `asc` }, + { indexKind: `basic`, direction: `desc` }, + { indexKind: `btree`, direction: `asc` }, + { indexKind: `btree`, direction: `desc` }, + ] as const)( + `does not read worse $indexKind index buckets in $direction order`, + async ({ indexKind, direction }) => { + const scenario = createOrderedPrefixRows( + { + leadingRejects: 2, + limit: 2, + extraBoundaryMatches: 1, + boundaryRejects: 2, + trailingRows: 40, + }, + direction, + ) + + const observed = await observeOrderedPrefix( + scenario.rows, + 2, + indexKind, + direction, + ) + + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + + for (const direction of [`asc`, `desc`] as const) { + const property = + direction === `asc` + ? `ordered-work.forward-prefix` + : `ordered-work.reverse-prefix` + const seed = direction === `asc` ? 1_780_103 : 1_780_101 + for (const campaign of orderedWorkCampaigns(property, seed)) { + fcTest.prop( + [ + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 0, max: 5 }), + fc.integer({ min: 0, max: 5 }), + fc.integer({ min: 0, max: 8 }), + fc.integer({ min: 0, max: 60 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + ], + campaign.options, + )( + `bounds ${direction} index reads at the sufficient bucket (${campaign.label})`, + async ( + leadingRejects, + limit, + extraBoundaryMatches, + boundaryRejects, + trailingRows, + indexKind, + ) => { + const scenario = createOrderedPrefixRows( + { + leadingRejects, + limit, + extraBoundaryMatches, + boundaryRejects, + trailingRows, + }, + direction, + ) + const observed = await observeOrderedPrefix( + scenario.rows, + limit, + indexKind, + direction, + ) + + expect(observed.keys).toEqual(scenario.expectedKeys) + expect(observed.sourceReads).toEqual(scenario.expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + } + } + + for (const direction of [`asc`, `desc`] as const) { + const property = + direction === `asc` + ? `ordered-work.forward-exhaustion` + : `ordered-work.reverse-exhaustion` + const seed = direction === `asc` ? 1_780_105 : 1_780_106 + for (const campaign of orderedWorkCampaigns(property, seed)) { + fcTest.prop( + [ + fc.integer({ min: 0, max: 60 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + ], + campaign.options, + )( + `reads each ${direction} bucket once before proving exhaustion (${campaign.label})`, + async (rowCount, indexKind) => { + const rows = Array.from( + { length: rowCount }, + (_, index): RankedRow => ({ + id: `rejected-${index.toString().padStart(2, `0`)}`, + rank: Math.floor(index / 2), + included: false, + }), + ).reverse() + const expectedSourceReads = [...rows] + .sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) + + const observed = await observeOrderedPrefix( + rows, + 1, + indexKind, + direction, + ) + + expect(observed.keys).toEqual([]) + expect(observed.sourceReads).toEqual(expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + } + } + + it(`reads the complete tied boundary when every candidate is tied`, async () => { + const rows = Array.from( + { length: 25 }, + (_, index): RankedRow => ({ + id: `tied-${index.toString().padStart(2, `0`)}`, + rank: 1, + included: index % 2 === 0, + }), + ).reverse() + + const observed = await observeOrderedPrefix(rows, 3) + expect(observed.keys).toEqual([`tied-00`, `tied-02`, `tied-04`]) + expect(observed.sourceReads).toEqual([ + ...rows.map(({ id }) => id).sort(comparePublicKeys), + `tied-00`, + `tied-02`, + `tied-04`, + ]) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }) + + it.each( + ([`basic`, `btree`] as const).flatMap((indexKind) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`one tie bucket`, `many buckets`] as const).map((bucketShape) => ({ + indexKind, + direction, + bucketShape, + })), + ), + ), + )( + `does exact unbounded work for $indexKind $direction order with $bucketShape`, + async ({ indexKind, direction, bucketShape }) => { + const rows: Array = + bucketShape === `one tie bucket` + ? [ + { id: `d`, rank: 1, included: true }, + { id: `b`, rank: 1, included: false }, + { id: `c`, rank: 1, included: true }, + { id: `a`, rank: 1, included: true }, + ] + : [ + { id: `d`, rank: 3, included: true }, + { id: `b`, rank: 1, included: false }, + { id: `e`, rank: 3, included: false }, + { id: `c`, rank: 2, included: true }, + { id: `a`, rank: 1, included: true }, + ] + const orderedRows = [...rows].sort((left, right) => { + const valueOrder = left.rank - right.rank + if (valueOrder !== 0) { + return direction === `asc` ? valueOrder : -valueOrder + } + return comparePublicKeys(left.id, right.id) + }) + const expectedKeys = orderedRows + .filter(({ included }) => included) + .map(({ id }) => id) + const expectedSourceReads = [ + ...orderedRows.map(({ id }) => id), + ...expectedKeys, + ] + + const observed = await observeOrderedPrefix( + rows, + undefined, + indexKind, + direction, + ) + + expect(observed.keys).toEqual(expectedKeys) + expect(observed.sourceReads).toEqual(expectedSourceReads) + expect(observed.valueReads).toBe(observed.expectedValueReads) + expect(observed.bucketReads).toBe(observed.expectedBucketReads) + expect(observed.cursorCalls).toBe(observed.expectedCursorCalls) + expect(observed.bucketYields).toBe(observed.expectedBucketYields) + expect(observed.unexpectedTraversalCalls).toBe(0) + expect(observed.keyComparisons).toBe(observed.expectedKeyComparisons) + expect(observed.totalOrderComparisons).toBe(0) + }, + ) + + it.each([ + { direction: `asc`, indexNulls: `first` }, + { direction: `desc`, indexNulls: `last` }, + ] as const)( + `stops Basic $direction traversal after a multi-value nullish tie`, + async ({ direction, indexNulls }) => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-basic-nullish-${direction}`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: true }, + ], + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BasicIndex, + options: { + compareOptions: { + direction: `asc`, + nulls: indexNulls, + stringSort: `locale`, + }, + }, + }) as BasicIndex + const readProbe = observeOrderedIndexReads(index, `basic`, direction) + + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(direction, `first`), + limit: 1, + })! + + expect(changes.map(({ key }) => key)).toEqual([`null`]) + // The two exact nullish values form one comparator bucket. Basic + // reads one worse value to close that group, but it must not scan the + // second worse value or construct either worse bucket. + expect(readProbe.getValueReads()).toBe(3) + expect(readProbe.getBucketReads()).toBe(2) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + } finally { + readProbe.restore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it(`keeps comparator-equivalent BTree values in one ordered tie class`, async () => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const rows: Array = [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-nullish-tie`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `last`), + limit: rows.length, + })! + + expect(changes.map(({ key }) => key)).toEqual([ + `one`, + `null`, + `undefined`, + ]) + } finally { + await collection.cleanup() + } + }) + + it(`does not reverse an index with incompatible null placement`, async () => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-null-placement`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(`desc`, `first`), + })! + + expect(changes.map(({ key }) => key)).toEqual([ + `null`, + `undefined`, + `one`, + ]) + } finally { + await collection.cleanup() + } + }) + + it.each(orderedIndexCompatibilityCases)( + `matches $indexKind index $indexDirection/nulls-$indexNulls to query $queryDirection/nulls-$queryNulls: $compatible`, + async ({ + indexKind, + indexDirection, + indexNulls, + queryDirection, + queryNulls, + compatible, + }) => { + type NullableRankedRow = Omit & { + rank: number | null | undefined + } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-index-compatibility-${indexDirection}-${indexNulls}-${queryDirection}-${queryNulls}`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `one`, rank: 1, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { + compareOptions: { + direction: indexDirection, + nulls: indexNulls, + stringSort: `locale`, + }, + }, + }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderBy(queryDirection, queryNulls), + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual( + compatible + ? queryNulls === `first` + ? [`null`, `undefined`, `one`] + : [`one`, `null`, `undefined`] + : undefined, + ) + } finally { + await collection.cleanup() + } + }, + ) + + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + domain: `signed number`, + tieKeys: [1, -2], + rejectedKey: -999, + laterKey: 999, + }, + { + domain: `NaN number`, + tieKeys: [Number.NaN, 2, -1], + rejectedKey: -999, + laterKey: 999, + }, + { + domain: `case-sensitive string`, + tieKeys: [`a`, `A`], + rejectedKey: `rejected`, + laterKey: `later`, + }, + { + domain: `non-ASCII string`, + tieKeys: [`é`, `e`, `Ω`, `ß`], + rejectedKey: `rejected`, + laterKey: `later`, + }, + { + domain: `mixed`, + tieKeys: [10, `2`, 2, `10`], + rejectedKey: `rejected`, + laterKey: `later`, + }, + ].map((keyCase) => ({ direction, ...keyCase })), + ), + )( + `fully refines a filtered $domain custom-index fallback in $direction order`, + async ({ direction, domain, tieKeys, rejectedKey, laterKey }) => { + const tieRank = 1 + const rejectedRank = direction === `asc` ? 0 : 2 + const laterRank = direction === `asc` ? 2 : 0 + const rows: Array = [ + { id: laterKey, rank: laterRank, included: true }, + ...tieKeys + .map((id) => ({ id, rank: tieRank, included: true })) + .reverse(), + { id: rejectedKey, rank: rejectedRank, included: false }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-index-fallback-${direction}-${domain}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + const requestedCounts: Array = [] + const customIndex = new Proxy(index, { + get(target, property) { + if ( + property === `orderedBuckets` || + property === `orderedBucketsReversed` + ) { + return undefined + } + const value = Reflect.get(target, property, target) as unknown + if ( + typeof value === `function` && + (property === `takeFromStart` || + property === `takeReversedFromEnd`) + ) { + return (count: number, ...args: Array) => { + requestedCounts.push(count) + return value.apply(target, [count, ...args]) + } + } + return typeof value === `function` ? value.bind(target) : value + }, + }) + collection.indexes.set(index.id, customIndex) + if (direction === `desc`) { + expect( + new ReverseIndex(customIndex).supportsOrderedBucketIteration, + ).toBe(false) + } + + const orderedTieKeys = [...tieKeys].sort(comparePublicKeys) + const indexTieKeys = + direction === `asc` ? orderedTieKeys : [...orderedTieKeys].reverse() + const indexScanKeys = [rejectedKey, ...indexTieKeys, laterKey] + const matchingIndexKeys = [...indexTieKeys, laterKey] + const rowsByKey = new Map(rows.map((row) => [row.id, row])) + let expectedTotalOrderComparisons = 0 + const expectedKeys = [...matchingIndexKeys] + .sort((left, right) => { + expectedTotalOrderComparisons++ + const leftRow = rowsByKey.get(left)! + const rightRow = rowsByKey.get(right)! + const rankOrder = leftRow.rank - rightRow.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left, right) + }) + .slice(0, 2) + + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(requestedCounts).toEqual([index.keyCount]) + expect(sourceReads).toEqual([ + ...indexScanKeys, + ...matchingIndexKeys, + ...expectedKeys, + ]) + expect(compareEntries).toHaveBeenCalledTimes( + expectedTotalOrderComparisons, + ) + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it.each( + ( + [ + { source: `no index`, IndexType: undefined }, + { source: `opaque BasicIndex`, IndexType: BasicIndex }, + { source: `opaque BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ source, IndexType }) => + ([`asc`, `desc`] as const).map((direction) => ({ + source, + IndexType, + direction, + })), + ), + )( + `does exact one-pass work for the $source fallback in $direction order`, + async ({ source, IndexType, direction }) => { + const rows: Array = [ + { + id: `later`, + rank: direction === `asc` ? 2 : 0, + included: true, + }, + { id: `é`, rank: 1, included: true }, + { id: `e`, rank: 1, included: true }, + { + id: `rejected`, + rank: direction === `asc` ? 0 : 2, + included: false, + }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-full-fallback-${source}-${direction}`, + getKey: (row) => row.id, + initialData: rows, + autoIndex: `off`, + }), + ) + + try { + await collection.preload() + if (IndexType) { + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + } + + let referenceCompilationReads = 0 + const referenceWhere = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + compileSingleRowExpression(referenceWhere) + + let compilationReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) compilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const expectedEntries = [...collection.entries()] + const enumeratedKeys: Array = [] + const originalEntries = collection.entries.bind(collection) + collection.entries = function* () { + for (const entry of originalEntries()) { + enumeratedKeys.push(entry[0]) + yield entry + } + } + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } + + let expectedTotalOrderComparisons = 0 + const expectedKeys = expectedEntries + .map(([, row]) => row) + .filter(({ included }) => included) + .sort((left, right) => { + expectedTotalOrderComparisons++ + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, 2) + .map(({ id }) => id) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where, + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(enumeratedKeys).toEqual(expectedEntries.map(([key]) => key)) + expect(sourceReads).toEqual([ + ...expectedEntries.map(([key]) => key), + ...expectedKeys, + ]) + expect(compilationReads).toBe(referenceCompilationReads) + expect(compareEntries).toHaveBeenCalledTimes( + expectedTotalOrderComparisons, + ) + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it(`does exact short-circuit work for a multi-term TotalOrder fallback`, async () => { + type MultiTermRow = RankedRow & { secondary: number } + const specs: Array = [ + { id: `d`, rank: 2, secondary: 1, included: true }, + { id: `b`, rank: 1, secondary: 2, included: true }, + { id: `a`, rank: 1, secondary: 2, included: true }, + { id: `c`, rank: 1, secondary: 1, included: true }, + { id: `hidden`, rank: 0, secondary: 0, included: false }, + ] + const reads = { rank: 0, secondary: 0, included: 0 } + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-multi-term-fallback`, + getKey: (row) => row.id, + initialData: specs, + autoIndex: `off`, + }), + ) + + try { + await collection.preload() + const originalEntries = collection.entries.bind(collection) + const storedRows = [...originalEntries()].map(([, value]) => value) + collection.entries = function* () { + for (const [key, value] of originalEntries()) { + yield [ + key, + new Proxy(value, { + get(target, property, receiver) { + if (property === `rank`) reads.rank++ + if (property === `secondary`) reads.secondary++ + if (property === `included`) reads.included++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + ] as const + } + } + reads.rank = 0 + reads.secondary = 0 + reads.included = 0 + + let referenceCompilationReads = 0 + compileSingleRowExpression( + new Proxy(new PropRef([`rank`]), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }), + ) + expect(referenceCompilationReads).toBeGreaterThan(0) + + let termCompilationReads = 0 + const trackedTerm = (propertyName: `rank` | `secondary`) => + new Proxy(new PropRef([propertyName]), { + get(target, property, receiver) { + if (property === `type`) termCompilationReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + const termComparisons: [number, number] = [0, 0] + const trackedCompareOptions = (term: 0 | 1): CompareOptions => + new Proxy( + { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `direction`) termComparisons[term]++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) + const order: OrderBy = [ + { + expression: trackedTerm(`rank`), + compareOptions: trackedCompareOptions(0), + }, + { + expression: trackedTerm(`secondary`), + compareOptions: trackedCompareOptions(1), + }, + ] + + let expectedComparisons = 0 + let expectedRankReads = 0 + let expectedSecondaryReads = 0 + let expectedKeyComparisons = 0 + const expectedKeys = storedRows + .filter(({ included }) => included) + .sort((left, right) => { + expectedComparisons++ + expectedRankReads += 2 + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) return rankOrder + expectedSecondaryReads += 2 + const secondaryOrder = left.secondary - right.secondary + if (secondaryOrder !== 0) return secondaryOrder + expectedKeyComparisons++ + return comparePublicKeys(left.id, right.id) + }) + .map(({ id }) => id) + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + keyComparisonCounter.count = 0 + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: order, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(termCompilationReads).toBe(referenceCompilationReads * 2) + expect(reads.included).toBe(specs.length) + expect(reads.rank).toBe(expectedRankReads) + expect(reads.secondary).toBe(expectedSecondaryReads) + expect(compareEntries).toHaveBeenCalledTimes(expectedComparisons) + expect(termComparisons).toEqual([ + expectedComparisons, + expectedSecondaryReads / 2, + ]) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }) + + it.each( + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { + capability: `neither iterator`, + exposeForward: false, + exposeReverse: false, + }, + { + capability: `the forward iterator only`, + exposeForward: true, + exposeReverse: false, + }, + { + capability: `the reverse iterator only`, + exposeForward: false, + exposeReverse: true, + }, + { + capability: `both iterators`, + exposeForward: true, + exposeReverse: true, + }, + ].map((capabilities) => ({ direction, ...capabilities })), + ), + )( + `trusts $capability for a custom index only when it serves $direction order`, + async ({ direction, exposeForward, exposeReverse }) => { + const rows: Array = [ + { + id: `later`, + rank: direction === `asc` ? 2 : 0, + included: true, + }, + { id: `tie-b`, rank: 1, included: true }, + { id: `tie-a`, rank: 1, included: true }, + { + id: `rejected`, + rank: direction === `asc` ? 0 : 2, + included: false, + }, + ] + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-capabilities-${direction}-${exposeForward}-${exposeReverse}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + const customIndex = new Proxy(index, { + get(target, property) { + if (property === `orderedBuckets` && !exposeForward) { + return undefined + } + if (property === `orderedBucketsReversed` && !exposeReverse) { + return undefined + } + const value = Reflect.get(target, property, target) as unknown + return typeof value === `function` ? value.bind(target) : value + }, + }) + collection.indexes.set(index.id, customIndex) + + const expectedKeys = rows + .filter(({ included }) => included) + .sort((left, right) => { + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, 2) + .map(({ id }) => id) + const usesLazyBuckets = + exposeForward && (direction === `asc` || exposeReverse) + expect( + new ReverseIndex(customIndex).supportsOrderedBucketIteration, + ).toBe(exposeForward && exposeReverse) + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + if (usesLazyBuckets) { + expect(compareEntries).not.toHaveBeenCalled() + } else { + expect(compareEntries).toHaveBeenCalled() + } + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it.each( + ( + [ + { name: `BasicIndex`, IndexType: BasicIndex }, + { name: `BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ name, IndexType }) => + ([`asc`, `desc`] as const).map((direction) => ({ + name, + IndexType, + direction, + })), + ), + )( + `fully refines a $name custom comparator in $direction order`, + async ({ name, IndexType, direction }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-comparator-${name}-${direction}`, + getKey: (row) => row.id, + initialData: [ + { id: `one`, rank: 1, included: true }, + { id: `two`, rank: 2, included: true }, + { id: `three`, rank: 3, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: 2, + })! + + expect(changes.map(({ key }) => key)).toEqual( + direction === `asc` ? [`one`, `two`] : [`three`, `two`], + ) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + for (const campaign of orderedWorkCampaigns( + `ordered-work.custom-comparator-fallback`, + 1_780_104, + )) { + fcTest.prop( + [ + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 2, + maxLength: 8, + }), + fc.integer({ min: 1, max: 8 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + fc.constantFrom(`asc`, `desc`), + ], + campaign.options, + )( + `fully refines generated custom comparator indexes (${campaign.label})`, + async (ranks, requestedLimit, indexKind, direction) => { + const rows = ranks.map( + (rank, index): RankedRow => ({ + id: `row-${index.toString().padStart(2, `0`)}`, + rank, + included: true, + }), + ) + const limit = Math.min(requestedLimit, rows.length) + const expectedKeys = [...rows] + .sort((left, right) => { + const rankOrder = left.rank - right.rank + if (rankOrder !== 0) { + return direction === `asc` ? rankOrder : -rankOrder + } + return comparePublicKeys(left.id, right.id) + }) + .slice(0, limit) + .map(({ id }) => id) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-custom-comparator-property-${Math.random()}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { + compareOptions: publicKeyIndexCompareOptions, + compareFn: (left: number, right: number) => right - left, + }, + }) + + const compareEntries = vi.spyOn( + TotalOrder.prototype, + `compareEntries`, + ) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(compareEntries).toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + } + + it(`groups comparator-equivalent values in every built-in index direction`, () => { + type TextRow = { id: string; value: string } + const rows: Array = [ + { id: `upper`, value: `A` }, + { id: `lower`, value: `a` }, + { id: `later`, value: `b` }, + ] + + for (const IndexType of [BasicIndex, BTreeIndex]) { + const index = new IndexType( + 1, + new PropRef([`value`]), + undefined, + { + compareFn: (left: string, right: string) => + left.toLowerCase().localeCompare(right.toLowerCase()), + }, + ) + index.build(rows.map((row) => [row.id, row])) + + expect( + [...index.orderedBuckets()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`lower`, `upper`], [`later`]]) + expect( + [...index.orderedBucketsReversed()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`later`], [`lower`, `upper`]]) + expect( + [...new ReverseIndex(index).orderedBuckets()].map(([, keys]) => + [...keys].sort(), + ), + ).toEqual([[`later`], [`lower`, `upper`]]) + + index.remove(`lower`, rows[1]) + expect([...index.equalityLookup(`A`)]).toEqual([`upper`]) + expect([...index.equalityLookup(`a`)]).toEqual([]) + expect( + [...index.orderedBuckets()].map(([, keys]) => [...keys].sort()), + ).toEqual([[`upper`], [`later`]]) + } + }) + + it.each([ + { + name: `BasicIndex`, + IndexType: BasicIndex, + direction: `asc`, + expectedKeys: [`a`, `z`], + }, + { + name: `BasicIndex`, + IndexType: BasicIndex, + direction: `desc`, + expectedKeys: [`m`, `a`], + }, + { + name: `BTreeIndex`, + IndexType: BTreeIndex, + direction: `asc`, + expectedKeys: [`a`, `z`], + }, + { + name: `BTreeIndex`, + IndexType: BTreeIndex, + direction: `desc`, + expectedKeys: [`m`, `a`], + }, + ] as const)( + `keeps the public-key suffix ascending for $name in $direction order`, + async ({ name, IndexType, direction, expectedKeys }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-key-suffix-${name}-${direction}`, + getKey: (row) => row.id, + initialData: [{ id: `m`, rank: 2, included: true }], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) + const z = collection.insert({ id: `z`, rank: 1, included: true }) + await z.isPersisted.promise + const a = collection.insert({ id: `a`, rank: 1, included: true }) + await a.isPersisted.promise + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: 2, + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it.each( + ( + [ + { name: `BasicIndex`, IndexType: BasicIndex }, + { name: `BTreeIndex`, IndexType: BTreeIndex }, + ] as const + ).flatMap(({ name, IndexType }) => + ([`asc`, `desc`] as const).flatMap((direction) => + [ + { domain: `signed number`, keys: [1, -2] }, + { domain: `NaN number`, keys: [Number.NaN, 2, -1] }, + { domain: `case-sensitive string`, keys: [`a`, `A`] }, + { domain: `non-ASCII string`, keys: [`é`, `e`, `Ω`, `ß`] }, + { domain: `mixed`, keys: [10, `2`, 2, `10`] }, + ].map(({ domain, keys }) => ({ + name, + IndexType, + direction, + domain, + keys, + })), + ), + ), + )( + `keeps $domain public keys in compareKeys order for $name in $direction order`, + async ({ name, IndexType, direction, keys }) => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-${name}-${direction}-${keys.join(`-`)}`, + getKey: (row) => row.id, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: IndexType, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) + for (const key of keys) { + const transaction = collection.insert({ + id: key, + rank: 1, + included: true, + }) + await transaction.isPersisted.promise + } + let expectedKeyComparisons = 0 + const expectedKeys = [...keys].sort((left, right) => { + expectedKeyComparisons++ + return comparePublicKeys(left, right) + }) + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + return originalGet(key) + } + + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + keyComparisonCounter.count = 0 + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit: keys.length, + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + + for (const campaign of orderedWorkCampaigns( + `ordered-work.public-key-suffix`, + 1_780_102, + )) { + fcTest.prop( + [ + fc.uniqueArray( + fc.oneof( + fc.integer({ min: -999, max: 999 }), + fc.constant(Number.NaN), + ), + { + minLength: 2, + maxLength: 8, + }, + ), + fc.integer({ min: 1, max: 16 }), + fc.constantFrom<`basic` | `btree`>(`basic`, `btree`), + fc.constantFrom(`asc`, `desc`), + fc.constantFrom<`string` | `number` | `mixed`>( + `string`, + `number`, + `mixed`, + ), + ], + campaign.options, + )( + `orders dynamic tie keys for every built-in path (${campaign.label})`, + async (keyNumbers, requestedLimit, indexKind, direction, keyDomain) => { + const keys: Array = + keyDomain === `string` + ? keyNumbers.map((key, index) => + index % 2 === 0 ? `key-${key}` : `Key-${key}`, + ) + : keyDomain === `number` + ? keyNumbers + : keyNumbers.flatMap((key) => [String(key), key]) + const limit = Math.min(requestedLimit, keys.length) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-key-property-${Math.random()}`, + getKey: (row) => row.id, + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) + for (const key of keys) { + const transaction = collection.insert({ + id: key, + rank: 1, + included: true, + }) + await transaction.isPersisted.promise + } + + const compareEntries = vi.spyOn( + TotalOrder.prototype, + `compareEntries`, + ) + try { + const changes = collection.currentStateAsChanges({ + orderBy: publicKeyOrderBy(direction), + limit, + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual( + [...keys].sort(comparePublicKeys).slice(0, limit), + ) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }, + ) + } + + it.each( + ([`basic`, `btree`] as const).flatMap((indexKind) => + ([`asc`, `desc`] as const).flatMap((direction) => + ([`string`, `nullish`] as const).map((orderDomain) => ({ + indexKind, + direction, + orderDomain, + })), + ), + ), + )( + `does exact optimized work for $indexKind $direction $orderDomain order values`, + async ({ indexKind, direction, orderDomain }) => { + type DomainRow = Omit & { + rank: string | number | null | undefined + } + const rows: Array = + orderDomain === `string` + ? [ + { id: `item-10`, rank: `item-10`, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + ] + : [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `two`, rank: 2, included: true }, + { id: `one`, rank: 1, included: true }, + ] + const expectedKeys = + orderDomain === `string` + ? direction === `asc` + ? [`item-2`, `item-10`] + : [`item-10`, `item-2`] + : direction === `asc` + ? [`one`, `two`, `null`, `undefined`] + : [`null`, `undefined`, `two`, `one`] + const indexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` as const }, + } satisfies CompareOptions + const queryCompareOptions = { + ...indexCompareOptions, + direction, + nulls: direction === `asc` ? (`last` as const) : (`first` as const), + } satisfies CompareOptions + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-domain-${indexKind}-${direction}-${orderDomain}`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { compareOptions: indexCompareOptions }, + }) as BasicIndex | BTreeIndex + const readProbe = observeOrderedIndexReads(index, indexKind, direction) + const sourceReads: Array = [] + let predicateReads = 0 + const originalGet = collection.get.bind(collection) + collection.get = (key) => { + sourceReads.push(key) + const value = originalGet(key) + return value === undefined + ? undefined + : new Proxy(value, { + get(target, property, receiver) { + if (property === `included`) predicateReads++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + } + const bucketCount = + orderDomain === `string` ? 2 : indexKind === `basic` ? 4 : 3 + + try { + keyComparisonCounter.count = 0 + const changes = collection.currentStateAsChanges({ + where: eq(new PropRef([`included`]), true), + orderBy: orderByWithOptions(queryCompareOptions), + optimizedOnly: true, + })! + + expect(changes.map(({ key }) => key)).toEqual(expectedKeys) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(predicateReads).toBe(rows.length) + expect(readProbe.getValueReads()).toBe(bucketCount) + expect(readProbe.getBucketReads()).toBe(bucketCount) + expect(readProbe.getCursorCalls()).toBe( + indexKind === `btree` ? bucketCount + 1 : 0, + ) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + expect(keyComparisonCounter.count).toBe( + orderDomain === `nullish` ? 1 : 0, + ) + } finally { + readProbe.restore() + } + } finally { + await collection.cleanup() + } + }, + ) + + it(`retains requested comparison metadata on an automatic index`, async () => { + type NullableRankedRow = Omit & { + rank: string | null | undefined + } + const compareOptions = { + direction: `desc`, + nulls: `first`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + } satisfies CompareOptions + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-auto-index-options`, + getKey: (row) => row.id, + initialData: [ + { id: `undefined`, rank: undefined, included: true }, + { id: `null`, rank: null, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + { id: `item-10`, rank: `item-10`, included: true }, + ], + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + }), + ) + + try { + await collection.preload() + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + try { + const changes = collection.currentStateAsChanges({ + orderBy: orderByWithOptions(compareOptions), + limit: 4, + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual([ + `null`, + `undefined`, + `item-10`, + `item-2`, + ]) + expect(compareEntries).not.toHaveBeenCalled() + } finally { + compareEntries.mockRestore() + } + } finally { + await collection.cleanup() + } + }) + + it.each(orderedStringCompatibilityCases)( + `matches a $indexKind index against $name in $queryDirection order: $compatible`, + async ({ indexKind, queryDirection, compareOptions, compatible }) => { + type TextRankedRow = Omit & { rank: string } + const indexCompareOptions = { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { numeric: true, sensitivity: `base` }, + } satisfies CompareOptions + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-string-options-${Math.random()}`, + getKey: (row) => row.id, + initialData: [ + { id: `item-10`, rank: `item-10`, included: true }, + { id: `item-2`, rank: `item-2`, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { + indexType: indexKind === `basic` ? BasicIndex : BTreeIndex, + options: { compareOptions: indexCompareOptions }, + }) + + const changes = collection.currentStateAsChanges({ + orderBy: orderByWithOptions(compareOptions), + optimizedOnly: true, + }) + + expect(changes?.map(({ key }) => key)).toEqual( + compatible + ? queryDirection === `asc` + ? [`item-2`, `item-10`] + : [`item-10`, `item-2`] + : undefined, + ) + } finally { + await collection.cleanup() + } + }, + ) +}) + +type SnapshotFixture = { + collection: CollectionImpl + snapshotRevisions: Array + replace: (row: RankedRow) => void +} + +function createSnapshotFixture( + initialRows: ReadonlyArray, +): SnapshotFixture { + let rows = new Map(initialRows.map((row) => [row.id, row])) + let revision = 0 + const snapshotRevisions: Array = [] + const collection = { + compareOptions: { stringSort: `lexical` }, + get _stateRevision() { + return revision + }, + currentStateAsChanges: (options: CurrentStateAsChangesOptions) => { + snapshotRevisions.push(revision) + return [...rows] + .filter(([, value]) => options.where === undefined || value.included) + .sort((left, right) => + left[1].rank === right[1].rank + ? left[0].localeCompare(right[0]) + : left[1].rank - right[1].rank, + ) + .map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + + return { + collection, + snapshotRevisions, + replace: (row) => { + rows = new Map(rows).set(row.id, row) + revision++ + }, + } +} + +function observeWindow( + window: WindowState, +) { + return { + localPrefixSize: window.localPrefixSize, + rowsNeeded: window.rowsNeeded(), + publication: window.publicationEntries().map(([key]) => key), + boundary: window.boundary(), + requestBoundary: window.requestBoundary(), + progressBoundary: window.progressBoundary(), + changes: window.reconcile(new Map()).map(({ key }) => key), + } +} + +function createCoveredWindow(fixture: SnapshotFixture, size: number) { + const window = new WindowState( + fixture.collection, + orderBy(`asc`), + eq(new PropRef([`included`]), true), + size, + ) + window.recordInitialCoverage(undefined, true) + return window +} + +it(`reuses one ordered source snapshot until the collection revision changes`, () => { + const fixture = createSnapshotFixture([ + { id: `a`, rank: 1, included: true }, + { id: `b`, rank: 2, included: true }, + { id: `hidden`, rank: 0, included: false }, + ]) + const window = createCoveredWindow(fixture, 2) + + expect(observeWindow(window)).toMatchObject({ + localPrefixSize: 2, + rowsNeeded: 0, + publication: [`a`, `b`], + }) + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(fixture.snapshotRevisions).toEqual([0]) + + fixture.replace({ id: `b`, rank: -1, included: true }) + + expect(observeWindow(window)).toMatchObject({ publication: [`b`, `a`] }) + expect(observeWindow(window)).toMatchObject({ publication: [`b`, `a`] }) + expect(fixture.snapshotRevisions).toEqual([0, 1]) +}) + +it(`does only exact predicate and boundary work when reusing an unbounded snapshot`, async () => { + const reads = { rank: 0, included: 0 } + const rows: Array = [`é`, `e`, `Ω`, `ß`, `A`].map((id) => ({ + id, + rank: 1, + included: true, + })) + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-unbounded-snapshot-reuse`, + getKey: (row) => row.id, + initialData: rows, + }), + ) + + try { + await collection.preload() + const index = collection.createIndex((row) => row.rank, { + indexType: BTreeIndex, + options: { compareOptions: publicKeyIndexCompareOptions }, + }) as BTreeIndex + const expectedKeys = rows.map(({ id }) => id).sort(comparePublicKeys) + const expectedKeyComparisons = Math.max(0, expectedKeys.length - 1) + + const readProbe = observeOrderedIndexReads(index, `btree`, `asc`) + const sourceReads: Array = [] + const originalGet = collection.get.bind(collection) + const observedValues = new Map< + Parameters[0], + NonNullable> + >() + collection.get = (key) => { + sourceReads.push(key) + const value = originalGet(key) + if (value === undefined) return + let observed = observedValues.get(key) + if (observed === undefined) { + observed = new Proxy(value, { + get(target, property, receiver) { + if (property === `rank`) reads.rank++ + if (property === `included`) reads.included++ + return Reflect.get(target, property, receiver) as unknown + }, + }) + observedValues.set(key, observed) + } + return observed + } + const compareEntries = vi.spyOn(TotalOrder.prototype, `compareEntries`) + const window = new WindowState( + collection, + publicKeyOrderBy(`asc`), + eq(new PropRef([`included`]), true), + 3, + ) + window.recordInitialCoverage(undefined, true) + + try { + keyComparisonCounter.count = 0 + reads.rank = 0 + reads.included = 0 + expect(observeWindow(window)).toMatchObject({ + publication: expectedKeys.slice(0, 3), + }) + expect(sourceReads).toEqual([...expectedKeys, ...expectedKeys]) + expect(readProbe.getValueReads()).toBe(1) + expect(readProbe.getBucketReads()).toBe(1) + expect(readProbe.getCursorCalls()).toBe(2) + expect(readProbe.getUnexpectedTraversalCalls()).toBe(0) + expect(keyComparisonCounter.count).toBe(expectedKeyComparisons) + expect(compareEntries).not.toHaveBeenCalled() + expect(reads.included).toBe(rows.length * 5) + expect(reads.rank).toBe(3) + + const firstReadCount = sourceReads.length + const firstValueReads = readProbe.getValueReads() + const firstBucketReads = readProbe.getBucketReads() + const firstCursorCalls = readProbe.getCursorCalls() + const firstKeyComparisons = keyComparisonCounter.count + const firstPredicateReads = reads.included + const firstOrderTermReads = reads.rank + + expect(observeWindow(window)).toMatchObject({ + publication: expectedKeys.slice(0, 3), + }) + expect(sourceReads).toHaveLength(firstReadCount) + expect(readProbe.getValueReads()).toBe(firstValueReads) + expect(readProbe.getBucketReads()).toBe(firstBucketReads) + expect(readProbe.getCursorCalls()).toBe(firstCursorCalls) + expect(keyComparisonCounter.count).toBe(firstKeyComparisons) + expect(compareEntries).not.toHaveBeenCalled() + expect(reads.included - firstPredicateReads).toBe(rows.length * 5) + expect(reads.rank - firstOrderTermReads).toBe(3) + } finally { + compareEntries.mockRestore() + readProbe.restore() + } + } finally { + await collection.cleanup() + } +}) + +it(`reuses a multi-term snapshot while extracting each boundary term once`, () => { + type MultiTermWindowRow = RankedRow & { secondary: number } + const reads = { rank: 0, secondary: 0 } + const row = ( + id: string, + rank: number, + secondary: number, + ): MultiTermWindowRow => ({ + id, + get rank() { + reads.rank++ + return rank + }, + get secondary() { + reads.secondary++ + return secondary + }, + included: true, + }) + const rows = new Map([ + [`a`, row(`a`, 1, 2)], + [`b`, row(`b`, 1, 3)], + [`c`, row(`c`, 2, 1)], + ]) + let snapshotCalls = 0 + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => { + snapshotCalls++ + return [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const order: OrderBy = [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + { + expression: new PropRef([`secondary`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ] + const window = new WindowState(collection, order, undefined, 2) + window.recordInitialCoverage(undefined, true) + + reads.rank = 0 + reads.secondary = 0 + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(snapshotCalls).toBe(1) + expect(reads).toEqual({ rank: 3, secondary: 3 }) + + expect(observeWindow(window)).toMatchObject({ publication: [`a`, `b`] }) + expect(snapshotCalls).toBe(1) + expect(reads).toEqual({ rank: 6, secondary: 6 }) +}) + +it(`scans each source row once when retaining additional-demand rows`, () => { + const rows = new Map([ + [`a`, { id: `a`, rank: 1, included: true }], + [`b`, { id: `b`, rank: 2, included: true }], + [`c`, { id: `c`, rank: 3, included: true }], + ]) + let snapshotCalls = 0 + let entryReads = 0 + let retentionChecks = 0 + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => { + snapshotCalls++ + return [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ) + }, + entries: function* () { + for (const entry of rows) { + entryReads++ + yield entry + } + }, + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, orderBy(`asc`), undefined, 1) + window.recordInitialCoverage(undefined, true) + + const reconcile = (publishedRows: CountingReadonlyMap) => { + entryReads = 0 + retentionChecks = 0 + const changes = window.reconcile(publishedRows, (candidate) => { + retentionChecks++ + return candidate.id === `c` + }) + expect(entryReads).toBe(rows.size) + expect(retentionChecks).toBe(rows.size) + expect(publishedRows.iterationReads).toBe(publishedRows.size) + expect(publishedRows.membershipReads).toBe(2) + return changes + } + + expect(reconcile(new CountingReadonlyMap()).map(({ key }) => key)).toEqual([ + `a`, + `c`, + ]) + expect(snapshotCalls).toBe(1) + expect( + reconcile( + new CountingReadonlyMap([ + [`a`, rows.get(`a`)!], + [`b`, rows.get(`b`)!], + ]), + ).map(({ type, key }) => `${type}:${key}`), + ).toEqual([`delete:b`, `insert:c`]) + expect(snapshotCalls).toBe(1) +}) + +it(`scans each side of a publication diff exactly once`, () => { + type RowWork = { + valueReads: number + keyReads: number + membershipReads: number + descriptorReads: number + } + const emptyRowWork = (): RowWork => ({ + valueReads: 0, + keyReads: 0, + membershipReads: 0, + descriptorReads: 0, + }) + const rowWork = { + published: emptyRowWork(), + desired: emptyRowWork(), + } + const countedRow = (row: RankedRow, side: keyof typeof rowWork): RankedRow => + new Proxy(row, { + get(target, property, receiver) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + rowWork[side].valueReads++ + } + return Reflect.get(target, property, receiver) as unknown + }, + ownKeys(target) { + rowWork[side].keyReads++ + return Reflect.ownKeys(target) + }, + has(target, property) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + rowWork[side].membershipReads++ + } + return Reflect.has(target, property) + }, + getOwnPropertyDescriptor(target, property) { + if (typeof property === `string` && Object.hasOwn(target, property)) { + rowWork[side].descriptorReads++ + } + return Reflect.getOwnPropertyDescriptor(target, property) + }, + }) + let unmatchedDesiredEntries = 0 + let maximumUnmatchedDesiredEntries = 0 + const publishedRows = new CountingReadonlyMap( + [ + [`a`, countedRow({ id: `a`, rank: 2, included: true }, `published`)], + [`b`, { id: `b`, rank: 2, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `published`)], + ], + undefined, + () => { + unmatchedDesiredEntries-- + }, + ) + const desiredRows = new CountingReadonlyMap( + [ + [`a`, countedRow({ id: `a`, rank: 1, included: true }, `desired`)], + [`c`, { id: `c`, rank: 3, included: true }], + [`d`, countedRow({ id: `d`, rank: 4, included: true }, `desired`)], + ], + () => { + unmatchedDesiredEntries++ + maximumUnmatchedDesiredEntries = Math.max( + maximumUnmatchedDesiredEntries, + unmatchedDesiredEntries, + ) + }, + ) + + expect( + diffPublications(publishedRows, desiredRows).map( + ({ type, key }) => `${type}:${key}`, + ), + ).toEqual([`update:a`, `delete:b`, `insert:c`]) + expect(maximumUnmatchedDesiredEntries).toBe(1) + expect(unmatchedDesiredEntries).toBe(0) + expect(publishedRows.iterationReads).toBe(publishedRows.size) + expect(publishedRows.membershipReads).toBe(desiredRows.size) + expect(publishedRows.valueReads).toBe(0) + expect(desiredRows.iterationReads).toBe(desiredRows.size) + expect(desiredRows.membershipReads).toBe(0) + expect(desiredRows.valueReads).toBe(publishedRows.size) + expect(rowWork).toEqual({ + published: { + valueReads: 5, + keyReads: 2, + membershipReads: 0, + descriptorReads: 6, + }, + desired: { + valueReads: 5, + keyReads: 2, + membershipReads: 5, + descriptorReads: 6, + }, + }) +}) + +it(`does one source-order comparison per row needed to close the boundary tie`, () => { + const reads = { rank: 0 } + const row = (id: string, rank: number): RankedRow => ({ + id, + get rank() { + reads.rank++ + return rank + }, + included: true, + }) + const rows = new Map([ + [`a`, row(`a`, 1)], + [`b`, row(`b`, 2)], + [`c`, row(`c`, 2)], + [`d`, row(`d`, 2)], + [`e`, row(`e`, 3)], + ]) + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, orderBy(`asc`), undefined, 2, true) + window.recordInitialCoverage(undefined, true) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + reads.rank = 0 + expect(window.publicationEntries().map(([key]) => key)).toEqual([ + `a`, + `b`, + `c`, + `d`, + ]) + expect(compareRows).toHaveBeenCalledTimes(3) + expect(reads.rank).toBe(6) + } finally { + compareRows.mockRestore() + } +}) + +it(`expands a source boundary through a comparator-equivalent string tie`, () => { + type CollatedRow = { id: string; value: string } + const reads = { value: 0 } + const row = (id: string, value: string): CollatedRow => ({ + id, + get value() { + reads.value++ + return value + }, + }) + const rows = new Map([ + [`plain`, row(`plain`, `e`)], + [`accent`, row(`accent`, `é`)], + [`later`, row(`later`, `z`)], + ]) + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const order: OrderBy = [ + { + expression: new PropRef([`value`]), + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` }, + }, + }, + ] + const window = new WindowState(collection, order, undefined, 1, true) + window.recordInitialCoverage(undefined, true) + expect( + window.totalOrder.compareRows(rows.get(`plain`)!, rows.get(`accent`)!), + ).toBe(0) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + reads.value = 0 + expect(window.publicationEntries().map(([key]) => key)).toEqual([ + `plain`, + `accent`, + ]) + expect(compareRows).toHaveBeenCalledTimes(2) + expect(reads.value).toBe(4) + } finally { + compareRows.mockRestore() + } +}) + +it.each([ + { + name: `one ascending term`, + direction: `asc` as const, + orderArity: 1 as const, + limit: 1, + sourceRows: [ + { id: `tie-a`, primary: `e`, secondary: 0 }, + { id: `tie-b`, primary: `é`, secondary: 0 }, + { id: `later`, primary: `z`, secondary: 0 }, + ], + expectedKeys: [`tie-a`, `tie-b`], + }, + { + name: `one descending term`, + direction: `desc` as const, + orderArity: 1 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `z`, secondary: 0 }, + { id: `tie-a`, primary: `e`, secondary: 0 }, + { id: `tie-b`, primary: `é`, secondary: 0 }, + { id: `later`, primary: `a`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, + { + name: `one ascending numeric term`, + direction: `asc` as const, + orderArity: 1 as const, + limit: 1, + sourceRows: [ + { id: `tie-a`, primary: 1, secondary: 0 }, + { id: `tie-b`, primary: 1, secondary: 0 }, + { id: `later`, primary: 2, secondary: 0 }, + ], + expectedKeys: [`tie-a`, `tie-b`], + }, + { + name: `one descending numeric term`, + direction: `desc` as const, + orderArity: 1 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: 3, secondary: 0 }, + { id: `tie-a`, primary: 1, secondary: 0 }, + { id: `tie-b`, primary: 1, secondary: 0 }, + { id: `later`, primary: 0, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, + { + name: `two ascending terms`, + direction: `asc` as const, + orderArity: 2 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `a`, secondary: 0 }, + { id: `tie-a`, primary: `b`, secondary: `e` }, + { id: `tie-b`, primary: `b`, secondary: `é` }, + { id: `later`, primary: `c`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, + { + name: `two descending terms`, + direction: `desc` as const, + orderArity: 2 as const, + limit: 2, + sourceRows: [ + { id: `prefix`, primary: `c`, secondary: 0 }, + { id: `tie-a`, primary: `b`, secondary: `e` }, + { id: `tie-b`, primary: `b`, secondary: `é` }, + { id: `later`, primary: `a`, secondary: 0 }, + ], + expectedKeys: [`prefix`, `tie-a`, `tie-b`], + }, +])( + `expands source ties for $name`, + ({ direction, orderArity, limit, sourceRows, expectedKeys }) => { + type SourceTieRow = { + id: string + primary: string | number + secondary: string | number + } + const termReads: [number, number] = [0, 0] + const innerComparisons: [number, number] = [0, 0] + const rows = new Map( + sourceRows.map((spec) => [ + spec.id, + { + id: spec.id, + get primary() { + termReads[0]++ + return spec.primary + }, + get secondary() { + termReads[1]++ + return spec.secondary + }, + }, + ]), + ) + const trackedCompareOptions = (term: 0 | 1): CompareOptions => + new Proxy( + { + direction, + nulls: direction === `asc` ? (`last` as const) : (`first` as const), + stringSort: `locale`, + locale: `en`, + localeOptions: { sensitivity: `base` as const }, + } satisfies CompareOptions, + { + get(target, property, receiver) { + if (property === `nulls`) innerComparisons[term]++ + return Reflect.get(target, property, receiver) as unknown + }, + }, + ) + const order: OrderBy = [ + { + expression: new PropRef([`primary`]), + compareOptions: trackedCompareOptions(0), + }, + ...(orderArity === 2 + ? [ + { + expression: new PropRef([`secondary`]), + compareOptions: trackedCompareOptions(1), + }, + ] + : []), + ] + const collection = { + _stateRevision: 0, + currentStateAsChanges: () => + [...rows].map( + ([key, value]): ChangeMessage => ({ + type: `insert`, + key, + value, + }), + ), + entries: () => rows.entries(), + get: (key: string) => rows.get(key), + } as unknown as CollectionImpl + const window = new WindowState(collection, order, undefined, limit, true) + window.recordInitialCoverage(undefined, true) + const compareRows = vi.spyOn(window.totalOrder, `compareRows`) + + try { + termReads[0] = 0 + termReads[1] = 0 + innerComparisons[0] = 0 + innerComparisons[1] = 0 + expect(window.publicationEntries().map(([key]) => key)).toEqual( + expectedKeys, + ) + expect(compareRows).toHaveBeenCalledTimes(2) + expect(termReads).toEqual([4, orderArity === 2 ? 2 : 0]) + expect(innerComparisons).toEqual([2, orderArity === 2 ? 1 : 0]) + } finally { + compareRows.mockRestore() + } + }, +) + +it(`compiles the ordered predicate once for the lifetime of a window`, () => { + const fixture = createSnapshotFixture([ + { id: `a`, rank: 1, included: true }, + { id: `hidden`, rank: 0, included: false }, + ]) + let referenceCompilationReads = 0 + const referenceWhere = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) referenceCompilationReads++ + return Reflect.get(target, property, receiver) + }, + }) + compileSingleRowExpression(referenceWhere) + expect(referenceCompilationReads).toBeGreaterThan(0) + + let compilationReads = 0 + const where = new Proxy(eq(new PropRef([`included`]), true), { + get(target, property, receiver) { + if (property === `type`) compilationReads++ + return Reflect.get(target, property, receiver) + }, + }) + const window = new WindowState(fixture.collection, orderBy(`asc`), where, 1) + const readsAfterConstruction = compilationReads + expect(readsAfterConstruction).toBe(referenceCompilationReads) + window.recordInitialCoverage(undefined, true) + + observeWindow(window) + observeWindow(window) + fixture.replace({ id: `a`, rank: 2, included: true }) + observeWindow(window) + + expect(compilationReads).toBe(readsAfterConstruction) +}) + +it(`invalidates the ordered snapshot after a committed collection write`, async () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `ordered-work-revision-write`, + getKey: (row) => row.id, + initialData: [ + { id: `a`, rank: 1, included: true }, + { id: `b`, rank: 2, included: true }, + ], + }), + ) + + try { + await collection.preload() + collection.createIndex((row) => row.rank, { indexType: BTreeIndex }) + const snapshotRevisions: Array = [] + const originalSnapshot = collection.currentStateAsChanges.bind(collection) + collection.currentStateAsChanges = (options) => { + snapshotRevisions.push(collection._stateRevision) + return originalSnapshot(options) + } + const window = new WindowState( + collection, + orderBy(`asc`), + eq(new PropRef([`included`]), true), + 2, + ) + window.recordInitialCoverage(undefined, true) + + expect(observeWindow(window).publication).toEqual([`a`, `b`]) + expect(observeWindow(window).publication).toEqual([`a`, `b`]) + const initialRevision = collection._stateRevision + expect(snapshotRevisions).toEqual([initialRevision]) + + collection.update(`b`, (draft) => { + draft.rank = -1 + }) + + expect(observeWindow(window).publication).toEqual([`b`, `a`]) + expect(observeWindow(window).publication).toEqual([`b`, `a`]) + expect(collection._stateRevision).toBeGreaterThan(initialRevision) + expect(snapshotRevisions).toEqual([ + initialRevision, + collection._stateRevision, + ]) + } finally { + await collection.cleanup() + } +}) + +for (const campaign of orderedWorkCampaigns( + `ordered-work.snapshot-reuse`, + 1_780_102, +)) { + fcTest.prop( + [ + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 1, + maxLength: 12, + }), + fc.integer({ min: 1, max: 8 }), + fc.array(fc.integer({ min: -20, max: 20 }), { + minLength: 0, + maxLength: 8, + }), + ], + campaign.options, + )( + `takes at most one ordered snapshot per source revision (${campaign.label})`, + (initialRanks, observationCount, replacementRanks) => { + const fixture = createSnapshotFixture( + initialRanks.map((rank, index) => ({ + id: `row-${index}`, + rank, + included: index % 3 !== 0, + })), + ) + const window = createCoveredWindow( + fixture, + Math.min(3, initialRanks.length), + ) + + for (let index = 0; index < observationCount; index++) { + observeWindow(window) + } + for (let index = 0; index < replacementRanks.length; index++) { + fixture.replace({ + id: `row-${index % initialRanks.length}`, + rank: replacementRanks[index]!, + included: index % 2 === 0, + }) + for (let repeat = 0; repeat < observationCount; repeat++) { + observeWindow(window) + } + } + + expect(fixture.snapshotRevisions).toEqual( + Array.from( + { length: replacementRanks.length + 1 }, + (_, index) => index, + ), + ) + }, + ) +} diff --git a/packages/db/tests/query/pagination-oracle.property.test.ts b/packages/db/tests/query/pagination-oracle.property.test.ts index a78e13156a..7338da3769 100644 --- a/packages/db/tests/query/pagination-oracle.property.test.ts +++ b/packages/db/tests/query/pagination-oracle.property.test.ts @@ -4,6 +4,7 @@ import { createCollection } from '../../src/collection/index.js' import { createDeferred } from '../../src/deferred.js' import { BTreeIndex } from '../../src/index.js' import { createLiveQueryCollection } from '../../src/query/live-query-collection.js' +import { eq } from '../../src/query/builder/functions.js' import { PropRef } from '../../src/query/ir.js' import { DeduplicatedLoadSubset } from '../../src/query/subset-dedupe.js' import { makeComparator } from '../../src/utils/comparison.js' @@ -14,6 +15,7 @@ import { import { evaluateReferenceExpression } from '../reference-expression.js' import { TraceAssertionError } from '../trace-runner.js' import { flushPromises, mockSyncCollectionOptions } from '../utils.js' +import type { Deferred } from '../../src/deferred.js' import type { LoadSubsetOptions, LoadSubsetResult } from '../../src/types.js' type PageRow = { @@ -322,13 +324,25 @@ const nullableCursorScenarioArbitrary: fc.Arbitrary = direction: fc.constantFrom(`asc` as const, `desc` as const), }) -const { multiplier, replaySeed } = readOracleRunConfig() +type CleanupTarget = { + cleanup: () => unknown +} + +async function cleanupAll( + ...targets: ReadonlyArray +): Promise { + const results = await Promise.allSettled( + targets.map((target) => Promise.resolve().then(() => target.cleanup())), + ) + const rejection = results.find( + (result): result is PromiseRejectedResult => result.status === `rejected`, + ) + if (rejection) throw rejection.reason +} + +const { multiplier, ...replay } = readOracleRunConfig() const orderedScenarioRuns = 12 * multiplier const transitionScenarioRuns = 8 * multiplier -const orderedScenarioRandomParameters = oracleRandomParameters( - orderedScenarioRuns, - replaySeed, -) let collectionSequence = 0 @@ -400,6 +414,56 @@ function withAppliedSubsetEvidence( }) } +function createConformingOrderedSource( + id: string, + rows: ReadonlyArray, + autoIndex: `eager` | `off` = `eager`, +) { + const requests: Array = [] + const delivered = new Set() + const source = createCollection({ + id, + getKey: (row) => row.id, + syncMode: `on-demand`, + startSync: true, + autoIndex, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ begin, write, commit, markReady }) => { + markReady() + return { + loadSubset: (options: LoadSubsetOptions) => { + requests.push(options) + const requested = rowsForLoadSubset(rows, options) + begin() + for (const row of requested) { + if (delivered.has(row.id)) continue + delivered.add(row.id) + write({ type: `insert`, value: row }) + } + const receipt = commit(options.signal) + const hasMore = options.cursor + ? rows.filter((row) => + Boolean( + evaluateReferenceExpression(options.cursor!.whereFrom, row), + ), + ).length > (options.limit ?? Number.POSITIVE_INFINITY) + : rows.length > + (options.offset ?? 0) + + (options.limit ?? Number.POSITIVE_INFINITY) + return Promise.resolve(receipt).then(() => ({ + hasMore, + appliedRowKeys: requested.map(({ id: key }) => key), + })) + }, + } + }, + }, + }) + + return { requests, source } +} + async function runPaginationScenario( scenario: PaginationScenario, ): Promise { @@ -438,8 +502,7 @@ async function runPaginationScenario( ) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -525,8 +588,7 @@ async function runMultiOrderScenario( throw new TraceAssertionError(0, error) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -614,8 +676,7 @@ async function runNullableCursorScenario( } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -687,8 +748,7 @@ async function runPaginationStateScenario( expectCurrentWindow(index + 1) } } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -794,8 +854,7 @@ async function runOnDemandPaginationScenario( for (const load of loads) expect(load.orderBy).toMatchObject(expectedOrderBy) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -888,9 +947,7 @@ async function expectOnDemandWindowsAreCompletionOrderIndependent( expect(Array.from(secondLive.values(), ({ id }) => id)).toEqual([1, 2, 3]) } finally { for (const request of pending) request.deferred.resolve() - firstLive.cleanup() - secondLive.cleanup() - source.cleanup() + await cleanupAll(firstLive, secondLive, source) } } @@ -1034,8 +1091,7 @@ async function runAdversarialOrderedProviderScenario(options: { // final refinement request. return [...loads] } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1253,8 +1309,7 @@ async function runPendingMutationScenario( } finally { for (const request of pending) request.deferred.resolve() await Promise.allSettled(outstanding) - await live.cleanup() - await source.cleanup() + await cleanupAll(live, source) } } @@ -1378,8 +1433,7 @@ async function runRejectedCursorRetryAfterMutation(): Promise { } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1522,8 +1576,7 @@ async function runPendingHistoryScenario( } finally { for (const request of pending) request.deferred.resolve() await Promise.allSettled(outstanding) - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } @@ -1619,12 +1672,279 @@ async function expectInflightRequestFillsNewWindow(): Promise { } } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } } describe(`pagination recomputation oracle`, () => { + it.each([ + { label: `Error`, reason: new Error(`first cleanup failed`) }, + { label: `undefined`, reason: undefined }, + { label: `null`, reason: null }, + { label: `false`, reason: false }, + { label: `zero`, reason: 0 }, + { label: `NaN`, reason: Number.NaN }, + { label: `empty string`, reason: `` }, + ])( + `observes $label cleanup failure after every teardown settles`, + async ({ reason: firstFailure }) => { + const secondFailure = new Error(`second cleanup failed`) + const firstFailureRelease = createDeferred() + const lastCleanupRelease = createDeferred() + const repeatedFirstFailureRelease = createDeferred() + const repeatedLastCleanupRelease = createDeferred() + const events: Array = [] + const unhandled: Array = [] + const recordUnhandled = (reason: unknown) => unhandled.push(reason) + const createTargets = ( + firstRelease: Deferred, + lastRelease: Deferred, + ): ReadonlyArray => [ + { + cleanup: async () => { + events.push(`first`) + await firstRelease.promise + throw firstFailure + }, + }, + { + cleanup: () => { + events.push(`second`) + throw secondFailure + }, + }, + { + cleanup: async () => { + events.push(`third`) + await lastRelease.promise + }, + }, + ] + const observeFirstFailure = (cleanup: Promise) => + cleanup.then( + () => { + throw new Error(`expected cleanup to reject`) + }, + (error: unknown) => expect(error).toBe(firstFailure), + ) + let cleanupFinished = false + process.on(`unhandledRejection`, recordUnhandled) + + try { + const cleanup = cleanupAll( + ...createTargets(firstFailureRelease, lastCleanupRelease), + ).finally(() => { + cleanupFinished = true + }) + const observedFailure = observeFirstFailure(cleanup) + + await flushPromises() + expect(events).toEqual([`first`, `second`, `third`]) + expect(cleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + firstFailureRelease.resolve() + await flushPromises() + expect(cleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + lastCleanupRelease.resolve() + await observedFailure + await flushPromises() + expect(cleanupFinished).toBe(true) + expect(unhandled).toEqual([]) + + let repeatedCleanupFinished = false + const repeatedCleanup = cleanupAll( + ...createTargets( + repeatedFirstFailureRelease, + repeatedLastCleanupRelease, + ), + ).finally(() => { + repeatedCleanupFinished = true + }) + const repeatedObservedFailure = observeFirstFailure(repeatedCleanup) + await flushPromises() + expect(events).toEqual([ + `first`, + `second`, + `third`, + `first`, + `second`, + `third`, + ]) + expect(repeatedCleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + repeatedFirstFailureRelease.resolve() + await flushPromises() + expect(repeatedCleanupFinished).toBe(false) + expect(unhandled).toEqual([]) + + repeatedLastCleanupRelease.resolve() + await repeatedObservedFailure + await flushPromises() + expect(repeatedCleanupFinished).toBe(true) + expect(unhandled).toEqual([]) + } finally { + firstFailureRelease.resolve() + lastCleanupRelease.resolve() + repeatedFirstFailureRelease.resolve() + repeatedLastCleanupRelease.resolve() + process.off(`unhandledRejection`, recordUnhandled) + } + }, + ) + + it(`refills a joined result window through a contract-compliant source`, async () => { + type ParentRow = { id: number; rank: number; groupId: number } + type ChildRow = { id: number; groupId: number } + const parents = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-joined-underfill-source-${collectionSequence++}`, + parents, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-joined-underfill-child-${collectionSequence++}`, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .innerJoin({ child: childSource }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([2, 3]) + expect(requests).toHaveLength(2) + expect(requests[0]?.limit).toBe(2) + expect(requests[1]?.cursor).toBeDefined() + } finally { + await cleanupAll(live, childSource, parentSource) + } + }) + + it(`loads the full ordered source when no continuation index exists`, async () => { + type ParentRow = { id: number; rank: number; groupId: number } + type ChildRow = { id: number; groupId: number } + const parents = [ + { id: 1, rank: 0, groupId: 1 }, + { id: 2, rank: 1, groupId: 2 }, + { id: 3, rank: 2, groupId: 3 }, + { id: 4, rank: 3, groupId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-no-index-underfill-source-${collectionSequence++}`, + parents, + `off`, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-no-index-underfill-child-${collectionSequence++}`, + initialData: [ + { id: 20, groupId: 2 }, + { id: 30, groupId: 3 }, + { id: 40, groupId: 4 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .innerJoin({ child: childSource }, ({ parent, child }) => + eq(parent.groupId, child.groupId), + ) + .orderBy(({ parent }) => parent.rank, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([2, 3]) + expect(requests).toHaveLength(1) + expect(requests[0]?.limit).toBeUndefined() + } finally { + await cleanupAll(live, childSource, parentSource) + } + }) + + it(`refines a joined foreign order term through the source tie class`, async () => { + type ParentRow = { id: number; sourceRank: number; childId: number } + type ChildRow = { id: number; score: number } + const parents = [ + { id: 1, sourceRank: 0, childId: 1 }, + { id: 2, sourceRank: 0, childId: 2 }, + { id: 3, sourceRank: 0, childId: 3 }, + { id: 4, sourceRank: 0, childId: 4 }, + ] satisfies ReadonlyArray + const { requests, source: parentSource } = createConformingOrderedSource( + `pagination-joined-foreign-order-source-${collectionSequence++}`, + parents, + ) + const childSource = createCollection( + mockSyncCollectionOptions({ + id: `pagination-joined-foreign-order-child-${collectionSequence++}`, + initialData: [ + { id: 1, score: 10 }, + { id: 2, score: 20 }, + { id: 3, score: 0 }, + { id: 4, score: 30 }, + ] satisfies ReadonlyArray, + getKey: (row: ChildRow) => row.id, + }), + ) + const live = createLiveQueryCollection((query) => + query + .from({ parent: parentSource }) + .leftJoin({ child: childSource }, ({ parent, child }) => + eq(parent.childId, child.id), + ) + .orderBy(({ parent }) => parent.sourceRank, `asc`) + .orderBy(({ child }) => child.score, `asc`) + .orderBy(({ parent }) => parent.id, `asc`) + .limit(2) + .select(({ parent }) => ({ id: parent.id })), + ) + + try { + await live.preload() + await flushPromises() + + expect(Array.from(live.values(), ({ id }) => id)).toEqual([3, 1]) + expect(requests).toHaveLength(2) + expect(requests[0]?.orderBy).toHaveLength(1) + expect(requests[1]?.cursor).toBeDefined() + } finally { + await cleanupAll(live, childSource, parentSource) + } + }) + it(`materializes an empty source window`, async () => { await runPaginationScenario({ ranks: [], @@ -1743,8 +2063,7 @@ describe(`pagination recomputation oracle`, () => { expect(requests[1]).toMatchObject({ limit: 2, offset: 0 }) expect(requests[1]?.cursor).toBeUndefined() } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -1799,8 +2118,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -1915,24 +2233,22 @@ describe(`pagination recomputation oracle`, () => { expectedCovering.slice(0, 1), ) } finally { - covered.cleanup() + await cleanupAll(covered) } if (releaseFirst === `covering`) { - covering.cleanup() + await cleanupAll(covering) expect(Array.from(narrower.values(), ({ id }) => id)).toEqual( expectedCovering.slice(0, 2), ) } else { - narrower.cleanup() + await cleanupAll(narrower) expect(Array.from(covering.values(), ({ id }) => id)).toEqual( expectedCovering, ) } } finally { - covering.cleanup() - narrower.cleanup() - source.cleanup() + await cleanupAll(covering, narrower, source) } }, ) @@ -2017,8 +2333,7 @@ describe(`pagination recomputation oracle`, () => { if (widened instanceof Promise) await widened expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2099,14 +2414,15 @@ describe(`pagination recomputation oracle`, () => { expect(refinement.options.limit).toBeUndefined() expect(refinement.options.offset).toBeUndefined() + const transportCount = pending.length const widened = live.utils.setWindow({ offset: 0, limit: 2 }) expect(widened).toBe(true) expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2]) + expect(pending).toHaveLength(transportCount) } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2201,7 +2517,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [multiOrderScenarioArbitrary], - oracleRandomParameters(orderedScenarioRuns, replaySeed), + oracleRandomParameters( + orderedScenarioRuns, + replay, + `pagination.multi-order`, + ), )( `matches multi-column nullable ordering for a random or replayed seed`, runMultiOrderScenario, @@ -2217,7 +2537,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [nullableCursorScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.nullable-cursor`, + ), )( `matches nullable cursor ordering while an async response is pending for a random or replayed seed`, runNullableCursorScenario, @@ -2335,8 +2659,7 @@ describe(`pagination recomputation oracle`, () => { await live.utils.setWindow({ offset: 0, limit: 3 }) expect(Array.from(live.values(), ({ id }) => id)).toEqual([1, 2, 9]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -2437,8 +2760,7 @@ describe(`pagination recomputation oracle`, () => { expect(loads.at(-1)).toMatchObject({ offset: 0, limit: 2 }) expect(loads.at(-1)?.cursor).toBeUndefined() } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) @@ -2571,8 +2893,7 @@ describe(`pagination recomputation oracle`, () => { ) } finally { for (const request of pending) request.deferred.resolve() - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) @@ -2654,7 +2975,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingMutationScenarioArbitrary, responseTimingArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.pending-mutation`, + ), )( `matches recomputation when source mutations cross a pending cursor response for a random or replayed seed`, runPendingMutationScenario, @@ -2675,7 +3000,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [pendingHistoryScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.pending-history`, + ), )( `matches recomputation across multi-action pending histories for a random or replayed seed`, runPendingHistoryScenario, @@ -2795,7 +3124,14 @@ describe(`pagination recomputation oracle`, () => { runPaginationScenario, ) - fcTest.prop([scenarioArbitrary], orderedScenarioRandomParameters)( + fcTest.prop( + [scenarioArbitrary], + oracleRandomParameters( + orderedScenarioRuns, + replay, + `pagination.ordered-window`, + ), + )( `matches full recomputation across ordered windows for a random or replayed seed`, runPaginationScenario, ) @@ -2810,7 +3146,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [stateScenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.window-transition`, + ), )( `matches full recomputation across source and window transitions for a random or replayed seed`, runPaginationStateScenario, @@ -2962,6 +3302,7 @@ describe(`pagination recomputation oracle`, () => { expect(loads).toHaveLength(2) expect(loads[1]?.limit).toBeUndefined() expect(loads[1]?.offset).toBeUndefined() + expect(loads[1]?.cursor).toBeUndefined() }) it.each([`continues`, `unknown`] as const)( @@ -2985,6 +3326,9 @@ describe(`pagination recomputation oracle`, () => { expect(loads[1]?.limit).toBeUndefined() expect(loads[1]?.offset).toBeUndefined() expect(loads.map(({ limit }) => limit)).toEqual([1, undefined, undefined]) + expect(loads.slice(1).every(({ cursor }) => cursor === undefined)).toBe( + true, + ) }, ) @@ -3070,8 +3414,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([3, 1]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }) @@ -3097,8 +3440,7 @@ describe(`pagination recomputation oracle`, () => { await live.preload() expect(Array.from(live.values(), ({ id }) => id)).toEqual([1]) } finally { - live.cleanup() - source.cleanup() + await cleanupAll(live, source) } }, ) @@ -3153,7 +3495,11 @@ describe(`pagination recomputation oracle`, () => { fcTest.prop( [scenarioArbitrary], - oracleRandomParameters(transitionScenarioRuns, replaySeed), + oracleRandomParameters( + transitionScenarioRuns, + replay, + `pagination.async-cursor`, + ), )( `matches full recomputation when exact async cursor loads widen ordered coverage for a random or replayed seed`, runOnDemandPaginationScenario, diff --git a/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts b/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts new file mode 100644 index 0000000000..3e4de6796d --- /dev/null +++ b/packages/db/tests/query/predicate-subtraction-oracle.property.test.ts @@ -0,0 +1,593 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { minusWherePredicates } from '../../src/query/predicate-utils' +import { Func, PropRef, Value } from '../../src/query/ir' +import { oraclePropertyOptions, oracleRuns } from '../oracle-config' +import type { BasicExpression } from '../../src/query/ir' + +type Field = `score` | `rank` + +type PredicateSpec = + | { kind: `eq`; field: Field; value: number | null } + | { + kind: `range` + field: Field + operator: `gt` | `gte` | `lt` | `lte` + value: number + } + | { kind: `in`; field: Field; values: Array } + | { kind: `not`; predicate: AtomicPredicateSpec } + | { + kind: `or` + left: AtomicPredicateSpec + right: AtomicPredicateSpec + } + +type AtomicPredicateSpec = Exclude + +type Association = `flat` | `left` | `right` +type ScenarioFamily = + | `general residuals` + | `ordered range overlap` + | `set overlap` + +interface DifferenceScenario { + family: ScenarioFamily + shared: Array + fromResidual: AtomicPredicateSpec + subtractResidual: AtomicPredicateSpec + fromAssociation: Association + subtractAssociation: Association + reverseFrom: boolean + reverseSubtract: boolean + duplicateFrom: boolean + duplicateSubtract: boolean +} + +type DifferenceOutcome = + | `successful narrowing` + | `unchanged fallback` + | `conservative bailout` + +type DifferenceObservation = `${ScenarioFamily} / ${DifferenceOutcome}` + +const finiteWorldProperty = `predicate-subtraction.finite-world` +const unboundedProperty = `predicate-subtraction.unbounded` +const duplicateProperty = `predicate-subtraction.duplicate-terms` + +const scalarArbitrary = fc.oneof( + fc.integer({ min: -2, max: 2 }), + fc.constant(null), +) +const fieldArbitrary = fc.constantFrom(`score`, `rank`) + +const atomicPredicateArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + kind: fc.constant(`eq` as const), + field: fieldArbitrary, + value: scalarArbitrary, + }), + fc.record({ + kind: fc.constant(`range` as const), + field: fieldArbitrary, + operator: fc.constantFrom<`gt` | `gte` | `lt` | `lte`>( + `gt`, + `gte`, + `lt`, + `lte`, + ), + value: fc.integer({ min: -2, max: 2 }), + }), + fc.record({ + kind: fc.constant(`in` as const), + field: fieldArbitrary, + values: fc.uniqueArray(scalarArbitrary, { minLength: 1, maxLength: 4 }), + }), +) + +const predicateArbitrary: fc.Arbitrary = fc.oneof( + atomicPredicateArbitrary, + atomicPredicateArbitrary.map((predicate) => ({ + kind: `not` as const, + predicate, + })), + fc + .tuple(atomicPredicateArbitrary, atomicPredicateArbitrary) + .map(([left, right]) => ({ kind: `or` as const, left, right })), +) + +const residualPairArbitrary = fc.oneof( + fc + .tuple(atomicPredicateArbitrary, atomicPredicateArbitrary) + .map(([fromResidual, subtractResidual]) => ({ + family: `general residuals` as const, + fromResidual, + subtractResidual, + })), + fc + .tuple(fieldArbitrary, fc.integer({ min: -2, max: 1 })) + .map(([field, boundary]) => ({ + family: `ordered range overlap` as const, + fromResidual: { + kind: `range` as const, + field, + operator: `gt` as const, + value: boundary, + }, + subtractResidual: { + kind: `range` as const, + field, + operator: `gt` as const, + value: boundary + 1, + }, + })), + fc + .tuple( + fieldArbitrary, + fc.uniqueArray(fc.integer({ min: -2, max: 2 }), { + minLength: 2, + maxLength: 4, + }), + ) + .map(([field, values]) => ({ + family: `set overlap` as const, + fromResidual: { kind: `in` as const, field, values }, + subtractResidual: { + kind: `in` as const, + field, + values: values.slice(1), + }, + })), +) + +const scenarioShapeArbitrary = fc.record({ + shared: fc.array(predicateArbitrary, { minLength: 1, maxLength: 3 }), + fromAssociation: fc.constantFrom(`flat`, `left`, `right`), + subtractAssociation: fc.constantFrom(`flat`, `left`, `right`), + reverseFrom: fc.boolean(), + reverseSubtract: fc.boolean(), + duplicateFrom: fc.boolean(), + duplicateSubtract: fc.boolean(), +}) + +const scenarioArbitrary: fc.Arbitrary = fc + .tuple(scenarioShapeArbitrary, residualPairArbitrary) + .map(([shape, residuals]) => ({ ...shape, ...residuals })) + +const refs: Record = { + score: new PropRef([`score`]), + rank: new PropRef([`rank`]), +} + +function value(input: unknown): Value { + return new Value(input) +} + +function call( + name: string, + ...args: Array +): BasicExpression { + return new Func(name, args) as BasicExpression +} + +function buildAtomic(spec: AtomicPredicateSpec): BasicExpression { + const ref = refs[spec.field] + if (spec.kind === `in`) { + return call(`in`, ref, value(spec.values)) + } + if (spec.kind === `range`) { + return call(spec.operator, ref, value(spec.value)) + } + return call(`eq`, ref, value(spec.value)) +} + +function buildPredicate(spec: PredicateSpec): BasicExpression { + if (spec.kind === `not`) { + return call(`not`, buildAtomic(spec.predicate)) + } + if (spec.kind === `or`) { + return call(`or`, buildAtomic(spec.left), buildAtomic(spec.right)) + } + return buildAtomic(spec) +} + +function predicateFields(spec: PredicateSpec): Array { + if (spec.kind === `not`) return [spec.predicate.field] + if (spec.kind === `or`) return [spec.left.field, spec.right.field] + return [spec.field] +} + +function scenarioFields(scenario: DifferenceScenario): Array { + return [ + ...scenario.shared.flatMap(predicateFields), + scenario.fromResidual.field, + scenario.subtractResidual.field, + ] +} + +function associateAnd( + terms: Array>, + association: Association, +): BasicExpression { + if (terms.length === 1) return terms[0]! + if (association === `flat`) return call(`and`, ...terms) + + if (association === `left`) { + return terms + .slice(1) + .reduce((left, right) => call(`and`, left, right), terms[0]!) + } + + return terms + .slice(0, -1) + .reduceRight((right, left) => call(`and`, left, right), terms.at(-1)!) +} + +function buildOperand( + sharedSpecs: Array, + residualSpec: AtomicPredicateSpec, + association: Association, + reverse: boolean, + duplicate: boolean, +): BasicExpression { + const shared = sharedSpecs.map(buildPredicate) + const residual = buildAtomic(residualSpec) + const terms = reverse ? [residual, ...shared] : [...shared, residual] + if (duplicate) terms.splice(1, 0, terms[0]!) + return associateAnd(terms, association) +} + +const finiteValues = [-3, -2, -1, 0, 1, 2, 3, null] +const finiteRows = finiteValues.flatMap((score) => + finiteValues.map((rank) => ({ score, rank })), +) + +function evaluatePredicate( + expression: BasicExpression, + row: Record, +): unknown { + if (expression.type === `val`) return expression.value + if (expression.type === `ref`) { + const [field, ...remainingPath] = expression.path + if (remainingPath.length > 0 || (field !== `score` && field !== `rank`)) { + throw new Error(`Unsupported reference path ${expression.path.join(`.`)}`) + } + return row[field] + } + + const args = expression.args.map((argument) => + evaluatePredicate(argument, row), + ) + const isUnknown = (candidate: unknown) => + candidate === null || candidate === undefined + switch (expression.name) { + case `and`: + return args.includes(false) ? false : args.some(isUnknown) ? null : true + case `or`: + return args.includes(true) ? true : args.some(isUnknown) ? null : false + case `not`: + return isUnknown(args[0]) ? null : !args[0] + case `eq`: + return isUnknown(args[0]) || isUnknown(args[1]) + ? null + : args[0] === args[1] + case `gt`: + case `gte`: + case `lt`: + case `lte`: { + if (isUnknown(args[0]) || isUnknown(args[1])) return null + const left = args[0] as number + const right = args[1] as number + if (expression.name === `gt`) return left > right + if (expression.name === `gte`) return left >= right + if (expression.name === `lt`) return left < right + return left <= right + } + case `in`: + if (isUnknown(args[0])) return null + return Array.isArray(args[1]) && args[1].includes(args[0]) + default: + throw new Error(`Unsupported predicate ${expression.name}`) + } +} + +function assertSemanticDifference( + scenario: DifferenceScenario, + override?: { result: BasicExpression | null }, +): void { + const difference = evaluateDifference(scenario) + const { requested, loaded } = difference + const result = override === undefined ? difference.result : override.result + + assertExpressionDifference(requested, loaded, result) +} + +function assertExpressionDifference( + requested: BasicExpression, + loaded: BasicExpression, + result: BasicExpression | null, +): void { + if (result === null) return + + for (const row of finiteRows) { + const expected = + evaluatePredicate(requested, row) === true && + evaluatePredicate(loaded, row) !== true + expect(evaluatePredicate(result, row) === true).toBe(expected) + } +} + +function assertUnboundedDifference(spec: PredicateSpec): void { + const loaded = buildPredicate(spec) + const result = minusWherePredicates(undefined, loaded) + assertExpressionDifference( + value(true) as BasicExpression, + loaded, + result, + ) +} + +function assertDuplicateTermDifference(field: Field, boundary: number): void { + const shared = buildAtomic({ + kind: `range`, + field, + operator: `gt`, + value: boundary, + }) + const nullableChoice = call( + `or`, + buildAtomic({ kind: `eq`, field, value: null }), + buildAtomic({ kind: `eq`, field, value: boundary + 1 }), + ) + const membership = buildAtomic({ + kind: `in`, + field, + values: [boundary + 1, boundary], + }) + const requested = call( + `and`, + shared, + nullableChoice, + membership, + buildAtomic({ + kind: `range`, + field, + operator: `gt`, + value: boundary - 1, + }), + ) + const loaded = call(`and`, shared, nullableChoice, membership, shared) + const result = minusWherePredicates(requested, loaded) + + assertExpressionDifference(requested, loaded, result) +} + +function evaluateDifference(scenario: DifferenceScenario): { + requested: BasicExpression + loaded: BasicExpression + result: BasicExpression | null +} { + const requested = buildOperand( + scenario.shared, + scenario.fromResidual, + scenario.fromAssociation, + scenario.reverseFrom, + scenario.duplicateFrom, + ) + const loaded = buildOperand( + scenario.shared, + scenario.subtractResidual, + scenario.subtractAssociation, + scenario.reverseSubtract, + scenario.duplicateSubtract, + ) + + return { + requested, + loaded, + result: minusWherePredicates(requested, loaded), + } +} + +function classifyDifferenceOutcome( + scenario: DifferenceScenario, +): DifferenceOutcome { + const { requested, result } = evaluateDifference(scenario) + if (result === null) return `conservative bailout` + + for (const row of finiteRows) { + if ( + (evaluatePredicate(requested, row) === true) !== + (evaluatePredicate(result, row) === true) + ) { + return `successful narrowing` + } + } + + return `unchanged fallback` +} + +function expectEveryDifferenceOutcome(parameters: { + numRuns: number + seed: number +}): void { + const counts = new Map() + + for (const scenario of fc.sample(scenarioArbitrary, parameters)) { + const observation: DifferenceObservation = `${scenario.family} / ${classifyDifferenceOutcome(scenario)}` + counts.set(observation, (counts.get(observation) ?? 0) + 1) + } + + const requiredObservations: Array = [ + `general residuals / unchanged fallback`, + `general residuals / conservative bailout`, + `ordered range overlap / successful narrowing`, + `set overlap / successful narrowing`, + ] + const diagnostics = `seed=${parameters.seed} counts=${JSON.stringify(Object.fromEntries(counts))}` + + for (const observation of requiredObservations) { + expect(counts.get(observation) ?? 0, diagnostics).toBeGreaterThanOrEqual(10) + } +} + +function calibrationScenario( + fromResidual: AtomicPredicateSpec, + subtractResidual: AtomicPredicateSpec, +): DifferenceScenario { + return { + family: `general residuals`, + shared: [], + fromResidual, + subtractResidual, + fromAssociation: `flat`, + subtractAssociation: `flat`, + reverseFrom: false, + reverseSubtract: false, + duplicateFrom: false, + duplicateSubtract: false, + } +} + +const outcomeCalibrations: Record = { + 'successful narrowing': calibrationScenario( + { kind: `range`, field: `score`, operator: `gt`, value: -1 }, + { kind: `range`, field: `score`, operator: `gt`, value: 0 }, + ), + 'unchanged fallback': calibrationScenario( + { kind: `eq`, field: `score`, value: 0 }, + { kind: `eq`, field: `score`, value: 1 }, + ), + 'conservative bailout': calibrationScenario( + { kind: `eq`, field: `score`, value: 0 }, + { kind: `eq`, field: `rank`, value: 0 }, + ), +} + +if (process.env.TANSTACK_DB_ORACLE_STATISTICS === `1`) { + fc.statistics( + scenarioArbitrary, + (scenario) => `${scenario.family} / ${classifyDifferenceOutcome(scenario)}`, + oraclePropertyOptions(1_000, finiteWorldProperty), + ) +} + +describe(`predicate subtraction oracle`, () => { + it(`resolves each generated reference path independently`, () => { + const row = { score: 1, rank: 2 } + + expect(evaluatePredicate(refs.score, row)).toBe(1) + expect(evaluatePredicate(refs.rank, row)).toBe(2) + }) + + it(`evaluates the Cartesian product of reference values`, () => { + const encodedRows = new Set( + finiteRows.map(({ score, rank }) => `${String(score)}:${String(rank)}`), + ) + + expect(finiteRows).toHaveLength(finiteValues.length ** 2) + expect(encodedRows).toHaveLength(finiteValues.length ** 2) + expect(finiteRows).toContainEqual({ score: -3, rank: null }) + expect(finiteRows).toContainEqual({ score: null, rank: -3 }) + }) + + it(`covers both reference paths in the fixed replay corpus`, () => { + const fields = new Set( + fc + .sample(scenarioArbitrary, { + numRuns: oracleRuns(250), + seed: 1777, + }) + .flatMap(scenarioFields), + ) + + expect(fields).toEqual(new Set([`score`, `rank`])) + }) + + it(`calibrates every subtraction outcome label`, () => { + for (const [expected, scenario] of Object.entries( + outcomeCalibrations, + ) as Array<[DifferenceOutcome, DifferenceScenario]>) { + expect(classifyDifferenceOutcome(scenario)).toBe(expected) + assertSemanticDifference(scenario) + } + }) + + it(`rejects a subtraction result with the wrong finite-world meaning`, () => { + const scenario = outcomeCalibrations[`successful narrowing`] + const { requested } = evaluateDifference(scenario) + + expect(() => + assertSemanticDifference(scenario, { result: requested }), + ).toThrow() + }) + + it(`calibrates runtime IN null semantics under NOT and OR`, () => { + const membership = buildAtomic({ + kind: `in`, + field: `score`, + values: [null, 1], + }) + const negated = call(`not`, membership) + const disjunction = call( + `or`, + negated, + buildAtomic({ kind: `eq`, field: `rank`, value: 2 }), + ) + + expect(evaluatePredicate(membership, { score: null, rank: 0 })).toBeNull() + expect(evaluatePredicate(membership, { score: 0, rank: 0 })).toBe(false) + expect(evaluatePredicate(negated, { score: 0, rank: 0 })).toBe(true) + expect(evaluatePredicate(disjunction, { score: null, rank: 0 })).toBeNull() + expect(evaluatePredicate(disjunction, { score: null, rank: 2 })).toBe(true) + }) + + fcTest.prop([scenarioArbitrary], { numRuns: oracleRuns(250), seed: 1777 })( + `preserves finite-world subtraction for a fixed replay corpus`, + assertSemanticDifference, + ) + + fcTest.prop( + [scenarioArbitrary], + oraclePropertyOptions(250, finiteWorldProperty), + )( + `preserves finite-world subtraction for a random or replayed seed`, + assertSemanticDifference, + ) + + fcTest.prop([predicateArbitrary], { numRuns: oracleRuns(100), seed: 1778 })( + `preserves unbounded subtraction across UNKNOWN rows for a fixed replay corpus`, + assertUnboundedDifference, + ) + + fcTest.prop( + [predicateArbitrary], + oraclePropertyOptions(100, unboundedProperty), + )( + `preserves unbounded subtraction across UNKNOWN rows for a random or replayed seed`, + assertUnboundedDifference, + ) + + fcTest.prop([fieldArbitrary, fc.integer({ min: -2, max: 2 })], { + numRuns: oracleRuns(100), + seed: 1779, + })( + `preserves duplicate common terms for a fixed replay corpus`, + assertDuplicateTermDifference, + ) + + fcTest.prop( + [fieldArbitrary, fc.integer({ min: -2, max: 2 })], + oraclePropertyOptions(100, duplicateProperty), + )( + `preserves duplicate common terms for a random or replayed seed`, + assertDuplicateTermDifference, + ) + + it(`covers every difference outcome in the fixed replay corpus`, () => { + expectEveryDifferenceOutcome({ + numRuns: oracleRuns(1_000), + seed: 1777, + }) + }) +}) diff --git a/packages/db/tests/query/predicate-utils.test.ts b/packages/db/tests/query/predicate-utils.test.ts index 6471950dee..a18b266733 100644 --- a/packages/db/tests/query/predicate-utils.test.ts +++ b/packages/db/tests/query/predicate-utils.test.ts @@ -10,6 +10,7 @@ import { unionWherePredicates, } from '../../src/query/predicate-utils' import { Func, PropRef, Value } from '../../src/query/ir' +import { evaluateReferenceExpression } from '../reference-expression' import type { BasicExpression, OrderBy, @@ -58,6 +59,10 @@ function or(...args: Array): Func { return func(`or`, ...args) } +function not(arg: BasicExpression): Func { + return func(`not`, arg) +} + function inOp(left: BasicExpression, values: Array): Func { return func(`in`, left, val(values)) } @@ -1194,11 +1199,22 @@ describe(`minusWherePredicates`, () => { const subtract = gt(ref(`age`), val(10)) const result = minusWherePredicates(undefined, subtract) - expect(result).toEqual({ - type: `func`, - name: `not`, - args: [subtract], - }) + expect(result).toBeNull() + }) + + it(`falls back before negating an IN predicate`, () => { + const subtract = inOp(ref(`status`), [`active`, null]) + + expect(minusWherePredicates(undefined, subtract)).toBeNull() + }) + + it(`falls back before negating an OR predicate`, () => { + const subtract = or( + eq(ref(`status`), val(`active`)), + eq(ref(`status`), val(null)), + ) + + expect(minusWherePredicates(undefined, subtract)).toBeNull() }) it(`should return empty set when from is subset of subtract`, () => { @@ -1431,6 +1447,87 @@ describe(`minusWherePredicates`, () => { }) describe(`common conditions`, () => { + it(`falls back before negating a nullable residual field`, () => { + const shared = lt(ref(`rank`), val(1)) + const requested = and(shared, shared) + const loaded = and(shared, shared, eq(ref(`score`), val(0))) + + expect(minusWherePredicates(requested, loaded)).toBeNull() + }) + + it(`removes only one matching occurrence for each common condition`, () => { + const score = ref(`score`) + const requested = and( + gt(score, val(0)), + or(eq(score, val(null)), eq(score, val(1))), + inOp(score, [1, 0]), + gt(score, val(-1)), + ) + const loaded = and( + gt(score, val(0)), + or(eq(score, val(null)), eq(score, val(1))), + inOp(score, [1, 0]), + gt(score, val(0)), + ) + + const result = minusWherePredicates(requested, loaded) + + expect(result).not.toBeNull() + for (const value of [-1, 0, 1, null]) { + const row = { score: value } + const expected = + evaluateReferenceExpression(requested, row) === true && + evaluateReferenceExpression(loaded, row) !== true + expect(evaluateReferenceExpression(result!, row)).toBe(expected) + } + }) + + it(`falls back when nested subtraction would negate an unknown value`, () => { + const score = ref(`score`) + const requested = eq(score, val(0)) + const loaded = and( + not(eq(score, val(-1))), + and(eq(score, val(0)), lt(score, val(1))), + ) + + expect(minusWherePredicates(requested, loaded)).toBeNull() + }) + + it(`falls back across nested equality, range, and NOT terms`, () => { + const score = ref(`score`) + const ranges = [gt, gte, lt, lte] + + for (const requestedValue of [-1, 0, 1]) { + const requested = eq(score, val(requestedValue)) + for (const excludedValue of [-1, 0, 1]) { + const negatedEquality = not(eq(score, val(excludedValue))) + for (const range of ranges) { + for (const boundary of [-1, 0, 1]) { + const rangePredicate = range(score, val(boundary)) + const loadedPredicates = [ + and( + negatedEquality, + and(eq(score, val(requestedValue)), rangePredicate), + ), + and( + and(negatedEquality, eq(score, val(requestedValue))), + rangePredicate, + ), + and( + rangePredicate, + and(negatedEquality, eq(score, val(requestedValue))), + ), + ] + + for (const loaded of loadedPredicates) { + expect(minusWherePredicates(requested, loaded)).toBeNull() + } + } + } + } + } + }) + it(`should handle common conditions: (age > 10 AND status = 'active') - (age > 20 AND status = 'active') = (age > 10 AND age <= 20 AND status = 'active')`, () => { const from = and( gt(ref(`age`), val(10)), diff --git a/packages/db/tests/query/scheduler.test.ts b/packages/db/tests/query/scheduler.test.ts index 3faf361648..f759bb1fbd 100644 --- a/packages/db/tests/query/scheduler.test.ts +++ b/packages/db/tests/query/scheduler.test.ts @@ -4,11 +4,13 @@ import { createLiveQueryCollection, eq, isNull } from '../../src/query/index.js' import { createTransaction } from '../../src/transactions.js' import { createOptimisticAction } from '../../src/optimistic-action.js' import { + Scheduler, getActivePublicationContext, transactionScopedScheduler, withPublicationContext, } from '../../src/scheduler.js' import { CollectionConfigBuilder } from '../../src/query/live/collection-config-builder.js' +import { CollectionSubscriber } from '../../src/query/live/collection-subscriber.js' import { mockSyncCollectionOptions, stripVirtualProps } from '../utils.js' import type { OutputWithVirtual } from '../utils.js' import type { FullSyncState } from '../../src/query/live/types.js' @@ -24,6 +26,17 @@ interface User { name: string } +const falsyListenerFailureCases = [ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `negative zero`, failure: -0 }, + { name: `bigint zero`, failure: 0n }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, +] + type UserWithVirtual = OutputWithVirtual interface Task { @@ -141,9 +154,677 @@ describe(`Collection publication scheduler context`, () => { expect(getActivePublicationContext()).toBeUndefined() expect(transactionScopedScheduler.hasPendingJobs(contextId!)).toBe(false) }) + + it(`preserves a falsy graph failure through a publication boundary`, () => { + let didThrow = false + let thrown: unknown + + try { + withPublicationContext(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: `failing`, + run: () => { + throw undefined + }, + }) + }) + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(thrown).toBeUndefined() + }) + + it(`attempts every clear listener and preserves its first failure`, () => { + const scheduler = new Scheduler() + const firstFailure = new Error(`first clear listener failed`) + const laterFailure = new Error(`later clear listener failed`) + const calls: Array = [] + let firstClear = true + let removeAdded: (() => void) | undefined + scheduler.onClear(() => { + calls.push(`first`) + if (!firstClear) return + removeSecond() + removeAdded ??= scheduler.onClear(() => calls.push(`added`)) + throw firstFailure + }) + const removeSecond = scheduler.onClear(() => { + calls.push(`second`) + if (firstClear) throw laterFailure + }) + + let thrown: unknown + try { + scheduler.clear(`context`) + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(calls).toEqual([`first`, `second`]) + + firstClear = false + expect(() => scheduler.clear(`next context`)).not.toThrow() + expect(calls).toEqual([`first`, `second`, `first`, `added`]) + removeAdded?.() + }) + + it.each([ + { source: `publication`, failureKind: `Error` }, + { source: `publication`, failureKind: `undefined` }, + { source: `graph`, failureKind: `Error` }, + { source: `graph`, failureKind: `undefined` }, + ] as const)( + `does not replace a $failureKind $source failure with a clear-listener failure`, + ({ source, failureKind }) => { + const primaryFailure = + failureKind === `Error` ? new Error(`${source} failed`) : undefined + const clearFailure = new Error(`clear listener failed`) + const laterClear = vi.fn() + const removeThrowingClear = transactionScopedScheduler.onClear(() => { + throw clearFailure + }) + const removeLaterClear = transactionScopedScheduler.onClear(laterClear) + + try { + let didThrow = false + let thrown: unknown + try { + withPublicationContext(() => { + if (source === `publication`) throw primaryFailure + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: `failing graph`, + run: () => { + throw primaryFailure + }, + }) + }) + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, primaryFailure)).toBe(true) + expect(laterClear).toHaveBeenCalledOnce() + } finally { + removeThrowingClear() + removeLaterClear() + } + }, + ) }) describe(`live query scheduler`, () => { + it(`delivers an ordinary source batch to its frozen listener snapshot`, async () => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const calls: Array = [] + const source = createCollection({ + id: `ordinary-listener-membership-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + let added: { unsubscribe: () => void } | undefined + const first = source.subscribeChanges(() => { + calls.push(`first`) + second.unsubscribe() + added ??= source.subscribeChanges(() => calls.push(`added`), { + includeInitialState: false, + }) + }) + const second = source.subscribeChanges(() => calls.push(`second`)) + + try { + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + commit() + expect(calls).toEqual([`first`, `second`]) + + begin() + write({ type: `insert`, value: { id: 2, name: `Grace` } }) + commit() + expect(calls).toEqual([`first`, `second`, `first`, `added`]) + } finally { + first.unsubscribe() + second.unsubscribe() + added?.unsubscribe() + await source.cleanup() + } + }) + + it(`delivers a layout-only batch to its frozen listener snapshot`, async () => { + type RankedUser = User & { rank: number } + const calls: Array = [] + const firstFailure = new Error(`first layout listener failed`) + const laterFailure = new Error(`later public listener failed`) + const graphJob = vi.fn(() => calls.push(`graph`)) + const source = createCollection( + mockSyncCollectionOptions({ + id: `layout-listener-membership-source`, + getKey: (user) => user.id, + initialData: [ + { id: 1, name: `Ada`, rank: 1 }, + { id: 2, name: `Grace`, rank: 2 }, + ], + }), + ) + const ordered = createLiveQueryCollection({ + id: `layout-listener-membership-ordered`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .orderBy(({ user }) => user.rank, `asc`) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + await ordered.preload() + expect(ordered.toArray.map(({ id }) => id)).toEqual([1, 2]) + let firstPublication = true + let addedLayout: (() => void) | undefined + let addedPublic: { unsubscribe: () => void } | undefined + const unsubscribeFirstLayout = ordered._subscribeLayoutChanges(() => { + calls.push(`layout:first`) + if (!firstPublication) return + unsubscribeSecondLayout() + secondPublic.unsubscribe() + addedLayout ??= ordered._subscribeLayoutChanges(() => + calls.push(`layout:added`), + ) + addedPublic ??= ordered.subscribeChanges( + () => calls.push(`public:added`), + { includeInitialState: false }, + ) + throw firstFailure + }) + const unsubscribeSecondLayout = ordered._subscribeLayoutChanges(() => { + calls.push(`layout:second`) + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: graphJob, + run: graphJob, + }) + }) + const firstPublic = ordered.subscribeChanges( + () => { + calls.push(`public:first`) + if (firstPublication) throw laterFailure + }, + { includeInitialState: false }, + ) + const secondPublic = ordered.subscribeChanges( + () => calls.push(`public:second`), + { + includeInitialState: false, + }, + ) + + try { + let thrown: unknown + try { + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: 1, name: `Ada`, rank: 3 }, + }) + source.utils.commit() + } catch (error) { + thrown = error + } + + expect(thrown).toBe(firstFailure) + expect(calls).toEqual([ + `layout:first`, + `layout:second`, + `public:first`, + `public:second`, + `graph`, + ]) + expect(graphJob).toHaveBeenCalledOnce() + expect(ordered.toArray.map(({ id }) => id)).toEqual([2, 1]) + + firstPublication = false + source.utils.begin() + source.utils.write({ + type: `update`, + value: { id: 1, name: `Ada`, rank: 0 }, + }) + expect(() => source.utils.commit()).not.toThrow() + expect(calls).toEqual([ + `layout:first`, + `layout:second`, + `public:first`, + `public:second`, + `graph`, + `layout:first`, + `layout:added`, + `public:first`, + `public:added`, + ]) + } finally { + unsubscribeFirstLayout() + unsubscribeSecondLayout() + addedLayout?.() + firstPublic.unsubscribe() + secondPublic.unsubscribe() + addedPublic?.unsubscribe() + await ordered.cleanup() + await source.cleanup() + } + }) + + it(`settles a dependent live query when an earlier source listener throws`, async () => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const listenerFailure = new Error(`source listener failed`) + const source = createCollection({ + id: `throwing-listener-live-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + () => { + throw listenerFailure + }, + { includeInitialState: false }, + ) + const live = createLiveQueryCollection({ + id: `throwing-listener-live-dependent`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + expect(() => commit()).toThrow(listenerFailure) + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + } finally { + throwingSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }) + + it.each(falsyListenerFailureCases)( + `preserves an exact $name row-listener failure after later delivery`, + async ({ name, failure }) => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + type UserObservation = { + changes: Array<{ + type: string + key: string | number + value: UserWithVirtual + previousValue: UserWithVirtual | undefined + }> + rows: Array + } + const sourceObservations: Array = [] + const dependentObservations: Array = [] + const snapshotUser = ({ + id, + name: userName, + $collectionId, + $key, + $origin, + $synced, + }: UserWithVirtual): UserWithVirtual => ({ + id, + name: userName, + $collectionId, + $key, + $origin, + $synced, + }) + const source = createCollection({ + id: `falsy-row-listener-${name.replaceAll(` `, `-`)}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + () => { + throw failure + }, + { includeInitialState: false }, + ) + const laterSubscription = source.subscribeChanges( + (changes) => { + sourceObservations.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotUser(value), + previousValue: + previousValue === undefined + ? undefined + : snapshotUser(previousValue), + })), + rows: [...source.state.values()].map(snapshotUser), + }) + }, + { includeInitialState: false }, + ) + const live = createLiveQueryCollection({ + id: `falsy-row-listener-dependent-${name.replaceAll(` `, `-`)}`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + let dependentSubscription: + | ReturnType + | undefined + + try { + await live.preload() + dependentSubscription = live.subscribeChanges( + (changes) => { + dependentObservations.push({ + changes: changes.map(({ type, key, value, previousValue }) => ({ + type, + key, + value: snapshotUser(value), + previousValue: + previousValue === undefined + ? undefined + : snapshotUser(previousValue), + })), + rows: [...live.state.values()].map(snapshotUser), + }) + }, + { includeInitialState: false }, + ) + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + let didThrow = false + let thrown: unknown + try { + commit() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + const expectedObservation = (collectionId: string): UserObservation => { + const row: UserWithVirtual = { + id: 1, + name: `Ada`, + $collectionId: collectionId, + $key: 1, + $origin: `remote`, + $synced: true, + } + return { + changes: [ + { + type: `insert`, + key: 1, + value: row, + previousValue: undefined, + }, + ], + rows: [row], + } + } + const expectedDependent = expectedObservation(live.id) + expect(sourceObservations).toEqual([expectedObservation(source.id)]) + expect(dependentObservations).toEqual([expectedDependent]) + expect([...live.state.values()].map(snapshotUser)).toEqual( + expectedDependent.rows, + ) + } finally { + throwingSubscription.unsubscribe() + laterSubscription.unsubscribe() + dependentSubscription?.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }, + ) + + it.each([ + { + name: `Error`, + failure: new Error(`filtered source listener failed`), + }, + ...falsyListenerFailureCases, + ])( + `preserves an exact $name filtered row-listener failure`, + async ({ name, failure }) => { + let begin!: () => void + let write!: (message: { type: `insert`; value: User }) => void + let commit!: () => void + const filteredCalls = vi.fn() + const laterListener = vi.fn() + const source = createCollection({ + id: `filtered-throwing-listener-source-${name.replaceAll(` `, `-`)}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + begin = actions.begin + write = actions.write + commit = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const throwingSubscription = source.subscribeChanges( + (changes) => { + filteredCalls(changes) + throw failure + }, + { + includeInitialState: false, + where: (user) => eq(user.name, `Ada`), + }, + ) + const laterSubscription = source.subscribeChanges(laterListener, { + includeInitialState: false, + }) + const live = createLiveQueryCollection({ + id: `filtered-throwing-listener-dependent-${name.replaceAll(` `, `-`)}`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + + try { + await live.preload() + begin() + write({ type: `insert`, value: { id: 1, name: `Ada` } }) + let didThrow = false + let thrown: unknown + try { + commit() + } catch (error) { + didThrow = true + thrown = error + } + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + expect(filteredCalls).toHaveBeenCalledOnce() + expect(filteredCalls.mock.calls[0]?.[0]).toEqual([ + expect.objectContaining({ type: `insert`, key: 1 }), + ]) + expect(laterListener).toHaveBeenCalledOnce() + expect(live.get(1)).toEqual(expect.objectContaining({ name: `Ada` })) + + begin() + write({ type: `insert`, value: { id: 2, name: `Grace` } }) + expect(() => commit()).not.toThrow() + expect(filteredCalls).toHaveBeenCalledOnce() + expect(laterListener).toHaveBeenCalledTimes(2) + expect(live.get(2)).toEqual(expect.objectContaining({ name: `Grace` })) + } finally { + throwingSubscription.unsubscribe() + laterSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }, + ) + + it(`keeps a nested ready failure when a later outer listener throws`, async () => { + let markInnerReady!: () => void + const readyFailure = new Error(`nested ready listener failed`) + const laterFailure = new Error(`later outer listener failed`) + const scheduledJob = vi.fn() + const inner = createCollection({ + id: `nested-ready-collision-inner`, + getKey: (user) => user.id, + sync: { + sync: ({ markReady }) => { + markInnerReady = markReady + }, + }, + }) + const innerFirst = inner.subscribeChanges(() => { + const contextId = getActivePublicationContext() + transactionScopedScheduler.schedule({ + contextId, + jobId: scheduledJob, + run: scheduledJob, + }) + }) + const innerSecond = inner.subscribeChanges(() => { + throw readyFailure + }) + + let beginOuter!: () => void + let writeOuter!: (message: { type: `insert`; value: User }) => void + let commitOuter!: () => void + const outer = createCollection({ + id: `nested-ready-collision-outer`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: (actions) => { + beginOuter = actions.begin + writeOuter = actions.write + commitOuter = () => { + actions.commit() + } + actions.markReady() + }, + }, + }) + const outerFirst = outer.subscribeChanges(() => markInnerReady()) + const outerSecond = outer.subscribeChanges(() => { + throw laterFailure + }) + + try { + beginOuter() + writeOuter({ type: `insert`, value: { id: 1, name: `Ada` } }) + expect(() => commitOuter()).toThrow(readyFailure) + expect(scheduledJob).toHaveBeenCalledOnce() + } finally { + outerFirst.unsubscribe() + outerSecond.unsubscribe() + innerFirst.unsubscribe() + innerSecond.unsubscribe() + await outer.cleanup() + await inner.cleanup() + } + }) + + it(`settles a dependent live query before a nested ready failure escapes`, async () => { + let markSourceReady: (() => void) | undefined + const listenerFailure = new Error(`source ready listener failed`) + const source = createCollection({ + id: `nested-ready-live-source`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: ({ begin, commit, markReady }) => { + begin() + commit() + markSourceReady = markReady + }, + }, + }) + const live = createLiveQueryCollection({ + id: `nested-ready-live-dependent`, + startSync: true, + query: (q) => + q + .from({ user: source }) + .select(({ user }) => ({ id: user.id, name: user.name })), + }) + const preload = live.preload() + const throwingSubscription = source.subscribeChanges(() => { + throw listenerFailure + }) + + try { + expect(live.status).toBe(`loading`) + expect(() => withPublicationContext(() => markSourceReady!())).toThrow( + listenerFailure, + ) + await expect(preload).resolves.toBeUndefined() + expect(source.status).toBe(`ready`) + expect(live.status).toBe(`ready`) + } finally { + throwingSubscription.unsubscribe() + await live.cleanup() + await source.cleanup() + } + }) + it(`runs the live query graph once per transaction that touches multiple collections`, async () => { const { users, tasks, assignments } = setupLiveQueryCollections(`single-batch`) @@ -596,6 +1277,213 @@ describe(`live query scheduler`, () => { maybeRunGraphSpy.mockRestore() }) + it.each([ + { name: `undefined`, failure: undefined }, + { name: `null`, failure: null }, + { name: `false`, failure: false }, + { name: `zero`, failure: 0 }, + { name: `empty string`, failure: `` }, + { name: `NaN`, failure: Number.NaN }, + ])(`preserves the first falsy graph-loader failure: $name`, ({ failure }) => { + const baseCollection = createCollection({ + id: `falsy-loader-users-${String(failure)}`, + getKey: (user) => user.id, + sync: { + sync: () => () => {}, + }, + }) + const builder = new CollectionConfigBuilder({ + id: `falsy-loader-builder-${String(failure)}`, + query: (q) => q.from({ user: baseCollection }), + }) + const contextId = Symbol(`falsy-loader-context`) + const laterLoader = vi.fn(() => true) + const config = { + begin: vi.fn(), + write: vi.fn(), + commit: vi.fn(), + markReady: vi.fn(), + truncate: vi.fn(), + } as unknown as Parameters[`sync`]>[0] + const syncState = { + messagesCount: 0, + subscribedToAllCollections: true, + unsubscribeCallbacks: new Set<() => void>(), + graph: { + pendingWork: () => false, + run: vi.fn(), + }, + inputs: {}, + pipeline: {}, + } as unknown as FullSyncState + const maybeRunGraphSpy = vi + .spyOn(builder, `maybeRunGraph`) + .mockImplementation((combinedLoader) => { + combinedLoader?.() + }) + + builder.currentSyncConfig = config + builder.currentSyncState = syncState + builder.scheduleGraphRun( + () => { + throw failure + }, + { contextId }, + ) + builder.scheduleGraphRun(laterLoader, { contextId }) + + let didThrow = false + let thrown: unknown + try { + transactionScopedScheduler.flush(contextId) + } catch (error) { + didThrow = true + thrown = error + } finally { + maybeRunGraphSpy.mockRestore() + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, failure)).toBe(true) + expect(laterLoader).toHaveBeenCalledOnce() + }) + + it(`attempts every repeated-alias source loader and preserves the first failure`, async () => { + const createSource = (name: string) => + createCollection({ + id: `source-loader-${name}`, + getKey: (user) => user.id, + startSync: true, + sync: { + sync: ({ markReady }) => { + markReady() + return () => {} + }, + }, + }) + const firstSource = createSource(`first`) + const secondSource = createSource(`second`) + const thirdSource = createSource(`third`) + const builder = new CollectionConfigBuilder({ + id: `source-loader-builder`, + query: (q) => + q.from({ root: firstSource }).select(({ root }) => ({ + id: root.id, + second: q + .from({ item: secondSource }) + .where(({ item }) => eq(item.id, root.id)), + third: q + .from({ item: thirdSource }) + .where(({ item }) => eq(item.id, root.id)), + })), + }) + type BuilderSyncConfig = Parameters< + ReturnType[`sync`][`sync`] + >[0] + const config = { + begin: vi.fn(), + write: vi.fn(), + commit: vi.fn(), + markReady: vi.fn(), + truncate: vi.fn(), + } as unknown as BuilderSyncConfig + const builderInternals = builder as unknown as { + graphCache: FullSyncState[`graph`] + inputsCache: FullSyncState[`inputs`] + pipelineCache: FullSyncState[`pipeline`] + collectionSources: Array<{ + sourceId: string + alias: string + collection: object + }> + subscribeToAllCollections: ( + syncConfig: typeof config, + state: FullSyncState, + ) => () => boolean + } + const syncState = { + messagesCount: 0, + unsubscribeCallbacks: new Set<() => void>(), + subscribedToAllCollections: false, + graph: builderInternals.graphCache, + inputs: builderInternals.inputsCache, + pipeline: builderInternals.pipelineCache, + } as unknown as FullSyncState + const sourceIdFor = (collection: object): string => { + const source = builderInternals.collectionSources.find( + (candidate) => candidate.collection === collection, + ) + if (!source) throw new Error(`Expected a lexical source`) + return source.sourceId + } + const firstSourceId = sourceIdFor(firstSource) + const secondSourceId = sourceIdFor(secondSource) + const thirdSourceId = sourceIdFor(thirdSource) + expect( + builderInternals.collectionSources.map(({ alias }) => alias), + ).toEqual([`root`, `item`, `item`]) + expect(new Set([firstSourceId, secondSourceId, thirdSourceId]).size).toBe(3) + const laterFailure = new Error(`later source failed`) + const loaderCalls: Array = [] + const loaderCallCounts = new Map() + const loadMoreSpy = vi + .spyOn(CollectionSubscriber.prototype, `loadMoreIfNeeded`) + .mockImplementation(function (this: unknown) { + const { sourceId } = this as { sourceId: string } + loaderCalls.push(sourceId) + loaderCallCounts.set( + sourceId, + (loaderCallCounts.get(sourceId) ?? 0) + 1, + ) + if (sourceId === firstSourceId) throw undefined + if (sourceId === secondSourceId) throw laterFailure + if (sourceId === thirdSourceId) return true + throw new Error(`Unexpected source: ${sourceId}`) + }) + + try { + builder.currentSyncConfig = config + builder.currentSyncState = syncState + const loadAllSources = builderInternals.subscribeToAllCollections( + config, + syncState, + ) + + let didThrow = false + let thrown: unknown + try { + loadAllSources() + } catch (error) { + didThrow = true + thrown = error + } + + expect(didThrow).toBe(true) + expect(Object.is(thrown, undefined)).toBe(true) + expect(loaderCalls).toEqual([ + firstSourceId, + secondSourceId, + thirdSourceId, + ]) + expect(loaderCallCounts).toEqual( + new Map([ + [firstSourceId, 1], + [secondSourceId, 1], + [thirdSourceId, 1], + ]), + ) + expect(loadMoreSpy).toHaveBeenCalledTimes(3) + } finally { + for (const unsubscribe of syncState.unsubscribeCallbacks) unsubscribe() + loadMoreSpy.mockRestore() + await Promise.all([ + firstSource.cleanup(), + secondSource.cleanup(), + thirdSource.cleanup(), + ]) + } + }) + it(`should handle optimistic mutations with nested left joins without scheduler errors`, async () => { // This test verifies that optimistic mutations on collections with nested live query // collections using left joins complete successfully without scheduler errors. diff --git a/packages/db/tests/query/subset-dedupe.test.ts b/packages/db/tests/query/subset-dedupe.test.ts index 5ea056658c..033a52f3da 100644 --- a/packages/db/tests/query/subset-dedupe.test.ts +++ b/packages/db/tests/query/subset-dedupe.test.ts @@ -4,8 +4,13 @@ import { cloneOptions, } from '../../src/query/subset-dedupe' import { Func, PropRef, Value } from '../../src/query/ir' +import { createCrossRealmUint8Array } from '../utils' import type { BasicExpression, OrderBy } from '../../src/query/ir' -import type { LoadSubsetOptions } from '../../src/types' +import type { + LoadSubsetFn, + LoadSubsetOptions, + LoadSubsetResult, +} from '../../src/types' // Helper functions to build expressions more easily function ref(path: string | Array): PropRef { @@ -40,11 +45,253 @@ function lte(left: BasicExpression, right: BasicExpression): Func { return new Func(`lte`, [left, right]) } -function not(expression: BasicExpression): Func { - return new Func(`not`, [expression]) -} - describe(`createDeduplicatedLoadSubset`, () => { + it(`does not let mutation rewrite settled large-binary coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const mutableToken = new Uint8Array(129).fill(1) + const demand = (token: Uint8Array): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(token)), + limit: 1, + }) + + deduplicated.loadSubset(demand(mutableToken)) + mutableToken.fill(2) + deduplicated.loadSubset(demand(new Uint8Array(129).fill(2))) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`does not let custom binary iteration alias intrinsic byte coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const customBytes = new Uint8Array([2]) + Object.defineProperty(customBytes, Symbol.iterator, { + value: function* () { + yield 1 + }, + }) + const demand = (token: Uint8Array): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(token)), + }) + + deduplicated.loadSubset(demand(new Uint8Array([1]))) + deduplicated.loadSubset(demand(customBytes)) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects binary proxies before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const bytes = new Proxy(new Uint8Array([2]), { + get: (target, key) => + key === Symbol.iterator + ? function* () { + yield 1 + } + : Reflect.get(target, key, target), + }) + + expect(() => + deduplicated.loadSubset({ where: eq(ref(`token`), val(bytes)) }), + ).toThrow(/Cannot snapshot binary equality value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`retains cross-realm binary coverage by acquired bytes`, () => { + const acquired: Array> = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquired.push( + Array.from(((options.where as Func).args[1] as Value).value), + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const bytes = createCrossRealmUint8Array([1]) + const demand = (): LoadSubsetOptions => ({ + where: eq(ref(`token`), val(bytes)), + }) + + deduplicated.loadSubset(demand()) + bytes[0] = 2 + deduplicated.loadSubset(demand()) + + expect(acquired).toEqual([[1], [2]]) + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`observes computed membership once for tracking and acquisition`, () => { + const first = new Uint8Array([1]) + const second = new Uint8Array([2]) + let observations = 0 + const candidates = new Proxy([first], { + getOwnPropertyDescriptor: (target, key) => { + const descriptor = Reflect.getOwnPropertyDescriptor(target, key) + if (key !== `0` || descriptor === undefined) return descriptor + observations += 1 + return { + ...descriptor, + value: observations === 1 ? first : second, + } + }, + }) + const acquired: Array = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquired.push( + ...( + ((options.where as Func).args[1] as Func).args[0] as Value< + Array + > + ).value, + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }) + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val([first])]), + ]), + }) + + expect(observations).toBe(1) + expect(acquired).toEqual([first]) + expect(loadSubset).toHaveBeenCalledTimes(1) + }) + + it(`rejects custom membership observation before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const candidates = [new Uint8Array([2])] + Object.defineProperty(candidates, Symbol.iterator, { + value: function* () { + yield new Uint8Array([1]) + }, + }) + + expect(() => + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }), + ).toThrow(/Cannot snapshot membership candidates/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`uses intrinsic Date state for tracking and adapter acquisition`, () => { + const acquiredDates: Array = [] + const loadSubset = vi.fn((options: LoadSubsetOptions) => { + acquiredDates.push( + ((options.where as Func).args[1] as Value).value.getTime(), + ) + return true as const + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const date = new Date(2) + let observedTime = 0 + Object.defineProperty(date, `getTime`, { + value: () => ++observedTime, + }) + const demand = (value: Date): LoadSubsetOptions => ({ + where: eq(ref(`date`), val(value)), + }) + + deduplicated.loadSubset(demand(date)) + deduplicated.loadSubset(demand(new Date(1))) + + expect(acquiredDates).toEqual([2, 1]) + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects constructor-shaped Temporal lookalikes before adapter entry`, () => { + class TemporalLookalike { + static from(): TemporalLookalike { + return new TemporalLookalike() + } + get [Symbol.toStringTag](): string { + return `Temporal.PlainDate` + } + toString(): string { + return `2024-01-15` + } + } + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + expect(() => + deduplicated.loadSubset({ + where: eq(ref(`date`), val(new TemporalLookalike())), + }), + ).toThrow(/Cannot snapshot Temporal.PlainDate equality value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`does not let mutation rewrite computed membership coverage`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const candidates = [new Uint8Array([1])] + const demand = (): LoadSubsetOptions => ({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val(candidates)]), + ]), + }) + + deduplicated.loadSubset(demand()) + candidates[0]![0] = 2 + deduplicated.loadSubset({ + where: new Func(`in`, [ + ref(`token`), + new Func(`coalesce`, [val([new Uint8Array([2])])]), + ]), + }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rejects unsupported relational coercion before adapter entry`, () => { + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const coercion = { [Symbol.toPrimitive]: () => 1 } + + expect(() => + deduplicated.loadSubset({ + where: gt(ref(`value`), val(coercion)), + }), + ).toThrow(/Cannot snapshot structural expression value/) + expect(loadSubset).not.toHaveBeenCalled() + }) + + it(`does not deduplicate structural predicates with different observable key order`, () => { + const left = Object.create(null) as Record + left.a = 1 + left.b = 2 + const right = Object.create(null) as Record + right.b = 2 + right.a = 1 + const loadSubset = vi.fn(() => true as const) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const expected = JSON.stringify(left) + const demand = (value: Record): LoadSubsetOptions => ({ + where: eq(new Func(`concat`, [val(value)]), val(expected)), + }) + + deduplicated.loadSubset(demand(left)) + deduplicated.loadSubset(demand(right)) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + it.each( [ { @@ -68,26 +315,33 @@ describe(`createDeduplicatedLoadSubset`, () => { })), ), )( - `invalidates $settlement $name settled coverage on unload`, + `invalidates $settlement $name settled coverage after its final owner unloads`, async ({ createOptions, settlement }) => { const loadSubset = vi.fn(() => settlement === `sync` ? (true as const) : Promise.resolve(), ) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) const owner = createOptions() + const peer = createOptions() await deduplicated.loadSubset(owner) - expect(deduplicated.loadSubset(createOptions())).toBe(true) + expect(deduplicated.loadSubset(peer)).toBe(true) expect(loadSubset).toHaveBeenCalledTimes(1) deduplicated.unloadSubset(owner) + const coOwner = createOptions() + expect(deduplicated.loadSubset(coOwner)).toBe(true) + expect(loadSubset).toHaveBeenCalledTimes(1) + + deduplicated.unloadSubset(peer) + deduplicated.unloadSubset(coOwner) await deduplicated.loadSubset(createOptions()) expect(loadSubset).toHaveBeenCalledTimes(2) }, ) - it(`bounds conservative adapter-wide invalidation to one refetch per revisited demand`, async () => { + it(`invalidates a released acquisition without erasing other exact owners`, async () => { const loadSubset = vi.fn(() => Promise.resolve()) const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) const demands = Array.from({ length: 6 }, (_, id) => ({ @@ -98,24 +352,24 @@ describe(`createDeduplicatedLoadSubset`, () => { for (const demand of demands) await deduplicated.loadSubset(demand) expect(loadSubset).toHaveBeenCalledTimes(demands.length) - // Core may delete rows owned by any remembered request when one collection - // owner leaves. Without adapter row provenance, preserving the other five - // request facts would be unsafe, so one release invalidates all six. + // A release invalidates broader coverage inferred from the combined + // request history, but each other physical acquisition still has a live + // exact owner and therefore retains its own evidence. deduplicated.unloadSubset(demands[0]!) for (const demand of demands.slice(1)) { - await deduplicated.loadSubset(demand) + expect(deduplicated.loadSubset(demand)).toBe(true) } - expect(loadSubset).toHaveBeenCalledTimes( - demands.length + demands.length - 1, - ) + expect(loadSubset).toHaveBeenCalledTimes(demands.length) - // Once those demands have rebuilt the cache, revisiting them is free again. - for (const demand of demands.slice(1)) { + await deduplicated.loadSubset(demands[0]!) + expect(loadSubset).toHaveBeenCalledTimes(demands.length + 1) + + // Once the released demand has rebuilt its acquisition, every exact owner + // can be revisited without transport. + for (const demand of demands) { expect(deduplicated.loadSubset(demand)).toBe(true) } - expect(loadSubset).toHaveBeenCalledTimes( - demands.length + demands.length - 1, - ) + expect(loadSubset).toHaveBeenCalledTimes(demands.length + 1) }) it(`does not restore invalidated coverage when unloaded work settles late`, async () => { @@ -137,6 +391,468 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(loadSubset).toHaveBeenCalledTimes(2) }) + it.each([`reset`, `rejection`] as const)( + `keeps newer exact in-flight work when an older owner unloads after %s`, + async (oldOutcome) => { + const pending: Array<{ + resolve: () => void + reject: (error: Error) => void + }> = [] + const loadSubset = vi.fn( + () => + new Promise((resolve, reject) => { + pending.push({ resolve, reject }) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + const oldLoad = deduplicated.loadSubset(reusedOptions) + + if (oldOutcome === `reset`) { + deduplicated.reset() + } else { + const rejected = expect(oldLoad).rejects.toThrow(`old failed`) + pending[0]!.reject(new Error(`old failed`)) + await rejected + } + + const freshLoad = deduplicated.loadSubset(reusedOptions) + deduplicated.unloadSubset(reusedOptions) + const peerLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + pending[1]!.resolve() + if (oldOutcome === `reset`) { + pending[0]!.resolve() + await oldLoad + } + await Promise.all([freshLoad, peerLoad]) + }, + ) + + it(`keeps newer settled exact work when a rejected older owner unloads late`, async () => { + const pending: Array<{ + resolve: () => void + reject: (error: Error) => void + }> = [] + const loadSubset = vi.fn( + () => + new Promise((resolve, reject) => { + pending.push({ resolve, reject }) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + const oldLoad = deduplicated.loadSubset(reusedOptions) + const rejected = expect(oldLoad).rejects.toThrow(`old failed`) + + pending[0]!.reject(new Error(`old failed`)) + await rejected + + const freshLoad = deduplicated.loadSubset(reusedOptions) + pending[1]!.resolve() + await freshLoad + + deduplicated.unloadSubset(reusedOptions) + const peerOptions = { limit: 2 } + expect(deduplicated.loadSubset(peerOptions)).toBe(true) + expect(loadSubset).toHaveBeenCalledTimes(2) + + deduplicated.unloadSubset(reusedOptions) + deduplicated.unloadSubset(peerOptions) + }) + + it.each([`sync`, `async`] as const)( + `does not retain exact evidence when its sole owner unloads during %s adapter entry`, + async (settlement) => { + const options = { limit: 2 } + const loadSubset = vi.fn(() => { + deduplicated.unloadSubset(options) + return settlement === `sync` ? (true as const) : Promise.resolve() + }) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset(options) + await deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }, + ) + + it(`keeps shared exact in-flight work while another logical owner remains`, async () => { + let resolveLoad: (() => void) | undefined + const loadSubset = vi.fn( + () => new Promise((resolve) => (resolveLoad = resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const firstOptions = { limit: 2 } + const first = deduplicated.loadSubset(firstOptions) + const second = deduplicated.loadSubset({ limit: 2 }) + + deduplicated.unloadSubset(firstOptions) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + resolveLoad?.() + await Promise.all([first, second, peer]) + }) + + it(`ignores an unload that has no matching logical owner`, async () => { + let resolveLoad: (() => void) | undefined + const loadSubset = vi.fn( + () => new Promise((resolve) => (resolveLoad = resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const load = deduplicated.loadSubset({ limit: 2 }) + + deduplicated.unloadSubset({ limit: 2 }) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(1) + resolveLoad?.() + await Promise.all([load, peer]) + }) + + it(`rolls back only the reservation whose adapter start throws`, async () => { + const pending: Array<() => void> = [] + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + throw new Error(`start failed`) + }) + .mockImplementation( + () => new Promise((resolve) => pending.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow(`start failed`) + const accepted = deduplicated.loadSubset(reusedOptions) + deduplicated.unloadSubset(reusedOptions) + const peer = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(3) + pending.forEach((resolve) => resolve()) + await Promise.all([accepted, peer]) + }) + + it(`does not cache synchronous work from before a reentrant reset`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return true + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`does not share pending work from before a reentrant reset`, async () => { + let resolveOld!: () => void + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return new Promise((resolve) => (resolveOld = resolve)) + }) + .mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + const oldLoad = deduplicated.loadSubset({ limit: 2 }) + const freshLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + resolveOld() + await Promise.all([oldLoad, freshLoad]) + }) + + it(`does not cache settled work from before a reentrant reset`, async () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return Promise.resolve() + }) + .mockResolvedValue(undefined) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset({ limit: 2 }) + await deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + }) + + it(`rolls back a stale reservation when adapter reset precedes a throw`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + throw new Error(`start failed after reset`) + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow( + `start failed after reset`, + ) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + + it(`rolls back the exact throw behind an older stale owner`, () => { + const loadSubset = vi + .fn() + .mockReturnValueOnce(true) + .mockImplementationOnce(() => { + throw new Error(`replacement start failed`) + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.reset() + expect(() => deduplicated.loadSubset(reusedOptions)).toThrow( + `replacement start failed`, + ) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + + it(`consumes a stale owner before releasing a fresh reused owner`, () => { + const loadSubset = vi + .fn() + .mockImplementationOnce(() => { + deduplicated.reset() + return true + }) + .mockReturnValue(true) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const reusedOptions = { limit: 2 } + + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset(reusedOptions)).toBe(true) + deduplicated.unloadSubset(reusedOptions) + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + + expect(loadSubset).toHaveBeenCalledTimes(3) + }) + + it(`does not share work reset while installing Promise handlers`, async () => { + let resolveOld!: () => void + class ResetOnThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + deduplicated.reset() + return super.then(onfulfilled, onrejected) + } + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + if (loadSubsetCalls === 1) { + return new ResetOnThenPromise((resolve) => { + resolveOld = resolve + }) + } + return Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + const oldLoad = deduplicated.loadSubset({ limit: 2 }) + const freshLoad = deduplicated.loadSubset({ limit: 2 }) + + expect(loadSubsetCalls).toBe(2) + resolveOld() + await Promise.all([oldLoad, freshLoad]) + }) + + it(`releases its abort lease when Promise handler installation throws`, async () => { + class ThrowOnThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + _onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + _onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + throw new Error(`then install failed`) + } + } + const signal = { + aborted: false, + reason: undefined, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as AbortSignal + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return loadSubsetCalls === 1 + ? new ThrowOnThenPromise((resolve) => resolve()) + : Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + const failedLoad = deduplicated.loadSubset({ limit: 2, signal }) + + expect(failedLoad).toBeInstanceOf(Promise) + await expect(failedLoad).rejects.toThrow(`then install failed`) + expect(signal.addEventListener).toHaveBeenCalledTimes(1) + expect(signal.removeEventListener).toHaveBeenCalledTimes(1) + + await deduplicated.loadSubset({ limit: 2 }) + expect(loadSubsetCalls).toBe(2) + }) + + it(`retains exact evidence when a Promise subclass settles during handler installation`, async () => { + class SynchronousThenPromise extends Promise { + static get [Symbol.species](): PromiseConstructor { + return Promise + } + + override then( + onfulfilled?: + | ((value: void) => TResult1 | PromiseLike) + | null, + _onrejected?: + | ((reason: unknown) => TResult2 | PromiseLike) + | null, + ): Promise { + return Promise.resolve(onfulfilled?.()) as Promise + } + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return loadSubsetCalls === 1 + ? new SynchronousThenPromise((resolve) => resolve()) + : Promise.resolve() + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await deduplicated.loadSubset({ limit: 2 }) + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(loadSubsetCalls).toBe(1) + }) + + it(`does not retain coverage when fulfilled result normalization throws`, async () => { + const resultError = new Error(`result read failed`) + const hostileResult = { + get hasMore(): boolean | undefined { + throw resultError + }, + } + const signal = { + aborted: false, + reason: undefined, + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + } as unknown as AbortSignal + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 ? hostileResult : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 2, signal })).rejects.toBe( + resultError, + ) + expect(signal.addEventListener).toHaveBeenCalledTimes(1) + expect(signal.removeEventListener).toHaveBeenCalledTimes(1) + + const retry = deduplicated.loadSubset({ limit: 2 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + + it(`does not retain coverage when row-key snapshotting throws`, async () => { + const resultError = new Error(`row-key snapshot failed`) + const hostileRowKeys = new Proxy>([1], { + get: (target, property, receiver) => { + if (property === Symbol.iterator) throw resultError + return Reflect.get(target, property, receiver) + }, + }) + const hostileResult: LoadSubsetResult = { + hasMore: false, + appliedRowKeys: hostileRowKeys, + } + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 ? hostileResult : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 2 })).rejects.toBe( + resultError, + ) + + const retry = deduplicated.loadSubset({ limit: 2 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + + it(`rejects sparse applied-row evidence without retaining coverage`, async () => { + const sparseRowKeys = new Array(1) + let loadSubsetCalls = 0 + const loadSubset: LoadSubsetFn = () => { + loadSubsetCalls += 1 + return Promise.resolve( + loadSubsetCalls === 1 + ? { hasMore: true, appliedRowKeys: sparseRowKeys } + : { hasMore: undefined }, + ) + } + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + + await expect(deduplicated.loadSubset({ limit: 1 })).rejects.toThrow( + `appliedRowKeys must contain only string or number keys`, + ) + + const retry = deduplicated.loadSubset({ limit: 1 }) + expect(retry).toBeInstanceOf(Promise) + await retry + expect(loadSubsetCalls).toBe(2) + }) + it(`shares in-flight work while any cancellation owner remains active`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined @@ -210,6 +926,30 @@ describe(`createDeduplicatedLoadSubset`, () => { await retry }) + it(`does not reuse an aborted in-flight lease while its work is still settling`, async () => { + const releases: Array<() => void> = [] + const loadSubset = vi.fn( + () => new Promise((resolve) => releases.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const owner = new AbortController() + const where = gt(ref(`age`), val(10)) + + const canceled = deduplicated.loadSubset({ + where, + signal: owner.signal, + }) + owner.abort() + + const retry = deduplicated.loadSubset({ where }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(retry).not.toBe(canceled) + + for (const release of releases) release() + await Promise.all([canceled, retry]) + }) + it(`keeps shared work active for a signal-less owner`, async () => { let resolveLoad: (() => void) | undefined let sharedSignal: AbortSignal | undefined @@ -240,6 +980,105 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(deduplicated.loadSubset({ where })).toBe(true) }) + it(`releases every owner from every in-flight lease when reset`, async () => { + const releases: Array<() => void> = [] + const sharedSignals: Array = [] + const loadSubset = vi.fn( + (options: LoadSubsetOptions) => + new Promise((resolve) => { + sharedSignals.push(options.signal) + releases.push(resolve) + }), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const owners = Array.from({ length: 4 }, () => new AbortController()) + const addSpies = owners.map((owner) => + vi.spyOn(owner.signal, `addEventListener`), + ) + const removeSpies = owners.map((owner) => + vi.spyOn(owner.signal, `removeEventListener`), + ) + + const loads = [ + deduplicated.loadSubset({ + where: gt(ref(`age`), val(10)), + signal: owners[0]!.signal, + }), + deduplicated.loadSubset({ + where: gt(ref(`age`), val(10)), + signal: owners[1]!.signal, + }), + deduplicated.loadSubset({ + where: lt(ref(`age`), val(0)), + signal: owners[2]!.signal, + }), + deduplicated.loadSubset({ + where: lt(ref(`age`), val(0)), + signal: owners[3]!.signal, + }), + ] + expect(loadSubset).toHaveBeenCalledTimes(2) + for (const addSpy of addSpies) expect(addSpy).toHaveBeenCalledOnce() + + deduplicated.reset() + + for (const removeSpy of removeSpies) + expect(removeSpy).toHaveBeenCalledOnce() + for (const signal of sharedSignals) expect(signal?.aborted).toBe(false) + for (const owner of owners) owner.abort() + for (const signal of sharedSignals) expect(signal?.aborted).toBe(false) + + for (const release of releases) release() + await Promise.all(loads) + for (const removeSpy of removeSpies) + expect(removeSpy).toHaveBeenCalledOnce() + }) + + it(`releases settled exact acquisition evidence when reset`, () => { + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: () => true, + }) + const retainedAcquisitions = () => + ( + deduplicated as unknown as { + exactAcquisitions: ReadonlyArray + } + ).exactAcquisitions.length + + expect(deduplicated.loadSubset({ limit: 2 })).toBe(true) + expect(retainedAcquisitions()).toBe(1) + + deduplicated.reset() + + expect(retainedAcquisitions()).toBe(0) + }) + + it(`starts new work immediately after reset and protects it from old completion`, async () => { + const releases: Array<() => void> = [] + const loadSubset = vi.fn( + () => new Promise((resolve) => releases.push(resolve)), + ) + const deduplicated = new DeduplicatedLoadSubset({ loadSubset }) + const where = gt(ref(`age`), val(10)) + + const oldLoad = deduplicated.loadSubset({ where }) + deduplicated.reset() + const currentLoad = deduplicated.loadSubset({ where }) + + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(currentLoad).not.toBe(oldLoad) + + releases[0]?.() + await oldLoad + + const joinedLoad = deduplicated.loadSubset({ where }) + expect(loadSubset).toHaveBeenCalledTimes(2) + expect(joinedLoad).toBe(currentLoad) + + releases[1]?.() + await Promise.all([currentLoad, joinedLoad]) + }) + it(`should call underlying loadSubset on first call`, async () => { let callCount = 0 const mockLoadSubset = () => { @@ -628,6 +1467,37 @@ describe(`createDeduplicatedLoadSubset`, () => { }) }) + it(`tracks the original demand while a narrowed transport is in flight`, async () => { + let resolveNarrowed: (() => void) | undefined + const calls: Array = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: (options) => { + calls.push(cloneOptions(options)) + if (calls.length === 1) return Promise.resolve() + return new Promise((resolve) => { + resolveNarrowed = resolve + }) + }, + }) + + await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) + + const wider = { where: gt(ref(`age`), val(10)) } + const first = deduplicated.loadSubset(wider) + const second = deduplicated.loadSubset(wider) + + expect(calls).toHaveLength(2) + expect(calls[1]?.where).toEqual( + and(gt(ref(`age`), val(10)), lte(ref(`age`), val(20))), + ) + expect(first).toBeInstanceOf(Promise) + expect(second).toBeInstanceOf(Promise) + + resolveNarrowed?.() + await Promise.all([first, second]) + expect(deduplicated.loadSubset(wider)).toBe(true) + }) + it(`should request only the difference for set predicates`, async () => { let callCount = 0 const calls: Array = [] @@ -783,11 +1653,11 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(1) // Second call: no where clause (all data) - // Should request all data except what we already loaded - // i.e. should request NOT (age > 20) + // The missing difference is not safe to express under three-valued + // logic, so the adapter receives the full all-data request. await deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ where: not(gt(ref(`age`), val(20))) }) + expect(calls[1]).toEqual({}) // After loading all data, subsequent calls should be deduplicated const result = await deduplicated.loadSubset({ @@ -797,6 +1667,37 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(2) }) + it(`retries a full-request fallback after transport failure`, async () => { + let rejectAllData: ((error: Error) => void) | undefined + const calls: Array = [] + const deduplicated = new DeduplicatedLoadSubset({ + loadSubset: (options) => { + calls.push(cloneOptions(options)) + if (calls.length === 1 || calls.length === 3) { + return Promise.resolve() + } + return new Promise((_resolve, reject) => { + rejectAllData = reject + }) + }, + }) + + await deduplicated.loadSubset({ where: gt(ref(`age`), val(20)) }) + + const failed = deduplicated.loadSubset({}) + const rejected = expect(failed).rejects.toThrow(`all-data failed`) + rejectAllData?.(new Error(`all-data failed`)) + await rejected + + expect(calls).toHaveLength(2) + expect(calls[1]).toEqual({}) + + await deduplicated.loadSubset({}) + expect(calls).toHaveLength(3) + expect(calls[2]).toEqual({}) + expect(deduplicated.loadSubset({})).toBe(true) + }) + describe(`hasLoadedAllData after loading filtered + unfiltered data`, () => { it(`should set hasLoadedAllData after a filtered load followed by an unfiltered load`, async () => { let callCount = 0 @@ -818,9 +1719,7 @@ describe(`createDeduplicatedLoadSubset`, () => { await deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ - where: not(inOp(ref(`task_id`), [`id1`, `id2`, `id3`])), - }) + expect(calls[1]).toEqual({}) const result = await deduplicated.loadSubset({}) expect(result).toBe(true) @@ -912,9 +1811,7 @@ describe(`createDeduplicatedLoadSubset`, () => { await deduplicated.loadSubset({}) - expect(calls[2]).toEqual({ - where: not(inOp(ref(`task_id`), [`uuid-1`, `uuid-2`])), - }) + expect(calls[2]).toEqual({}) expect((deduplicated as any).hasLoadedAllData).toBe(true) expect((deduplicated as any).unlimitedWhere).toBeUndefined() @@ -975,11 +1872,8 @@ describe(`createDeduplicatedLoadSubset`, () => { const secondAllDataLoad = deduplicated.loadSubset({}) expect(callCount).toBe(2) - expect(calls[1]).toEqual({ - where: not(eq(ref(`task_id`), val(`uuid-1`))), - }) - expect(firstAllDataLoad).toBeInstanceOf(Promise) - expect(secondAllDataLoad).toBeInstanceOf(Promise) + expect(calls[1]).toEqual({}) + expect(secondAllDataLoad).toBe(firstAllDataLoad) resolveAllDataLoad?.() await firstAllDataLoad @@ -1012,23 +1906,12 @@ describe(`createDeduplicatedLoadSubset`, () => { expect(callCount).toBe(10) // Now load all data (no WHERE clause) - // This should send NOT(IN(...)) to the backend but track as "all data loaded" + // The adapter receives the full request because NOT(IN(...)) would drop + // rows whose task_id is null under three-valued logic. await deduplicated.loadSubset({}) expect(callCount).toBe(11) - // The load request should be NOT(IN(task_id, [all accumulated uuids])) - const loadWhere = calls[10]!.where as any - expect(loadWhere.name).toBe(`not`) - expect(loadWhere.args[0].name).toBe(`in`) - expect(loadWhere.args[0].args[0].path).toEqual([`task_id`]) - const loadedUuids = ( - loadWhere.args[0].args[1].value as Array - ).sort() - const expectedUuids = Array.from( - { length: 10 }, - (_, i) => `uuid-${i}`, - ).sort() - expect(loadedUuids).toEqual(expectedUuids) + expect(calls[10]).toEqual({}) // Critical: after loading all data, subsequent requests should be deduplicated const result1 = await deduplicated.loadSubset({ diff --git a/packages/db/tests/query/subset-error-matrix.test.ts b/packages/db/tests/query/subset-error-matrix.test.ts index a7fbe87649..5e8cbd393f 100644 --- a/packages/db/tests/query/subset-error-matrix.test.ts +++ b/packages/db/tests/query/subset-error-matrix.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { BTreeIndex } from '../../src/indexes/btree-index.js' +import { SyncCleanupError } from '../../src/errors.js' import { createEffect, createLiveQueryCollection, eq } from '../../src/index.js' import { mockSyncCollectionOptions } from '../utils.js' @@ -362,7 +363,7 @@ describe(`loadSubset failure matrix`, () => { ) it.each(cleanupFailureCases)( - `does not mistake an unreported cleanup failure for a source error: $name`, + `reports obsolete-demand cleanup failure without failing the source commit: $name`, async ({ consumer, failure }) => { const suffix = `${consumer}-${ failure === undefined @@ -394,7 +395,7 @@ describe(`loadSubset failure matrix`, () => { }, }, }) - const sourceErrors: Array = [] + const sourceErrors: Array = [] const effect = consumer === `effect` ? createEffect({ @@ -434,10 +435,22 @@ describe(`loadSubset failure matrix`, () => { thrown = error } - expect(didThrow).toBe(true) - expect(Object.is(thrown, failure)).toBe(true) - expect(sourceErrors).toEqual([]) - if (live) expect(live.utils.lastSubsetError).toBeUndefined() + await flushFailures() + + expect(didThrow).toBe(false) + expect(thrown).toBeUndefined() + if (effect) { + expect(sourceErrors).toHaveLength(1) + expect(sourceErrors[0]?.message).toBe(String(failure)) + expect(effect.disposed).toBe(true) + } else { + expect(sourceErrors).toEqual([]) + } + if (live) { + expect(live.utils.hasSubsetError).toBe(true) + expect(Object.is(live.utils.lastSubsetError, failure)).toBe(true) + expect(live.status).toBe(`ready`) + } } finally { if (effect) await effect.dispose() if (live) await live.cleanup() @@ -446,4 +459,72 @@ describe(`loadSubset failure matrix`, () => { } }, ) + + it(`retries live cleanup after an undefined failure survives demand retirement`, async () => { + const parent = createStaticSource(`undefined-cleanup-retry-parent`, [row]) + let unloadCount = 0 + const child = createCollection({ + id: `undefined-cleanup-retry-child`, + getKey: (item) => item.id, + syncMode: `on-demand`, + autoIndex: `eager`, + defaultIndexType: BTreeIndex, + sync: { + sync: ({ markReady }) => { + markReady() + return { + loadSubset: () => true, + unloadSubset: () => { + unloadCount++ + if (unloadCount <= 2) throw undefined + }, + } + }, + }, + }) + const live = createLiveQueryCollection((q) => + q + .from({ item: parent }) + .leftJoin({ child }, ({ item, child: childRow }) => + eq(item.id, childRow.parentId), + ), + ) + const originalQueueMicrotask = globalThis.queueMicrotask + const queuedMicrotasks: Array<() => void> = [] + + try { + await live.preload() + + parent.utils.begin() + parent.utils.write({ type: `delete`, value: row }) + parent.utils.commit() + await flushFailures() + + expect(unloadCount).toBe(1) + expect(live.utils.hasSubsetError).toBe(true) + expect(live.utils.lastSubsetError).toBeUndefined() + + globalThis.queueMicrotask = (callback) => { + queuedMicrotasks.push(callback) + } + await live.cleanup() + expect(unloadCount).toBe(2) + expect(queuedMicrotasks).toHaveLength(1) + + let cleanupError: unknown + try { + queuedMicrotasks[0]!() + } catch (error) { + cleanupError = error + } + expect(cleanupError).toBeInstanceOf(SyncCleanupError) + expect((cleanupError as Error).message).toContain(`error: undefined`) + + await live.cleanup() + expect(unloadCount).toBe(3) + } finally { + globalThis.queueMicrotask = originalQueueMicrotask + await Promise.all([live.cleanup(), parent.cleanup(), child.cleanup()]) + } + }) }) diff --git a/packages/db/tests/query/window-state.test.ts b/packages/db/tests/query/window-state.test.ts index 2828c5bafa..98afb999ee 100644 --- a/packages/db/tests/query/window-state.test.ts +++ b/packages/db/tests/query/window-state.test.ts @@ -234,10 +234,129 @@ describe(`WindowState`, () => { window.recordLocalRequestSatisfaction(requestedPrefix) expect(window.localPrefixSize).toBe(Math.min(requestedPrefix, 3)) - expect(window.coversActiveWindow).toBe(requestedPrefix <= 3) + expect(window.coversActiveWindow).toBe(false) + expect(window.satisfiesActiveWindow).toBe(requestedPrefix <= 3) expect(window.requestBoundary()).toBeUndefined() expect(window.progressBoundary()?.key).toBe(Math.min(requestedPrefix, 3)) expect(window.requiresPrefixRefresh).toBe(true) + + if (requestedPrefix > 3) { + expect(window.settleLocalRequestAfterNoProgress()).toBe(true) + expect(window.satisfiesActiveWindow).toBe(true) + } + + window.ensureSize(requestedPrefix + 1) + expect(window.satisfiesActiveWindow).toBe(false) + }, + ) + + it(`refreshes an outcome-free window after shrinking and regrowing`, () => { + const window = new WindowState( + mockCollection([ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ]), + [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + undefined, + 4, + ) + + window.recordLocalRequestSatisfaction(4) + expect(window.settleLocalRequestAfterNoProgress()).toBe(true) + expect(window.satisfiesActiveWindow).toBe(true) + expect(window.coversRetainedWindow).toBe(false) + + window.ensureSize(2) + expect(window.satisfiesActiveWindow).toBe(true) + + window.ensureSize(3) + expect(window.satisfiesActiveWindow).toBe(false) + expect(window.requestBoundary()).toBeUndefined() + expect(window.coverageRevision).toBe(1) + + window.recordLocalRequestSatisfaction(3) + expect(window.satisfiesActiveWindow).toBe(true) + + window.ensureSize(2) + window.ensureSize(3) + expect(window.satisfiesActiveWindow).toBe(false) + expect(window.coverageRevision).toBe(2) + }) + + it.each([ + { + transition: `coverage reset`, + apply: (window: WindowState) => window.resetCoverage(), + expectedCoverage: false, + expectedSatisfaction: false, + }, + { + transition: `continuing authoritative result`, + apply: (window: WindowState) => + window.recordContinuationCoverage( + [], + false, + 4, + window.coverageRevision, + ), + expectedCoverage: false, + expectedSatisfaction: false, + }, + { + transition: `exhausted authoritative result`, + apply: (window: WindowState) => + window.recordContinuationCoverage([], true, 4, window.coverageRevision), + expectedCoverage: true, + expectedSatisfaction: true, + }, + { + transition: `prefix-invalidating live change`, + apply: (window: WindowState) => + window.admitChanges([ + { + type: `delete`, + key: 1, + value: { id: 1, rank: 1 }, + }, + ]), + expectedCoverage: false, + expectedSatisfaction: false, + }, + ])( + `clears local outcome-free satisfaction after $transition`, + ({ apply, expectedCoverage, expectedSatisfaction }) => { + const window = new WindowState( + mockCollection([ + { id: 1, rank: 1 }, + { id: 2, rank: 2 }, + { id: 3, rank: 3 }, + ]), + [ + { + expression: new PropRef([`rank`]), + compareOptions: { direction: `asc`, nulls: `last` }, + }, + ], + undefined, + 4, + ) + + window.recordLocalRequestSatisfaction(4) + window.settleLocalRequestAfterNoProgress() + expect(window.satisfiesActiveWindow).toBe(true) + expect(window.coversActiveWindow).toBe(false) + + apply(window) + + expect(window.coversActiveWindow).toBe(expectedCoverage) + expect(window.satisfiesActiveWindow).toBe(expectedSatisfaction) + expect(window.settleLocalRequestAfterNoProgress()).toBe(false) }, ) }) diff --git a/packages/db/tests/transactions.test.ts b/packages/db/tests/transactions.test.ts index d77e196005..e6179c99eb 100644 --- a/packages/db/tests/transactions.test.ts +++ b/packages/db/tests/transactions.test.ts @@ -216,6 +216,173 @@ describe(`Transactions`, () => { transaction.isPersisted.promise.catch(() => {}) expect(transaction.state).toBe(`failed`) }) + it(`keeps a persisting transaction failed when rollback wins`, async () => { + let releasePersistence!: () => void + const persistence = new Promise((resolve) => { + releasePersistence = resolve + }) + const collection = createCollection<{ id: number }>({ + id: `persisting-rollback-wins`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const transaction = createTransaction({ + autoCommit: false, + mutationFn: () => persistence, + }) + + try { + transaction.mutate(() => collection.insert({ id: 1 })) + const persisted = transaction.isPersisted.promise.then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + const commit = transaction.commit() + expect(transaction.state).toBe(`persisting`) + + transaction.rollback() + expect(transaction.state).toBe(`failed`) + + releasePersistence() + await expect(commit).resolves.toBe(transaction) + expect(await persisted).toEqual({ + status: `rejected`, + reason: undefined, + }) + expect(transaction.state).toBe(`failed`) + expect(transaction.error).toBeUndefined() + } finally { + releasePersistence() + await collection.cleanup() + } + }) + it.each([ + [`Error`, (): unknown => new Error(`late persistence rejection`)], + [`undefined`, (): unknown => undefined], + [`false`, (): unknown => false], + [`zero`, (): unknown => 0], + [`NaN`, (): unknown => Number.NaN], + [`string`, (): unknown => `late persistence rejection`], + [`object`, (): unknown => ({ late: true })], + ] as const)( + `ignores a late %s persistence rejection after rollback wins`, + async (reasonName, createReason) => { + type Row = { id: number; owner: string } + let rejectPersistence!: (reason: unknown) => void + const persistence = new Promise((_resolve, reject) => { + rejectPersistence = reject + }) + const collection = createCollection({ + id: `late-persistence-rejection-${reasonName}`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const batches: Array> = [] + const subscription = collection.subscribeChanges( + (changes) => { + batches.push( + changes.map(({ type, key }) => ({ + type, + key, + })), + ) + }, + { includeInitialState: false }, + ) + const first = createTransaction({ + autoCommit: false, + mutationFn: () => persistence, + }) + const second = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + + try { + const persisted = first.isPersisted.promise.then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + first.mutate(() => collection.insert({ id: 1, owner: `first` })) + const commit = first.commit().then( + (value) => ({ status: `fulfilled` as const, value }), + (reason: unknown) => ({ status: `rejected` as const, reason }), + ) + + first.rollback() + second.mutate(() => collection.insert({ id: 1, owner: `second` })) + rejectPersistence(createReason()) + + const commitOutcome = await commit + expect(commitOutcome.status).toBe(`fulfilled`) + if (commitOutcome.status === `fulfilled`) { + expect(commitOutcome.value).toBe(first) + } + expect(await persisted).toEqual({ + status: `rejected`, + reason: undefined, + }) + expect(first.state).toBe(`failed`) + expect(first.error).toBeUndefined() + expect(second.state).toBe(`pending`) + expect(collection.get(1)).toEqual({ + id: 1, + owner: `second`, + $collectionId: collection.id, + $key: 1, + $origin: `local`, + $synced: false, + }) + expect(batches).toEqual([ + [{ type: `insert`, key: 1 }], + [{ type: `delete`, key: 1 }], + [{ type: `insert`, key: 1 }], + ]) + } finally { + rejectPersistence(new Error(`test cleanup`)) + if (second.state === `pending`) { + second.rollback({ isSecondaryRollback: true }) + } + subscription.unsubscribe() + await collection.cleanup() + } + }, + ) + it(`keeps repeated rollback from affecting newer transactions`, async () => { + type Row = { id: number; owner: string } + const collection = createCollection({ + id: `repeated-rollback-is-terminal`, + getKey: (item) => item.id, + sync: { sync: () => {} }, + }) + const first = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + const second = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + + try { + void first.isPersisted.promise.catch(() => undefined) + first.mutate(() => collection.insert({ id: 1, owner: `first` })) + first.rollback() + + second.mutate(() => collection.insert({ id: 1, owner: `second` })) + expect(second.state).toBe(`pending`) + + expect(first.rollback()).toBe(first) + expect(first.state).toBe(`failed`) + expect(second.state).toBe(`pending`) + expect(collection.get(1)).toMatchObject({ id: 1, owner: `second` }) + } finally { + if (second.state === `pending`) { + second.rollback({ isSecondaryRollback: true }) + } + await collection.cleanup() + } + }) it(`should rollback if the mutationFn throws an error`, async () => { const transaction = createTransaction({ mutationFn: async () => { diff --git a/packages/db/tests/utils.test.ts b/packages/db/tests/utils.test.ts index 0cfc2b27a2..170853787a 100644 --- a/packages/db/tests/utils.test.ts +++ b/packages/db/tests/utils.test.ts @@ -1,23 +1,43 @@ import { describe, expect, it } from 'vitest' import { Temporal } from 'temporal-polyfill' +import packageJson from '../package.json' import { deepEquals } from '../src/utils' import { isPromiseLike } from '../src/utils/type-guards' -import { oracleRandomParameters, readOracleRunConfig } from './oracle-config' +import { + oracleRandomParameters, + readOracleRunConfig, + validateOraclePropertyRegistry, +} from './oracle-config' describe(`oracle run configuration`, () => { - it(`reads the multiplier and replay seed from an explicit environment`, () => { + it(`runs the predicate subtraction oracle in the oracle campaign`, () => { + expect(packageJson.scripts[`test:oracles`]).toContain( + `tests/query/predicate-subtraction-oracle.property.test.ts`, + ) + }) + + it(`reads the multiplier and replay coordinates from an explicit environment`, () => { expect( readOracleRunConfig({ TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: `100`, TANSTACK_DB_ORACLE_SEED: `-42`, + TANSTACK_DB_ORACLE_PATH: `1:0:2`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, }), - ).toEqual({ multiplier: 100, replaySeed: -42 }) + ).toEqual({ + multiplier: 100, + replaySeed: -42, + replayPath: `1:0:2`, + replayProperty: `coverage-registry.claim-churn`, + }) }) - it(`uses one run multiplier and no replay seed by default`, () => { + it(`uses one run multiplier and no replay coordinates by default`, () => { expect(readOracleRunConfig({})).toEqual({ multiplier: 1, replaySeed: undefined, + replayPath: undefined, + replayProperty: undefined, }) }) @@ -27,6 +47,51 @@ describe(`oracle run configuration`, () => { [{ TANSTACK_DB_ORACLE_RUNS_MULTIPLIER: ` ` }, `positive integer`], [{ TANSTACK_DB_ORACLE_SEED: `1.5` }, `must be an integer`], [{ TANSTACK_DB_ORACLE_SEED: ` ` }, `must be an integer`], + [{ TANSTACK_DB_ORACLE_PATH: `1:0` }, `requires TANSTACK_DB_ORACLE_SEED`], + [ + { + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `requires TANSTACK_DB_ORACLE_PATH`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: ` `, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `must be non-empty`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:-1`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `colon-separated nonnegative integers`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:0`, + }, + `requires TANSTACK_DB_ORACLE_PROPERTY`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PATH: `1:0`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.typo`, + }, + `unknown oracle property`, + ], + [ + { + TANSTACK_DB_ORACLE_SEED: `42`, + TANSTACK_DB_ORACLE_PROPERTY: `coverage-registry.claim-churn`, + }, + `requires TANSTACK_DB_ORACLE_PATH`, + ], ] satisfies ReadonlyArray, string]>)( `rejects invalid environment values`, (environment, message) => { @@ -34,11 +99,39 @@ describe(`oracle run configuration`, () => { }, ) - it(`adds a seed only for replay runs`, () => { - expect(oracleRandomParameters(40, undefined)).toEqual({ numRuns: 40 }) - expect(oracleRandomParameters(40, -42)).toEqual({ + it(`rejects duplicate registered property names`, () => { + expect(() => + validateOraclePropertyRegistry([`one.property`, `one.property`]), + ).toThrow(`duplicate oracle property`) + }) + + it(`adds a shrink path only to its named property`, () => { + const ordinaryRun = { + replaySeed: undefined, + replayPath: undefined, + replayProperty: undefined, + } + const replayRun = { + replaySeed: -42, + replayPath: `1:0:2`, + replayProperty: `coverage-registry.claim-churn`, + } + + expect( + oracleRandomParameters(40, ordinaryRun, `coverage-registry.claim-churn`), + ).toEqual({ numRuns: 40 }) + expect( + oracleRandomParameters(40, replayRun, `coverage-registry.state-machine`), + ).toEqual({ + numRuns: 40, + seed: -42, + }) + expect( + oracleRandomParameters(40, replayRun, `coverage-registry.claim-churn`), + ).toEqual({ numRuns: 40, seed: -42, + path: `1:0:2`, }) }) }) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index b31408d0b4..c5cd7fa0b5 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -1,3 +1,4 @@ +import { runInNewContext } from 'node:vm' import { expect } from 'vitest' import { BTreeIndex } from '../src/indexes/btree-index' import { withCollectionConfigFactory } from '../src/client' @@ -10,34 +11,12 @@ import type { import type { IndexConstructor } from '../src/indexes/base-index' import type { WithVirtualProps } from '../src/virtual-props.js' -type OracleEnvironment = Record - -export function readOracleRunConfig( - environment: OracleEnvironment = process.env, -): { multiplier: number; replaySeed: number | undefined } { - const multiplierValue = environment.TANSTACK_DB_ORACLE_RUNS_MULTIPLIER ?? `1` - const multiplier = Number(multiplierValue) - if (!Number.isSafeInteger(multiplier) || multiplier < 1) { - throw new Error( - `TANSTACK_DB_ORACLE_RUNS_MULTIPLIER must be a positive integer`, - ) - } - - const seedValue = environment.TANSTACK_DB_ORACLE_SEED - if (seedValue === undefined) return { multiplier, replaySeed: undefined } - - const replaySeed = Number(seedValue) - if (!Number.isSafeInteger(replaySeed)) { - throw new Error(`TANSTACK_DB_ORACLE_SEED must be an integer`) - } - return { multiplier, replaySeed } -} - -export function oracleRandomParameters( - numRuns: number, - replaySeed: number | undefined, -): { numRuns: number; seed?: number } { - return replaySeed === undefined ? { numRuns } : { numRuns, seed: replaySeed } +export function createCrossRealmUint8Array( + values: ReadonlyArray, +): Uint8Array { + return runInNewContext(`new Uint8Array(values)`, { + values: Array.from(values), + }) as Uint8Array } export type OutputWithVirtual< diff --git a/packages/electric-db-collection/CHANGELOG.md b/packages/electric-db-collection/CHANGELOG.md index e87bc98987..db2dfe74bc 100644 --- a/packages/electric-db-collection/CHANGELOG.md +++ b/packages/electric-db-collection/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/electric-db-collection +## 0.4.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.4.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.4.5 ### Patch Changes diff --git a/packages/electric-db-collection/package.json b/packages/electric-db-collection/package.json index 0f5118b513..6aa1cba220 100644 --- a/packages/electric-db-collection/package.json +++ b/packages/electric-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electric-db-collection", - "version": "0.4.5", + "version": "0.4.7", "description": "ElectricSQL collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/electric-db-collection/src/applied-commit-capture.ts b/packages/electric-db-collection/src/applied-commit-capture.ts new file mode 100644 index 0000000000..2b38728fb1 --- /dev/null +++ b/packages/electric-db-collection/src/applied-commit-capture.ts @@ -0,0 +1,67 @@ +import type { SyncAppliedReceipt } from '@tanstack/db' + +export type AppliedCommitCapture = { + wait: () => Promise + dispose: () => void +} + +export type AppliedCommitCaptureRegistry = { + capture: (signal?: AbortSignal) => AppliedCommitCapture + record: (receipt: SyncAppliedReceipt) => void + readonly activeCount: number +} + +/** + * Captures every asynchronous commit receipt produced during an Electric + * request. A capture is sealed before waiting, so later stream work cannot + * become part of an already-settled request. + */ +export function createAppliedCommitCaptureRegistry( + onActiveCountChange?: (activeCount: number) => void, +): AppliedCommitCaptureRegistry { + const activeCaptures = new Set>>() + const notifyActiveCount = () => onActiveCountChange?.(activeCaptures.size) + + return { + capture: (signal) => { + const receipts = new Set>() + let active = true + const dispose = () => { + if (!active) return + active = false + signal?.removeEventListener(`abort`, dispose) + activeCaptures.delete(receipts) + notifyActiveCount() + } + + activeCaptures.add(receipts) + notifyActiveCount() + if (signal?.aborted) { + dispose() + } else { + signal?.addEventListener(`abort`, dispose, { once: true }) + } + + return { + wait: async () => { + dispose() + await Promise.all(receipts) + }, + dispose, + } + }, + record: (receipt) => { + if (receipt === true || activeCaptures.size === 0) return + + for (const receipts of activeCaptures) { + receipts.add(receipt) + } + // A receipt can reject before its request Promise settles. Observe it + // now while retaining the original Promise for every capture to await. + void receipt.catch(() => undefined) + }, + get activeCount() { + return activeCaptures.size + }, + } +} diff --git a/packages/electric-db-collection/src/electric.ts b/packages/electric-db-collection/src/electric.ts index e47d1fc076..226df9da4c 100644 --- a/packages/electric-db-collection/src/electric.ts +++ b/packages/electric-db-collection/src/electric.ts @@ -8,6 +8,7 @@ import { Store } from '@tanstack/store' import DebugModule from 'debug' import { DeduplicatedLoadSubset, + SyncTransactionAbortedError, and, withCollectionConfigFactory, } from '@tanstack/db' @@ -17,6 +18,7 @@ import { TimeoutWaitingForMatchError, TimeoutWaitingForTxIdError, } from './errors' +import { createAppliedCommitCaptureRegistry } from './applied-commit-capture' import { compileSQL } from './sql-compiler' import { addTagToIndex, @@ -85,6 +87,8 @@ export interface ElectricTestHooks { * Allows tests to pause and validate snapshot phase before atomic swap completes */ beforeMarkingReady?: () => Promise + /** Reports the number of active on-demand applied-receipt captures. */ + onActiveCommitCapturesChange?: (activeCount: number) => void } /** @@ -527,8 +531,7 @@ function createLoadSubsetDedupe>({ begin, write, commit, - getCommitCursor, - waitForCommitsAfter, + captureCommits, collectionId, encodeColumnName, signal, @@ -543,8 +546,10 @@ function createLoadSubsetDedupe>({ metadata: Record }) => void commit: (signal?: AbortSignal) => SyncAppliedReceipt - getCommitCursor: () => number - waitForCommitsAfter: (cursor: number) => Promise + captureCommits: (signal?: AbortSignal) => { + wait: () => Promise + dispose: () => void + } collectionId?: string /** * Optional function to encode column names (e.g., camelCase to snake_case). @@ -553,7 +558,6 @@ function createLoadSubsetDedupe>({ encodeColumnName?: ColumnEncoder /** * Abort signal to check if the stream has been aborted during cleanup. - * When aborted, errors from requestSnapshot are silently ignored. */ signal: AbortSignal }): DeduplicatedLoadSubset | null { @@ -561,6 +565,39 @@ function createLoadSubsetDedupe>({ return null } + const combineAbortSignals = ( + ...signals: Array + ): { signal: AbortSignal; cleanup: () => void } => { + const uniqueSignals = Array.from( + new Set( + signals.filter( + (candidate): candidate is AbortSignal => candidate !== undefined, + ), + ), + ) + if (uniqueSignals.length === 1) { + return { signal: uniqueSignals[0]!, cleanup: () => {} } + } + + const controller = new AbortController() + const abort = () => controller.abort() + for (const candidate of uniqueSignals) { + if (candidate.aborted) { + abort() + } else { + candidate.addEventListener(`abort`, abort, { once: true }) + } + } + return { + signal: controller.signal, + cleanup: () => { + for (const candidate of uniqueSignals) { + candidate.removeEventListener(`abort`, abort) + } + }, + } + } + const compileOptions = encodeColumnName ? { encodeColumnName } : undefined const logPrefix = collectionId ? `[${collectionId}] ` : `` @@ -578,14 +615,26 @@ function createLoadSubsetDedupe>({ } const loadSubset = async (opts: LoadSubsetOptions) => { - const commitCursor = getCommitCursor() - if (opts.signal?.aborted) return + const isAborted = (): boolean => + signal.aborted || opts.signal?.aborted === true + const throwIfCollectionAborted = () => { + if (signal.aborted) { + throw new SyncTransactionAbortedError() + } + } + const throwIfAborted = () => { + if (isAborted()) { + throw new SyncTransactionAbortedError() + } + } + throwIfAborted() if (isBufferingInitialSync()) { const snapshotParams = compileSQL(opts, compileOptions) try { const { data: rows } = await stream.fetchSnapshot(snapshotParams) - if (opts.signal?.aborted || !isBufferingInitialSync()) { + throwIfAborted() + if (!isBufferingInitialSync()) { debug(`${logPrefix}Ignoring snapshot - sync completed while fetching`) return } @@ -599,11 +648,16 @@ function createLoadSubsetDedupe>({ metadata: { ...row.headers }, }) } - await commit(opts.signal) + const commitSignal = combineAbortSignals(signal, opts.signal) + try { + await commit(commitSignal.signal) + } finally { + commitSignal.cleanup() + } debug(`${logPrefix}Applied snapshot with ${rows.length} rows`) } } catch (error) { - if (opts.signal?.aborted) return + throwIfAborted() if (handleSnapshotError(error, `fetchSnapshot`)) { return } @@ -625,10 +679,34 @@ function createLoadSubsetDedupe>({ // long-poll requests promptly. Bound the wait so on-demand live queries don't // remain loading until the long-poll naturally times out. // If the refresh fails or times out, we fall through to requestSnapshot which - // still works. + // still works. Cleanup or request cancellation ends the wait without starting + // a snapshot that no current demand can use. if (stream.isUpToDate) { let timeoutId: ReturnType | undefined + let removeAbortListeners = () => {} try { + const abortSignals = new Set( + [signal, opts.signal].filter( + (candidate): candidate is AbortSignal => candidate !== undefined, + ), + ) + const aborted = new Promise((resolve) => { + const onAbort = () => resolve() + if (Array.from(abortSignals).some((candidate) => candidate.aborted)) { + resolve() + return + } + + abortSignals.forEach((candidate) => + candidate.addEventListener(`abort`, onAbort, { once: true }), + ) + removeAbortListeners = () => { + abortSignals.forEach((candidate) => + candidate.removeEventListener(`abort`, onAbort), + ) + } + }) + await Promise.race([ stream.forceDisconnectAndRefresh(), new Promise((resolve) => { @@ -637,8 +715,10 @@ function createLoadSubsetDedupe>({ FORCE_DISCONNECT_AND_REFRESH_TIMEOUT_MS, ) }), + aborted, ]) } catch (error) { + throwIfAborted() if (handleSnapshotError(error, `forceDisconnectAndRefresh`)) { return } @@ -647,11 +727,12 @@ function createLoadSubsetDedupe>({ error, ) } finally { + removeAbortListeners() clearTimeout(timeoutId) } } - if (opts.signal?.aborted) return + throwIfAborted() // Upstream limitation: ShapeStream.requestSnapshot() publishes its rows // through the stream callback before its Promise resolves. It accepts no @@ -659,45 +740,62 @@ function createLoadSubsetDedupe>({ // aborted request can already have installed rows before the check below. // Full request-scoped cancellation requires support in the Electric client; // matching snapshots by parameters is unsafe for overlapping equal requests. + const commitCapture = captureCommits(signal) try { - if (cursor) { - const whereCurrentOpts: LoadSubsetOptions = { - where: where ? and(where, cursor.whereCurrent) : cursor.whereCurrent, - orderBy, - } - const whereCurrentParams = compileSQL( - whereCurrentOpts, - compileOptions, - ) + try { + if (cursor) { + const whereCurrentOpts: LoadSubsetOptions = { + where: where + ? and(where, cursor.whereCurrent) + : cursor.whereCurrent, + orderBy, + } + const whereCurrentParams = compileSQL( + whereCurrentOpts, + compileOptions, + ) - const whereFromOpts: LoadSubsetOptions = { - where: where ? and(where, cursor.whereFrom) : cursor.whereFrom, - orderBy, - limit, - } - const whereFromParams = compileSQL(whereFromOpts, compileOptions) + const whereFromOpts: LoadSubsetOptions = { + where: where ? and(where, cursor.whereFrom) : cursor.whereFrom, + orderBy, + limit, + } + const whereFromParams = compileSQL(whereFromOpts, compileOptions) - debug(`${logPrefix}Requesting cursor.whereCurrent snapshot (all ties)`) - debug( - `${logPrefix}Requesting cursor.whereFrom snapshot (with limit ${limit})`, - ) + debug( + `${logPrefix}Requesting cursor.whereCurrent snapshot (all ties)`, + ) + debug( + `${logPrefix}Requesting cursor.whereFrom snapshot (with limit ${limit})`, + ) - await Promise.all([ - stream.requestSnapshot(whereCurrentParams), - stream.requestSnapshot(whereFromParams), - ]) - } else { - const snapshotParams = compileSQL(opts, compileOptions) - await stream.requestSnapshot(snapshotParams) - } - } catch (error) { - if (opts.signal?.aborted) return - if (handleSnapshotError(error, `requestSnapshot`)) { - return + const requestResults = await Promise.allSettled([ + stream.requestSnapshot(whereCurrentParams), + stream.requestSnapshot(whereFromParams), + ]) + const failedRequest = requestResults.find( + (result) => result.status === `rejected`, + ) + if (failedRequest) throw failedRequest.reason + } else { + const snapshotParams = compileSQL(opts, compileOptions) + await stream.requestSnapshot(snapshotParams) + } + } catch (error) { + if (signal.aborted) { + throw new SyncTransactionAbortedError() + } + if (handleSnapshotError(error, `requestSnapshot`)) { + return + } + throw error } - throw error + throwIfCollectionAborted() + await commitCapture.wait() + throwIfCollectionAborted() + } finally { + commitCapture.dispose() } - await waitForCommitsAfter(commitCursor) } return new DeduplicatedLoadSubset({ loadSubset }) @@ -1501,26 +1599,14 @@ function createElectricSync>( collection, metadata, } = params - let commitSequence = 0 - const pendingAppliedReceipts = new Map>() + const commitCaptures = createAppliedCommitCaptureRegistry( + testHooks?.onActiveCommitCapturesChange, + ) const commit = (signal?: AbortSignal): SyncAppliedReceipt => { - const sequence = ++commitSequence const applied = commitSyncTransaction(signal) - if (applied === true) { - return true - } - pendingAppliedReceipts.set(sequence, applied) - const removeReceipt = () => pendingAppliedReceipts.delete(sequence) - void applied.then(removeReceipt, removeReceipt) + commitCaptures.record(applied) return applied } - const waitForCommitsAfter = async (cursor: number): Promise => { - await Promise.all( - Array.from(pendingAppliedReceipts, ([sequence, applied]) => - sequence > cursor ? applied : undefined, - ), - ) - } const readPersistedResumeState = (): ElectricResumeState | undefined => { const persistedResumeState = metadata?.collection.get(`electric:resume`) return parseElectricResumeState(persistedResumeState) @@ -1573,19 +1659,21 @@ function createElectricSync>( // Abort controller for the stream - wraps the signal if provided const abortController = new AbortController() + let removeShapeAbortListener = () => {} if (shapeOptions.signal) { - shapeOptions.signal.addEventListener( - `abort`, - () => { - abortController.abort() - }, - { - once: true, - }, - ) + const abortFromShapeSignal = () => abortController.abort() if (shapeOptions.signal.aborted) { abortController.abort() + } else { + shapeOptions.signal.addEventListener(`abort`, abortFromShapeSignal, { + once: true, + }) + removeShapeAbortListener = () => + shapeOptions.signal?.removeEventListener( + `abort`, + abortFromShapeSignal, + ) } } @@ -1766,8 +1854,7 @@ function createElectricSync>( begin, write, commit, - getCommitCursor: () => commitSequence, - waitForCommitsAfter, + captureCommits: commitCaptures.capture, collectionId, // Pass the columnMapper's encode function to transform column names // (e.g., camelCase to snake_case) when compiling SQL for subset queries @@ -1986,7 +2073,7 @@ function createElectricSync>( // Commit the atomic swap stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) // Exit buffering phase by marking that we've received up-to-date // isBufferingInitialSync() will now return false @@ -2000,12 +2087,12 @@ function createElectricSync>( // Both up-to-date and subset-end trigger a commit if (transactionStarted) { stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) transactionStarted = false } else if (commitPoint === `up-to-date` && metadata) { begin() stageResumeMetadata() - applied = commit() + applied = commit(abortController.signal) } } const readyErrorVersion = streamErrorVersion @@ -2068,6 +2155,7 @@ function createElectricSync>( cleanup: () => { // Unsubscribe from the stream unsubscribeStream() + removeShapeAbortListener() // Abort the abort controller to stop the stream abortController.abort() // Reset deduplication tracking so collection can load fresh data if restarted diff --git a/packages/electric-db-collection/tests/applied-commit-capture.test.ts b/packages/electric-db-collection/tests/applied-commit-capture.test.ts new file mode 100644 index 0000000000..88f6dd3bbe --- /dev/null +++ b/packages/electric-db-collection/tests/applied-commit-capture.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it, vi } from 'vitest' +import { createAppliedCommitCaptureRegistry } from '../src/applied-commit-capture' + +function createDeferred() { + let resolve!: (value: T | PromiseLike) => void + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise + reject = rejectPromise + }) + return { promise, resolve, reject } +} + +describe(`applied commit capture`, () => { + it(`waits for every recorded receipt before settling`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const first = createDeferred() + const second = createDeferred() + registry.record(first.promise) + registry.record(second.promise) + + const wait = capture.wait() + second.resolve() + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + + await expect( + Promise.race([wait.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + expect(registry.activeCount).toBe(0) + + first.resolve() + await wait + }) + + it(`seals the receipt set before waiting`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const lateReceipt = createDeferred() + + const wait = capture.wait() + registry.record(lateReceipt.promise) + + expect(registry.activeCount).toBe(0) + await expect(wait).resolves.toBeUndefined() + lateReceipt.resolve() + }) + + it.each([`first`, `second`] as const)( + `propagates a settled %s receipt failure`, + async (failedReceipt) => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const first = createDeferred() + const second = createDeferred() + const failure = new Error(`${failedReceipt} receipt failed`) + registry.record(first.promise) + registry.record(second.promise) + + if (failedReceipt === `first`) { + first.reject(failure) + second.resolve() + } else { + first.resolve() + second.reject(failure) + } + + await expect(capture.wait()).rejects.toBe(failure) + expect(registry.activeCount).toBe(0) + }, + ) + + it(`observes a receipt failure before waiting begins`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + const receipt = createDeferred() + const failure = new Error(`receipt failed before wait`) + registry.record(receipt.promise) + + receipt.reject(failure) + await new Promise((resolve) => setTimeout(resolve, 0)) + + await expect(capture.wait()).rejects.toBe(failure) + }) + + it(`records one receipt for every concurrent capture`, async () => { + const registry = createAppliedCommitCaptureRegistry() + const firstCapture = registry.capture() + const secondCapture = registry.capture() + const receipt = createDeferred() + const failure = new Error(`shared receipt failed`) + registry.record(receipt.promise) + receipt.reject(failure) + + const errors = await Promise.all([ + firstCapture.wait().catch((error: unknown) => error), + secondCapture.wait().catch((error: unknown) => error), + ]) + expect(errors).toEqual([failure, failure]) + expect(registry.activeCount).toBe(0) + }) + + it(`disposes a capture as soon as its lifetime signal aborts`, () => { + const registry = createAppliedCommitCaptureRegistry() + const controller = new AbortController() + const addSpy = vi.spyOn(controller.signal, `addEventListener`) + const removeSpy = vi.spyOn(controller.signal, `removeEventListener`) + registry.capture(controller.signal) + + expect(registry.activeCount).toBe(1) + expect(addSpy).toHaveBeenCalledOnce() + + controller.abort() + + expect(registry.activeCount).toBe(0) + expect(removeSpy).toHaveBeenCalledOnce() + }) + + it(`does not retain a capture for an already-aborted lifetime`, () => { + const registry = createAppliedCommitCaptureRegistry() + const controller = new AbortController() + controller.abort() + const addSpy = vi.spyOn(controller.signal, `addEventListener`) + + registry.capture(controller.signal) + + expect(registry.activeCount).toBe(0) + expect(addSpy).not.toHaveBeenCalled() + }) + + it.each([`wait`, `dispose`] as const)( + `removes a capture after %s`, + async (settlement) => { + const registry = createAppliedCommitCaptureRegistry() + const capture = registry.capture() + + if (settlement === `wait`) await capture.wait() + else capture.dispose() + + expect(registry.activeCount).toBe(0) + }, + ) +}) diff --git a/packages/electric-db-collection/tests/electric-live-query.test.ts b/packages/electric-db-collection/tests/electric-live-query.test.ts index 572fb90bee..a07592cc3b 100644 --- a/packages/electric-db-collection/tests/electric-live-query.test.ts +++ b/packages/electric-db-collection/tests/electric-live-query.test.ts @@ -8,10 +8,15 @@ import { lt, } from '@tanstack/db' import { electricCollectionOptions } from '../src/electric' +import { + projectRetainedRowKeys, + projectTransportLoads, +} from '../../db/tests/load-subset-full-flow-model' import type { ElectricCollectionUtils } from '../src/electric' import type { Collection } from '@tanstack/db' import type { Message } from '@electric-sql/client' import type { StandardSchemaV1 } from '@standard-schema/spec' +import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' // Sample user type for tests type User = { @@ -1206,9 +1211,9 @@ describe(`Electric Collection - loadSubset deduplication`, () => { // Wait for the existing live query to re-request data after truncate await new Promise((resolve) => setTimeout(resolve, 0)) - // Truncate replays the exact demand once. Electric does not yet return an - // applied outcome, so the empty local prefix then requests one refill. - expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) + // Truncate replays the exact demand once. Releasing the old acquisition + // must not discard that replacement while it is still owned. + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2) // Create the same live query again after reset // This should NOT be deduped because the reset cleared the deduplication state, @@ -1226,9 +1231,9 @@ describe(`Electric Collection - loadSubset deduplication`, () => { await new Promise((resolve) => setTimeout(resolve, 0)) - // Should have more calls - the different query triggered a new request - // 1 initial + 1 replay + 1 outcome-free refill + 1 new query = 4 - expect(mockRequestSnapshot).toHaveBeenCalledTimes(4) + // The different query triggers one more physical request. + // 1 initial + 1 replay + 1 new query = 3 + expect(mockRequestSnapshot).toHaveBeenCalledTimes(3) }) it(`should deduplicate unlimited queries regardless of orderBy`, async () => { @@ -1316,4 +1321,105 @@ describe(`Electric Collection - loadSubset deduplication`, () => { // Still 2 calls - third was covered by the union of first two expect(mockRequestSnapshot).toHaveBeenCalledTimes(2) }) + + it(`matches the shared remount history after final-owner release`, async () => { + const electricCollection = createElectricCollectionWithSyncMode(`on-demand`) + const row = sampleUsers[0]! + const history: Array = [ + { + type: `requestDemand`, + ownerId: `owner-1`, + sessionId: `session-1`, + sourceId: `electric-users`, + demandId: `active-users`, + attemptId: `attempt-1`, + alreadyAborted: false, + }, + ] + const createLive = (id: string) => + createLiveQueryCollection({ + id, + startSync: true, + query: (q) => + q + .from({ user: electricCollection }) + .where(({ user }) => eq(user.active, true)), + }) + simulateInitialSync([]) + mockRequestSnapshot.mockResolvedValue({ + data: [ + { + headers: { operation: `insert` }, + key: row.id, + value: row, + }, + ], + }) + const first = createLive(`electric-conformance-first`) + let second: ReturnType | undefined + + try { + await first.preload() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-1`, + sourceId: `electric-users`, + demandId: `active-users`, + attemptId: `attempt-1`, + rowKeys: [String(row.id)], + }) + expect(first.toArray.map(({ id }) => String(id))).toEqual([ + String(row.id), + ]) + + await first.cleanup() + history.push( + { + type: `releaseDemand`, + ownerId: `owner-1`, + sourceId: `electric-users`, + demandId: `active-users`, + attemptId: `attempt-1`, + }, + { + type: `restartSession`, + previousSessionId: `session-1`, + nextSessionId: `session-2`, + }, + { + type: `requestDemand`, + ownerId: `owner-2`, + sessionId: `session-2`, + sourceId: `electric-users`, + demandId: `active-users`, + attemptId: `attempt-2`, + alreadyAborted: false, + }, + ) + + second = createLive(`electric-conformance-second`) + await second.preload() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-2`, + sourceId: `electric-users`, + demandId: `active-users`, + attemptId: `attempt-2`, + rowKeys: [String(row.id)], + }) + + expect(mockRequestSnapshot).toHaveBeenCalledTimes( + projectTransportLoads(history), + ) + expect(second.toArray.map(({ id }) => String(id))).toEqual( + projectRetainedRowKeys(history), + ) + } finally { + await Promise.all([ + first.cleanup(), + second?.cleanup(), + electricCollection.cleanup(), + ]) + } + }) }) diff --git a/packages/electric-db-collection/tests/electric.test.ts b/packages/electric-db-collection/tests/electric.test.ts index 552cbd8021..8f5ce237f5 100644 --- a/packages/electric-db-collection/tests/electric.test.ts +++ b/packages/electric-db-collection/tests/electric.test.ts @@ -2,11 +2,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ShapeStream } from '@electric-sql/client' import { CollectionImpl, + IR, createCollection, createTransaction, } from '@tanstack/db' import { persistedCollectionOptions } from '../../db-sqlite-persistence-core/src' -import { electricCollectionOptions, isChangeMessage } from '../src/electric' +import { + ELECTRIC_TEST_HOOKS, + electricCollectionOptions, + isChangeMessage, +} from '../src/electric' import { stripVirtualProps } from '../../db/tests/utils' import type { ElectricCollectionUtils } from '../src/electric' import type { @@ -26,12 +31,15 @@ const NativeAbortController = globalThis.AbortController function createDeferred(): { promise: Promise resolve: (value: T | PromiseLike) => void + reject: (reason?: unknown) => void } { let resolve!: (value: T | PromiseLike) => void - const promise = new Promise((resolvePromise) => { + let reject!: (reason?: unknown) => void + const promise = new Promise((resolvePromise, rejectPromise) => { resolve = resolvePromise + reject = rejectPromise }) - return { promise, resolve } + return { promise, resolve, reject } } // Mock the ShapeStream module @@ -2659,6 +2667,53 @@ describe(`Electric Integration`, () => { // Tests for syncMode configuration describe(`syncMode configuration`, () => { + const createOnDemandCollection = (id: string) => + createCollection( + electricCollectionOptions({ + id, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + it(`removes the external shape abort listener across cleanup and restart`, async () => { + const externalAbort = new NativeAbortController() + const addSpy = vi.spyOn(externalAbort.signal, `addEventListener`) + const removeSpy = vi.spyOn(externalAbort.signal, `removeEventListener`) + const testCollection = createCollection( + electricCollectionOptions({ + id: `shape-signal-listener-cleanup-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: externalAbort.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + await testCollection.cleanup() + const subscription = testCollection.subscribeChanges(() => {}) + await testCollection.cleanup() + subscription.unsubscribe() + + const addedListeners = addSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + const removedListeners = removeSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + expect(addedListeners).toHaveLength(2) + expect(removedListeners).toEqual(addedListeners) + }) + it(`should not request snapshots during subscription in eager mode`, () => { vi.clearAllMocks() @@ -2749,213 +2804,1576 @@ describe(`Electric Integration`, () => { } }) - it(`should refresh the stream before requesting on-demand snapshots when already up-to-date`, async () => { - vi.clearAllMocks() - - const config = { - id: `on-demand-refresh-before-snapshot-test`, - shapeOptions: { - url: `http://test-url`, - params: { - table: `test_table`, - }, - }, - syncMode: `on-demand` as const, - getKey: (item: Row) => item.id as number, - startSync: true, - } - - const testCollection = createCollection(electricCollectionOptions(config)) - - mockStream.isUpToDate = true - - await testCollection._sync.loadSubset({ limit: 10 }) - - expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(1) - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - const refreshCall = - mockForceDisconnectAndRefresh.mock.invocationCallOrder[0]! - const snapshotCall = mockRequestSnapshot.mock.invocationCallOrder[0]! - expect(refreshCall).toBeLessThan(snapshotCall) - }) - - it(`should fall through to requestSnapshot when forceDisconnectAndRefresh fails`, async () => { - vi.clearAllMocks() - - const config = { - id: `on-demand-refresh-fallthrough-test`, - shapeOptions: { - url: `http://test-url`, - params: { - table: `test_table`, - }, - }, - syncMode: `on-demand` as const, - getKey: (item: Row) => item.id as number, - startSync: true, - } - - const testCollection = createCollection(electricCollectionOptions(config)) - - mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockImplementationOnce(() => { - return Promise.reject(new Error(`PauseLock held`)) - }) + it.each([ + { abortPhase: `before-publication`, result: `empty` }, + { abortPhase: `before-publication`, result: `rows` }, + { abortPhase: `after-publication`, result: `empty` }, + { abortPhase: `after-publication`, result: `rows` }, + ] as const)( + `keeps an on-demand $result result applied $abortPhase cancellation but retries the canceled demand`, + async ({ abortPhase, result }) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-${abortPhase}-abort-boundary-test`, + ) + const abortController = new AbortController() - await testCollection._sync.loadSubset({ limit: 10 }) + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) - expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(1) - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - }) + if (abortPhase === `before-publication`) abortController.abort() + subscriber([ + ...(result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Applied on-demand row` }, + headers: { operation: `insert` as const }, + }, + ] + : []), + { headers: { control: `subset-end` } }, + ]) + await vi.waitFor(() => + expect(testCollection.has(2)).toBe(result === `rows`), + ) + if (abortPhase === `after-publication`) abortController.abort() + request.resolve() + + await expect(loadError).resolves.toBeUndefined() + if (result === `rows`) { + expect(stripVirtualProps(testCollection.get(2))).toEqual({ + id: 2, + name: `Applied on-demand row`, + }) + } + await load - it(`should request the snapshot after the refresh timeout and ignore late fulfillment`, async () => { - vi.useFakeTimers() - try { - let resolveRefresh: () => void = () => {} - const refresh = new Promise((resolve) => { - resolveRefresh = resolve - }) - mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + subscriber([{ headers: { control: `subset-end` } }]) + await retry + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + it.each([ + { cancellationSource: `collection`, requestOutcome: `fulfillment` }, + { cancellationSource: `collection`, requestOutcome: `rejection` }, + { cancellationSource: `cleanup`, requestOutcome: `fulfillment` }, + { cancellationSource: `cleanup`, requestOutcome: `rejection` }, + ] as const)( + `rejects with AbortError when $cancellationSource cancellation ends an active on-demand request before $requestOutcome`, + async ({ cancellationSource, requestOutcome }) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() const testCollection = createCollection( electricCollectionOptions({ - id: `on-demand-refresh-timeout-fulfillment-test`, + id: `on-demand-${cancellationSource}-active-request-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, + signal: collectionAbortController.signal, }, syncMode: `on-demand`, getKey: (item: Row) => item.id as number, startSync: true, }), ) + const failure = new Error(`request failed after cancellation`) - let loadSettled = false - const load = Promise.resolve( - testCollection._sync.loadSubset({ limit: 10 }), - ).then(() => { - loadSettled = true - }) - - await vi.advanceTimersByTimeAsync(249) - expect(mockRequestSnapshot).not.toHaveBeenCalled() - expect(loadSettled).toBe(false) - - await vi.advanceTimersByTimeAsync(1) - await load - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - expect(loadSettled).toBe(true) - await testCollection.cleanup() - expect(vi.getTimerCount()).toBe(0) - - resolveRefresh() - await refresh - await Promise.resolve() - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - } finally { - vi.useRealTimers() - } - }) - - it(`should handle late refresh rejection after requesting the snapshot`, async () => { - vi.useFakeTimers() - try { - let rejectRefresh: (error: Error) => void = () => {} - const refresh = new Promise((_resolve, reject) => { - rejectRefresh = reject - }) - mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } + if (requestOutcome === `rejection`) request.reject(failure) + else request.resolve() + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + it.each([`collection`, `cleanup`] as const)( + `cancels a parked on-demand commit after %s cancellation`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() const testCollection = createCollection( electricCollectionOptions({ - id: `on-demand-refresh-timeout-rejection-test`, + id: `on-demand-${cancellationSource}-parked-commit-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, + signal: collectionAbortController.signal, }, syncMode: `on-demand`, getKey: (item: Row) => item.id as number, startSync: true, }), ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) - const load = testCollection._sync.loadSubset({ limit: 10 }) - await vi.advanceTimersByTimeAsync(250) - await load - - rejectRefresh(new Error(`late refresh failure`)) - await expect(refresh).rejects.toThrow(`late refresh failure`) - await Promise.resolve() - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - } finally { - vi.useRealTimers() - } - }) - - it(`should clear the refresh timeout when refresh settles early`, async () => { - vi.useFakeTimers() - try { - mockStream.isUpToDate = true - mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Canceled parked row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }, + ) + it.each([`collection`, `cleanup`] as const)( + `disposes the active commit capture during %s cancellation while the request remains pending`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const activeCaptureCounts: Array = [] const testCollection = createCollection( electricCollectionOptions({ - id: `on-demand-refresh-clears-timeout-test`, + id: `on-demand-${cancellationSource}-pending-capture-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, + signal: collectionAbortController.signal, }, syncMode: `on-demand`, getKey: (item: Row) => item.id as number, startSync: true, + [ELECTRIC_TEST_HOOKS]: { + onActiveCommitCapturesChange: (activeCount) => + activeCaptureCounts.push(activeCount), + }, }), ) - await testCollection._sync.loadSubset({ limit: 10 }) + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + expect(activeCaptureCounts.at(-1)).toBe(1) - expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) - expect(vi.getTimerCount()).toBe(0) - } finally { - vi.useRealTimers() - } - }) + if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + await testCollection.cleanup() + } - it(`should fetch snapshots in progressive mode when loadSubset is called before sync completes`, async () => { - vi.clearAllMocks() + expect(activeCaptureCounts.at(-1)).toBe(0) + request.resolve() + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) - mockSubscribe.mockImplementation((_callback) => { - return () => {} + it(`waits for a successful on-demand commit to apply`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-successful-parked-commit-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, }) - mockRequestSnapshot.mockResolvedValue(undefined) - mockFetchSnapshot.mockResolvedValue({ - metadata: {}, - data: [ + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ { key: `2`, - value: { id: 2, name: `Snapshot User` }, + value: { id: 2, name: `Applied parked row` }, headers: { operation: `insert` }, }, - ], - }) + { headers: { control: `subset-end` } }, + ]) + request.resolve() - const config = { - id: `progressive-snapshot-test`, - shapeOptions: { - url: `http://test-url`, - params: { - table: `test_table`, - }, - }, - syncMode: `progressive` as const, - getKey: (item: Row) => item.id as number, - startSync: true, + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([load.then(() => `load-settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + expect(testCollection.has(2)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await load + expect(stripVirtualProps(testCollection.get(2))).toEqual({ + id: 2, + name: `Applied parked row`, + }) + } finally { + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() } + }) - const testCollection = createCollection(electricCollectionOptions(config)) + it(`retains every applied receipt until the on-demand request settles`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-retained-receipts-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Applied row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + subscriber([ + { + key: `4`, + value: { id: 4, name: `Canceled row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + expect(testCollection._state.pendingSyncedTransactions).toHaveLength(2) + const canceledReceipt = + testCollection._state.pendingSyncedTransactions[1]! + testCollection._state.cancelPendingSyncedTransaction(canceledReceipt) + await Promise.resolve() + + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(true) + expect(testCollection.has(4)).toBe(false) + await load.catch(() => undefined) + + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + await retry + } finally { + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }) + + it(`rejects when collection cancellation lands after request fulfillment but before applied settlement`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-post-request-collection-cancel-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + + request.resolve() + queueMicrotask(() => collectionAbortController.abort()) + + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + await testCollection.cleanup() + } + }) + + it.each([`external abort`, `cleanup`] as const)( + `prefers %s over an already-rejected applied receipt`, + async (cancellationSource) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const collectionAbortController = new AbortController() + const receiptFailure = new Error(`applied receipt failed`) + const options = electricCollectionOptions({ + id: `on-demand-pre-wait-collection-cancel-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const commitMock = vi.fn(() => Promise.reject(receiptFailure)) + const controls = options.sync.sync({ + collection: { + id: options.id, + status: `loading`, + getKeyFromItem: (item: Row) => item.id, + }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + + try { + const load = Promise.resolve(controls.loadSubset({ limit: 10 })) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Rejected receipt row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + + if (cancellationSource === `external abort`) { + collectionAbortController.abort() + } else { + controls.cleanup?.() + } + request.resolve() + + await expect(load).rejects.toMatchObject({ name: `AbortError` }) + } finally { + collectionAbortController.abort() + request.resolve() + controls.cleanup?.() + } + }, + ) + + it.each([`success`, `rejection`, `cancellation`] as const)( + `removes the on-demand request lease listener after %s`, + async (settlement) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-request-listener-${settlement}-test`, + ) + const abortController = new AbortController() + const addSpy = vi.spyOn(abortController.signal, `addEventListener`) + const removeSpy = vi.spyOn( + abortController.signal, + `removeEventListener`, + ) + const failure = new Error(`request failed`) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (settlement === `cancellation`) abortController.abort() + subscriber([{ headers: { control: `subset-end` } }]) + if (settlement === `rejection`) request.reject(failure) + else request.resolve() + + if (settlement === `rejection`) { + await expect(loadError).resolves.toBe(failure) + } else { + await expect(loadError).resolves.toBeUndefined() + } + await load.catch(() => undefined) + + const addedListeners = addSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + const removedListeners = removeSpy.mock.calls + .filter(([type]) => type === `abort`) + .map(([, listener]) => listener) + expect(addedListeners.length).toBeGreaterThan(0) + expect(removedListeners).toEqual(addedListeners) + } finally { + addSpy.mockRestore() + removeSpy.mockRestore() + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + + it(`keeps on-demand coverage when cancellation happens after settlement`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-post-settlement-abort-test`, + ) + const abortController = new AbortController() + const options = { limit: 10, signal: abortController.signal } + + try { + const load = Promise.resolve(testCollection._sync.loadSubset(options)) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Settled on-demand row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + request.resolve() + await load + + abortController.abort() + expect(testCollection.has(2)).toBe(true) + await testCollection._sync.loadSubset({ limit: 10 }) + expect(mockRequestSnapshot).toHaveBeenCalledOnce() + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }) + + it.each([ + { cancellation: `none`, result: `empty` }, + { cancellation: `none`, result: `rows` }, + { cancellation: `request`, result: `empty` }, + { cancellation: `request`, result: `rows` }, + ] as const)( + `propagates an on-demand request error with $result after $cancellation cancellation`, + async ({ cancellation, result }) => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-${cancellation}-${result}-request-error-test`, + ) + const abortController = new AbortController() + const failure = new Error(`request failed`) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + if (cancellation === `request`) abortController.abort() + subscriber([ + ...(result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Partial on-demand row` }, + headers: { operation: `insert` as const }, + }, + ] + : []), + { headers: { control: `subset-end` } }, + ]) + request.reject(failure) + + await expect(loadError).resolves.toBe(failure) + expect(testCollection.has(2)).toBe(result === `rows`) + await load.catch(() => undefined) + + const retry = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + subscriber([{ headers: { control: `subset-end` } }]) + await retry + } finally { + abortController.abort() + request.resolve() + await testCollection.cleanup() + } + }, + ) + + it(`does not fulfill a failed on-demand request before its published receipt applies`, async () => { + const request = createDeferred() + mockRequestSnapshot.mockReturnValueOnce(request.promise) + const testCollection = createOnDemandCollection( + `on-demand-parked-request-error-test`, + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + const abortController = new AbortController() + const failure = new Error(`request failed`) + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledOnce(), + ) + subscriber([ + { + key: `2`, + value: { id: 2, name: `Parked on-demand row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + abortController.abort() + request.reject(failure) + + await expect(loadError).resolves.toBe(failure) + expect(testCollection.has(2)).toBe(false) + + persistence.resolve() + await transaction.isPersisted.promise + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await load.catch(() => undefined) + } finally { + abortController.abort() + request.resolve() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }) + + it.each([`whereCurrent`, `whereFrom`] as const)( + `waits for the cursor sibling after $failedRequest rejects`, + async (failedRequest) => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-${failedRequest}-error-test`, + ) + const abortController = new AbortController() + const failure = new Error(`${failedRequest} request failed`) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + signal: abortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + abortController.abort() + const failed = + failedRequest === `whereCurrent` ? whereCurrent : whereFrom + const sibling = + failedRequest === `whereCurrent` ? whereFrom : whereCurrent + failed.reject(failure) + + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([loadError.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + subscriber([ + { + key: `2`, + value: { id: 2, name: `Late cursor row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `subset-end` } }, + ]) + sibling.resolve() + + await expect(loadError).resolves.toBe(failure) + await vi.waitFor(() => expect(testCollection.has(2)).toBe(true)) + await load.catch(() => undefined) + } finally { + abortController.abort() + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }, + ) + + it.each([`whereCurrent`, `whereFrom`] as const)( + `uses stable cursor error priority when $firstFailure rejects first`, + async (firstFailure) => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-${firstFailure}-first-double-error-test`, + ) + const currentFailure = new Error(`whereCurrent request failed`) + const fromFailure = new Error(`whereFrom request failed`) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + + const first = + firstFailure === `whereCurrent` ? whereCurrent : whereFrom + const second = + firstFailure === `whereCurrent` ? whereFrom : whereCurrent + first.reject( + firstFailure === `whereCurrent` ? currentFailure : fromFailure, + ) + + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([loadError.then(() => `settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + second.reject( + firstFailure === `whereCurrent` ? fromFailure : currentFailure, + ) + await expect(loadError).resolves.toBe(currentFailure) + await load.catch(() => undefined) + } finally { + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }, + ) + + it(`waits for both cursor snapshot requests before settling`, async () => { + const whereCurrent = createDeferred() + const whereFrom = createDeferred() + mockRequestSnapshot + .mockReturnValueOnce(whereCurrent.promise) + .mockReturnValueOnce(whereFrom.promise) + const testCollection = createOnDemandCollection( + `on-demand-cursor-all-requests-test`, + ) + const id = new IR.PropRef([`id`]) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + orderBy: [ + { + expression: id, + compareOptions: { + direction: `asc`, + nulls: `last`, + stringSort: `lexical`, + }, + }, + ], + cursor: { + whereCurrent: new IR.Func(`eq`, [id, new IR.Value(1)]), + whereFrom: new IR.Func(`gt`, [id, new IR.Value(1)]), + lastKey: 1, + }, + }), + ) + await vi.waitFor(() => + expect(mockRequestSnapshot).toHaveBeenCalledTimes(2), + ) + + whereCurrent.resolve() + const nextTurn = new Promise<`next-turn`>((resolve) => + setTimeout(() => resolve(`next-turn`), 0), + ) + await expect( + Promise.race([load.then(() => `load-settled` as const), nextTurn]), + ).resolves.toBe(`next-turn`) + + whereFrom.resolve() + await load + } finally { + whereCurrent.resolve() + whereFrom.resolve() + await testCollection.cleanup() + } + }) + + it(`should refresh the stream before requesting on-demand snapshots when already up-to-date`, async () => { + vi.clearAllMocks() + + const config = { + id: `on-demand-refresh-before-snapshot-test`, + shapeOptions: { + url: `http://test-url`, + params: { + table: `test_table`, + }, + }, + syncMode: `on-demand` as const, + getKey: (item: Row) => item.id as number, + startSync: true, + } + + const testCollection = createCollection(electricCollectionOptions(config)) + + mockStream.isUpToDate = true + + await testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(1) + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + const refreshCall = + mockForceDisconnectAndRefresh.mock.invocationCallOrder[0]! + const snapshotCall = mockRequestSnapshot.mock.invocationCallOrder[0]! + expect(refreshCall).toBeLessThan(snapshotCall) + }) + + it(`should fall through to requestSnapshot when forceDisconnectAndRefresh fails`, async () => { + vi.clearAllMocks() + + const config = { + id: `on-demand-refresh-fallthrough-test`, + shapeOptions: { + url: `http://test-url`, + params: { + table: `test_table`, + }, + }, + syncMode: `on-demand` as const, + getKey: (item: Row) => item.id as number, + startSync: true, + } + + const testCollection = createCollection(electricCollectionOptions(config)) + + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockImplementationOnce(() => { + return Promise.reject(new Error(`PauseLock held`)) + }) + + await testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(1) + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + }) + + it(`should request the snapshot after the refresh timeout and ignore late fulfillment`, async () => { + vi.useFakeTimers() + try { + let resolveRefresh: () => void = () => {} + const refresh = new Promise((resolve) => { + resolveRefresh = resolve + }) + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-timeout-fulfillment-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + let loadSettled = false + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ).then(() => { + loadSettled = true + }) + + await vi.advanceTimersByTimeAsync(249) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(loadSettled).toBe(false) + + await vi.advanceTimersByTimeAsync(1) + await load + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + expect(loadSettled).toBe(true) + await testCollection.cleanup() + expect(vi.getTimerCount()).toBe(0) + + resolveRefresh() + await refresh + await Promise.resolve() + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it(`should handle late refresh rejection after requesting the snapshot`, async () => { + vi.useFakeTimers() + try { + let rejectRefresh: (error: Error) => void = () => {} + const refresh = new Promise((_resolve, reject) => { + rejectRefresh = reject + }) + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-timeout-rejection-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + const load = testCollection._sync.loadSubset({ limit: 10 }) + await vi.advanceTimersByTimeAsync(250) + await load + + rejectRefresh(new Error(`late refresh failure`)) + await expect(refresh).rejects.toThrow(`late refresh failure`) + await Promise.resolve() + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it(`should cancel a pending refresh wait when the collection is cleaned up`, async () => { + vi.useFakeTimers() + const refresh = createDeferred() + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) + + const testCollection = createOnDemandCollection( + `on-demand-refresh-cleanup-test`, + ) + + let loadSettled = false + const load = Promise.resolve( + testCollection._sync.loadSubset({ limit: 10 }), + ).finally(() => { + loadSettled = true + }) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + await testCollection.cleanup() + await vi.advanceTimersByTimeAsync(0) + + expect(loadSettled).toBe(true) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + + refresh.resolve() + await refresh.promise + await load.catch(() => undefined) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + } finally { + refresh.resolve() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + + it(`rejects buffered snapshot publication after adapter cleanup`, async () => { + const snapshot = createDeferred<{ + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + mockFetchSnapshot.mockReturnValueOnce(snapshot.promise) + const options = electricCollectionOptions({ + id: `progressive-snapshot-cleanup-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const begin = vi.fn() + const write = vi.fn() + const commit = vi.fn(() => true as const) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin, + write, + commit, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!controls || typeof controls === `function` || !controls.loadSubset) { + throw new Error(`Expected progressive sync controls`) + } + + const load = Promise.resolve(controls.loadSubset({ limit: 10 })) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + controls.cleanup?.() + snapshot.resolve({ + data: [ + { + key: `1`, + value: { id: 1, name: `Late snapshot user` }, + headers: { operation: `insert` }, + }, + ], + }) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + + expect(begin).not.toHaveBeenCalled() + expect(write).not.toHaveBeenCalled() + expect(commit).not.toHaveBeenCalled() + await load.catch(() => undefined) + }) + + it.each([ + { syncMode: `on-demand`, signalSource: `collection` }, + { syncMode: `on-demand`, signalSource: `request` }, + { syncMode: `progressive`, signalSource: `collection` }, + { syncMode: `progressive`, signalSource: `request` }, + ] as const)( + `rejects before starting $syncMode work when the $signalSource signal is already aborted`, + async ({ syncMode, signalSource }) => { + mockStream.isUpToDate = true + const abortController = new AbortController() + abortController.abort() + const testCollection = createCollection( + electricCollectionOptions({ + id: `${syncMode}-${signalSource}-already-aborted-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: + signalSource === `collection` + ? abortController.signal + : undefined, + }, + syncMode, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + await expect( + testCollection._sync.loadSubset({ + limit: 10, + signal: + signalSource === `request` ? abortController.signal : undefined, + }), + ).rejects.toMatchObject({ name: `AbortError` }) + + expect(mockForceDisconnectAndRefresh).not.toHaveBeenCalled() + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(mockFetchSnapshot).not.toHaveBeenCalled() + await testCollection.cleanup() + }, + ) + + it(`retries immediately after the requesting demand is aborted`, async () => { + vi.useFakeTimers() + const refresh = createDeferred() + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) + + const testCollection = createOnDemandCollection( + `on-demand-refresh-abort-retry-test`, + ) + const abortController = new AbortController() + let abortedLoadSettled = false + const abortedLoad = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: abortController.signal, + }), + ).finally(() => { + abortedLoadSettled = true + }) + const abortedLoadError = abortedLoad.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + abortController.abort() + + expect(mockRequestSnapshot).not.toHaveBeenCalled() + + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + const retry = testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockForceDisconnectAndRefresh).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(0) + + expect(abortedLoadSettled).toBe(true) + await expect(abortedLoadError).resolves.toMatchObject({ + name: `AbortError`, + }) + expect(vi.getTimerCount()).toBe(0) + + await retry + + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + await testCollection.cleanup() + await abortedLoad.catch(() => undefined) + } finally { + refresh.resolve() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + + it.each([ + { cancellationSource: `request`, order: `rejection-first` }, + { cancellationSource: `request`, order: `cancellation-first` }, + { cancellationSource: `collection`, order: `rejection-first` }, + { cancellationSource: `collection`, order: `cancellation-first` }, + ] as const)( + `prefers AbortError for $cancellationSource cancellation in $order order`, + async ({ cancellationSource, order }) => { + vi.useFakeTimers() + let rejectRefresh: (error: Error) => void = () => {} + const refresh = new Promise((_resolve, reject) => { + rejectRefresh = reject + }) + const request = new AbortController() + let testCollection: + | ReturnType + | undefined + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + testCollection = createOnDemandCollection( + `on-demand-refresh-${cancellationSource}-race-test`, + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + let cleanup: Promise | undefined + const cancel = () => { + if (cancellationSource === `request`) { + request.abort() + } else { + cleanup = testCollection?.cleanup() + } + } + const reject = () => rejectRefresh(new Error(`refresh failed`)) + if (order === `rejection-first`) { + reject() + cancel() + } else { + cancel() + reject() + } + await cleanup + + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + await load.catch(() => undefined) + } finally { + request.abort() + await testCollection?.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }, + ) + + it.each([ + { cancellationSource: `request`, lateSettlement: `fulfillment` }, + { cancellationSource: `request`, lateSettlement: `rejection` }, + { cancellationSource: `collection`, lateSettlement: `fulfillment` }, + { cancellationSource: `collection`, lateSettlement: `rejection` }, + ] as const)( + `keeps $cancellationSource cancellation final after late refresh $lateSettlement`, + async ({ cancellationSource, lateSettlement }) => { + vi.useFakeTimers() + let resolveRefresh: () => void = () => {} + let rejectRefresh: (error: Error) => void = () => {} + const refresh = new Promise((resolve, reject) => { + resolveRefresh = resolve + rejectRefresh = reject + }) + const refreshOutcome = refresh.then( + () => `fulfilled` as const, + () => `rejected` as const, + ) + const request = new AbortController() + let testCollection: + | ReturnType + | undefined + + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh) + testCollection = createOnDemandCollection( + `on-demand-refresh-${cancellationSource}-late-${lateSettlement}-test`, + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + await Promise.resolve() + let cleanup: Promise | undefined + if (cancellationSource === `request`) { + request.abort() + } else { + cleanup = testCollection.cleanup() + } + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + await cleanup + expect(mockRequestSnapshot).not.toHaveBeenCalled() + + if (lateSettlement === `fulfillment`) { + resolveRefresh() + } else { + rejectRefresh(new Error(`late refresh failure`)) + } + await expect(refreshOutcome).resolves.toBe( + lateSettlement === `fulfillment` ? `fulfilled` : `rejected`, + ) + await Promise.resolve() + + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + expect(mockRequestSnapshot).not.toHaveBeenCalled() + expect(vi.getTimerCount()).toBe(0) + await load.catch(() => undefined) + } finally { + request.abort() + resolveRefresh() + await testCollection?.cleanup() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }, + ) + + it.each([ + `refresh`, + `rejection`, + `timeout`, + `request`, + `collection`, + ] as const)( + `removes every abort listener when %s settles the refresh wait`, + async (settlement) => { + vi.useFakeTimers() + const refresh = createDeferred() + const request = new AbortController() + const added: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const removed: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const originalAdd = AbortSignal.prototype.addEventListener + const originalRemove = AbortSignal.prototype.removeEventListener + mockStream.isUpToDate = true + if (settlement === `refresh`) { + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + } else if (settlement === `rejection`) { + mockForceDisconnectAndRefresh.mockRejectedValueOnce( + new Error(`refresh failed`), + ) + } else { + mockForceDisconnectAndRefresh.mockReturnValueOnce(refresh.promise) + } + const testCollection = createOnDemandCollection( + `on-demand-refresh-${settlement}-listener-cleanup-test`, + ) + + const addSpy = vi + .spyOn(AbortSignal.prototype, `addEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) added.push({ signal: this, listener }) + return originalAdd.call(this, type, listener, options) + }) + const removeSpy = vi + .spyOn(AbortSignal.prototype, `removeEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + options, + ) { + if (type === `abort`) removed.push({ signal: this, listener }) + return originalRemove.call(this, type, listener, options) + }) + + try { + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 10, + signal: request.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + if (settlement === `timeout`) { + await vi.advanceTimersByTimeAsync(250) + } else if (settlement === `request`) { + request.abort() + } else if (settlement === `collection`) { + await testCollection.cleanup() + } + + if (settlement === `request` || settlement === `collection`) { + await expect(loadError).resolves.toMatchObject({ + name: `AbortError`, + }) + } else { + await expect(loadError).resolves.toBeUndefined() + } + expect(vi.getTimerCount()).toBe(0) + + expect(added.length).toBeGreaterThan(0) + for (const installed of added) { + expect( + removed.some( + (candidate) => + candidate.signal === installed.signal && + candidate.listener === installed.listener, + ), + ).toBe(true) + } + await load.catch(() => undefined) + } finally { + request.abort() + refresh.resolve() + await testCollection.cleanup() + addSpy.mockRestore() + removeSpy.mockRestore() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }, + ) + + it(`should clear the refresh timeout when refresh settles early`, async () => { + vi.useFakeTimers() + try { + mockStream.isUpToDate = true + mockForceDisconnectAndRefresh.mockResolvedValueOnce(undefined) + + const testCollection = createCollection( + electricCollectionOptions({ + id: `on-demand-refresh-clears-timeout-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + }, + syncMode: `on-demand`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + + await testCollection._sync.loadSubset({ limit: 10 }) + + expect(mockRequestSnapshot).toHaveBeenCalledTimes(1) + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it(`should fetch snapshots in progressive mode when loadSubset is called before sync completes`, async () => { + vi.clearAllMocks() + + mockSubscribe.mockImplementation((_callback) => { + return () => {} + }) + mockRequestSnapshot.mockResolvedValue(undefined) + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot User` }, + headers: { operation: `insert` }, + }, + ], + }) + + const config = { + id: `progressive-snapshot-test`, + shapeOptions: { + url: `http://test-url`, + params: { + table: `test_table`, + }, + }, + syncMode: `progressive` as const, + getKey: (item: Row) => item.id as number, + startSync: true, + } + + const testCollection = createCollection(electricCollectionOptions(config)) expect(testCollection.status).toBe(`loading`) // Not ready yet @@ -2979,25 +4397,190 @@ describe(`Electric Integration`, () => { }) }) - it(`ignores a progressive snapshot after its subset request is aborted`, async () => { - mockFetchSnapshot.mockReset() - let resolveSnapshot!: (value: { - metadata: Record - data: Array<{ - key: string - value: Row - headers: { operation: `insert` } - }> - }) => void - mockFetchSnapshot.mockReturnValue( - new Promise((resolve) => { - resolveSnapshot = resolve - }), - ) - mockSubscribe.mockImplementation(() => () => {}) - const testCollection = createCollection( - electricCollectionOptions({ - id: `progressive-aborted-snapshot-test`, + it.each([ + { signalSource: `request`, result: `empty` }, + { signalSource: `request`, result: `rows` }, + { signalSource: `collection`, result: `empty` }, + { signalSource: `collection`, result: `rows` }, + ] as const)( + `rejects a progressive $result snapshot when the $signalSource signal aborts before application`, + async ({ signalSource, result }) => { + const snapshot = createDeferred<{ + metadata: Record + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + mockFetchSnapshot.mockReturnValue(snapshot.promise) + mockSubscribe.mockImplementation(() => () => {}) + const collectionAbortController = new AbortController() + const requestAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `progressive-${signalSource}-${result}-abort-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const abortController = + signalSource === `request` + ? requestAbortController + : collectionAbortController + + try { + expect(mockFetchSnapshot).not.toHaveBeenCalled() + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + expect(mockFetchSnapshot).toHaveBeenCalledOnce() + abortController.abort() + snapshot.resolve({ + metadata: {}, + data: + result === `rows` + ? [ + { + key: `2`, + value: { id: 2, name: `Obsolete snapshot` }, + headers: { operation: `insert` }, + }, + ] + : [], + }) + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + requestAbortController.abort() + snapshot.resolve({ metadata: {}, data: [] }) + await testCollection.cleanup() + } + }, + ) + + it.each([ + { cancellationSource: `request`, requestSignal: `present` }, + { cancellationSource: `collection`, requestSignal: `present` }, + { cancellationSource: `collection`, requestSignal: `absent` }, + { cancellationSource: `cleanup`, requestSignal: `present` }, + ] as const)( + `rejects a progressive snapshot when $cancellationSource cancellation occurs with the request signal $requestSignal while its commit is parked`, + async ({ cancellationSource, requestSignal }) => { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Obsolete snapshot` }, + headers: { operation: `insert` }, + }, + ], + }) + mockSubscribe.mockImplementation(() => () => {}) + const collectionAbortController = new AbortController() + const testCollection = createCollection( + electricCollectionOptions({ + id: `progressive-parked-abort-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }), + ) + const persistence = createDeferred() + const transaction = createTransaction({ + mutationFn: () => persistence.promise, + }) + const requestAbortController = new AbortController() + const abortController = + cancellationSource === `request` + ? requestAbortController + : collectionAbortController + + try { + transaction.mutate(() => + testCollection.insert({ id: 3, name: `Local row` }), + ) + const load = Promise.resolve( + testCollection._sync.loadSubset({ + limit: 1, + signal: + requestSignal === `present` + ? requestAbortController.signal + : undefined, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => + expect(mockFetchSnapshot).toHaveBeenCalledOnce(), + ) + await Promise.resolve() + await Promise.resolve() + + expect(testCollection.has(2)).toBe(false) + if (cancellationSource === `cleanup`) { + await testCollection.cleanup() + } else { + abortController.abort() + } + persistence.resolve() + await transaction.isPersisted.promise + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + + expect(testCollection.has(2)).toBe(false) + await load.catch(() => undefined) + } finally { + abortController.abort() + persistence.resolve() + await transaction.isPersisted.promise.catch(() => undefined) + await testCollection.cleanup() + } + }, + ) + + it.each([`fetch`, `commit`] as const)( + `propagates an uncanceled progressive %s error`, + async (failurePhase) => { + const failure = new Error(`${failurePhase} failed`) + if (failurePhase === `fetch`) { + mockFetchSnapshot.mockRejectedValue(failure) + } else { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + } + const options = electricCollectionOptions({ + id: `progressive-${failurePhase}-error-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, @@ -3005,53 +4588,254 @@ describe(`Electric Integration`, () => { syncMode: `progressive`, getKey: (item: Row) => item.id as number, startSync: true, - }), - ) - const abortController = new AbortController() + }) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: + failurePhase === `commit` + ? vi.fn(() => Promise.reject(failure)) + : vi.fn(() => true as const), + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } - try { - expect(mockFetchSnapshot).not.toHaveBeenCalled() - const load = testCollection._sync.loadSubset({ - limit: 1, - signal: abortController.signal, + try { + await expect( + Promise.resolve(controls.loadSubset({ limit: 1 })), + ).rejects.toBe(failure) + } finally { + controls.cleanup?.() + } + }, + ) + + it.each([ + { failurePhase: `fetch`, signalSource: `request`, order: `cancel-first` }, + { failurePhase: `fetch`, signalSource: `request`, order: `error-first` }, + { + failurePhase: `fetch`, + signalSource: `collection`, + order: `cancel-first`, + }, + { + failurePhase: `fetch`, + signalSource: `collection`, + order: `error-first`, + }, + { + failurePhase: `commit`, + signalSource: `request`, + order: `cancel-first`, + }, + { failurePhase: `commit`, signalSource: `request`, order: `error-first` }, + { + failurePhase: `commit`, + signalSource: `collection`, + order: `cancel-first`, + }, + { + failurePhase: `commit`, + signalSource: `collection`, + order: `error-first`, + }, + ] as const)( + `prefers AbortError when $signalSource cancellation races a progressive $failurePhase error in $order order`, + async ({ failurePhase, signalSource, order }) => { + const failure = new Error(`${failurePhase} failed`) + const fetch = createDeferred<{ + metadata: Record + data: Array<{ + key: string + value: Row + headers: { operation: `insert` } + }> + }>() + const commit = createDeferred() + if (failurePhase === `fetch`) { + mockFetchSnapshot.mockReturnValue(fetch.promise) + } else { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + } + const collectionAbortController = new AbortController() + const requestAbortController = new AbortController() + const options = electricCollectionOptions({ + id: `progressive-${failurePhase}-${signalSource}-${order}-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, }) - expect(mockFetchSnapshot).toHaveBeenCalledOnce() - expect(testCollection.has(2)).toBe(false) - abortController.abort() - resolveSnapshot({ + const commitMock = + failurePhase === `commit` + ? vi.fn(() => commit.promise) + : vi.fn(() => true as const) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } + const abortController = + signalSource === `request` + ? requestAbortController + : collectionAbortController + const load = Promise.resolve( + controls.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + + try { + if (failurePhase === `commit`) { + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + } + if (order === `cancel-first`) abortController.abort() + if (failurePhase === `fetch`) fetch.reject(failure) + else commit.reject(failure) + if (order === `error-first`) abortController.abort() + + await expect(loadError).resolves.toMatchObject({ name: `AbortError` }) + await load.catch(() => undefined) + } finally { + collectionAbortController.abort() + requestAbortController.abort() + fetch.resolve({ metadata: {}, data: [] }) + commit.resolve() + controls.cleanup?.() + } + }, + ) + + it.each([`request`, `collection`, `cleanup`] as const)( + `keeps a progressive snapshot applied before %s cancellation`, + async (cancellationSource) => { + mockFetchSnapshot.mockResolvedValue({ metadata: {}, data: [ { key: `2`, - value: { id: 2, name: `Obsolete snapshot` }, + value: { id: 2, name: `Applied snapshot` }, headers: { operation: `insert` }, }, ], }) - if (load instanceof Promise) await load - - expect(testCollection.has(2)).toBe(false) - } finally { - resolveSnapshot({ metadata: {}, data: [] }) - await testCollection.cleanup() - } - }) - - it(`does not publish a progressive snapshot aborted while its commit is parked`, async () => { - mockFetchSnapshot.mockResolvedValue({ - metadata: {}, - data: [ - { - key: `2`, - value: { id: 2, name: `Obsolete snapshot` }, - headers: { operation: `insert` }, + const requestAbortController = new AbortController() + const collectionAbortController = new AbortController() + const stagedRows: Array = [] + const appliedRows: Array = [] + let cleanup = () => {} + const options = electricCollectionOptions({ + id: `progressive-applied-before-${cancellationSource}-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, }, - ], - }) - mockSubscribe.mockImplementation(() => () => {}) - const testCollection = createCollection( - electricCollectionOptions({ - id: `progressive-parked-abort-test`, + syncMode: `progressive`, + getKey: (item: Row) => item.id as number, + startSync: true, + }) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn((change: { value: Row }) => + stagedRows.push(change.value), + ), + commit: vi.fn(() => { + appliedRows.push(...stagedRows) + if (cancellationSource === `request`) { + requestAbortController.abort() + } else if (cancellationSource === `collection`) { + collectionAbortController.abort() + } else { + cleanup() + } + return true as const + }), + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } + cleanup = controls.cleanup ?? (() => {}) + + try { + await expect( + Promise.resolve( + controls.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ), + ).resolves.toBeUndefined() + expect(appliedRows).toEqual([{ id: 2, name: `Applied snapshot` }]) + } finally { + controls.cleanup?.() + } + }, + ) + + it.each([`success`, `rejection`, `request-abort`, `cleanup`] as const)( + `removes combined commit abort listeners after %s`, + async (settlement) => { + mockFetchSnapshot.mockResolvedValue({ + metadata: {}, + data: [ + { + key: `2`, + value: { id: 2, name: `Snapshot row` }, + headers: { operation: `insert` }, + }, + ], + }) + const commit = createDeferred() + const requestAbortController = new AbortController() + const options = electricCollectionOptions({ + id: `progressive-combined-listener-${settlement}-test`, shapeOptions: { url: `http://test-url`, params: { table: `test_table` }, @@ -3059,40 +4843,180 @@ describe(`Electric Integration`, () => { syncMode: `progressive`, getKey: (item: Row) => item.id as number, startSync: true, - }), - ) - const persistence = createDeferred() - const transaction = createTransaction({ - mutationFn: () => persistence.promise, - }) - const abortController = new AbortController() + }) + const commitMock = vi.fn(() => commit.promise) + const controls = options.sync.sync({ + collection: { id: options.id, status: `loading` }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !controls || + typeof controls === `function` || + !controls.loadSubset + ) { + throw new Error(`Expected progressive sync controls`) + } - try { - transaction.mutate(() => - testCollection.insert({ id: 3, name: `Local row` }), - ) - const load = testCollection._sync.loadSubset({ - limit: 1, - signal: abortController.signal, + const added: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const removed: Array<{ + signal: AbortSignal + listener: EventListenerOrEventListenerObject + }> = [] + const originalAdd = AbortSignal.prototype.addEventListener + const originalRemove = AbortSignal.prototype.removeEventListener + const addSpy = vi + .spyOn(AbortSignal.prototype, `addEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + listenerOptions, + ) { + if (type === `abort`) added.push({ signal: this, listener }) + return originalAdd.call(this, type, listener, listenerOptions) + }) + const removeSpy = vi + .spyOn(AbortSignal.prototype, `removeEventListener`) + .mockImplementation(function ( + this: AbortSignal, + type, + listener, + listenerOptions, + ) { + if (type === `abort`) removed.push({ signal: this, listener }) + return originalRemove.call(this, type, listener, listenerOptions) + }) + const failure = new Error(`commit failed`) + + try { + const load = Promise.resolve( + controls.loadSubset({ + limit: 1, + signal: requestAbortController.signal, + }), + ) + const loadError = load.then( + () => undefined, + (error: unknown) => error, + ) + await vi.waitFor(() => expect(commitMock).toHaveBeenCalledOnce()) + + if (settlement === `request-abort`) { + requestAbortController.abort() + } else if (settlement === `cleanup`) { + controls.cleanup?.() + } + if (settlement === `rejection`) commit.reject(failure) + else commit.resolve() + + if (settlement === `rejection`) { + await expect(loadError).resolves.toBe(failure) + } else { + await expect(loadError).resolves.toBeUndefined() + } + await load.catch(() => undefined) + + for (const installed of added) { + expect(removed).toContainEqual(installed) + } + } finally { + addSpy.mockRestore() + removeSpy.mockRestore() + requestAbortController.abort() + commit.resolve() + controls.cleanup?.() + } + }, + ) + + it.each( + ([`progressive atomic-swap`, `metadata-only`] as const).flatMap( + (commitPath) => + ([`external abort`, `cleanup`] as const).map( + (cancellationSource) => [commitPath, cancellationSource] as const, + ), + ), + )( + `binds the %s commit to collection lifetime through %s`, + (commitPath, cancellationSource) => { + const collectionAbortController = new AbortController() + const receipt = createDeferred() + const metadataHarness = createInMemorySyncMetadataApi() + const isProgressive = commitPath === `progressive atomic-swap` + let commitSignal: AbortSignal | undefined + const options = electricCollectionOptions({ + id: `${commitPath.replaceAll(` `, `-`)}-commit-signal-test`, + shapeOptions: { + url: `http://test-url`, + params: { table: `test_table` }, + signal: collectionAbortController.signal, + }, + syncMode: isProgressive ? `progressive` : `eager`, + getKey: (item: Row) => item.id as number, + startSync: true, }) - await vi.waitFor(() => expect(mockFetchSnapshot).toHaveBeenCalledOnce()) - await Promise.resolve() - await Promise.resolve() + const commitMock = vi.fn((signal?: AbortSignal) => { + commitSignal = signal + return receipt.promise + }) + const controls = options.sync.sync({ + collection: { + id: options.id, + status: `loading`, + getKeyFromItem: (item: Row) => item.id, + }, + begin: vi.fn(), + write: vi.fn(), + commit: commitMock, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + metadata: isProgressive ? undefined : metadataHarness.api, + } as never) + if (!controls || typeof controls === `function`) { + throw new Error(`Expected sync controls`) + } - expect(testCollection.has(2)).toBe(false) - abortController.abort() - persistence.resolve() - await transaction.isPersisted.promise - if (load instanceof Promise) await load + try { + if (isProgressive) { + subscriber([ + { + key: `2`, + value: { id: 2, name: `Buffered row` }, + headers: { operation: `insert` }, + }, + { headers: { control: `up-to-date` } }, + ]) + } else { + subscriber([{ headers: { control: `up-to-date` } }]) + } - expect(testCollection.has(2)).toBe(false) - } finally { - abortController.abort() - persistence.resolve() - await transaction.isPersisted.promise.catch(() => undefined) - await testCollection.cleanup() - } - }) + expect(commitMock).toHaveBeenCalledOnce() + expect(commitSignal).toBeDefined() + expect(commitSignal?.aborted).toBe(false) + + if (cancellationSource === `external abort`) { + collectionAbortController.abort() + } else { + controls.cleanup?.() + } + + expect(commitSignal?.aborted).toBe(true) + } finally { + collectionAbortController.abort() + receipt.resolve() + controls.cleanup?.() + } + }, + ) it(`should not request snapshots when loadSubset is called in eager mode`, async () => { vi.clearAllMocks() diff --git a/packages/electron-db-sqlite-persistence/CHANGELOG.md b/packages/electron-db-sqlite-persistence/CHANGELOG.md index e0e58b37dd..0d625c1398 100644 --- a/packages/electron-db-sqlite-persistence/CHANGELOG.md +++ b/packages/electron-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/electron-db-sqlite-persistence +## 0.1.32 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.1.31 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.1.30 ### Patch Changes diff --git a/packages/electron-db-sqlite-persistence/package.json b/packages/electron-db-sqlite-persistence/package.json index b7134749e2..ef4d4fcb9c 100644 --- a/packages/electron-db-sqlite-persistence/package.json +++ b/packages/electron-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/electron-db-sqlite-persistence", - "version": "0.1.30", + "version": "0.1.32", "description": "Electron SQLite persisted collection bridge for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/expo-db-sqlite-persistence/CHANGELOG.md b/packages/expo-db-sqlite-persistence/CHANGELOG.md index f7dbde7b80..c59705795b 100644 --- a/packages/expo-db-sqlite-persistence/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/expo-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md index 508b133904..835b6088d8 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/CHANGELOG.md @@ -1,5 +1,21 @@ # @tanstack/expo-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/expo-db-sqlite-persistence@0.2.20 + +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/expo-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json index a323c10556..c874b5be68 100644 --- a/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json +++ b/packages/expo-db-sqlite-persistence/e2e/expo-runtime-app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/expo-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.32", "main": "index.js", "scripts": { "start": "expo start", diff --git a/packages/expo-db-sqlite-persistence/package.json b/packages/expo-db-sqlite-persistence/package.json index 867f5c0206..414efa6300 100644 --- a/packages/expo-db-sqlite-persistence/package.json +++ b/packages/expo-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/expo-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/node-db-sqlite-persistence/CHANGELOG.md b/packages/node-db-sqlite-persistence/CHANGELOG.md index 2dc6946565..391f0eb7f8 100644 --- a/packages/node-db-sqlite-persistence/CHANGELOG.md +++ b/packages/node-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/node-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/node-db-sqlite-persistence/package.json b/packages/node-db-sqlite-persistence/package.json index 87616d066b..f1cb10f54c 100644 --- a/packages/node-db-sqlite-persistence/package.json +++ b/packages/node-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/node-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Node SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/offline-transactions/CHANGELOG.md b/packages/offline-transactions/CHANGELOG.md index 50db918180..f2fc2fa41b 100644 --- a/packages/offline-transactions/CHANGELOG.md +++ b/packages/offline-transactions/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/offline-transactions +## 1.0.53 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 1.0.52 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 1.0.51 ### Patch Changes diff --git a/packages/offline-transactions/package.json b/packages/offline-transactions/package.json index 5194a4365e..cec8ca3078 100644 --- a/packages/offline-transactions/package.json +++ b/packages/offline-transactions/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/offline-transactions", - "version": "1.0.51", + "version": "1.0.53", "description": "Offline-first transaction capabilities for TanStack DB", "author": "TanStack", "license": "MIT", diff --git a/packages/powersync-db-collection/CHANGELOG.md b/packages/powersync-db-collection/CHANGELOG.md index 71d9229252..43d1dbe735 100644 --- a/packages/powersync-db-collection/CHANGELOG.md +++ b/packages/powersync-db-collection/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/powersync-db-collection +## 0.1.66 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.1.65 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.64 ### Patch Changes diff --git a/packages/powersync-db-collection/package.json b/packages/powersync-db-collection/package.json index bff245fe88..cf2ecc9d5e 100644 --- a/packages/powersync-db-collection/package.json +++ b/packages/powersync-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/powersync-db-collection", - "version": "0.1.64", + "version": "0.1.66", "description": "PowerSync collection for TanStack DB", "author": "POWERSYNC", "license": "MIT", diff --git a/packages/powersync-db-collection/src/internal.ts b/packages/powersync-db-collection/src/internal.ts new file mode 100644 index 0000000000..c7122835f6 --- /dev/null +++ b/packages/powersync-db-collection/src/internal.ts @@ -0,0 +1,5 @@ +export const POWERSYNC_TEST_HOOKS = Symbol(`powerSyncTestHooks`) + +export type PowerSyncTestHooks = { + getDemandCount: () => number +} diff --git a/packages/powersync-db-collection/src/powersync.ts b/packages/powersync-db-collection/src/powersync.ts index 76b8dedd3e..736caaecf5 100644 --- a/packages/powersync-db-collection/src/powersync.ts +++ b/packages/powersync-db-collection/src/powersync.ts @@ -5,6 +5,7 @@ import { PendingOperationStore } from './PendingOperationStore' import { PowerSyncTransactor } from './PowerSyncTransactor' import { DEFAULT_BATCH_SIZE } from './definitions' import { asPowerSyncRecord, mapOperation } from './helpers' +import { POWERSYNC_TEST_HOOKS } from './internal' import { convertTableToSchema } from './schema' import { serializeForSQLite } from './serialization' import type { @@ -323,6 +324,7 @@ function createPowerSyncCollectionConfig< let disposeTracking: | ((options?: { context?: LockContext }) => Promise) | null = null + let trackingSetup: Promise | null = null if (syncMode === `eager`) { return runEagerSync() @@ -337,6 +339,13 @@ function createPowerSyncCollectionConfig< async function safelyDisposeTracking( context?: LockContext, ): Promise { + // Cleanup can race trigger creation. Wait until the disposer has been + // published so an abort cannot strand a freshly-created trigger. + const setup = trackingSetup + if (setup) { + await setup.catch(() => undefined) + } + const dispose = disposeTracking if (!dispose) { return @@ -346,6 +355,25 @@ function createPowerSyncCollectionConfig< await dispose(context ? { context } : undefined) } + async function establishTracking( + options: Parameters[0], + appliedReceipts: Array, + ): Promise { + const setup = (async () => { + const dispose = await createDiffTrigger(options, appliedReceipts) + disposeTracking = dispose + })() + trackingSetup = setup + + try { + await setup + } finally { + if (trackingSetup === setup) { + trackingSetup = null + } + } + } + async function createDiffTrigger( options: { setupContext?: LockContext @@ -398,6 +426,17 @@ function createPowerSyncCollectionConfig< } async function flushDiffRecords(): Promise { + // PowerSync can notify after creating the tracking table but before its + // create call returns. Preserve that notification until the disposer, + // which proves the trigger is usable, has been published. + const setup = trackingSetup + if (setup) { + await setup.catch(() => undefined) + } + if (!disposeTracking) { + return + } + const ignoredReceipts: Array = [] await database .writeTransaction(async (context) => { @@ -512,10 +551,15 @@ function createPowerSyncCollectionConfig< let onUnload: CleanupFn | void | null = null start(async () => { - onUnload = await restConfig.onLoad?.() + const cleanup = await restConfig.onLoad?.() + if (abortController.signal.aborted) { + cleanup?.() + return + } + onUnload = cleanup const appliedReceipts: Array = [] - disposeTracking = await createDiffTrigger( + await establishTracking( { // Initial eager hydration must make the source usable before // PowerSync can persist a mutation queued during startup. @@ -562,110 +606,160 @@ function createPowerSyncCollectionConfig< // On-demand mode. // Registers a diff trigger for the active WHERE expressions. function runOnDemandSync() { - const unloadSubsetCallbacks = new Map() + type DemandRecord = { + options: LoadSubsetOptions + state: `provisional` | `active` | `released` | `failed` + cleanup?: CleanupFn + } + type PendingRelease = { + options: LoadSubsetOptions + failures: number + } + + const demands = new Map() const releasedSubsets = new WeakSet() + const pendingReleases: Array = [] let stopped = false + let lifecycleGeneration = 0 + let trackingRevision = 0 + let reconciledTrackingRevision = 0 + let rebuildPromise: Promise | null = null + let drainingReleases = false + let releaseRetryTimer: ReturnType | undefined const hasStopped = () => stopped - start().catch((error) => + const startup = start() + void startup.catch((error) => database.logger.error( `Could not start syncing process for ${viewName} into ${trackedTableName}`, error, ), ) - // Tracks all active WHERE expressions for on-demand sync filtering. - // Each loadSubset call pushes its predicate; unloadSubset removes it. - const activeWhereExpressions: Array = [] - - const loadSubset = async ( - options?: LoadSubsetOptions, - ): Promise => { - if (hasStopped()) return - const appliedReceipts: Array = [] - - if (options) { - activeWhereExpressions.push(options.where) - const cleanup = await restConfig.onLoadSubset?.(options) - if (hasStopped()) { - cleanup?.() - return - } - if (cleanup) { - if (releasedSubsets.has(options) || options.signal?.aborted) { - cleanup() - } else { - unloadSubsetCallbacks.set(options, cleanup) - } - } - } + const activeWhereExpressions = () => + Array.from(demands.values()) + .filter((demand) => demand.state === `active`) + .map((demand) => demand.options.where) + + // One reconciliation owns every queued revision so callers cannot + // settle against a stale trigger configuration. + const reconcileTracking = async (): Promise => { + while ( + !hasStopped() && + reconciledTrackingRevision !== trackingRevision + ) { + const generation = lifecycleGeneration + const revision = trackingRevision + const isCurrent = () => + !hasStopped() && + lifecycleGeneration === generation && + trackingRevision === revision + const appliedReceipts: Array = [] - // No predicates remain, so stop tracking entirely. Both calls are no-ops - // when no tracking table is currently active. - if (activeWhereExpressions.length === 0) { await database.writeLock(async (ctx) => { + if (!isCurrent()) return await flushDiffRecordsWithContext(ctx, appliedReceipts) + if (!isCurrent()) return await safelyDisposeTracking(ctx) + if (!isCurrent()) return + + const active = activeWhereExpressions() + if (active.length === 0) return + const combinedWhere = + active.length === 1 + ? active[0] + : or(active[0], active[1], ...active.slice(2)) + const compiledNewData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'NEW.data' }, + ) + const compiledOldData = compileSQLite( + { where: combinedWhere }, + { jsonColumn: 'OLD.data' }, + ) + const compiledView = compileSQLite({ where: combinedWhere }) + const newDataWhenClause = toInlinedWhereClause(compiledNewData) + const oldDataWhenClause = toInlinedWhereClause(compiledOldData) + const viewWhereClause = toInlinedWhereClause(compiledView) + + await establishTracking( + { + setupContext: ctx, + when: { + [DiffTriggerOperation.INSERT]: newDataWhenClause, + [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, + [DiffTriggerOperation.DELETE]: oldDataWhenClause, + }, + writeType: (rowId: string) => + collection.has(rowId) ? `update` : `insert`, + batchQuery: ( + lockContext: LockContext, + batchSize: number, + cursor: number, + ) => + lockContext.getAll( + `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, + [batchSize, cursor], + ), + }, + appliedReceipts, + ) + if (!isCurrent()) await safelyDisposeTracking(ctx) }) await Promise.all(appliedReceipts) - return + if (isCurrent()) { + reconciledTrackingRevision = revision + } } + } - const combinedWhere = - activeWhereExpressions.length === 1 - ? activeWhereExpressions[0] - : or( - activeWhereExpressions[0], - activeWhereExpressions[1], - ...activeWhereExpressions.slice(2), - ) + const rebuildTracking = (): Promise => { + rebuildPromise ??= reconcileTracking().finally(() => { + rebuildPromise = null + }) + return rebuildPromise + } - const compiledNewData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'NEW.data' }, - ) + const loadSubset = async ( + options: LoadSubsetOptions, + ): Promise => { + if (hasStopped()) return + // Never create a trigger that has no observer to drain its diff table. + await startup + if ( + hasStopped() || + releasedSubsets.has(options) || + options.signal?.aborted + ) { + return + } - const compiledOldData = compileSQLite( - { where: combinedWhere }, - { jsonColumn: 'OLD.data' }, - ) + const demand: DemandRecord = { options, state: `provisional` } + demands.set(options, demand) + try { + const cleanup = await restConfig.onLoadSubset?.(options) + if (cleanup) demand.cleanup = cleanup + } catch (error) { + demand.state = `failed` + demands.delete(options) + throw error + } - const compiledView = compileSQLite({ where: combinedWhere }) - - const newDataWhenClause = toInlinedWhereClause(compiledNewData) - const oldDataWhenClause = toInlinedWhereClause(compiledOldData) - const viewWhereClause = toInlinedWhereClause(compiledView) - - await database.writeLock(async (ctx) => { - // Replace any active tracking with one covering the new set of - // predicates. - await flushDiffRecordsWithContext(ctx, appliedReceipts) - await safelyDisposeTracking(ctx) - - disposeTracking = await createDiffTrigger( - { - setupContext: ctx, - when: { - [DiffTriggerOperation.INSERT]: newDataWhenClause, - [DiffTriggerOperation.UPDATE]: `(${newDataWhenClause}) OR (${oldDataWhenClause})`, - [DiffTriggerOperation.DELETE]: oldDataWhenClause, - }, - writeType: (rowId: string) => - collection.has(rowId) ? `update` : `insert`, - batchQuery: ( - lockContext: LockContext, - batchSize: number, - cursor: number, - ) => - lockContext.getAll( - `SELECT * FROM ${viewName} WHERE ${viewWhereClause} LIMIT ? OFFSET ?`, - [batchSize, cursor], - ), - }, - appliedReceipts, - ) - }) - await Promise.all(appliedReceipts) + if ( + hasStopped() || + releasedSubsets.has(options) || + options.signal?.aborted || + demands.get(options) !== demand + ) { + demand.state = `released` + demands.delete(options) + demand.cleanup?.() + return + } + + demand.state = `active` + trackingRevision++ + await rebuildTracking() } const toInlinedWhereClause = (compiled: { @@ -680,73 +774,145 @@ function createPowerSyncCollectionConfig< ) } - const unloadSubset = async (options: LoadSubsetOptions) => { - releasedSubsets.add(options) - unloadSubsetCallbacks.get(options)?.() - unloadSubsetCallbacks.delete(options) - - const idx = activeWhereExpressions.indexOf(options.where) - if (idx !== -1) { - activeWhereExpressions.splice(idx, 1) - } - - // Evict rows that were exclusively loaded by the departing predicate. - // These are rows matching the departing WHERE that are no longer covered - // by any remaining active predicate. + const performPhysicalRelease = async ( + options: LoadSubsetOptions, + ): Promise => { const compiledDeparting = compileSQLite({ where: options.where }) const departingWhereSQL = toInlinedWhereClause(compiledDeparting) + let rowsToEvict: Array<{ id: string }> + for (;;) { + if (hasStopped()) return + const revision = trackingRevision + const active = activeWhereExpressions() + let evictionSQL: string + if (active.length === 0) { + evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` + } else { + const combinedRemaining = + active.length === 1 + ? active[0]! + : or(active[0], active[1], ...active.slice(2)) + const compiledRemaining = compileSQLite({ + where: combinedRemaining, + }) + const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) + evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + } - let evictionSQL: string - if (activeWhereExpressions.length === 0) { - evictionSQL = `SELECT id FROM ${viewName} WHERE ${departingWhereSQL}` - } else { - const combinedRemaining = - activeWhereExpressions.length === 1 - ? activeWhereExpressions[0]! - : or( - activeWhereExpressions[0], - activeWhereExpressions[1], - ...activeWhereExpressions.slice(2), - ) - const compiledRemaining = compileSQLite({ - where: combinedRemaining, - }) - const remainingWhereSQL = toInlinedWhereClause(compiledRemaining) - evictionSQL = `SELECT id FROM ${viewName} WHERE (${departingWhereSQL}) AND NOT (${remainingWhereSQL})` + rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) + if (hasStopped()) return + if (trackingRevision === revision) break } - - const rowsToEvict = await database.getAll<{ id: string }>(evictionSQL) if (rowsToEvict.length > 0) { begin() for (const { id } of rowsToEvict) { write({ type: `delete`, key: id }) } - // Eviction does not establish new subset coverage. Keep trigger - // replacement in the same unload turn even when this delete waits - // behind a persisting mutation; the later load tracks its own - // establishing receipts. void commit() } + await rebuildTracking() + } - // Recreate the diff trigger for the remaining active WHERE expressions. - await loadSubset() + function scheduleReleaseDrain(delay = 0): void { + if (hasStopped() || drainingReleases || releaseRetryTimer) return + if (delay > 0) { + releaseRetryTimer = setTimeout(() => { + releaseRetryTimer = undefined + void drainReleases() + }, delay) + return + } + void drainReleases() + } + + async function drainReleases(): Promise { + if (hasStopped() || drainingReleases) return + drainingReleases = true + let retryDelay = 0 + try { + while (!hasStopped() && pendingReleases.length > 0) { + const pending = pendingReleases[0]! + try { + await performPhysicalRelease(pending.options) + pendingReleases.shift() + } catch (error) { + pending.failures++ + retryDelay = Math.min(1000 * 2 ** (pending.failures - 1), 30000) + database.logger.error( + `Could not release subset tracking for ${viewName}; retrying`, + error, + ) + break + } + } + } finally { + drainingReleases = false + } + if (pendingReleases.length > 0) scheduleReleaseDrain(retryDelay) + } + + const unloadSubset = (options: LoadSubsetOptions): void => { + releasedSubsets.add(options) + const demand = demands.get(options) + if ( + !demand || + demand.state === `released` || + demand.state === `failed` + ) { + return + } + + const wasActive = demand.state === `active` + demand.state = `released` + demands.delete(options) + if (wasActive) trackingRevision++ + try { + demand.cleanup?.() + } catch (error) { + database.logger.error( + `Could not clean up subset hook for ${viewName}`, + error, + ) + } + + if (wasActive) { + pendingReleases.push({ options, failures: 0 }) + scheduleReleaseDrain() + } } markReady() return { + [POWERSYNC_TEST_HOOKS]: { + getDemandCount: () => demands.size, + }, cleanup: () => { stopped = true + lifecycleGeneration++ + trackingRevision++ + clearTimeout(releaseRetryTimer) + releaseRetryTimer = undefined database.logger.info( `Sync has been stopped for ${viewName} into ${trackedTableName}`, ) abortController.abort() - for (const cleanup of unloadSubsetCallbacks.values()) cleanup() - unloadSubsetCallbacks.clear() - activeWhereExpressions.length = 0 + for (const demand of demands.values()) { + try { + demand.cleanup?.() + } catch (error) { + database.logger.error( + `Could not clean up subset hook for ${viewName}`, + error, + ) + } + demand.state = `released` + } + demands.clear() + pendingReleases.length = 0 }, loadSubset: (options: LoadSubsetOptions) => loadSubset(options), - unloadSubset: (options: LoadSubsetOptions) => unloadSubset(options), + unloadSubset, } } }, diff --git a/packages/powersync-db-collection/tests/load-hooks.test.ts b/packages/powersync-db-collection/tests/load-hooks.test.ts index cc428816e8..b5094f6156 100644 --- a/packages/powersync-db-collection/tests/load-hooks.test.ts +++ b/packages/powersync-db-collection/tests/load-hooks.test.ts @@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' import { createCollection, createLiveQueryCollection, eq } from '@tanstack/db' +import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' @@ -91,6 +92,34 @@ describe(`Sync Streams`, () => { expect(collection.status).toBe(`error`) }) + it(`eager mode: releases a load hook that resolves after cleanup`, async () => { + const db = await createDatabase() + const releaseLoad = pDefer() + const loadStarted = pDefer() + const cleanupLoad = vi.fn() + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(async () => {}) + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + onLoad: async () => { + loadStarted.resolve() + await releaseLoad.promise + return cleanupLoad + }, + }), + ) + + await loadStarted.promise + collection.cleanup() + releaseLoad.resolve() + + await vi.waitFor(() => expect(cleanupLoad).toHaveBeenCalledOnce()) + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + it(`on-demand mode: should call onLoadSubset/onUnloadSubset for each live query`, async () => { const db = await createDatabase() await createTestProducts(db) diff --git a/packages/powersync-db-collection/tests/on-demand-sync.test.ts b/packages/powersync-db-collection/tests/on-demand-sync.test.ts index 8d1dc34122..84ee7cad25 100644 --- a/packages/powersync-db-collection/tests/on-demand-sync.test.ts +++ b/packages/powersync-db-collection/tests/on-demand-sync.test.ts @@ -1,7 +1,9 @@ import { randomUUID } from 'node:crypto' import { tmpdir } from 'node:os' import { PowerSyncDatabase, Schema, Table, column } from '@powersync/node' +import { fc, test as fcTest } from '@fast-check/vitest' import { + IR, and, createCollection, createLiveQueryCollection, @@ -12,8 +14,17 @@ import { lt, or, } from '@tanstack/db' +import pDefer from 'p-defer' import { describe, expect, it, onTestFinished, vi } from 'vitest' import { powerSyncCollectionOptions } from '../src' +import { POWERSYNC_TEST_HOOKS } from '../src/internal' +import { + projectRetainedRowKeys, + projectTransportLoads, +} from '../../db/tests/load-subset-full-flow-model' +import type { PowerSyncTestHooks } from '../src/internal' +import type { LoadSubsetFullFlowEvent } from '../../db/tests/load-subset-full-flow-model' +import type { Scheduler } from 'fast-check' const APP_SCHEMA = new Schema({ products: new Table({ @@ -56,6 +67,118 @@ describe(`On-Demand Sync Mode`, () => { `) } + type ProductRow = { + id: string + name: string + price: number + category: string + } + + type StagedChange = { + type: `insert` | `update` | `delete` + value?: ProductRow + key?: string + } + + type ControlledReceipt = { + promise: Promise + resolve: () => void + reject: (reason: unknown) => void + } + + async function startAppliedOutcomeLoad( + source: `rows` | `empty`, + syncBatchSize?: number, + receiptMode: `controlled` | `immediate` = `controlled`, + ) { + const db = await createDatabase() + await createTestProducts(db) + const category = source === `rows` ? `electronics` : `furniture` + const authoritativeRows = await db.getAll( + `SELECT id, name, price, category FROM products WHERE category = ?`, + [category], + ) + const receipts: Array = [] + const readableRows = new Map() + let stagedChanges: Array = [] + const applyChanges = (changes: Array) => { + for (const change of changes) { + if (change.type === `delete`) { + if (!change.key) throw new Error(`Delete requires a key`) + readableRows.delete(change.key) + } else { + if (!change.value) throw new Error(`Write requires a value`) + readableRows.set(change.value.id, change.value) + } + } + } + const commit = vi.fn(() => { + const changes = stagedChanges + stagedChanges = [] + if (receiptMode === `immediate`) { + applyChanges(changes) + return true + } + const receipt = pDefer() + receipts.push(receipt) + return receipt.promise.then(() => applyChanges(changes)) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + ...(syncBatchSize === undefined ? {} : { syncBatchSize }), + }) + const sync = config.sync.sync({ + collection: { + status: `ready`, + has: (key: string) => readableRows.has(key), + }, + begin: vi.fn(() => { + stagedChanges = [] + }), + write: vi.fn((change: StagedChange) => { + stagedChanges.push(change) + }), + commit, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + let settled = false + const where = new IR.Func(`eq`, [ + new IR.PropRef([`category`]), + new IR.Value(category), + ]) + const observed = Promise.resolve(sync.loadSubset({ where })).then( + () => { + settled = true + return { status: `fulfilled` } as const + }, + (reason: unknown) => { + settled = true + return { status: `rejected`, reason } as const + }, + ) + + return { + authoritativeRows, + readableRows, + receipts, + observed, + isSettled: () => settled, + cleanup: async () => { + receipts.forEach((receipt) => receipt.resolve()) + sync.cleanup?.() + await observed + }, + } + } + it(`should not load any data initially in on-demand mode`, async () => { const db = await createDatabase() await createTestProducts(db) @@ -217,6 +340,194 @@ describe(`On-Demand Sync Mode`, () => { } }) + it.each([ + { source: `rows`, settlement: `fulfill` }, + { source: `empty`, settlement: `fulfill` }, + { source: `rows`, settlement: `reject` }, + { source: `empty`, settlement: `reject` }, + ] as const)( + `settles a $source subset only through an applied $settlement outcome`, + async ({ source, settlement }) => { + const harness = await startAppliedOutcomeLoad(source) + const receiptFailure = new Error(`applied receipt failed`) + expect(harness.authoritativeRows.length > 0).toBe(source === `rows`) + + try { + await vi.waitFor(() => expect(harness.receipts).toHaveLength(1)) + expect(harness.isSettled()).toBe(false) + expect(harness.readableRows.size).toBe(0) + + if (settlement === `reject`) { + harness.receipts[0]!.reject(receiptFailure) + } else { + harness.receipts[0]!.resolve() + } + + const result = await harness.observed + if (settlement === `reject`) { + expect(result).toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect(harness.readableRows.size).toBe(0) + } else { + expect(result).toEqual({ status: `fulfilled` }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } + } finally { + await harness.cleanup() + } + }, + ) + + it(`waits for every applied receipt before fulfilling a multi-batch subset`, async () => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + + for (const [index, receipt] of harness.receipts.entries()) { + receipt.resolve() + await vi.waitFor(() => + expect(harness.readableRows.size).toBe( + Math.min(index + 1, harness.authoritativeRows.length), + ), + ) + if (index < harness.receipts.length - 1) { + expect(harness.isSettled()).toBe(false) + } + } + await expect(harness.observed).resolves.toEqual({ + status: `fulfilled`, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }) + + it.each([ + { receiptIndex: 0 }, + { receiptIndex: 1 }, + { receiptIndex: 2 }, + { receiptIndex: 3 }, + ])( + `keeps applied receipt $receiptIndex independent in a multi-batch subset`, + async ({ receiptIndex }) => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const receiptFailure = new Error(`applied receipt ${receiptIndex} failed`) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + expect(receiptIndex).toBeLessThan(harness.receipts.length) + + harness.receipts.forEach((receipt, index) => { + if (index !== receiptIndex) receipt.resolve() + }) + const expectedRows = harness.authoritativeRows.filter( + (_row, index) => index !== receiptIndex, + ) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe(expectedRows.length), + ) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + expect(harness.isSettled()).toBe(false) + + harness.receipts[receiptIndex]!.reject(receiptFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }, + ) + + it.each([ + { receiptIndex: 0 }, + { receiptIndex: 1 }, + { receiptIndex: 2 }, + { receiptIndex: 3 }, + ])( + `fails fast at applied receipt $receiptIndex while later receipts remain pending`, + async ({ receiptIndex }) => { + const harness = await startAppliedOutcomeLoad(`rows`, 1) + const receiptFailure = new Error( + `applied receipt ${receiptIndex} failed before its suffix settled`, + ) + + try { + await vi.waitFor(() => + expect(harness.receipts).toHaveLength( + harness.authoritativeRows.length + 1, + ), + ) + expect(receiptIndex).toBeLessThan(harness.receipts.length) + + harness.receipts + .slice(0, receiptIndex) + .forEach((receipt) => receipt.resolve()) + const expectedRows = harness.authoritativeRows.slice(0, receiptIndex) + await vi.waitFor(() => + expect(harness.readableRows.size).toBe(expectedRows.length), + ) + + harness.receipts[receiptIndex]!.reject(receiptFailure) + await expect(harness.observed).resolves.toEqual({ + status: `rejected`, + reason: receiptFailure, + }) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(expectedRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }, + ) + + it.each([`rows`, `empty`] as const)( + `accepts an immediate applied outcome for a %s subset`, + async (source) => { + const harness = await startAppliedOutcomeLoad( + source, + undefined, + `immediate`, + ) + + try { + expect(harness.authoritativeRows.length > 0).toBe(source === `rows`) + await expect(harness.observed).resolves.toEqual({ + status: `fulfilled`, + }) + expect(harness.receipts).toHaveLength(0) + expect( + Array.from(harness.readableRows.values(), (row) => row.name).sort(), + ).toEqual(harness.authoritativeRows.map((row) => row.name).sort()) + } finally { + await harness.cleanup() + } + }, + ) + it(`should reactively update live query when new matching data is inserted into SQLite`, async () => { const db = await createDatabase() await createTestProducts(db) @@ -1792,6 +2103,113 @@ describe(`On-Demand Sync Mode`, () => { { timeout: 2000 }, ) }) + + it(`matches the shared remount history after final-owner release`, async () => { + const db = await createDatabase() + await createTestProducts(db) + const expectedRowKeys = ( + await db.getAll<{ id: string }>( + `SELECT id FROM products WHERE category = 'electronics'`, + ) + ) + .map(({ id }) => String(id)) + .sort() + expect(expectedRowKeys).toHaveLength(3) + let transportLoads = 0 + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset: () => { + transportLoads++ + }, + }), + ) + await collection.stateWhenReady() + const createLive = () => + createLiveQueryCollection({ + query: (q) => + q + .from({ product: collection }) + .where(({ product }) => eq(product.category, `electronics`)), + }) + const first = createLive() + let second: ReturnType | undefined + const history: Array = [ + { + type: `requestDemand`, + ownerId: `owner-1`, + sessionId: `session-1`, + sourceId: `powersync-products`, + demandId: `electronics`, + attemptId: `attempt-1`, + alreadyAborted: false, + }, + ] + + try { + await first.preload() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-1`, + sourceId: `powersync-products`, + demandId: `electronics`, + attemptId: `attempt-1`, + rowKeys: expectedRowKeys, + }) + expect(first.toArray.map(({ id }) => String(id)).sort()).toEqual( + projectRetainedRowKeys(history), + ) + + await first.cleanup() + history.push( + { + type: `releaseDemand`, + ownerId: `owner-1`, + sourceId: `powersync-products`, + demandId: `electronics`, + attemptId: `attempt-1`, + }, + { + type: `restartSession`, + previousSessionId: `session-1`, + nextSessionId: `session-2`, + }, + { + type: `requestDemand`, + ownerId: `owner-2`, + sessionId: `session-2`, + sourceId: `powersync-products`, + demandId: `electronics`, + attemptId: `attempt-2`, + alreadyAborted: false, + }, + ) + await vi.waitFor(() => expect(collection.size).toBe(0)) + + second = createLive() + await second.preload() + const reloadedKeys = second.toArray.map(({ id }) => String(id)).sort() + history.push({ + type: `applyAuthoritativeRows`, + ownerId: `owner-2`, + sourceId: `powersync-products`, + demandId: `electronics`, + attemptId: `attempt-2`, + rowKeys: expectedRowKeys, + }) + + expect(transportLoads).toBe(projectTransportLoads(history)) + expect(reloadedKeys).toEqual(projectRetainedRowKeys(history)) + } finally { + await Promise.all([ + first.cleanup(), + second?.cleanup(), + collection.cleanup(), + ]) + } + }) }) describe(`Overlapping data across queries`, () => { @@ -2229,6 +2647,1076 @@ describe(`On-Demand Sync Mode`, () => { }) } + function queueWriteLocks( + db: PowerSyncDatabase, + scheduler?: Scheduler, + invocationOrder?: Array, + ) { + const queued: Array<() => Promise> = [] + vi.spyOn(db, `writeLock`).mockImplementation( + (callback) => + new Promise((resolve, reject) => { + let started = false + const label = `write-lock-${queued.length + 1}` + const run = async () => { + if (started) return + started = true + invocationOrder?.push(label) + try { + const result = await callback({} as never) + resolve(result as never) + } catch (error) { + reject(error) + } + } + queued.push(run) + if (scheduler) { + void scheduler.schedule(Promise.resolve(), label).then(run) + } + }) as never, + ) + return queued + } + + async function startConcurrentLifecycleHarness(scheduler?: Scheduler) { + const db = await createDatabase() + const hooks: Array>> = [] + const hookCleanups: Array> = [] + const onLoadSubset = vi.fn(() => { + const hook = pDefer() + hooks.push(hook) + const cleanup = vi.fn() + hookCleanups.push(cleanup) + return hook.promise.then(() => cleanup) + }) + const queuedLocks = queueWriteLocks(db, scheduler) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + const trackingHandles: Array<{ + when: Record<`INSERT` | `UPDATE` | `DELETE`, string> + dispose: ReturnType + }> = [] + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockImplementation(({ when }) => { + const dispose = vi.fn(() => Promise.resolve()) + trackingHandles.push({ + when: when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + dispose, + }) + return Promise.resolve(dispose) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const first = { where: eq(`category`, `electronics`) } + const second = { where: eq(`category`, `clothing`) } + const loadSubset = sync.loadSubset + const unloadSubset = sync.unloadSubset + + return { + sync, + loadSubset, + unloadSubset, + first, + second, + hooks, + hookCleanups, + queuedLocks, + createDiffTrigger, + trackingHandles, + cleanup: async () => { + hooks.forEach((hook) => hook.resolve()) + sync.cleanup?.() + await Promise.all(queuedLocks.map((run) => run())) + }, + } + } + + type ScheduledSecondOutcome = + | `activate` + | `reject` + | `release-during-hook` + | `release-after-publication` + | `cleanup-during-hook` + | `cleanup-after-publication` + + async function drainScheduledLifecycle(scheduler: Scheduler) { + let quietTurns = 0 + while (quietTurns < 2) { + if (scheduler.count() > 0) { + quietTurns = 0 + await scheduler.waitAll() + } else { + quietTurns++ + await Promise.resolve() + } + } + } + + async function expectScheduledLifecycleMatches( + scheduler: Scheduler, + secondOutcome: ScheduledSecondOutcome, + expectedActionOrder?: ReadonlyArray, + ) { + const harness = await startConcurrentLifecycleHarness(scheduler) + const hookFailure = new Error(`scheduled hook failure`) + const actionOrder: Array = [] + let firstError: unknown + let secondError: unknown + + const firstLoad = Promise.resolve(harness.loadSubset(harness.first)) + .then(() => undefined) + .catch((error: unknown) => { + firstError = error + }) + let secondLoad: Promise | undefined + + try { + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve(harness.loadSubset(harness.second)) + .then(() => undefined) + .catch((error: unknown) => { + secondError = error + }) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + + const schedule = (label: string, action: () => void) => { + void scheduler.schedule(Promise.resolve(), label).then(() => { + actionOrder.push(label) + action() + }) + } + const endsInRelease = secondOutcome.startsWith(`release-`) + const endsInCleanup = secondOutcome.startsWith(`cleanup-`) + const actsAfterPublication = secondOutcome.endsWith(`after-publication`) + + if (secondOutcome === `reject`) { + schedule(`reject-second-hook`, () => + harness.hooks[1]!.reject(hookFailure), + ) + } else { + schedule(`resolve-second-hook`, () => harness.hooks[1]!.resolve()) + if (secondOutcome === `release-during-hook`) { + schedule(`release-second-demand`, () => + harness.unloadSubset(harness.second), + ) + } else if (secondOutcome === `cleanup-during-hook`) { + schedule(`cleanup-sync`, () => harness.sync.cleanup?.()) + } + } + + await scheduler.waitFor(Promise.all([firstLoad, secondLoad])) + await drainScheduledLifecycle(scheduler) + if (actsAfterPublication) { + if (endsInRelease) { + harness.unloadSubset(harness.second) + } else { + harness.sync.cleanup?.() + } + await drainScheduledLifecycle(scheduler) + } + + if (expectedActionOrder) { + expect(actionOrder).toEqual(expectedActionOrder) + } + + expect(firstError).toBeUndefined() + expect(secondError).toBe( + secondOutcome === `reject` ? hookFailure : undefined, + ) + expect(harness.hookCleanups[0]).toHaveBeenCalledTimes( + endsInCleanup ? 1 : 0, + ) + expect(harness.hookCleanups[1]).toHaveBeenCalledTimes( + endsInRelease || endsInCleanup ? 1 : 0, + ) + + const liveTracking = harness.trackingHandles.filter( + ({ dispose }) => dispose.mock.calls.length === 0, + ) + if (endsInCleanup) { + expect(liveTracking).toEqual([]) + return + } + + expect(liveTracking).toHaveLength(1) + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + const finalClause = liveTracking[0]!.when[operation] + expect(finalClause).toContain(`electronics`) + if (secondOutcome === `activate`) { + expect(finalClause).toContain(`clothing`) + } else { + expect(finalClause).not.toContain(`clothing`) + } + } + if (secondOutcome === `reject`) { + expect( + harness.trackingHandles.every(({ when }) => + ([`INSERT`, `UPDATE`, `DELETE`] as const).every( + (operation) => !when[operation].includes(`clothing`), + ), + ), + ).toBe(true) + } + } finally { + await harness.cleanup() + if (scheduler.count() > 0) await scheduler.waitAll() + await Promise.allSettled([firstLoad, secondLoad]) + } + } + + it(`does not acquire a subset released while tracking startup is suspended`, async () => { + const db = await createDatabase() + const onLoadSubset = vi.fn() + const createDiffTrigger = vi.spyOn(db.triggers, `createDiffTrigger`) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const abortController = new AbortController() + const request = { + where: eq(`category`, `electronics`), + signal: abortController.signal, + } + const load = sync.loadSubset(request) + + // Release the request before start() crosses its first async boundary. + abortController.abort() + sync.unloadSubset?.(request) + + try { + await load + + expect(onLoadSubset).not.toHaveBeenCalled() + expect(createDiffTrigger).not.toHaveBeenCalled() + } finally { + sync.cleanup?.() + } + }) + + it.each([`reject`, `release`] as const)( + `keeps an active rebuild current when a provisional hook will %s`, + async (secondOutcome) => { + const harness = await startConcurrentLifecycleHarness() + const hookFailure = new Error(`second hook failed`) + let firstSettled = false + let secondLoad: Promise | undefined + + try { + const firstLoad = Promise.resolve( + harness.loadSubset(harness.first), + ).then(() => { + firstSettled = true + }) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve(harness.loadSubset(harness.second)).then( + () => undefined, + ) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + + await harness.queuedLocks[0]!() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(firstSettled).toBe(true) + expect(harness.createDiffTrigger).toHaveBeenCalledOnce() + const when = harness.createDiffTrigger.mock.calls[0]?.[0].when + expect(when?.INSERT).toContain(`electronics`) + expect(when?.INSERT).not.toContain(`clothing`) + + if (secondOutcome === `reject`) { + harness.hooks[1]!.reject(hookFailure) + await expect(secondLoad).rejects.toBe(hookFailure) + } else { + harness.unloadSubset(harness.second) + harness.hooks[1]!.resolve() + await secondLoad + } + + await firstLoad + expect(harness.queuedLocks).toHaveLength(1) + expect(harness.hookCleanups[1]).toHaveBeenCalledTimes( + secondOutcome === `release` ? 1 : 0, + ) + } finally { + await harness.cleanup() + await secondLoad?.catch(() => undefined) + } + }, + ) + + it(`does not settle a superseded rebuild before its replacement publishes`, async () => { + const harness = await startConcurrentLifecycleHarness() + let firstSettled = false + let secondSettled = false + let firstLoad: Promise | undefined + let secondLoad: Promise | undefined + + try { + firstLoad = Promise.resolve(harness.loadSubset(harness.first)).then( + () => { + firstSettled = true + }, + ) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(1)) + harness.hooks[0]!.resolve() + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(1)) + + secondLoad = Promise.resolve(harness.loadSubset(harness.second)).then( + () => { + secondSettled = true + }, + ) + await vi.waitFor(() => expect(harness.hooks).toHaveLength(2)) + harness.hooks[1]!.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(secondSettled).toBe(false) + + await harness.queuedLocks[0]!() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(firstSettled).toBe(false) + expect(secondSettled).toBe(false) + expect(harness.createDiffTrigger).not.toHaveBeenCalled() + + await vi.waitFor(() => expect(harness.queuedLocks).toHaveLength(2)) + await harness.queuedLocks[1]!() + await Promise.all([firstLoad, secondLoad]) + + expect(harness.queuedLocks).toHaveLength(2) + expect(harness.createDiffTrigger).toHaveBeenCalledOnce() + const when = harness.createDiffTrigger.mock.calls[0]?.[0].when + expect(when?.INSERT).toContain(`electronics`) + expect(when?.INSERT).toContain(`clothing`) + } finally { + await harness.cleanup() + await Promise.all([ + firstLoad?.catch(() => undefined), + secondLoad?.catch(() => undefined), + ]) + } + }) + + it(`disposes superseded tracking before its replacement starts`, async () => { + const db = await createDatabase() + const hooks: Array>> = [] + const onLoadSubset = vi.fn(() => { + const hook = pDefer() + hooks.push(hook) + return hook.promise.then(() => vi.fn()) + }) + const queuedLocks = queueWriteLocks(db) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + + const triggerStarted = pDefer() + const finishTrigger = pDefer() + const staleDispose = vi.fn(() => Promise.resolve()) + const currentDispose = vi.fn(() => Promise.resolve()) + const triggerClauses: Array< + Record<`INSERT` | `UPDATE` | `DELETE`, string> + > = [] + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockImplementation(async ({ when }) => { + triggerClauses.push( + when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + ) + if (triggerClauses.length === 1) { + triggerStarted.resolve() + await finishTrigger.promise + return staleDispose + } + return currentDispose + }) + + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const first = { where: eq(`category`, `electronics`) } + const second = { where: eq(`category`, `clothing`) } + let firstLoad: Promise | undefined + let secondLoad: Promise | undefined + + try { + firstLoad = Promise.resolve(sync.loadSubset(first)).then( + () => undefined, + ) + await vi.waitFor(() => expect(hooks).toHaveLength(1)) + hooks[0]!.resolve() + await vi.waitFor(() => expect(queuedLocks).toHaveLength(1)) + + const staleRebuild = queuedLocks[0]!() + await triggerStarted.promise + + secondLoad = Promise.resolve(sync.loadSubset(second)).then( + () => undefined, + ) + await vi.waitFor(() => expect(hooks).toHaveLength(2)) + hooks[1]!.resolve() + await new Promise((resolve) => setTimeout(resolve, 0)) + + finishTrigger.resolve() + await staleRebuild + + expect(staleDispose).toHaveBeenCalledOnce() + expect(createDiffTrigger).toHaveBeenCalledOnce() + + await vi.waitFor(() => expect(queuedLocks).toHaveLength(2)) + await queuedLocks[1]!() + await Promise.all([firstLoad, secondLoad]) + + expect(createDiffTrigger).toHaveBeenCalledTimes(2) + expect(currentDispose).not.toHaveBeenCalled() + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + expect(triggerClauses[1]![operation]).toContain(`electronics`) + expect(triggerClauses[1]![operation]).toContain(`clothing`) + } + } finally { + hooks.forEach((hook) => hook.resolve()) + sync.cleanup?.() + await Promise.all(queuedLocks.map((run) => run())) + await Promise.allSettled([firstLoad, secondLoad]) + } + }) + + for (const secondOutcome of [ + `activate`, + `reject`, + `release-during-hook`, + `release-after-publication`, + `cleanup-during-hook`, + `cleanup-after-publication`, + ] as const) { + fcTest.prop([fc.scheduler()], { numRuns: 8 })( + `keeps tracking coherent when concurrent lifecycle tasks end in ${secondOutcome}`, + async (scheduler) => { + await expectScheduledLifecycleMatches(scheduler, secondOutcome) + }, + 15_000, + ) + } + + it.each([ + { + name: `release before hook resolution`, + outcome: `release-during-hook` as const, + order: [3, 2, 1], + expectedActionOrder: [`release-second-demand`, `resolve-second-hook`], + }, + { + name: `hook resolution before release`, + outcome: `release-during-hook` as const, + order: [2, 3, 1, 4], + expectedActionOrder: [`resolve-second-hook`, `release-second-demand`], + }, + { + name: `cleanup before hook resolution`, + outcome: `cleanup-during-hook` as const, + order: [3, 2, 1], + expectedActionOrder: [`cleanup-sync`, `resolve-second-hook`], + }, + { + name: `hook resolution before cleanup`, + outcome: `cleanup-during-hook` as const, + order: [2, 3, 1], + expectedActionOrder: [`resolve-second-hook`, `cleanup-sync`], + }, + ])( + `keeps tracking coherent when $name`, + async ({ outcome, order, expectedActionOrder }) => { + await expectScheduledLifecycleMatches( + fc.schedulerFor(order), + outcome, + expectedActionOrder, + ) + }, + ) + + it.each([ + { + name: `the stopped callback runs before the restarted callback`, + order: [1, 2], + expectedInvocationOrder: [`write-lock-1`, `write-lock-2`], + }, + { + name: `the restarted callback runs before the stopped callback`, + order: [2, 1], + expectedInvocationOrder: [`write-lock-2`, `write-lock-1`], + }, + ])( + `keeps a restarted sync isolated when $name`, + async ({ order, expectedInvocationOrder }) => { + const scheduler = fc.schedulerFor(order) + const db = await createDatabase() + const invocationOrder: Array = [] + queueWriteLocks(db, scheduler, invocationOrder) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + + const hookCleanups: Array> = [] + const onLoadSubset = vi.fn(() => { + const cleanup = vi.fn() + hookCleanups.push(cleanup) + return cleanup + }) + const trackingHandles: Array<{ + when: Record<`INSERT` | `UPDATE` | `DELETE`, string> + dispose: ReturnType + }> = [] + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockImplementation(({ when }) => { + const dispose = vi.fn(() => Promise.resolve()) + trackingHandles.push({ + when: when as Record<`INSERT` | `UPDATE` | `DELETE`, string>, + dispose, + }) + return Promise.resolve(dispose) + }) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const startSync = () => { + const started = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !started || + typeof started === `function` || + !started.loadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + return started + } + + const stoppedSync = startSync() + let stoppedSettled = false + let restartedSettled = false + const stoppedLoad = Promise.resolve( + stoppedSync.loadSubset!({ + where: eq(`category`, `electronics`), + }), + ).then(() => { + stoppedSettled = true + }) + let restartedSync: ReturnType | undefined + let restartedLoad: Promise | undefined + let restartedCleaned = false + let usingFakeTimers = false + + try { + await vi.waitFor(() => expect(scheduler.count()).toBe(1)) + stoppedSync.cleanup?.() + + restartedSync = startSync() + restartedLoad = Promise.resolve( + restartedSync.loadSubset!({ + where: eq(`category`, `clothing`), + }), + ).then(() => { + restartedSettled = true + }) + await vi.waitFor(() => expect(scheduler.count()).toBe(2)) + expect(stoppedSettled).toBe(false) + expect(restartedSettled).toBe(false) + + await scheduler.waitOne() + const stoppedRunsFirst = order[0] === 1 + await vi.waitFor(() => { + expect(stoppedSettled).toBe(stoppedRunsFirst) + expect(restartedSettled).toBe(!stoppedRunsFirst) + }) + + await scheduler.waitFor(Promise.all([stoppedLoad, restartedLoad])) + await drainScheduledLifecycle(scheduler) + + expect(invocationOrder).toEqual(expectedInvocationOrder) + expect(hookCleanups[0]).toHaveBeenCalledOnce() + expect(hookCleanups[1]).not.toHaveBeenCalled() + expect(createDiffTrigger).toHaveBeenCalledOnce() + expect(trackingHandles).toHaveLength(1) + expect(trackingHandles[0]!.dispose).not.toHaveBeenCalled() + for (const operation of [`INSERT`, `UPDATE`, `DELETE`] as const) { + expect(trackingHandles[0]!.when[operation]).toContain(`clothing`) + expect(trackingHandles[0]!.when[operation]).not.toContain( + `electronics`, + ) + } + + vi.useFakeTimers() + usingFakeTimers = true + restartedSync.cleanup?.() + restartedSync.cleanup?.() + restartedCleaned = true + await vi.runAllTimersAsync() + expect(hookCleanups[1]).toHaveBeenCalledOnce() + expect(trackingHandles[0]!.dispose).toHaveBeenCalledOnce() + vi.useRealTimers() + usingFakeTimers = false + } finally { + if (usingFakeTimers) vi.useRealTimers() + stoppedSync.cleanup?.() + if (!restartedCleaned) restartedSync?.cleanup?.() + if (scheduler.count() > 0) await scheduler.waitAll() + await Promise.allSettled([stoppedLoad, restartedLoad]) + } + }, + ) + + it(`does not start queued tracking after collection cleanup`, async () => { + const db = await createDatabase() + const queued = pDefer() + let runQueuedWriteLock!: () => Promise + vi.spyOn(db, `writeLock`).mockImplementation( + (callback) => + new Promise((resolve, reject) => { + runQueuedWriteLock = async () => { + try { + await callback({} as never) + resolve(undefined as never) + } catch (error) { + reject(error) + } + } + queued.resolve() + }) as never, + ) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const load = sync.loadSubset({ where: eq(`category`, `electronics`) }) + await queued.promise + sync.cleanup?.() + await runQueuedWriteLock() + await load + + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + + it(`does not retain a predicate whose load hook rejects`, async () => { + const db = await createDatabase() + const hookFailure = new Error(`subset hook failed`) + const onLoadSubset = vi + .fn() + .mockRejectedValueOnce(hookFailure) + .mockRejectedValueOnce(hookFailure) + .mockRejectedValueOnce(hookFailure) + .mockResolvedValueOnce(undefined) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + const { getDemandCount } = ( + sync as typeof sync & { + [POWERSYNC_TEST_HOOKS]: PowerSyncTestHooks + } + )[POWERSYNC_TEST_HOOKS] + + try { + for (const category of [`electronics`, `clothing`, `outdoors`]) { + await expect( + sync.loadSubset({ where: eq(`category`, category) }), + ).rejects.toBe(hookFailure) + expect(getDemandCount()).toBe(0) + } + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + + const when = createDiffTrigger.mock.calls.at(-1)?.[0].when + expect(when?.INSERT).toContain(`clothing`) + expect(when?.INSERT).not.toContain(`electronics`) + } finally { + sync.cleanup?.() + } + }) + + it(`does not publish a provisional hook through another active demand`, async () => { + const db = await createDatabase() + const firstHook = pDefer() + const onLoadSubset = vi + .fn() + .mockReturnValueOnce(firstHook.promise) + .mockResolvedValueOnce(undefined) + const createDiffTrigger = vi + .spyOn(db.triggers, `createDiffTrigger`) + .mockResolvedValue(vi.fn()) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + onLoadSubset, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if (!sync || typeof sync === `function` || !sync.loadSubset) { + throw new Error(`Expected on-demand sync controls`) + } + + const provisional = sync.loadSubset({ + where: eq(`category`, `electronics`), + }) + await vi.waitFor(() => expect(onLoadSubset).toHaveBeenCalledTimes(1)) + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + + const when = createDiffTrigger.mock.calls.at(-1)?.[0].when + expect(when?.INSERT).toContain(`clothing`) + expect(when?.INSERT).not.toContain(`electronics`) + + firstHook.resolve() + await provisional + sync.cleanup?.() + }) + + it(`hands subset release to the adapter without returning a promise`, async () => { + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + vi.spyOn(db, `getAll`).mockResolvedValue([]) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const request = { where: eq(`category`, `electronics`) } + + await sync.loadSubset(request) + const release = ( + sync.unloadSubset as (options: typeof request) => unknown + )(request) + try { + expect(release).toBeUndefined() + } finally { + await Promise.resolve(release) + sync.cleanup?.() + } + }) + + it(`retries physical subset release after asynchronous adapter failure`, async () => { + vi.useFakeTimers() + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + const getAll = vi + .spyOn(db, `getAll`) + .mockRejectedValueOnce(new Error(`transient eviction failure`)) + .mockResolvedValueOnce([]) + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write: vi.fn(), + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const request = { where: eq(`category`, `electronics`) } + + try { + await sync.loadSubset(request) + expect(sync.unloadSubset(request)).toBeUndefined() + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(1)) + + await vi.advanceTimersByTimeAsync(1000) + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(2)) + } finally { + sync.cleanup?.() + await vi.runOnlyPendingTimersAsync() + vi.useRealTimers() + } + }) + + it(`recomputes eviction when another demand activates during release`, async () => { + const db = await createDatabase() + vi.spyOn(db.triggers, `createDiffTrigger`).mockResolvedValue(vi.fn()) + const firstEviction = pDefer>() + const getAll = vi + .spyOn(db, `getAll`) + .mockReturnValueOnce(firstEviction.promise) + .mockResolvedValueOnce([]) + const write = vi.fn() + const config = powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + syncMode: `on-demand`, + }) + const sync = config.sync.sync({ + collection: { status: `ready`, has: () => false }, + begin: vi.fn(), + write, + commit: () => true, + markReady: vi.fn(), + markError: vi.fn(), + truncate: vi.fn(), + } as never) + if ( + !sync || + typeof sync === `function` || + !sync.loadSubset || + !sync.unloadSubset + ) { + throw new Error(`Expected on-demand sync controls`) + } + const departing = { where: eq(`category`, `electronics`) } + + try { + await sync.loadSubset(departing) + sync.unloadSubset(departing) + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(1)) + + await sync.loadSubset({ where: eq(`category`, `clothing`) }) + firstEviction.resolve([{ id: `row-now-owned-by-clothing` }]) + + await vi.waitFor(() => expect(getAll).toHaveBeenCalledTimes(2)) + expect(write).not.toHaveBeenCalledWith({ + type: `delete`, + key: `row-now-owned-by-clothing`, + }) + } finally { + firstEviction.resolve([]) + sync.cleanup?.() + } + }) + + it(`flushes eager changes that arrive before the tracking handle is published`, async () => { + const db = await createDatabase() + await createTestProducts(db) + + let flushTrackingChanges: + | ((event: { changedTables: Array }) => Promise | void) + | undefined + vi.spyOn(db, `onChangeWithCallback`).mockImplementation((handler) => { + flushTrackingChanges = handler?.onChange + return () => {} + }) + + const triggerCreated = pDefer() + const publishTrackingHandle = pDefer() + const createDiffTrigger = db.triggers.createDiffTrigger.bind(db.triggers) + vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation( + async (options) => { + const dispose = await createDiffTrigger(options) + triggerCreated.resolve() + await publishTrackingHandle.promise + return dispose + }, + ) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + }), + ) + onTestFinished(() => collection.cleanup()) + + await triggerCreated.promise + await db.execute(` + INSERT INTO products (id, name, price, category) + VALUES ('during-startup', 'During startup', 300, 'electronics') + `) + + expect(flushTrackingChanges).toBeDefined() + const flush = Promise.resolve( + flushTrackingChanges!({ + changedTables: [collection.utils.getMeta().trackedTableName], + }), + ) + await new Promise((resolve) => setTimeout(resolve, 0)) + + publishTrackingHandle.resolve() + await Promise.all([flush, collection.stateWhenReady()]) + + expect(collection.get(`during-startup`)?.name).toBe(`During startup`) + }) + + it(`does not create tracking when change observation fails to start`, async () => { + const db = await createDatabase() + const startupError = new Error(`change observation failed`) + vi.spyOn(db.logger, `error`).mockImplementation(() => {}) + const consoleError = vi + .spyOn(console, `error`) + .mockImplementation(() => {}) + onTestFinished(() => consoleError.mockRestore()) + vi.spyOn(db, `onChangeWithCallback`).mockImplementation(() => { + throw startupError + }) + const createDiffTrigger = vi.spyOn(db.triggers, `createDiffTrigger`) + + const collection = makeCollection(db) + onTestFinished(() => collection.cleanup()) + await collection.stateWhenReady() + + const query = categoryQuery(collection, `electronics`) + onTestFinished(() => query.cleanup()) + + await expect(query.preload()).rejects.toBe(startupError) + expect(createDiffTrigger).not.toHaveBeenCalled() + }) + + it(`disposes tracking that finishes starting during collection cleanup`, async () => { + const db = await createDatabase() + vi.spyOn(db, `onChangeWithCallback`).mockImplementation(() => () => {}) + + const triggerStarted = pDefer() + const finishTrigger = pDefer() + const dispose = vi.fn(async () => {}) + vi.spyOn(db.triggers, `createDiffTrigger`).mockImplementation( + async () => { + triggerStarted.resolve() + await finishTrigger.promise + return dispose + }, + ) + + const collection = createCollection( + powerSyncCollectionOptions({ + database: db, + table: APP_SCHEMA.props.products, + }), + ) + + await triggerStarted.promise + collection.cleanup() + finishTrigger.resolve() + + await vi.waitFor(() => { + expect(dispose).toHaveBeenCalledTimes(1) + }) + }) + it(`should start tracking again when a subset is loaded after every subset was unloaded`, async () => { const db = await createDatabase() await createTestProducts(db) diff --git a/packages/query-db-collection/CHANGELOG.md b/packages/query-db-collection/CHANGELOG.md index 8add837808..fcbbaa4ffa 100644 --- a/packages/query-db-collection/CHANGELOG.md +++ b/packages/query-db-collection/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/query-db-collection +## 1.2.12 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 1.2.11 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 1.2.10 ### Patch Changes diff --git a/packages/query-db-collection/package.json b/packages/query-db-collection/package.json index 2ad2b4856f..c067e7fcae 100644 --- a/packages/query-db-collection/package.json +++ b/packages/query-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/query-db-collection", - "version": "1.2.10", + "version": "1.2.12", "description": "TanStack Query collection for TanStack DB", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/query-db-collection/src/query.ts b/packages/query-db-collection/src/query.ts index 11d498b3e7..cd6b52537f 100644 --- a/packages/query-db-collection/src/query.ts +++ b/packages/query-db-collection/src/query.ts @@ -1575,14 +1575,20 @@ export function queryCollectionOptions( newItemsMap.forEach((newItem, key) => { const owners = getPersistedOwners(key) - if (!owners.has(hashedQueryKey)) { + const addsOwner = !owners.has(hashedQueryKey) + const insertsRow = !currentSyncedItems.has(key) + if (addsOwner) { owners.add(hashedQueryKey) - setPersistedOwners(key, owners) } addRowOwner(key, hashedQueryKey) - if (!currentSyncedItems.has(key)) { + if (insertsRow) { write({ type: `insert`, value: newItem }) } + if (addsOwner || insertsRow) { + // An insert clears stale metadata for its key. Stage ownership + // afterward so rows and ownership commit as one state change. + setPersistedOwners(key, owners) + } }) const applied = commit(signal) @@ -1946,6 +1952,12 @@ export function queryCollectionOptions( unsubscribePendingReadyListeners(hashedQueryKey) } + // Refcounts are explicit ownership tokens. A cache event can remove the + // observer while an active acquisition still owns this query. + if (refcount > 0) { + return + } + const hasListeners = observer?.hasListeners() ?? false if (hasListeners) { @@ -1955,16 +1967,6 @@ export function queryCollectionOptions( return } - // No listeners means the query is truly idle. - // Even if refcount > 0, we treat hasListeners as authoritative to prevent leaks. - // This can happen if subscriptions are GC'd without calling unloadSubset. - if (refcount > 0) { - console.warn( - `[cleanupQueryIfIdle] Invariant violation: refcount=${refcount} but no listeners. Cleaning up to prevent leak.`, - { hashedQueryKey }, - ) - } - if ( effectivePersistedGcTime !== undefined && metadata && diff --git a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts index 4dddbf3564..d684f61584 100644 --- a/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts +++ b/packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts @@ -1,7 +1,7 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { QueryClient } from '@tanstack/query-core' +import { QueryClient, hashKey } from '@tanstack/query-core' import { createCollection, eq } from '@tanstack/db' -import { expectAssertionFailure } from '../../db/tests/expected-failure.js' +import { createDeferred } from '../../db/src/deferred.js' import { TraceAssertionError } from '../../db/tests/trace-runner.js' import { queryCollectionOptions } from '../src/query.js' import type { Collection, SyncMetadataApi } from '@tanstack/db' @@ -28,9 +28,10 @@ type MetadataRecorder = { type OwnershipFixtureOptions = { id: string - results: Array> + results: Array | Promise>> syncMode?: `eager` | `on-demand` metadataRecorder?: MetadataRecorder + setupMetadata?: (metadata: SyncMetadataApi) => void } type OwnershipFixture = { @@ -140,82 +141,6 @@ function assertCheckpoint( } } -function asRecords({ - actual, - expected, -}: { - actual: unknown - expected: unknown -}): - | { - observed: Record - wanted: Record - } - | undefined { - if ( - !actual || - typeof actual !== `object` || - !expected || - typeof expected !== `object` - ) { - return undefined - } - - return { - observed: actual as Record, - wanted: expected as Record, - } -} - -function classifyInsertedOwnerMetadataLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - Array.isArray(observed.persistedOwners) && - observed.persistedOwners.length === 0 && - Array.isArray(observed.metadataSetKeys) && - observed.metadataSetKeys.length === 1 && - observed.metadataSetKeys[0] === shared.id && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 1 && - typeof wanted.persistedOwners[0] === `string` && - Array.isArray(wanted.metadataSetKeys) && - wanted.metadataSetKeys.length === 1 && - wanted.metadataSetKeys[0] === shared.id - ) -} - -function sameArray(actual: unknown, expected: unknown): boolean { - return ( - Array.isArray(actual) && - Array.isArray(expected) && - actual.length === expected.length && - actual.every((value, index) => value === expected[index]) - ) -} - -function classifyPersistedBaselineLoss(difference: { - actual: unknown - expected: unknown -}): boolean { - const records = asRecords(difference) - if (!records) return false - const { observed, wanted } = records - return ( - sameArray(observed.liveOwners, wanted.liveOwners) && - sameArray(observed.persistedOwners, wanted.insertedOwners) && - Array.isArray(observed.insertedOwners) && - observed.insertedOwners.length === 0 && - Array.isArray(wanted.persistedOwners) && - wanted.persistedOwners.length === 2 && - sameArray(observed.metadataSetKeys, wanted.metadataSetKeys) - ) -} - function recordMetadataWrites( metadata: SyncMetadataApi, recorder: MetadataRecorder, @@ -246,10 +171,13 @@ function createOwnershipFixture({ results, syncMode = `on-demand`, metadataRecorder, + setupMetadata, }: OwnershipFixtureOptions): OwnershipFixture { const queryClient = createQueryClient() const queryFn = vi.fn<() => Promise>>() - results.forEach((result) => queryFn.mockResolvedValueOnce(result)) + results.forEach((result) => + queryFn.mockImplementationOnce(() => Promise.resolve(result)), + ) queryFn.mockRejectedValue(new Error(`Unexpected ownership-oracle refetch`)) const baseOptions = queryCollectionOptions({ id, @@ -262,8 +190,9 @@ function createOwnershipFixture({ }) const maps = inspectOwnershipMaps(baseOptions) const originalSync = baseOptions.sync + let pendingSetupMetadata = setupMetadata const collection = createCollection( - metadataRecorder + metadataRecorder || setupMetadata ? { ...baseOptions, sync: { @@ -271,12 +200,17 @@ function createOwnershipFixture({ if (!params.metadata) { throw new Error(`Sync metadata API is unavailable`) } + if (pendingSetupMetadata) { + params.begin() + pendingSetupMetadata(params.metadata) + params.commit() + pendingSetupMetadata = undefined + } return originalSync.sync({ ...params, - metadata: recordMetadataWrites( - params.metadata, - metadataRecorder, - ), + metadata: metadataRecorder + ? recordMetadataWrites(params.metadata, metadataRecorder) + : params.metadata, }) }, }, @@ -602,10 +536,58 @@ describe(`query collection ownership lifecycle oracle`, () => { } }) - it(`#1656 keeps the first persisted owner when a second query inserts another row`, async () => { + it(`keeps an active on-demand owner when its cache entry is removed`, async () => { + const id = `ownership-active-cache-removal` + const { collection, maps, queryClient } = createOwnershipFixture({ + id, + results: [[shared]], + }) + const subset = { where: eq(`category`, `detail`) } + + await collection._sync.loadSubset(subset) + const queryHash = onlyOwner(maps, shared.id) + const subscription = collection.subscribeChanges(() => {}) + subscription.unsubscribe() + assertCheckpoint(0, collection.subscriberCount, 0) + + const warning = vi.spyOn(console, `warn`).mockImplementation(() => {}) + try { + queryClient.removeQueries({ queryKey: [id] }) + + assertCheckpoint( + 1, + { + rows: collectionRows(collection), + owners: ownersOf(maps, shared.id), + ownedRows: rowsOwnedBy(maps, queryHash), + }, + { + rows: [shared.id], + owners: [queryHash], + ownedRows: [shared.id], + }, + ) + expect(warning).not.toHaveBeenCalled() + } finally { + warning.mockRestore() + } + + collection._sync.unloadSubset(subset) + assertCheckpoint( + 2, + { + rows: collectionRows(collection), + ownershipRows: maps.rowToQueries.size, + ownershipQueries: maps.queryToRows.size, + }, + { rows: [], ownershipRows: 0, ownershipQueries: 0 }, + ) + }) + + it(`keeps every persisted owner when overlapping queries insert rows`, async () => { const metadataRecorder: MetadataRecorder = { rowWrites: [] } const { collection, maps } = createOwnershipFixture({ - id: `ownership-persisted-baseline-1656`, + id: `ownership-persisted-baseline`, results: [[shared], [shared, listOnly]], metadataRecorder, }) @@ -614,59 +596,41 @@ describe(`query collection ownership lifecycle oracle`, () => { await collection._sync.loadSubset(detailSubset) const detailHash = onlyOwner(maps, shared.id) - // The production metadata API records the owner write, but the insert's - // commit currently loses it. Accept only that exact #1656 boundary. - const assertInsertedOwnerPersists = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 0, - { - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, - ) - }), - { checkpoint: 0, classify: classifyInsertedOwnerMetadataLoss }, + assertCheckpoint( + 0, + { + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { persistedOwners: [detailHash], metadataSetKeys: [shared.id] }, ) - await assertInsertedOwnerPersists() await collection._sync.loadSubset(listSubset) const listHash = otherOwner(maps, shared.id, detailHash) - // A second insert loses its own owner and rebuilds the persisted baseline - // with only the later query, while the in-memory ownership remains sound. - const assertPersistedBaselineSurvives = expectAssertionFailure( - () => - Promise.resolve().then(() => { - assertCheckpoint( - 1, - { - liveOwners: ownersOf(maps, shared.id), - persistedOwners: persistedOwners( - collection._state.syncedMetadata, - shared.id, - ), - insertedOwners: persistedOwners( - collection._state.syncedMetadata, - listOnly.id, - ), - metadataSetKeys: setMetadataKeys(metadataRecorder), - }, - { - liveOwners: sorted([detailHash, listHash]), - persistedOwners: sorted([detailHash, listHash]), - insertedOwners: [listHash], - metadataSetKeys: [listOnly.id, shared.id], - }, - ) - }), - { checkpoint: 1, classify: classifyPersistedBaselineLoss }, + assertCheckpoint( + 1, + { + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + insertedOwners: persistedOwners( + collection._state.syncedMetadata, + listOnly.id, + ), + metadataSetKeys: setMetadataKeys(metadataRecorder), + }, + { + liveOwners: sorted([detailHash, listHash]), + persistedOwners: sorted([detailHash, listHash]), + insertedOwners: [listHash], + metadataSetKeys: [listOnly.id, shared.id], + }, ) - await assertPersistedBaselineSurvives() collection._sync.unloadSubset(listSubset) assertCheckpoint( @@ -686,4 +650,89 @@ describe(`query collection ownership lifecycle oracle`, () => { }, ) }) + + it(`restages an existing persisted owner when its absent row is inserted`, async () => { + const id = `ownership-existing-metadata-before-insert` + const queryHash = hashKey([id]) + const result = createDeferred>() + let setupCalls = 0 + const { collection, maps, queryFn } = createOwnershipFixture({ + id, + syncMode: `eager`, + results: [result.promise, [{ ...shared, name: `Restarted` }]], + setupMetadata: (metadata) => { + setupCalls += 1 + metadata.row.set(shared.id, { + queryCollection: { owners: { [queryHash]: true } }, + }) + }, + }) + + expect(queryFn).toHaveBeenCalledTimes(1) + assertCheckpoint( + 0, + { + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + }, + { + rows: [], + liveOwners: [], + persistedOwners: [queryHash], + }, + ) + + result.resolve([shared]) + await collection.stateWhenReady() + assertCheckpoint( + 1, + { + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + }, + { + rows: [shared.id], + liveOwners: [queryHash], + persistedOwners: [queryHash], + }, + ) + + await collection.cleanup() + assertCheckpoint(2, collection.status, `cleaned-up`) + await collection.preload() + await vi.waitFor(() => { + expect(queryFn).toHaveBeenCalledTimes(2) + expect(collection.get(shared.id)?.name).toBe(`Restarted`) + }) + assertCheckpoint( + 3, + { + status: collection.status, + fetches: queryFn.mock.calls.length, + rows: collectionRows(collection), + liveOwners: ownersOf(maps, shared.id), + persistedOwners: persistedOwners( + collection._state.syncedMetadata, + shared.id, + ), + setupCalls, + }, + { + status: `ready`, + fetches: 2, + rows: [shared.id], + liveOwners: [queryHash], + persistedOwners: [queryHash], + setupCalls: 1, + }, + ) + }) }) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index b6f8266813..9f4c38f70a 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -7401,11 +7401,7 @@ describe(`QueryCollection`, () => { } }) - it(`should reset refcount after query GC and reload (stale refcount bug)`, async () => { - // This test catches Bug 2: stale refcounts after GC/remove - // When TanStack Query GCs a query, the refcount should be cleaned up - // Otherwise, reloading the same subset will start with a stale count - + it(`should reload a released subset without retaining a stale refcount`, async () => { const baseQueryKey = [`stale-refcount-test`] const items: Array = [ { id: `1`, name: `Item 1`, category: `A` }, @@ -7443,13 +7439,17 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Force GC by calling removeQueries (simulates gcTime expiry) + // Release the first acquisition before its cache entry is removed. + // Cache events do not revoke active collection ownership. + await query1.cleanup() + await vi.waitFor(() => { + expect(collection.size).toBe(0) + }) + + // Force GC by calling removeQueries (simulates gcTime expiry). queryClient.removeQueries({ queryKey: baseQueryKey }) await flushPromises() - // BUG: queryRefCounts still has stale count, wasn't cleaned up by cleanupQuery - // When we load again, the refcount will be wrong (starts at 1 instead of 0, or accumulates) - // Reload the same query const query2 = createLiveQueryCollection({ query: (q) => @@ -7466,14 +7466,11 @@ describe(`QueryCollection`, () => { expect(collection.size).toBe(2) }) - // Cleanup - this should properly decrement from 1 to 0 and clean up + // Cleanup should decrement the new acquisition from one to zero. await query2.cleanup() await vi.waitFor(() => { - expect(collection.size).toBe(0) // Should be cleaned up + expect(collection.size).toBe(0) }) - - // BUG SYMPTOM: If refcount was stale (e.g. was 2, decremented to 1), - // the observer won't be destroyed and data won't be cleaned up }) it(`should handle mount/unmount/remount without breaking cache (destroyed observer bug)`, async () => { diff --git a/packages/react-db/CHANGELOG.md b/packages/react-db/CHANGELOG.md index 940786b42b..205e5023f1 100644 --- a/packages/react-db/CHANGELOG.md +++ b/packages/react-db/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/react-db +## 0.3.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.3.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.3.5 ### Patch Changes diff --git a/packages/react-db/package.json b/packages/react-db/package.json index 0dd6d60e6a..b9612dc394 100644 --- a/packages/react-db/package.json +++ b/packages/react-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-db", - "version": "0.3.5", + "version": "0.3.7", "description": "React integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/react-native-db-sqlite-persistence/CHANGELOG.md b/packages/react-native-db-sqlite-persistence/CHANGELOG.md index 46539d7520..1282f97f79 100644 --- a/packages/react-native-db-sqlite-persistence/CHANGELOG.md +++ b/packages/react-native-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/react-native-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/react-native-db-sqlite-persistence/package.json b/packages/react-native-db-sqlite-persistence/package.json index a3ef7dc90d..c3549d5f03 100644 --- a/packages/react-native-db-sqlite-persistence/package.json +++ b/packages/react-native-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/react-native-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "React Native and Expo SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/rxdb-db-collection/CHANGELOG.md b/packages/rxdb-db-collection/CHANGELOG.md index 9343913263..1e8d68589f 100644 --- a/packages/rxdb-db-collection/CHANGELOG.md +++ b/packages/rxdb-db-collection/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/rxdb-db-collection +## 0.1.94 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.1.93 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.92 ### Patch Changes diff --git a/packages/rxdb-db-collection/package.json b/packages/rxdb-db-collection/package.json index 97cf5ba13e..36387acf49 100644 --- a/packages/rxdb-db-collection/package.json +++ b/packages/rxdb-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/rxdb-db-collection", - "version": "0.1.92", + "version": "0.1.94", "description": "Reactive, Offline-First adapter for TanStack DB using RxDB. Sync, Replication and Local-First support.", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/solid-db/CHANGELOG.md b/packages/solid-db/CHANGELOG.md index 81957f00cc..c4fa5b37a0 100644 --- a/packages/solid-db/CHANGELOG.md +++ b/packages/solid-db/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/react-db +## 0.2.42 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.2.41 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.2.40 ### Patch Changes diff --git a/packages/solid-db/package.json b/packages/solid-db/package.json index d0af6c4225..cff5ce25ff 100644 --- a/packages/solid-db/package.json +++ b/packages/solid-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/solid-db", - "version": "0.2.40", + "version": "0.2.42", "description": "Solid integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/svelte-db/CHANGELOG.md b/packages/svelte-db/CHANGELOG.md index 86729d8162..3a205b7f2a 100644 --- a/packages/svelte-db/CHANGELOG.md +++ b/packages/svelte-db/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/svelte-db +## 0.3.7 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.3.6 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.3.5 ### Patch Changes diff --git a/packages/svelte-db/package.json b/packages/svelte-db/package.json index c8c5a1e717..1e5857494f 100644 --- a/packages/svelte-db/package.json +++ b/packages/svelte-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/svelte-db", - "version": "0.3.5", + "version": "0.3.7", "description": "Svelte integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/packages/tauri-db-sqlite-persistence/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/CHANGELOG.md index 9456964a9a..4b20263184 100644 --- a/packages/tauri-db-sqlite-persistence/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/tauri-db-sqlite-persistence +## 0.2.20 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.20 + +## 0.2.19 + +### Patch Changes + +- Updated dependencies []: + - @tanstack/db-sqlite-persistence-core@0.2.19 + ## 0.2.18 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md index f654a6452e..c1fa5008e2 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md +++ b/packages/tauri-db-sqlite-persistence/e2e/app/CHANGELOG.md @@ -1,5 +1,21 @@ # @tanstack/tauri-db-sqlite-persistence-e2e-app +## 0.0.32 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + - @tanstack/tauri-db-sqlite-persistence@0.2.20 + +## 0.0.31 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + - @tanstack/tauri-db-sqlite-persistence@0.2.19 + ## 0.0.30 ### Patch Changes diff --git a/packages/tauri-db-sqlite-persistence/e2e/app/package.json b/packages/tauri-db-sqlite-persistence/e2e/app/package.json index 26ae159194..9da824e803 100644 --- a/packages/tauri-db-sqlite-persistence/e2e/app/package.json +++ b/packages/tauri-db-sqlite-persistence/e2e/app/package.json @@ -1,7 +1,7 @@ { "name": "@tanstack/tauri-db-sqlite-persistence-e2e-app", "private": true, - "version": "0.0.30", + "version": "0.0.32", "type": "module", "scripts": { "build": "vite build", diff --git a/packages/tauri-db-sqlite-persistence/package.json b/packages/tauri-db-sqlite-persistence/package.json index 416536bf85..0d6ffdf8f4 100644 --- a/packages/tauri-db-sqlite-persistence/package.json +++ b/packages/tauri-db-sqlite-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/tauri-db-sqlite-persistence", - "version": "0.2.18", + "version": "0.2.20", "description": "Tauri SQLite persisted collection adapter for TanStack DB", "author": "TanStack Team", "license": "MIT", diff --git a/packages/trailbase-db-collection/CHANGELOG.md b/packages/trailbase-db-collection/CHANGELOG.md index 4d153b4893..ef587279e9 100644 --- a/packages/trailbase-db-collection/CHANGELOG.md +++ b/packages/trailbase-db-collection/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/trailbase-db-collection +## 0.1.106 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.1.105 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.104 ### Patch Changes diff --git a/packages/trailbase-db-collection/package.json b/packages/trailbase-db-collection/package.json index ab06ace3f9..3b9bc9f010 100644 --- a/packages/trailbase-db-collection/package.json +++ b/packages/trailbase-db-collection/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/trailbase-db-collection", - "version": "0.1.104", + "version": "0.1.106", "description": "TrailBase collection for TanStack DB", "author": "Sebastian Jeltsch", "license": "MIT", diff --git a/packages/vue-db/CHANGELOG.md b/packages/vue-db/CHANGELOG.md index 2d0753fbd5..a152ff8b93 100644 --- a/packages/vue-db/CHANGELOG.md +++ b/packages/vue-db/CHANGELOG.md @@ -1,5 +1,19 @@ # @tanstack/vue-db +## 0.1.9 + +### Patch Changes + +- Updated dependencies [[`bbc9edf`](https://github.com/TanStack/db/commit/bbc9edf28ef707c8fb3c451fe2c4724039a0037a)]: + - @tanstack/db@0.8.7 + +## 0.1.8 + +### Patch Changes + +- Updated dependencies [[`ae2fe74`](https://github.com/TanStack/db/commit/ae2fe74e4cb4e74500a90034e6db7987bbd90bd8)]: + - @tanstack/db@0.8.6 + ## 0.1.7 ### Patch Changes diff --git a/packages/vue-db/package.json b/packages/vue-db/package.json index 92f9567971..e93d027804 100644 --- a/packages/vue-db/package.json +++ b/packages/vue-db/package.json @@ -1,6 +1,6 @@ { "name": "@tanstack/vue-db", - "version": "0.1.7", + "version": "0.1.9", "description": "Vue integration for @tanstack/db", "author": "Kyle Mathews", "license": "MIT", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 943e48209a..ce1a271cf6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -148,10 +148,10 @@ importers: specifier: ^20.3.16 version: 20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(@angular/platform-browser@20.3.16(@angular/common@20.3.16(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1))(rxjs@7.8.2))(@angular/core@20.3.17(@angular/compiler@20.3.16)(rxjs@7.8.2)(zone.js@0.15.1)))(rxjs@7.8.2) '@tanstack/angular-db': - specifier: ^0.1.86 + specifier: ^0.1.88 version: link:../../../packages/angular-db '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db rxjs: specifier: ^7.8.2 @@ -209,19 +209,19 @@ importers: examples/electron/offline-first: dependencies: '@tanstack/electron-db-sqlite-persistence': - specifier: ^0.1.30 + specifier: ^0.1.32 version: link:../../../packages/electron-db-sqlite-persistence '@tanstack/node-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.20 version: link:../../../packages/node-db-sqlite-persistence '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -300,19 +300,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.20 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -397,19 +397,19 @@ importers: specifier: 11.4.1 version: 11.4.1(react-native@0.79.6(@babel/core@7.29.0)(@types/react@19.2.13)(react@19.0.0)) '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-native-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.20 version: link:../../../packages/react-native-db-sqlite-persistence '@tanstack/react-query': specifier: ^5.90.20 @@ -482,10 +482,10 @@ importers: examples/react/next-ssr-e2e: dependencies: '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db next: specifier: ^16.3.1 @@ -516,19 +516,19 @@ importers: examples/react/offline-transactions: dependencies: '@tanstack/browser-db-sqlite-persistence': - specifier: ^0.2.18 + specifier: ^0.2.20 version: link:../../../packages/browser-db-sqlite-persistence '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/offline-transactions': - specifier: ^1.0.51 + specifier: ^1.0.53 version: link:../../../packages/offline-transactions '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-query': specifier: ^5.90.20 @@ -586,10 +586,10 @@ importers: examples/react/paced-mutations-demo: dependencies: '@tanstack/db': - specifier: ^0.8.5 + specifier: ^0.8.7 version: link:../../../packages/db '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db mitt: specifier: ^3.0.1 @@ -626,10 +626,10 @@ importers: specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -759,7 +759,7 @@ importers: examples/react/start-ssr-e2e: dependencies: '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -805,16 +805,16 @@ importers: examples/react/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/react-db': - specifier: ^0.3.5 + specifier: ^0.3.7 version: link:../../../packages/react-db '@tanstack/react-router': specifier: ^1.159.5 @@ -823,7 +823,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.104 + specifier: ^0.1.106 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6 @@ -926,16 +926,16 @@ importers: examples/solid/todo: dependencies: '@tanstack/electric-db-collection': - specifier: ^0.4.5 + specifier: ^0.4.7 version: link:../../../packages/electric-db-collection '@tanstack/query-core': specifier: ^5.90.20 version: 5.90.20 '@tanstack/query-db-collection': - specifier: ^1.2.10 + specifier: ^1.2.12 version: link:../../../packages/query-db-collection '@tanstack/solid-db': - specifier: ^0.2.40 + specifier: ^0.2.42 version: link:../../../packages/solid-db '@tanstack/solid-router': specifier: ^1.159.5 @@ -944,7 +944,7 @@ importers: specifier: ^1.159.5 version: 1.159.5(@tanstack/react-router@1.159.5(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(solid-js@1.9.11)(vite-plugin-solid@2.11.10(@testing-library/jest-dom@6.9.1)(solid-js@1.9.11)(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)))(vite@7.3.2(@types/node@25.2.2)(jiti@2.6.1)(lightningcss@1.30.2)(sass@1.90.0)(terser@5.44.0)(tsx@4.21.0)(yaml@2.8.1)) '@tanstack/trailbase-db-collection': - specifier: ^0.1.104 + specifier: ^0.1.106 version: link:../../../packages/trailbase-db-collection cors: specifier: ^2.8.6