diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 77931a27..f7baa7f8 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -1,87 +1,91 @@ -export const CI_TEST_GROUPS = [ - "cli-smoke", - "core-regression", - "storage-and-schema", - "llm-clients-and-auth", - "packaging-and-workflow", -]; - -export const CI_TEST_MANIFEST = [ - { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, - { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, - { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, - { group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" }, - { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, - { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, - { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, - { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, - { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, - { group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, - { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, - { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, - // Issue #598 regression tests - { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, - { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, - // Issue #629 batch embedding fix - { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, - // Issue #665 bulkStore tests - // Issue #690 cross-call batch accumulator tests - { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, - // Issue #665 bulkStore tests (from upstream) - { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, - { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, - // Issue #680 regression tests (from upstream) - { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, - // Issue #606 SDK migration Bug 2 regression tests - { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, - // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback - { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, - // Issue #736 recall governance - isRecallUsed() unit tests - { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, - // Issue #492 agentId validation tests - { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, - { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, - // Tier 1 memory counter fix - { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, -]; - -export function getEntriesForGroup(group) { - if (!CI_TEST_GROUPS.includes(group)) { - throw new Error(`Unknown CI test group: ${group}`); - } - - return CI_TEST_MANIFEST.filter((entry) => entry.group === group); -} +export const CI_TEST_GROUPS = [ + "cli-smoke", + "core-regression", + "storage-and-schema", + "llm-clients-and-auth", + "packaging-and-workflow", +]; + +export const CI_TEST_MANIFEST = [ + { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" }, + { group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, + { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, + { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, + { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" }, + { group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" }, + { group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" }, + { group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] }, + { group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] }, + { group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] }, + // Issue #598 regression tests + { group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" }, + { group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" }, + // Issue #629 batch embedding fix + { group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" }, + // Issue #665 bulkStore tests + // Issue #690 cross-call batch accumulator tests + { group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] }, + // Issue #665 bulkStore tests (from upstream) + { group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] }, + // Issue #665 bulkStore tests (from upstream) + { group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] }, + { group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] }, + // Issue #680 regression tests (from upstream) + { group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] }, + // Issue #606 SDK migration Bug 2 regression tests + { group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" }, + // PR #713 inference regression tests - inferProviderFromBaseURL + model fallback + { group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" }, + // Issue #736 recall governance - isRecallUsed() unit tests + { group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] }, + // Issue #492 agentId validation tests + { group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] }, + // Tier 1 memory counter fix + { group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] }, + // Issue #693 extraction write validation tests + { group: "core-regression", runner: "node", file: "test/extraction-validation.test.mjs", args: ["--test"] }, + { group: "core-regression", runner: "node", file: "test/dedup-false-alarm.test.mjs", args: ["--test"] }, +]; + +export function getEntriesForGroup(group) { + if (!CI_TEST_GROUPS.includes(group)) { + throw new Error(`Unknown CI test group: ${group}`); + } + + return CI_TEST_MANIFEST.filter((entry) => entry.group === group); +} \ No newline at end of file diff --git a/src/memory-categories.ts b/src/memory-categories.ts index 7edc7f53..3bff1b06 100644 --- a/src/memory-categories.ts +++ b/src/memory-categories.ts @@ -76,6 +76,29 @@ export type ExtractionStats = { superseded?: number; // temporal fact replacements }; +/** + * Payload delivered to `ExtractPersistOptions.onExtractionValidationFailed` + * when the number of entries actually written to the store differs from + * the number of candidates produced by the LLM. + * + * @see ExtractPersistOptions.onExtractionValidationFailed + */ +export type ExtractionValidation = { + /** Number of candidates the LLM intended to create (createEntries.length) */ + expected: number; + /** Number of rows actually written (countAfter - countBefore) */ + actual: number; + /** + * expected - actual; positive = under-write (SIGKILL/OOM partial write, fewer rows), + * negative = over-write (concurrent session ADDED rows before our countAfter, more rows). + * positive (under-write): callback invoked; if abortOnExtractionMismatch=true throws. + * negative (over-write): always logged as WARNING and never throws. + */ + mismatch: number; + /** Session key passed to extractAndPersist */ + sessionKey: string; +}; + /** Validate and normalize a category string. */ export function normalizeCategory(raw: string): MemoryCategory | null { const lower = raw.toLowerCase().trim(); diff --git a/src/smart-extractor.ts b/src/smart-extractor.ts index 11354ae6..3637c9d5 100644 --- a/src/smart-extractor.ts +++ b/src/smart-extractor.ts @@ -25,6 +25,7 @@ import { type DedupDecision, type DedupResult, type ExtractionStats, + type ExtractionValidation, type MemoryCategory, ALWAYS_MERGE_CATEGORIES, MERGE_SUPPORTED_CATEGORIES, @@ -273,6 +274,25 @@ export interface ExtractPersistOptions { * - pass a non-empty array to restrict reads to those scopes */ scopeFilter?: string[]; + /** + * Callback invoked when the number of entries actually written to the store + * differs from the number of LLM-generated candidates. + * + * This detects write-path anomalies including: + * - SIGKILL / OOM during bulkStore (partial write) + * - Concurrent compactor deletions (count window between bulkStore and count()) + * + * The callback is NOT invoked when createEntries is empty (no-op write). + * The mismatch field: positive = under-write, negative = over-write (rare). + */ + onExtractionValidationFailed?: (validation: ExtractionValidation) => void; + /** + * When `true`, a positive mismatch (actual < expected, under-write) throws an Error + * that aborts the extraction, signaling the caller to handle the failure. + * When `false` (default), the callback is invoked but extraction proceeds. + * Does not affect negative mismatch (over-write, rare) - always logs and calls callback. + */ + abortOnExtractionMismatch?: boolean; } export class SmartExtractor { @@ -441,7 +461,102 @@ export class SmartExtractor { } if (createEntries.length > 0) { - await this.store.bulkStore(createEntries); + // Phase 1: Verify write success via count-before/after. + // bulkStore is atomic per entry (randomUUID + table.add), so any + // discrepancy here indicates partial failure (SIGKILL/OOM). + // LIMITATION: This count-diff approach is only reliable under single- + // process assumption (no concurrent compactor/session writes during + // extractAndPersist). Concurrent writes inflate actualCreated and can + // cause false negatives (mismatch not detected). This is acceptable for + // Phase 1 since plugin architecture guarantees single-process extraction. + // Phase 2 will address concurrent-safe validation (UUID list compare): + // - Collect UUIDs of entries passed to bulkStore + // - After bulkStore, query store for those UUIDs to verify each exists + // - This catches both SIGKILL/OOM partial writes AND concurrent compactor deletions + // + // LIMITATION (F2): This approach CANNOT detect SIGKILL/OOM during bulkStore. + // If process dies during bulkStore, countAfter never runs, no validation triggered. + // Phase 2 will address this via UUID verification after the write. + // + // LIMITATION (MR3): Each extraction calls store.count() twice, which may be O(N) in LanceDB + // at scale. This is a performance tradeoff for validation reliability. + // + // NOTE: supersede entries are included in createEntries and therefore counted + // in expectedCreated - the count check validates them normally (F4). + let countValidationFailed = false; + let actualCreated: number; + let bulkStoreErr: unknown = null; + + const countBefore = await this.store.count(); + try { + await this.store.bulkStore(createEntries); + } catch (err) { + bulkStoreErr = err; + } + const countAfter = await this.store.count(); + + if (bulkStoreErr) { + // F1 FIX: bulkStore failure is a real write failure — throw to abort extraction. + // Do NOT swallow it like a count() failure. The write did not succeed. + throw new Error( + "memory-pro: smart-extractor: bulkStore failed during extraction write validation: " + + String(bulkStoreErr), + ); + } + + try { + actualCreated = countAfter - countBefore; + } catch (err) { + // F3: count() failed — we cannot verify the write, but it may have succeeded. + // Skip validation rather than abort extraction. + this.log( + "memory-pro: smart-extractor: count() failed after bulkStore: " + + String(err) + " — skipping validation, assuming write succeeded.", + ); + countValidationFailed = true; + actualCreated = createEntries.length; // assume success + } + const expectedCreated = createEntries.length; + + if (!countValidationFailed && actualCreated !== expectedCreated) { + const mismatch = expectedCreated - actualCreated; + + const validation: ExtractionValidation = { + expected: expectedCreated, + actual: actualCreated, + mismatch, + sessionKey, + }; + // F3 FIX: support both sync and async callbacks via Promise.resolve().then() + Promise.resolve() + .then(() => options.onExtractionValidationFailed?.(validation)) + .catch((cbErr) => { + this.log( + "memory-pro: smart-extractor: onExtractionValidationFailed callback threw: " + String(cbErr), + ); + }); + + // F1: when no callback and mismatch > 0, log at minimum so production is not silent + if (mismatch > 0 && !options.onExtractionValidationFailed) { + this.log( + "memory-pro: smart-extractor: extraction mismatch: expected=" + expectedCreated + ", actual=" + actualCreated + " (diff=" + mismatch + "). Provide onExtractionValidationFailed callback or set abortOnExtractionMismatch=true to handle this.", + ); + } + + // F6: when mismatch > 0 and abortOnExtractionMismatch is true, throw to abort + if (mismatch > 0 && options.abortOnExtractionMismatch === true) { + throw new Error( + "memory-pro: smart-extractor: extraction aborted: " + mismatch + " entries failed to persist (expected=" + expectedCreated + ", actual=" + actualCreated + ")", + ); + } + + // F6: when mismatch < 0 (over-write), always log as WARNING, never throw + if (mismatch < 0) { + this.log( + "memory-pro: smart-extractor: WARNING - over-write detected: " + Math.abs(mismatch) + " more entries persisted than expected (expected=" + expectedCreated + ", actual=" + actualCreated + "). This is rare and may indicate a concurrent compactor or duplicate session run.", + ); + } + } } return stats; diff --git a/test/dedup-false-alarm.test.mjs b/test/dedup-false-alarm.test.mjs new file mode 100644 index 00000000..7c6c8088 --- /dev/null +++ b/test/dedup-false-alarm.test.mjs @@ -0,0 +1,174 @@ +/** + * P0 驗證測試:確認 bulkStore 不會因 batchDedup 去重 + * + * 問題:是否 bulkStore 內部呼叫 batchDedup,導致 near-duplicate entries + * 被錯誤過濾,造成 countAfter - countBefore < createEntries.length? + * + * 測試策略: + * 1. 直接構造兩個 cosine similarity = 0.95 的向量(保證是 near-duplicate) + * 2. 用 batchDedup 確認它們確實被視為 duplicates + * 3. 透過 bulkStore 寫入這兩個 entry + * 4. 驗證 count 增加 2(而非 1)→ 確認 bulkStore 不去重 + */ + +import { describe, it, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { MemoryStore } = jiti("../src/store.ts"); +const { batchDedup } = jiti("../src/batch-dedup.ts"); + +/** 256-dim vector with cosine similarity to baseVec controlled by parameter. + * Uses a fixed seed so the output is deterministic across test runs. */ +function makeNearDuplicateVector(baseVec, similarity = 0.95) { + const scale = similarity; + const dim = baseVec.length; + // Use a fixed seed so this function is deterministic (not random each call) + const orth = Array.from({ length: dim }, (_, i) => (i % 2 === 0 ? 0.5 : -0.5)); + const baseNormSq = orth.reduce((s, v, i) => s + v * v, 0); + const proj = orth.reduce((s, v, i) => s + v * baseVec[i], 0) / (baseNormSq || 1); + for (let i = 0; i < dim; i++) orth[i] -= proj * baseVec[i]; + const orthNorm = Math.sqrt(Math.max(0, orth.reduce((s, v) => s + v * v, 0))); + if (orthNorm > 0) for (let i = 0; i < dim; i++) orth[i] /= orthNorm; + const orthNorm2 = Math.sqrt(Math.max(0, orth.reduce((s, v) => s + v * v, 0))); + if (orthNorm2 > 0) for (let i = 0; i < dim; i++) orth[i] /= orthNorm2; + const orthScale = Math.sqrt(Math.max(0, 1 - scale * scale)); + return baseVec.map((v, i) => v * scale + orth[i] * orthScale); +} + +/** Two identical vectors — maximum similarity */ +function makeIdenticalVector(dim = 256) { + const rng = makeRng(42); + return Array.from({ length: dim }, () => rng()); +} + +/** Seeded LCG RNG for deterministic vectors */ +function makeRng(seed) { + let s = seed >>> 0; + return () => { + s = (Math.imul(1664525, s) + 1013904223) >>> 0; + return s / 0x100000000; + }; +} + +/** Cosine similarity between two vectors */ +function cosineSimilarity(a, b) { + let dot = 0, normA = 0, normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + return dot / (Math.sqrt(normA) * Math.sqrt(normB)); +} + +const TEST_DB_PREFIX = "/tmp/test-dedup-p0-"; + +describe("P0: bulkStore does NOT deduplicate near-duplicate entries", () => { + /** @type {MemoryStore} */ + let store; + let dbPath; + + afterEach(async () => { + if (store) { + try { await store.deleteAll("test-session"); } catch {} + try { await store.destroy(); } catch {} + } + }); + + it("bulkStore writes both near-duplicate entries (cosine = 0.95)", async () => { + // Step 1: Create base vector and a near-duplicate (cosine = 0.95) + const dim = 256; + const baseVec = makeIdenticalVector(dim); + const dupVec = makeNearDuplicateVector(baseVec, 0.95); + + const cosSim = cosineSimilarity(baseVec, dupVec); + console.log(`[P0] cosine similarity between base and near-duplicate: ${cosSim.toFixed(4)}`); + assert(cosSim > 0.94, `Near-duplicate should have cosine > 0.94, got ${cosSim}`); + + // Step 2: Verify batchDedup marks one as duplicate + const dedupResult = batchDedup( + ["abstract one", "abstract two"], + [baseVec, dupVec], + 0.85 // default threshold + ); + console.log(`[P0] batchDedup: ${dedupResult.inputCount} → ${dedupResult.outputCount}, duplicates=${JSON.stringify(dedupResult.duplicateIndices)}`); + assert(dedupResult.outputCount < dedupResult.inputCount, + `batchDedup should mark one as duplicate (input=${dedupResult.inputCount}, output=${dedupResult.outputCount})`); + + // Step 3: Create MemoryStore and write both via bulkStore + dbPath = TEST_DB_PREFIX + Date.now() + "-1"; + store = new MemoryStore({ dbPath, vectorDim: dim }); + const countBefore = await store.count(); + + await store.bulkStore([ + { + text: "Meeting attendance — quarterly business review", + vector: baseVec, + category: "fact", + scope: "test-session", + importance: 0.5, + metadata: JSON.stringify({ l0_abstract: "abstract one" }), + }, + { + text: "Quarterly business review with team lead", + vector: dupVec, + category: "fact", + scope: "test-session", + importance: 0.5, + metadata: JSON.stringify({ l0_abstract: "abstract two" }), + }, + ]); + + const countAfter = await store.count(); + const delta = countAfter - countBefore; + + console.log(`[P0 result] countBefore=${countBefore}, countAfter=${countAfter}, delta=${delta}`); + + // KEY ASSERTION: delta should be exactly 2 — bulkStore does NOT dedupe + assert.strictEqual(delta, 2, + `bulkStore should write both entries (delta=2), got delta=${delta}. ` + + `If delta=1, bulkStore is internally deduplicating near-duplicate entries — this is a P0 bug.`); + }); + + it("bulkStore writes all 5 entries even when batchDedup would reduce them to 1", async () => { + const dim = 256; + + // Create 5 identical vectors — batchDedup with threshold 0.85 will keep only 1 + const baseVec = makeIdenticalVector(dim); + const vectors = Array.from({ length: 5 }, () => baseVec); + + const dedupResult = batchDedup( + Array(5).fill("abstract"), + vectors, + 0.85 + ); + console.log(`[P0 batch] batchDedup: 5 identical vectors → ${dedupResult.outputCount} survivors`); + assert(dedupResult.outputCount < 5, + "Sanity: 5 identical vectors should produce < 5 survivors in batchDedup"); + + // Now write all 5 via bulkStore + dbPath = TEST_DB_PREFIX + Date.now() + "-2"; + store = new MemoryStore({ dbPath, vectorDim: dim }); + const countBefore = await store.count(); + + await store.bulkStore(vectors.map((v, i) => ({ + text: `Event ${i + 1}`, + vector: v, + category: "fact", + scope: "test-session", + importance: 0.5, + metadata: JSON.stringify({ l0_abstract: `abstract ${i}` }), + }))); + + const countAfter = await store.count(); + const delta = countAfter - countBefore; + + console.log(`[P0 batch result] countBefore=${countBefore}, countAfter=${countAfter}, delta=${delta}`); + + // KEY ASSERTION: all 5 should be written despite batchDedup saying they're duplicates + assert.strictEqual(delta, 5, + `bulkStore should write all 5 entries even if batchDedup would drop 4 of them (delta=5), got delta=${delta}`); + }); +}); diff --git a/test/extraction-validation.test.mjs b/test/extraction-validation.test.mjs new file mode 100644 index 00000000..58dc9354 --- /dev/null +++ b/test/extraction-validation.test.mjs @@ -0,0 +1,582 @@ +/** + * Test: Extraction Write Validation (Issue #693) + * + * Tests the countBefore/countAfter validation logic in extractAndPersist(). + * + * Key challenge: SmartExtractor's `batchDedup` uses cosine similarity (threshold 0.85) + * on embedded candidate abstracts to filter near-duplicates BEFORE the dedup pipeline. + * Simple charCodeAt-based vectors all score >0.85 similarity (common English letters). + * + * Solution: `DeterministicRandomEmbedder` generates 256-dim vectors using a + * seeded RNG (Mulberry32) keyed on the text content. Each distinct text produces + * a unique vector with cosine similarity < 0.85, ensuring candidates survive batchDedup. + * + * Validates: + * T1. Normal extraction: expected === actual, mismatch = 0, callback NOT triggered + * T2. Empty extraction: skipped, mismatch undefined (validation skipped) + * T3. Partial bulkStore failure: actual < expected → mismatch > 0, callback triggered + * T4. Post-write deletion (compactor race): actual < expected → mismatch > 0 + * T5. Callback is optional — no error if omitted even on mismatch + * T6. Multiple extractions each get independent validation state + */ + +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; +import jitiFactory from "jiti"; + +const jiti = jitiFactory(import.meta.url, { interopDefault: true }); +const { SmartExtractor } = jiti("../src/smart-extractor.ts"); + +// ============================================================================ +// Helpers +// ============================================================================ + +/** + * Mulberry32 seeded PRNG — fast, deterministic, good distribution. + * Returns a new RNG function seeded from an integer. + */ +function makeRng(seed) { + let s = seed >>> 0; + return () => { + s |= 0; + s = (s + 0x6d2b79f5) | 0; + let t = Math.imul(s ^ (s >>> 15), 1 | s); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Deterministic random embedder. + * + * Produces 256-dimensional vectors using a seeded RNG keyed on the text. + * Different texts → different seeds → vectors with cosine similarity < 0.85, + * ensuring candidates survive SmartExtractor's internal batchDedup filter. + * + * This lets us test with 2+ candidates without fighting the dedup logic. + */ +function makeDeterministicEmbedder() { + return { + async embed(text) { + const seed = [...text].reduce((acc, c) => acc + c.charCodeAt(0), 0) >>> 0; + const rng = makeRng(seed === 0 ? 1 : seed); + return Array.from({ length: 256 }, () => rng()); + }, + async embedBatch(texts) { + return texts.map((t) => { + const seed = [...t].reduce((acc, c) => acc + c.charCodeAt(0), 0) >>> 0; + const rng = makeRng(seed === 0 ? 1 : seed); + return Array.from({ length: 256 }, () => rng()); + }); + }, + }; +} + +/** + * Mock LLM — returns configurable candidates and "create" for all dedup decisions. + * The "create" decision ensures candidates progress through the dedup pipeline + * and reach bulkStore without special handling (no handleSupersede, handleMerge, etc.). + */ +function makeLlm(candidates = []) { + return { + async completeJson(_prompt, mode) { + if (mode === "extract-candidates") return { memories: candidates }; + if (mode === "dedup-decision") return { decision: "create", reason: "no match" }; + if (mode === "merge-memory") return candidates[0] ?? null; + return null; + }, + }; +} + +/** + * Mock store with configurable write behavior. + * + * Config: + * initialCount — starting row count (default 0) + * dropLastN — silently drop last N entries on bulkStore (partial write failure) + * bulkStoreThrows — throw on bulkStore (total write failure) + */ +function makeStore(config = {}) { + const { initialCount = 0, dropLastN = 0, bulkStoreThrows = false } = config; + let rowCount = initialCount; + const entries = []; + + const store = { + _config: config, + + async count() { return rowCount; }, + + async vectorSearch() { return []; }, + + async store(entry) { + rowCount++; + entries.push({ action: "store", id: entry.id ?? "?" }); + return { ...entry, id: "direct-id-" + entries.length }; + }, + + async bulkStore(batchEntries) { + if (bulkStoreThrows) throw new Error("bulkStore simulated failure"); + const stored = dropLastN > 0 + ? batchEntries.slice(0, batchEntries.length - dropLastN) + : batchEntries; + for (let i = 0; i < stored.length; i++) { + rowCount++; + entries.push({ action: "bulkStore", id: stored[i].id ?? "bulk-" + i }); + } + if (dropLastN > 0) entries.push({ action: "bulkStore_dropped", count: dropLastN }); + return stored.map((e, i) => ({ ...e, id: "bulk-id-" + i })); + }, + + async update(_id, _patch, _scopeFilter) { + entries.push({ action: "update", id: _id }); + }, + + async getById() { return null; }, + + async delete(_id) { + rowCount = Math.max(0, rowCount - 1); + entries.push({ action: "delete", id: _id }); + }, + + get entries() { return [...entries]; }, + get rowCount() { return rowCount; }, + reset() { rowCount = initialCount; entries.length = 0; }, + }; + return store; +} + +function makeExtractor(embedder, llm, store, config = {}) { + return new SmartExtractor(store, embedder, llm, { + user: "User", + extractMinMessages: 1, + extractMaxChars: 8000, + defaultScope: "global", + log() {}, + debugLog() {}, + ...config, + }); +} + +// ============================================================================ +// Tests +// ============================================================================ + +describe("Issue #693: Extraction write validation", () => { + + // -------------------------------------------------------------------------- + // T1: Normal extraction — expected === actual, mismatch = 0, no callback + // -------------------------------------------------------------------------- + it("T1: normal extraction passes validation with mismatch=0", async () => { + const embedder = makeDeterministicEmbedder(); + // Two semantically different abstracts → low cosine similarity → both survive batchDedup + const llm = makeLlm([ + { + category: "preferences", + abstract: "User prefers dark mode interface settings for eye comfort during coding", + overview: "Display preference", + content: "The user prefers dark mode interface settings on their workstation for reduced eye strain during extended coding sessions.", + }, + { + category: "entities", + abstract: "User works at Acme Corporation headquarters in the R&D division", + overview: "Employment information", + content: "The user is employed as a senior software engineer at Acme Corporation's research and development headquarters facility.", + }, + ]); + const store = makeStore({ initialCount: 0 }); + const extractor = makeExtractor(embedder, llm, store); + + let callbackInvoked = false; + let receivedValidation = null; + + const stats = await extractor.extractAndPersist( + "I use dark mode at work and work at Acme Corp", + "session-t1", + { + onExtractionValidationFailed(validation) { + callbackInvoked = true; + receivedValidation = validation; + }, + } + ); + + assert.strictEqual(stats.created, 2, "both candidates should be created"); + assert.strictEqual( + stats.validationMismatch, + undefined, + "validationMismatch should be undefined (not in public ExtractionStats)" + ); + assert.strictEqual(callbackInvoked, false, "callback should NOT be triggered"); + assert.strictEqual(store.rowCount, 2, "both entries should be written"); + assert.strictEqual(receivedValidation, null, "no validation object received"); + }); + + // -------------------------------------------------------------------------- + // T2: Empty extraction — no bulkStore, validation skipped + // -------------------------------------------------------------------------- + it("T2: empty extraction skips validation", async () => { + const embedder = makeDeterministicEmbedder(); + const llm = makeLlm([]); + const store = makeStore({ initialCount: 5 }); + const extractor = makeExtractor(embedder, llm, store); + + let callbackInvoked = false; + + const stats = await extractor.extractAndPersist( + "nothing to extract here", + "session-t2", + { onExtractionValidationFailed() { callbackInvoked = true; } } + ); + + assert.strictEqual(stats.created, 0, "no entries created"); + assert.strictEqual( + stats.validationMismatch, + undefined, + "validationMismatch should be undefined for empty extraction (validation skipped)" + ); + assert.strictEqual(callbackInvoked, false, "callback should NOT fire"); + assert.strictEqual(store.rowCount, 5, "pre-existing count unchanged"); + }); + + // -------------------------------------------------------------------------- + // T3: Partial bulkStore failure — actual < expected → mismatch > 0 + // -------------------------------------------------------------------------- + it("T3: partial bulkStore failure triggers mismatch > 0", async () => { + const embedder = makeDeterministicEmbedder(); + // 3 candidates → dropLastN=1 → actual=2, mismatch=1 + const llm = makeLlm([ + { + category: "preferences", + abstract: "User prefers dark mode interface settings for eye comfort during coding", + overview: "Display preference", + content: "The user prefers dark mode interface settings on their workstation for reduced eye strain during extended coding sessions.", + }, + { + category: "preferences", + abstract: "User prefers light theme when editing documents and writing emails", + overview: "Display preference", + content: "In contrast to dark mode, the user prefers light theme when editing documents and writing emails in their daily productivity workflow.", + }, + { + category: "entities", + abstract: "User works at Acme Corporation headquarters in the R&D division", + overview: "Employment information", + content: "The user is employed as a senior software engineer at Acme Corporation's research and development headquarters facility.", + }, + ]); + const store = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractor = makeExtractor(embedder, llm, store); + + let callbackInvoked = false; + let receivedValidation = null; + + const stats = await extractor.extractAndPersist( + "I use dark mode for coding but light theme for writing emails at Acme Corp", + "session-t3", + { + onExtractionValidationFailed(validation) { + callbackInvoked = true; + receivedValidation = validation; + }, + } + ); + + // Expected = 3, Actual = 2 (dropLastN=1), Mismatch = 1 + // Note: validationMismatch is NOT written to stats (removed from public API) + // Only the callback receives the mismatch information + assert.strictEqual(callbackInvoked, true, "callback SHOULD be triggered"); + assert.ok(receivedValidation); + assert.strictEqual(receivedValidation.expected, 3); + assert.strictEqual(receivedValidation.actual, 2); + assert.strictEqual(receivedValidation.mismatch, 1); + assert.strictEqual(receivedValidation.sessionKey, "session-t3"); + assert.strictEqual(store.rowCount, 2, "only 2 rows written"); + }); + + // -------------------------------------------------------------------------- + // T4: Post-write deletion (compactor race) — actual < expected + // -------------------------------------------------------------------------- + it("T4: post-write deletion triggers mismatch > 0 (compactor race)", async () => { + const embedder = makeDeterministicEmbedder(); + // 2 candidates → compactor deletes 1 after bulkStore → actual=1, mismatch=1 + const llm = makeLlm([ + { + category: "cases", + abstract: "User completed initial setup wizard on first launch of the application", + overview: "Setup wizard completion", + content: "The user has successfully completed the initial setup wizard and application onboarding process during their first launch of the software application on their primary workstation.", + }, + { + category: "cases", + abstract: "User configured notification preferences including email and push alerts", + overview: "Notification settings configuration", + content: "Following the initial setup, the user proceeded to configure various notification preferences including email alerts, desktop push notifications, and mobile synchronization settings.", + }, + ]); + const store = makeStore({ initialCount: 0 }); + const extractor = makeExtractor(embedder, llm, store); + + // Simulate compactor deleting 1 entry after bulkStore succeeds + const originalBulkStore = store.bulkStore.bind(store); + store.bulkStore = async (entries) => { + const result = await originalBulkStore(entries); + await store.delete("bulk-id-0"); // compactor race: delete first entry + return result; + }; + + let callbackInvoked = false; + let receivedValidation = null; + + const stats = await extractor.extractAndPersist( + "I completed setup and configured notification preferences", + "session-t4", + { + onExtractionValidationFailed(validation) { + callbackInvoked = true; + receivedValidation = validation; + }, + } + ); + + // Expected = 2, Actual = 1 (compactor deleted 1), Mismatch = 1 + // Note: validationMismatch is NOT written to stats (removed from public API) + assert.strictEqual(callbackInvoked, true, "callback SHOULD be triggered"); + assert.ok(receivedValidation); + assert.strictEqual(receivedValidation.expected, 2); + assert.strictEqual(receivedValidation.actual, 1); + assert.strictEqual(receivedValidation.mismatch, 1); + assert.strictEqual(receivedValidation.sessionKey, "session-t4"); + assert.strictEqual(store.rowCount, 1, "1 row remaining after deletion"); + }); + + // -------------------------------------------------------------------------- + // T5: Callback is optional — no error if omitted + // -------------------------------------------------------------------------- + it("T5: callback is optional — no error if omitted even on mismatch", async () => { + const embedder = makeDeterministicEmbedder(); + // 2 candidates that survive batchDedup, with dropLastN=1 + const llm = makeLlm([ + { + category: "events", + abstract: "User attended quarterly business review meeting with the team lead", + overview: "Meeting attendance", + content: "The quarterly business review meeting was attended by the user along with their direct team members to discuss ongoing project status and future planning initiatives.", + }, + { + category: "events", + abstract: "User participated in a formal code review session with constructive feedback", + overview: "Code review participation", + content: "The user actively participated in a formal code review session where they provided constructive feedback on pull request implementations and discussed architectural decision implications.", + }, + ]); + const store = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractor = makeExtractor(embedder, llm, store); + + // Should NOT throw even though mismatch occurs and callback is absent + // Note: validationMismatch is NOT written to stats (exposed only via callback) + await extractor.extractAndPersist( + "User said: I attended the quarterly business review and participated in a code review", + "session-t5", + {} // no callback + ); + assert.strictEqual(store.rowCount, 1, "1 row written despite mismatch (dropLastN=1)"); + }); + + // -------------------------------------------------------------------------- + // T6: Multiple extractions — independent validation state + // -------------------------------------------------------------------------- + it("T6: multiple extractions each get independent validation", async () => { + const embedder = makeDeterministicEmbedder(); + + // First extraction: normal (no mismatch) + const llm1 = makeLlm([{ + category: "events", + abstract: "User attended quarterly business review meeting with the team lead", + overview: "Meeting", + content: "The quarterly business review meeting was attended by the user along with their direct team members to discuss ongoing project status and future planning initiatives.", + }]); + const store1 = makeStore({ initialCount: 0 }); + const extractor1 = makeExtractor(embedder, llm1, store1); + store1.reset(); + + const validations = []; + + const stats1 = await extractor1.extractAndPersist( + "User said: I attended the quarterly business review", + "session-multi-1", + { onExtractionValidationFailed(v) { validations.push(v); } } + ); + + assert.strictEqual(stats1.validationMismatch, undefined, "first: validationMismatch undefined"); + assert.strictEqual(validations.length, 0, "first: no callback fired"); + + // Second extraction: partial write failure (dropLastN=1) → mismatch=1 + const llm2 = makeLlm([ + { + category: "events", + abstract: "User attended quarterly business review meeting with the team lead", + overview: "Meeting", + content: "The quarterly business review meeting was attended by the user along with their direct team members to discuss ongoing project status and future planning initiatives.", + }, + { + category: "events", + abstract: "User participated in a formal code review session with constructive feedback", + overview: "Code review", + content: "The user actively participated in a formal code review session where they provided constructive feedback on pull request implementations and discussed architectural decision implications.", + }, + ]); + const store2 = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractor2 = makeExtractor(embedder, llm2, store2); + + const stats2 = await extractor2.extractAndPersist( + "User said: I attended a quarterly meeting and participated in a code review", + "session-multi-2", + { onExtractionValidationFailed(v) { validations.push(v); } } + ); + + // Second extraction: 2 candidates, dropLastN=1 → actual=1, expected=2, mismatch=1 + // Note: validationMismatch is NOT written to stats (removed from public API) + // Only the callback receives the mismatch + assert.strictEqual(validations.length, 1, "second: callback fired once"); + assert.strictEqual(validations[0].sessionKey, "session-multi-2"); + assert.strictEqual(validations[0].mismatch, 1); + }); + + + // -------------------------------------------------------------------------- + // T7: abortOnExtractionMismatch behavior + negative mismatch + // -------------------------------------------------------------------------- + it("T7: abortOnExtractionMismatch=true throws on under-write, mismatch<0 logs only", async () => { + const embedder = makeDeterministicEmbedder(); + const llmA = makeLlm([{ category: "events", abstract: "User attended strategic planning", overview: "Planning", content: "Strategic planning session attended by user for Q3 roadmap." }]); + const storeA = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractorA = makeExtractor(embedder, llmA, storeA); + let caught = null; + try { + await extractorA.extractAndPersist("User said: I attended strategic planning", "session-abort", { abortOnExtractionMismatch: true }); + } catch (err) { caught = err; } + assert.ok(caught !== null, "T7a: should throw when abortOnExtractionMismatch=true"); + assert.ok(caught.message.includes("extraction aborted"), "T7a: error mentions extraction aborted"); + + const embedderB = makeDeterministicEmbedder(); + const llmB = makeLlm([{ category: "events", abstract: "User attended standup", overview: "Standup", content: "Team standup attended by user." }]); + const storeB = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractorB = makeExtractor(embedderB, llmB, storeB); + let callbackCalled = false; + const statsB = await extractorB.extractAndPersist("User said: standup", "session-callback", { abortOnExtractionMismatch: false, onExtractionValidationFailed() { callbackCalled = true; } }); + assert.strictEqual(callbackCalled, true, "T7b: callback invoked despite no throw"); + assert.strictEqual(statsB.created, 1, "T7b: one entry written despite mismatch"); + + const embedderC = makeDeterministicEmbedder(); + const llmC = makeLlm([{ category: "events", abstract: "User attended architecture review", overview: "Architecture", content: "Architecture review attended." }]); + const storeC = makeStore({ initialCount: 0 }); + const extractorC = makeExtractor(embedderC, llmC, storeC); + const statsC = await extractorC.extractAndPersist("User said: architecture review", "session-normal", {}); + assert.strictEqual(statsC.created, 1, "T7c: normal extraction completed"); + }); + + // -------------------------------------------------------------------------- + // T8: Negative mismatch (over-write) — actual > expected + // -------------------------------------------------------------------------- + it("T8: negative mismatch logs WARNING and never throws", async () => { + const embedder = makeDeterministicEmbedder(); + const llm = makeLlm([{ + category: "events", abstract: "User attended sprint planning for Q3", + overview: "Planning", content: "Sprint planning session attended." }]); + const store = makeStore({ initialCount: 3 }); + const extractor = makeExtractor(embedder, llm, store); + + // Simulate concurrent compactor race: countBefore returns the pre-write + // baseline (3), countAfter returns +2 extra because the compactor deleted + // 1 row and another session inserted 3 rows during our bulkStore window. + // actualCreated = (3 + 2) - 3 = 2, expectedCreated = 1, mismatch = -1 + let callCount = 0; + const originalCount = store.count.bind(store); + store.count = async () => { + callCount++; + // First call (countBefore): rowCount = 3, no extra offset + // Second call (countAfter): rowCount = 4, compactor/session added +2 more + const raw = await originalCount(); + return callCount === 1 ? raw : raw + 2; + }; + + let callbackInvoked = false; + let receivedMismatch = null; + const stats = await extractor.extractAndPersist( + "User said: sprint planning", "session-t8", + { + abortOnExtractionMismatch: true, + onExtractionValidationFailed(validation) { + callbackInvoked = true; + receivedMismatch = validation.mismatch; + }, + }, + ); + + // Negative mismatch: actualCreated (2) > expectedCreated (1) → mismatch = -1 + // abortOnExtractionMismatch=true does NOT throw for negative mismatch + assert.strictEqual(callbackInvoked, true, "T8: callback invoked for negative mismatch"); + assert.ok(receivedMismatch < 0, "T8: mismatch should be negative, got " + receivedMismatch); + assert.strictEqual(stats.created, 1, "T8: extraction completed with 1 entry"); + }); + + // -------------------------------------------------------------------------- + // T9: Callback throws — extraction should complete + // -------------------------------------------------------------------------- + it("T9: callback throwing does not abort extraction", async () => { + const embedder = makeDeterministicEmbedder(); + const llm = makeLlm([ + { category: "preferences", abstract: "User prefers dark mode", overview: "Display", content: "Dark mode." }, + { category: "preferences", abstract: "User prefers quiet workspace", overview: "Workspace", content: "Quiet." }, + ]); + const store = makeStore({ initialCount: 0, dropLastN: 1 }); + const extractor = makeExtractor(embedder, llm, store); + + let callbackThrew = false; + const stats = await extractor.extractAndPersist( + "User preferences", "session-t9", + { + onExtractionValidationFailed() { + callbackThrew = true; + throw new Error("callback threw"); + }, + }, + ); + + assert.strictEqual(callbackThrew, true, "T9: callback was invoked and threw"); + assert.strictEqual(stats.created, 2, "T9: stats shows 2 candidates"); + assert.strictEqual(store.rowCount, 1, "T9: store has 1 entry despite mismatch"); + }); + + // -------------------------------------------------------------------------- + // T10: Async callback support — async callback completes without aborting + // -------------------------------------------------------------------------- + it("T10: async callback support", async () => { + const embedder = makeDeterministicEmbedder(); + const llm = makeLlm([ + { category: "events", abstract: "User attended team sync", overview: "Sync", content: "Team sync." }, + ]); + const store = makeStore({ initialCount: 0, dropLastN: 1 }); // → mismatch = 1 + const extractor = makeExtractor(embedder, llm, store); + + let asyncCallbackCompleted = false; + const stats = await extractor.extractAndPersist( + "User attended team sync", "session-t10", + { + async onExtractionValidationFailed(validation) { + // Simulate async work (e.g., HTTP call to alerting service) + await new Promise((resolve) => setTimeout(resolve, 10)); + asyncCallbackCompleted = true; + }, + }, + ); + + // Wait for async callback to complete + await new Promise((resolve) => setTimeout(resolve, 50)); + + assert.strictEqual(asyncCallbackCompleted, true, "T10: async callback was invoked and completed"); + assert.strictEqual(stats.created, 1, "T10: extraction completed with 1 entry"); + assert.strictEqual(store.rowCount, 0, "T10: store has 0 entries (dropLastN=1 removed the only entry)"); + }); + +}); diff --git a/test/preference-slots.test.mjs b/test/preference-slots.test.mjs index 1849ef31..4e20fabe 100644 --- a/test/preference-slots.test.mjs +++ b/test/preference-slots.test.mjs @@ -125,6 +125,7 @@ function makeGuardExtractor({ vectorSearchResults, onDedupCalled }) { stored.push(...entries); return entries; }, + async count() { return stored.length; }, }; const embedder = { async embed() { @@ -241,6 +242,7 @@ test("dedup guard: non-preference category -> skips guard, goes to LLM", async ( }, async store() {}, async bulkStore() { return []; }, + async count() { return 0; }, }; const embedder = { async embed() { return [0.1, 0.2, 0.3]; }, diff --git a/test/smart-extractor-batch-embed.test.mjs b/test/smart-extractor-batch-embed.test.mjs index ea0120b0..634f291e 100644 --- a/test/smart-extractor-batch-embed.test.mjs +++ b/test/smart-extractor-batch-embed.test.mjs @@ -99,6 +99,9 @@ function makeStore() { async getById(_id, _scopeFilter) { return null; }, + async count() { + return entries.length; + }, get entries() { return [...entries]; }, diff --git a/test/smart-extractor-bulk-store-edge-cases.test.mjs b/test/smart-extractor-bulk-store-edge-cases.test.mjs index 54153158..2eb5ac50 100644 --- a/test/smart-extractor-bulk-store-edge-cases.test.mjs +++ b/test/smart-extractor-bulk-store-edge-cases.test.mjs @@ -61,6 +61,7 @@ class MockStore { async vectorSearch() { return []; } async getById() { return null; } + async count() { return 0; } } // ============================================================ diff --git a/test/smart-extractor-bulk-store.test.mjs b/test/smart-extractor-bulk-store.test.mjs index cef140c5..b39c06f0 100644 --- a/test/smart-extractor-bulk-store.test.mjs +++ b/test/smart-extractor-bulk-store.test.mjs @@ -66,6 +66,7 @@ class MockStore { async vectorSearch() { return []; } async getById() { return null; } + async count() { return 0; } } // ============================================================ diff --git a/test/smart-extractor-scope-filter.test.mjs b/test/smart-extractor-scope-filter.test.mjs index adef26da..488fb12f 100644 --- a/test/smart-extractor-scope-filter.test.mjs +++ b/test/smart-extractor-scope-filter.test.mjs @@ -13,6 +13,7 @@ function makeExtractor(scopeFilters) { }, async store() {}, async bulkStore() {}, + async count() { return 0; }, }; const embedder = {