Skip to content

feat(extract): Phase 1 — extraction write validation with count-before/after (issue #693) #747

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jlin53882 wants to merge 8 commits into
base: master
Choose a base branch
from
Open

feat(extract): Phase 1 — extraction write validation with count-before/after (issue #693) #747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b66fb0a
feat(extract): Phase 1 extraction write validation (Issue #693)
jlin53882 May 9, 2026
bd174ca
fix(extract): address PR747 review comments (F1-F6, MR1-MR4)
jlin53882 May 10, 2026
78a3818
fix(extract): MR1 use ExtractionValidation type, F1 add default log
penggaolai May 12, 2026
42add5b
fix(extract): PR747 all review fixes from 83a2322
penggaolai May 12, 2026
4566ef1
fix(ci): add missing Issue #665 comment
penggaolai May 12, 2026
8e13892
fix(extract): F1 bulkStore error isolation + T8 negative mismatch tes…
jlin53882 May 14, 2026
fe238e0
fix(extract): MR1/MR2/MR3 review fixes for PR #747
jlin53882 May 14, 2026
d64f178
fix(extract): F2/F3/F5/F6/Nice-to-have review fixes for PR #747
jlin53882 May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 91 additions & 87 deletions scripts/ci-test-manifest.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,87 +1,91 @@
export const CI_TEST_GROUPS = [
"cli-smoke",
"core-regression",
"storage-and-schema",
"llm-clients-and-auth",
"packaging-and-workflow",
];

export const CI_TEST_MANIFEST = [
{ group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" },
{ group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" },
{ group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" },
{ group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] },
{ group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" },
{ group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] },
{ group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] },
{ group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] },
{ group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] },
// Issue #598 regression tests
{ group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" },
// Issue #629 batch embedding fix
{ group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" },
// Issue #665 bulkStore tests
// Issue #690 cross-call batch accumulator tests
{ group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] },
// Issue #665 bulkStore tests (from upstream)
{ group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] },
// Issue #680 regression tests (from upstream)
{ group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] },
// Issue #606 SDK migration Bug 2 regression tests
{ group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" },
// PR #713 inference regression tests - inferProviderFromBaseURL + model fallback
{ group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" },
// Issue #736 recall governance - isRecallUsed() unit tests
{ group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] },
// Issue #492 agentId validation tests
{ group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] },
// Tier 1 memory counter fix
{ group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] },
];

export function getEntriesForGroup(group) {
if (!CI_TEST_GROUPS.includes(group)) {
throw new Error(`Unknown CI test group: ${group}`);
}

return CI_TEST_MANIFEST.filter((entry) => entry.group === group);
}
export const CI_TEST_GROUPS = [
"cli-smoke",
"core-regression",
"storage-and-schema",
"llm-clients-and-auth",
"packaging-and-workflow",
];

export const CI_TEST_MANIFEST = [
{ group: "llm-clients-and-auth", runner: "node", file: "test/embedder-error-hints.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/cjk-recursion-regression.test.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/migrate-legacy-schema.test.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/config-session-strategy-migration.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/scope-access-undefined.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/reflection-bypass-hook.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-scope-filter.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/store-empty-scope-filter.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/storage-path-normalization.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/auto-recall-timeout.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] },
{ group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" },
{ group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/per-agent-auto-recall.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" },
{ group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" },
{ group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" },
{ group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] },
{ group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/smart-metadata-v2.mjs" },
{ group: "storage-and-schema", runner: "node", file: "test/vector-search-cosine.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/context-support-e2e.mjs" },
{ group: "core-regression", runner: "node", file: "test/temporal-facts.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/memory-update-supersede.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/memory-upgrader-diagnostics.test.mjs" },
{ group: "llm-clients-and-auth", runner: "node", file: "test/llm-api-key-client.test.mjs", args: ["--test"] },
{ group: "llm-clients-and-auth", runner: "node", file: "test/llm-oauth-client.test.mjs", args: ["--test"] },
{ group: "llm-clients-and-auth", runner: "node", file: "test/cli-oauth-login.test.mjs", args: ["--test"] },
{ group: "packaging-and-workflow", runner: "node", file: "test/workflow-fork-guards.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/clawteam-scope.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/cross-process-lock.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/lock-stress-test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/lock-release-on-error.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/preference-slots.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/is-latest-auto-supersede.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/temporal-awareness.test.mjs", args: ["--test"] },
// Issue #598 regression tests
{ group: "core-regression", runner: "node", file: "test/store-serialization.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/mmr-tiny.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/access-tracker-retry.test.mjs" },
{ group: "core-regression", runner: "node", file: "test/embedder-cache.test.mjs" },
// Issue #629 batch embedding fix
{ group: "llm-clients-and-auth", runner: "node", file: "test/embedder-ollama-batch-routing.test.mjs" },
// Issue #665 bulkStore tests
// Issue #690 cross-call batch accumulator tests
{ group: "storage-and-schema", runner: "node", file: "test/issue-690-cross-call-batch.test.mjs", args: ["--test"] },
// Issue #665 bulkStore tests (from upstream)
{ group: "storage-and-schema", runner: "node", file: "test/bulk-store.test.mjs", args: ["--test"] },
// Issue #665 bulkStore tests (from upstream)
{ group: "storage-and-schema", runner: "node", file: "test/bulk-store-edge-cases.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store.test.mjs", args: ["--test"] },
{ group: "storage-and-schema", runner: "node", file: "test/smart-extractor-bulk-store-edge-cases.test.mjs", args: ["--test"] },
// Issue #680 regression tests (from upstream)
{ group: "core-regression", runner: "node", file: "test/memory-reflection-issue680-tdd.test.mjs", args: ["--test"] },
// Issue #606 SDK migration Bug 2 regression tests
{ group: "core-regression", runner: "node", file: "test/issue606_sdk-migration.test.mjs" },
// PR #713 inference regression tests - inferProviderFromBaseURL + model fallback
{ group: "core-regression", runner: "node", file: "test/infer-provider-from-baseurl.test.mjs" },
// Issue #736 recall governance - isRecallUsed() unit tests
{ group: "core-regression", runner: "node", file: "test/is-recall-used.test.mjs", args: ["--test"] },
// Issue #492 agentId validation tests
{ group: "core-regression", runner: "node", file: "test/agentid-validation.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/command-reflection-guard.test.mjs", args: ["--test"] },
// Tier 1 memory counter fix
{ group: "core-regression", runner: "node", file: "test/tier1-counters.test.mjs", args: ["--test"] },
// Issue #693 extraction write validation tests
{ group: "core-regression", runner: "node", file: "test/extraction-validation.test.mjs", args: ["--test"] },
{ group: "core-regression", runner: "node", file: "test/dedup-false-alarm.test.mjs", args: ["--test"] },
];

export function getEntriesForGroup(group) {
if (!CI_TEST_GROUPS.includes(group)) {
throw new Error(`Unknown CI test group: ${group}`);
}

return CI_TEST_MANIFEST.filter((entry) => entry.group === group);
}
23 changes: 23 additions & 0 deletions src/memory-categories.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,29 @@ export type ExtractionStats = {
superseded?: number; // temporal fact replacements
};

/**
* Payload delivered to `ExtractPersistOptions.onExtractionValidationFailed`
* when the number of entries actually written to the store differs from
* the number of candidates produced by the LLM.
*
* @see ExtractPersistOptions.onExtractionValidationFailed
*/
export type ExtractionValidation = {
/** Number of candidates the LLM intended to create (createEntries.length) */
expected: number;
/** Number of rows actually written (countAfter - countBefore) */
actual: number;
/**
* expected - actual; positive = under-write (SIGKILL/OOM partial write, fewer rows),
* negative = over-write (concurrent session ADDED rows before our countAfter, more rows).
* positive (under-write): callback invoked; if abortOnExtractionMismatch=true throws.
* negative (over-write): always logged as WARNING and never throws.
*/
mismatch: number;
/** Session key passed to extractAndPersist */
sessionKey: string;
};

/** Validate and normalize a category string. */
export function normalizeCategory(raw: string): MemoryCategory | null {
const lower = raw.toLowerCase().trim();
Expand Down
Loading
Loading