diff --git a/src/cli/service.ts b/src/cli/service.ts index dbe5567..522983e 100644 --- a/src/cli/service.ts +++ b/src/cli/service.ts @@ -174,6 +174,9 @@ export class NmgService { #nodeSummaryProvider: NodeSummaryProvider | undefined | null; readonly #nodeSummaryDrains = new Set(); readonly #embeddingDrains = new Set(); + /** Project roots whose tessera SimHash backfill pass already ran. One batched + * pass per root per process: rows are stamped once, then never revisited. */ + readonly #tesseraBackfillRoots = new Set(); readonly #stgSyncTimes = new WeakMap>(); #shutdownRequested = false; readonly #maintenanceJobs = new Map(); @@ -1414,7 +1417,9 @@ export class NmgService { // Tessera (bookmark) source: independent of projectDir — tesserae live in the // shared store and are searched alongside memory. Snippet relocation to a // line needs the project root (to read the current file), so the resolver - // is applied here, not in the store. + // is applied here, not in the store. The root also unlocks the one-time + // legacy-fingerprint backfill. + this.#backfillTesseraSimhashes(projectDir); const tesseraHits = this.#resolveTesseraLines( this.#searchTesseraSource(query, options.limit ?? 8), projectDir, @@ -1542,6 +1547,48 @@ export class NmgService { })); } + /** One batched SimHash backfill pass per project root: stamp the drift + * fingerprint onto tesserae written before it existed (or whose file was + * unreadable at write time). Runs on searches that know the project root, + * so legacy bookmarks converge to drift tolerance without a manual + * migration; update-at-most-once in the store means a re-run can never + * overwrite a stamped fingerprint. Per-row failures (file gone, transient + * writer contention) skip that row only; the root is marked done once a + * pass has nothing left to stamp, so an interrupted pass is retried by a + * later search instead of being lost for the process lifetime. */ + #backfillTesseraSimhashes(projectDir: string | undefined): void { + if (!projectDir) return; + const root = resolve(projectDir); + if (this.#tesseraBackfillRoots.has(root)) return; + try { + const store = this.#getStore(); + const missing = store.tesseraeMissingSimhash(); + let unstamped = 0; + for (const row of missing) { + try { + const content = readFileSafe(resolve(root, row.path)); + if (content === null) { + unstamped += 1; + continue; + } + if (!store.updateTesseraSimhash(row.id, simhashToHex(simhash64(content)))) { + unstamped += 1; + } + } catch { + unstamped += 1; // file vanished mid-pass, or a transient writer conflict + } + } + // Done only when this pass had nothing stampable left (all rows already + // stamped, or every remaining row is unreadable). A pass cut short by + // contention stays pending and retries on the next search. + if (missing.length === 0 || missing.length === unstamped) { + this.#tesseraBackfillRoots.add(root); + } + } catch { + // backfill is pure drift-tolerance maintenance, never a search failure + } + } + /** Resolve tessera snippets to current line numbers against a project root. * Best-effort: file missing/unreadable or snippet absent marks stale. */ #resolveTesseraLines(tesserae: TesseraHit[], projectRoot?: string): TesseraHit[] { @@ -1861,9 +1908,19 @@ export class NmgService { .finally(() => this.#embeddingDrains.delete(store)); } - /** Reason to report when the embedding provider is cooling down after a - * failure, or undefined when provider calls may proceed. */ + /** Reason to report when the embedding provider is degraded, or undefined + * when provider calls may proceed (or no provider was ever configured — + * plain lexical is then the normal mode, not a degradation). Two failure + * shapes are covered: a construction failure (provider configured but + * unusable — e.g. a missing API key, no client object exists) is a + * persistent state with no cooldown, so it must be reported directly or + * every search would silently claim a healthy lexical mode; a runtime + * provider failure (429, network) arms the cooldown and is reported until + * it elapses. */ #embeddingDegradedReason(): string | undefined { + if (this.#embeddingClient === null && this.#configuredProvider() !== null) { + return this.#embeddingError ?? "embedding provider configured but unavailable (missing key?)"; + } if (this.#embeddingCooldownUntil <= Date.now()) return undefined; return this.#embeddingError ?? "embedding provider unavailable (cooling down)"; } diff --git a/src/core/store/retrieval.ts b/src/core/store/retrieval.ts index abf602e..098df98 100644 --- a/src/core/store/retrieval.ts +++ b/src/core/store/retrieval.ts @@ -1710,6 +1710,33 @@ export function withRetrieval(Base: TBase) { })); } + /** Lazy SimHash backfill: stamp a 16-hex drift fingerprint onto a tessera + * written before the fingerprint existed. Update-at-most-once — an + * already-stamped row is never rewritten (a fingerprint is the file as it + * was at write time, not a moving baseline). Returns true when stamped. */ + updateTesseraSimhash(id: string, fileSimhash: string): boolean { + if (!/^[0-9a-f]{16}$/u.test(fileSimhash)) return false; + const result = this.db + .prepare( + `UPDATE tesserae SET file_simhash = ? + WHERE id = ? AND file_simhash IS NULL`, + ) + .run(fileSimhash, id); + return Number(result.changes) === 1; + } + + /** Tesserae still missing a drift fingerprint, oldest first. Bounded: the + * backfill caller stamps what it can in one pass. */ + tesseraeMissingSimhash(limit = 200): Array<{ id: string; path: string }> { + const rows = this.db + .prepare( + `SELECT id, path FROM tesserae + WHERE file_simhash IS NULL ORDER BY created_at LIMIT ?`, + ) + .all(Math.max(1, Math.min(limit, 1_000))) as Array<{ id: string; path: string }>; + return rows; + } + searchByVector( query: string, queryVector: readonly number[], diff --git a/src/core/store/schema.ts b/src/core/store/schema.ts index aa5e95d..f1ca304 100644 --- a/src/core/store/schema.ts +++ b/src/core/store/schema.ts @@ -767,6 +767,29 @@ export function ensureTesseraColumns(db: DatabaseSync): void { if (!columns.has("file_simhash")) { db.exec("ALTER TABLE tesserae ADD COLUMN file_simhash TEXT"); } + ensureTesseraFtsSynced(db); +} + +/** Self-heal for the external-content tesserae FTS index: rows can end up in + * the content table without a matching index entry when they were written by + * an older build whose schema predates the sync triggers (observed live: 12 + * of 45 bookmarks were invisible to tesserae search and any UPDATE touching + * them failed with SQLITE_CORRUPT because the trigger's FTS5 'delete' found + * no docsize entry). 'rebuild' re-derives the whole index from the content + * table, which is always authoritative. Cheap to check (two COUNTs at open); + * the rebuild itself only runs on real divergence. */ +function ensureTesseraFtsSynced(db: DatabaseSync): void { + try { + const contentCount = (db.prepare("SELECT COUNT(*) AS n FROM tesserae").get() as Row).n; + const docsizeCount = (db.prepare("SELECT COUNT(*) AS n FROM tesserae_fts_docsize").get() as Row) + .n; + if (contentCount !== docsizeCount) { + db.exec("INSERT INTO tesserae_fts(tesserae_fts) VALUES('rebuild')"); + } + } catch { + // A diverged index degrades search to the memory source only; never block + // store open on index maintenance. + } } export function ensureDeltaColumns(db: DatabaseSync): void { diff --git a/src/integration/agent-surface.ts b/src/integration/agent-surface.ts index 3cc8f03..5398f44 100644 --- a/src/integration/agent-surface.ts +++ b/src/integration/agent-surface.ts @@ -137,6 +137,9 @@ export function renderCompactSearchSurface( }); return [ options.preamble, + context.retrieval?.degraded + ? `[degraded retrieval: mode=${context.retrieval.mode}; reason=${context.retrieval.reason ?? "unknown"}] — results are lexical-only; vector ranking is unavailable until the embedding provider recovers.` + : "", ...lines, ...tesseraLines, context.logicalChainCount > 0 diff --git a/src/integration/search-projection.ts b/src/integration/search-projection.ts index dc2e4e3..e90ef7b 100644 --- a/src/integration/search-projection.ts +++ b/src/integration/search-projection.ts @@ -37,6 +37,10 @@ export interface CompactSearchContext { /** Tessera (bookmark) hits — file locations attached to memories, resolved to * lines (or marked stale) by the service layer. */ tesserae?: TesseraHit[]; + /** Retrieval health: present and `degraded: true` when results came back + * lexically despite a configured provider (cooldown, missing key, index + * not ready). Absent for callers that predate the field. */ + retrieval?: MemoryContext["retrieval"]; } /** Agent-facing search projection. Exact records and evidence remain behind `nmg get`. */ @@ -58,6 +62,9 @@ export function compactSearchContext(context: MemoryContext): CompactSearchConte activeGraphId: context.activeGraph?.id ?? null, deferredMemoryIds: context.progressiveDisclosure?.deferredMemoryIds ?? [], ...(context.tesserae && context.tesserae.length > 0 ? { tesserae: context.tesserae } : {}), + // Degraded retrieval must survive projection — a silently healthy-looking + // lexical surface is how a dead provider went unnoticed for weeks. + ...(context.retrieval?.degraded ? { retrieval: context.retrieval } : {}), }; } diff --git a/tests/cli/process.test.ts b/tests/cli/process.test.ts index 04feba5..756dd4a 100644 --- a/tests/cli/process.test.ts +++ b/tests/cli/process.test.ts @@ -72,13 +72,17 @@ test("search compact JSON exposes bounded headers without exact evidence", () => "--data-dir", directory, ]); + // A clean environment (no embedding provider configured) keeps plain + // lexical the normal mode: no retrieval metadata is projected. A configured + // provider that fails to construct is the explicit-degradation case and is + // covered separately in service.test.ts. const compact = runLauncher([ "search", "Durable detail", "--compact-json", "--data-dir", directory, - ]) as { + ], { NMG_EMBED_PROVIDER: "", NMG_EMBED_BASE_URL: "" } ) as { candidates: Array<{ id: string; preview: string; chains: string[] }>; logicalChainCount: number; activeGraphId: string | null; diff --git a/tests/cli/service.test.ts b/tests/cli/service.test.ts index 4700911..b398172 100644 --- a/tests/cli/service.test.ts +++ b/tests/cli/service.test.ts @@ -1823,6 +1823,36 @@ test("a configured-but-unreachable embedding provider degrades search to lexical } }); +test("a configured provider whose client cannot construct (missing key) degrades search explicitly", async () => { + const directory = mkdtempSync(join(tmpdir(), "nmg-cli-embed-nokey-")); + const service = new NmgService({ + databasePath: join(directory, "nmg.sqlite"), + environment: { NMG_EMBED_PROVIDER: "gemini" }, // no NMG_EMBED_API_KEY + }); + try { + await service.invoke("remember", { + statement: "User prefers Chinese explanations.", + nodeName: "Language preference", + memoryType: "preference", + }); + // Construction fails persistently (no client object, no cooldown), so the + // old behavior was a silent healthy-looking lexical mode. It must now be + // an explicit degradation carrying the construction error. + const searched = await service.invoke("search", { query: "Chinese explanations" }); + assert.equal(searched.results.length, 1); + assert.equal(searched.retrieval?.mode, "lexical"); + assert.equal(searched.retrieval?.degraded, true); + assert.match(searched.retrieval?.reason ?? "", /api key/iu); + const status = await service.invoke("status"); + assert.equal(status.embedding.configured, true); + assert.equal(status.embedding.indexId, null); + assert.match(String(status.embedding.reason ?? ""), /api key/iu); + } finally { + service.close(); + removeTempDirectory(directory); + } +}); + test("opt-in embedding auto-sync makes remembered records available to hybrid search", async () => { const directory = mkdtempSync(join(tmpdir(), "nmg-cli-embedding-auto-sync-")); const server = createServer((request, response) => { diff --git a/tests/core/tesserae-simhash.test.ts b/tests/core/tesserae-simhash.test.ts index 8ae0fd5..2590ce9 100644 --- a/tests/core/tesserae-simhash.test.ts +++ b/tests/core/tesserae-simhash.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import test from "node:test"; import { NmgService } from "../../src/cli/service.ts"; @@ -25,7 +25,16 @@ async function withProject( await run(service, projectDir); } finally { service.close(); - rmSync(root, { recursive: true, force: true }); + // Windows keeps SQLite file handles briefly alive after close; a second + // service in a test widens that window, so removal tolerates it. + for (let attempt = 0; attempt < 10; attempt += 1) { + try { + rmSync(root, { recursive: true, force: true }); + break; + } catch { + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); + } + } } } @@ -163,3 +172,176 @@ test("tessera simhash: unrelated content rewrite reports stale, not relocated", assert.ok(hit.stale, "honestly stale after unrelated rewrite"); }); }); + +test("tessera simhash: legacy rows without a fingerprint are backfilled by the first search", async () => { + await withProject(async (service, projectDir) => { + // Write a real tessera, then strip its fingerprint to simulate a legacy + // row from before ticket 8 (the shape of every pre-existing bookmark). + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "alpha.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + const { DatabaseSync } = await import("node:sqlite"); + const db = new DatabaseSync(join(dirname(projectDir), "nmg.sqlite")); + db.prepare("UPDATE tesserae SET file_simhash = NULL").run(); + db.close(); + + // First search while the file still exists: the backfill pass stamps the + // fingerprint from current content, and the hit carries it immediately. + const first = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const before = first.tesserae?.find((tessera: { path: string }) => tessera.path === "alpha.ts"); + assert.ok(before, "legacy tessera surfaces on the backfilling search"); + assert.match(String(before.fileSimhash ?? ""), /^[0-9a-f]{16}$/u, "backfilled on first search"); + + // Now the file moves: the backfilled fingerprint enables SimHash recovery. + mkdirSync(join(projectDir, "lib"), { recursive: true }); + writeFileSync( + join(projectDir, "lib", "alpha.ts"), + [ + "export const alpha = 1;", + "// The indexing pipeline drains embedding batches.", + "export function beta() { return alpha; }", + ].join("\n"), + ); + rmSync(join(projectDir, "alpha.ts")); + + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + const hit = searched.tesserae?.find( + (tessera: { path: string }) => tessera.path === "lib/alpha.ts", + ); + assert.ok(hit, "backfilled legacy tessera survives a file move"); + assert.ok(hit.relocated, "recovered via the backfilled drift fingerprint"); + // A second search still works — the stamp is never rewritten or doubled. + const again = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + assert.ok( + again.tesserae?.some((tessera: { path: string }) => tessera.path === "lib/alpha.ts"), + "second search still surfaces the relocated hit", + ); + }); +}); + +test("tessera simhash: backfill skips files that are unreadable, stamping nothing", async () => { + await withProject(async (service, projectDir) => { + await service.invoke("remember", { + statement: "alpha.ts pipeline drains batches", + nodeName: "alpha pipeline", + projectDir, + tesserae: [ + { + path: "gone.ts", + snippet: "The indexing pipeline drains embedding batches.", + label: "pipeline note", + }, + ], + }); + const { DatabaseSync } = await import("node:sqlite"); + const db = new DatabaseSync(join(dirname(projectDir), "nmg.sqlite")); + db.prepare("UPDATE tesserae SET file_simhash = NULL").run(); + db.close(); + + const searched = await service.invoke("search", { + query: "pipeline drains batches", + projectDir, + }); + // Search must not fail; the hit surfaces stale (file unreadable), and no + // fingerprint was invented for a file that cannot be read. + const hit = searched.tesserae?.find((tessera: { path: string }) => tessera.path === "gone.ts"); + assert.ok(hit, "unbackfillable tessera still surfaces"); + assert.ok(hit.stale, "honestly stale when the file is gone"); + assert.equal(hit.fileSimhash, undefined, "no fingerprint invented for a missing file"); + }); +}); + +test("tessera fts: rows missing from the fts index are rebuilt and then backfilled", async () => { + const { DatabaseSync } = await import("node:sqlite"); + const root = mkdtempSync(join(tmpdir(), "nmg-tessera-fts-heal-")); + const databasePath = join(root, "nmg.sqlite"); + const projectDir = join(root, "project"); + mkdirSync(projectDir, { recursive: true }); + writeFileSync( + join(projectDir, "alpha.ts"), + [ + "export const alpha = 1;", + "// The indexing pipeline drains embedding batches.", + "export function beta() { return alpha; }", + ].join("\n"), + ); + try { + // Simulate the live divergence observed in the LTG store: a row written by + // an older build whose schema predates the FTS sync triggers. Content row + // exists; the index (created later by the current schema) starts empty, so + // the row is invisible to tesserae search and any UPDATE touching it fails + // with SQLITE_CORRUPT from the trigger's FTS5 'delete'. + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + CREATE TABLE tesserae ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + snippet TEXT NOT NULL, + label TEXT NOT NULL DEFAULT '', + kind TEXT, + memory_id TEXT, + created_at TEXT NOT NULL, + file_simhash TEXT + ); + `); + legacy + .prepare( + `INSERT INTO tesserae (id, path, snippet, label, kind, memory_id, created_at, file_simhash) + VALUES (?, ?, ?, 'pipeline note', NULL, NULL, ?, NULL)`, + ) + .run("55555555-5555-4555-8555-555555555555", "alpha.ts", "The indexing pipeline drains embedding batches.", new Date().toISOString()); + legacy.close(); + + // Open with the current schema: FTS + triggers are created now, and the + // open-time self-heal must detect content/index divergence and rebuild. + const healed = new NmgService({ databasePath, environment: {} }); + try { + const found = (await healed.invoke("search", { + query: "pipeline drains batches", + projectDir, + })) as { tesserae?: Array<{ path: string; fileSimhash?: string }> }; + const hit = found.tesserae?.find((tessera) => tessera.path === "alpha.ts"); + assert.ok(hit, "orphaned tessera is searchable again after the open-time rebuild"); + assert.match(String(hit?.fileSimhash ?? ""), /^[0-9a-f]{16}$/u, "backfill stamps after heal"); + + const check = new DatabaseSync(databasePath, { readOnly: true }); + const docsize = ( + check.prepare("SELECT COUNT(*) AS n FROM tesserae_fts_docsize").get() as { n: number } + ).n; + const content = ( + check.prepare("SELECT COUNT(*) AS n FROM tesserae").get() as { n: number } + ).n; + check.close(); + assert.equal(docsize, content, "fts index converges to the content table"); + } finally { + healed.close(); + } + } finally { + for (let attempt = 0; attempt < 10; attempt += 1) { + try { + rmSync(root, { recursive: true, force: true }); + break; + } catch { + await new Promise((resolveWait) => setTimeout(resolveWait, 100)); + } + } + } +});