fix(tesserae): legacy SimHash backfill + open-time FTS self-heal + explicit embedding degradation

src/cli/service.ts

@@ -174,6 +174,9 @@
   #nodeSummaryProvider: NodeSummaryProvider | undefined | null;
   readonly #nodeSummaryDrains = new Set<NmgStore>();
   readonly #embeddingDrains = new Set<NmgStore>();
+  /** Project roots whose tessera SimHash backfill pass already ran. One batched
+   *  pass per root per process: rows are stamped once, then never revisited. */
+  readonly #tesseraBackfillRoots = new Set<string>();
   readonly #stgSyncTimes = new WeakMap<NmgStore, Map<string, number>>();
   #shutdownRequested = false;
   readonly #maintenanceJobs = new Map<NmgStore, NodeJS.Immediate>();
@@ -1133,123 +1136,123 @@
     this.#maintenanceJobs.set(store, job);
   }
 
   #resolveRemember(params: NmgResolveRememberParams): NmgMethodResult["resolveRemember"] {
     const stores = params.projectDir
       ? [this.#getStgStore(params.projectDir, params.sessionId), this.#getStore()]
       : [this.#getStore()];
     if (params.action === "forget") {
       const store = stores.find(
         (candidate) => candidate.getMemory(params.memoryId, params.sessionId) !== null,
       );
       if (!store) {
         throw new NmgProtocolError("INVALID_PARAMS", `memory ${params.memoryId} does not exist`);
       }
       return {
         action: "forget",
         memoryId: params.memoryId,
         deleted: store.deleteMemory(params.memoryId) !== null,
       };
     }
     if (params.action === "resolve" || params.action === "reopen") {
       const store = stores.find(
         (candidate) => candidate.getMemory(params.memoryId, params.sessionId) !== null,
       );
       if (!store) {
         throw new NmgProtocolError("INVALID_PARAMS", `memory ${params.memoryId} does not exist`);
       }
       const state = store.setMemoryResolution(
         params.memoryId,
         params.action === "resolve" ? "resolved" : "reopened",
         {
           relatedMemoryIds: params.relatedMemoryIds,
           reason: params.reason,
         },
       );
       return { action: params.action, ...state };
     }
     const store = stores.find(
       (candidate) => candidate.getMemory(params.newMemoryId, params.sessionId) !== null,
     );
     if (!store) {
       throw new NmgProtocolError(
         "INVALID_PARAMS",
         `new memory ${params.newMemoryId} does not exist`,
       );
     }
     const newer = store.getMemory(params.newMemoryId, params.sessionId);
     if (params.action === "relate") {
       const related = store.getMemory(params.relatedMemoryId, params.sessionId);
       if (!newer || !related) {
         throw new NmgProtocolError(
           "INVALID_PARAMS",
           "relation targets must exist in the same LTG or session-owned STG store",
         );
       }
       if (
         ["conflict", "refines", "same_entity"].includes(params.relationJudgement) &&
         !scopesOverlap(newer.scope, related.scope)
       ) {
         throw new NmgProtocolError(
           "INVALID_PARAMS",
           `${params.relationJudgement} requires non-conflicting scope; use distinct for different entities or retain both memories`,
         );
       }
       if (params.relationJudgement === "conflict" && !validityIntervalsOverlap(newer, related)) {
         throw new NmgProtocolError(
           "INVALID_PARAMS",
           "conflict requires overlapping validity; sequential values should remain temporal states or use supersede",
         );
       }
       const relationType = {
         conflict: "contradicts",
         distinct: "distinct_from",
         refines: "refines",
         related: "related_to",
         same_entity: "same_as",
       } as const;
       const proposal = store.proposeSemanticRelation({
         sourceNodeId: newer.nodeId,
         targetNodeId: related.nodeId,
         relationType: relationType[params.relationJudgement],
         evidenceMemoryIds: [newer.id, related.id],
         confidence: params.confidence,
       });
       return {
         action: "relate",
         newMemoryId: newer.id,
         relatedMemoryId: related.id,
         proposal,
       };
     }
     const stale = store.getMemory(params.supersededMemoryId, params.sessionId);
     if (!newer || !stale) {
       throw new NmgProtocolError(
         "INVALID_PARAMS",
         "supersession targets must exist in the same LTG or session-owned STG store",
       );
     }
     if (!sameScope(newer.scope, stale.scope)) {
       throw new NmgProtocolError(
         "INVALID_PARAMS",
         "supersession requires identical scope; use distinct memories for different scopes",
       );
     }
     store.recordFeedback({
       sessionId: params.sessionId,
       supersede: {
         newMemoryId: params.newMemoryId,
         supersededMemoryId: params.supersededMemoryId,
         reason: params.reason,
       },
     });
     return {
       action: "supersede",
       newMemoryId: params.newMemoryId,
       supersededMemoryId: params.supersededMemoryId,
       applied:
         store.getMemory(params.supersededMemoryId, params.sessionId)?.status === "superseded",
     };
   }
 
   #recordClaimOutcomes(
     params: NmgRecordClaimOutcomesParams,
@@ -1414,7 +1417,9 @@
     // Tessera (bookmark) source: independent of projectDir — tesserae live in the
     // shared store and are searched alongside memory. Snippet relocation to a
     // line needs the project root (to read the current file), so the resolver
-    // is applied here, not in the store.
+    // is applied here, not in the store. The root also unlocks the one-time
+    // legacy-fingerprint backfill.
+    this.#backfillTesseraSimhashes(projectDir);
     const tesseraHits = this.#resolveTesseraLines(
       this.#searchTesseraSource(query, options.limit ?? 8),
       projectDir,
@@ -1542,6 +1547,48 @@
     }));
   }
 
+  /** One batched SimHash backfill pass per project root: stamp the drift
+   *  fingerprint onto tesserae written before it existed (or whose file was
+   *  unreadable at write time). Runs on searches that know the project root,
+   *  so legacy bookmarks converge to drift tolerance without a manual
+   *  migration; update-at-most-once in the store means a re-run can never
+   *  overwrite a stamped fingerprint. Per-row failures (file gone, transient
+   *  writer contention) skip that row only; the root is marked done once a
+   *  pass has nothing left to stamp, so an interrupted pass is retried by a
+   *  later search instead of being lost for the process lifetime. */
+  #backfillTesseraSimhashes(projectDir: string | undefined): void {
+    if (!projectDir) return;
+    const root = resolve(projectDir);
+    if (this.#tesseraBackfillRoots.has(root)) return;
+    try {
+      const store = this.#getStore();
+      const missing = store.tesseraeMissingSimhash();
+      let unstamped = 0;
+      for (const row of missing) {
+        try {
+          const content = readFileSafe(resolve(root, row.path));
+          if (content === null) {
+            unstamped += 1;
+            continue;
+          }
+          if (!store.updateTesseraSimhash(row.id, simhashToHex(simhash64(content)))) {
+            unstamped += 1;
+          }
+        } catch {
+          unstamped += 1; // file vanished mid-pass, or a transient writer conflict
+        }
+      }
+      // Done only when this pass had nothing stampable left (all rows already
+      // stamped, or every remaining row is unreadable). A pass cut short by
+      // contention stays pending and retries on the next search.
+      if (missing.length === 0 || missing.length === unstamped) {
+        this.#tesseraBackfillRoots.add(root);
+      }
+    } catch {
+      // backfill is pure drift-tolerance maintenance, never a search failure
+    }
+  }
+
   /** Resolve tessera snippets to current line numbers against a project root.
    *  Best-effort: file missing/unreadable or snippet absent marks stale. */
   #resolveTesseraLines(tesserae: TesseraHit[], projectRoot?: string): TesseraHit[] {
@@ -1861,9 +1908,19 @@
       .finally(() => this.#embeddingDrains.delete(store));
   }
 
-  /** Reason to report when the embedding provider is cooling down after a
-   *  failure, or undefined when provider calls may proceed. */
+  /** Reason to report when the embedding provider is degraded, or undefined
+   *  when provider calls may proceed (or no provider was ever configured —
+   *  plain lexical is then the normal mode, not a degradation). Two failure
+   *  shapes are covered: a construction failure (provider configured but
+   *  unusable — e.g. a missing API key, no client object exists) is a
+   *  persistent state with no cooldown, so it must be reported directly or
+   *  every search would silently claim a healthy lexical mode; a runtime
+   *  provider failure (429, network) arms the cooldown and is reported until
+   *  it elapses. */
   #embeddingDegradedReason(): string | undefined {
+    if (this.#embeddingClient === null && this.#configuredProvider() !== null) {
+      return this.#embeddingError ?? "embedding provider configured but unavailable (missing key?)";
+    }
     if (this.#embeddingCooldownUntil <= Date.now()) return undefined;
     return this.#embeddingError ?? "embedding provider unavailable (cooling down)";
   }

src/core/store/retrieval.ts

@@ -1710,6 +1710,33 @@ export function withRetrieval<TBase extends Constructor>(Base: TBase) {
       }));
     }
 
+    /** Lazy SimHash backfill: stamp a 16-hex drift fingerprint onto a tessera
+     *  written before the fingerprint existed. Update-at-most-once — an
+     *  already-stamped row is never rewritten (a fingerprint is the file as it
+     *  was at write time, not a moving baseline). Returns true when stamped. */
+    updateTesseraSimhash(id: string, fileSimhash: string): boolean {
+      if (!/^[0-9a-f]{16}$/u.test(fileSimhash)) return false;
+      const result = this.db
+        .prepare(
+          `UPDATE tesserae SET file_simhash = ?
+           WHERE id = ? AND file_simhash IS NULL`,
+        )
+        .run(fileSimhash, id);
+      return Number(result.changes) === 1;
+    }
+
+    /** Tesserae still missing a drift fingerprint, oldest first. Bounded: the
+     *  backfill caller stamps what it can in one pass. */
+    tesseraeMissingSimhash(limit = 200): Array<{ id: string; path: string }> {
+      const rows = this.db
+        .prepare(
+          `SELECT id, path FROM tesserae
+           WHERE file_simhash IS NULL ORDER BY created_at LIMIT ?`,
+        )
+        .all(Math.max(1, Math.min(limit, 1_000))) as Array<{ id: string; path: string }>;
+      return rows;
+    }
+
     searchByVector(
       query: string,
       queryVector: readonly number[],

src/core/store/schema.ts

@@ -767,6 +767,29 @@ export function ensureTesseraColumns(db: DatabaseSync): void {
   if (!columns.has("file_simhash")) {
     db.exec("ALTER TABLE tesserae ADD COLUMN file_simhash TEXT");
   }
+  ensureTesseraFtsSynced(db);
+}
+
+/** Self-heal for the external-content tesserae FTS index: rows can end up in
+ *  the content table without a matching index entry when they were written by
+ *  an older build whose schema predates the sync triggers (observed live: 12
+ *  of 45 bookmarks were invisible to tesserae search and any UPDATE touching
+ *  them failed with SQLITE_CORRUPT because the trigger's FTS5 'delete' found
+ *  no docsize entry). 'rebuild' re-derives the whole index from the content
+ *  table, which is always authoritative. Cheap to check (two COUNTs at open);
+ *  the rebuild itself only runs on real divergence. */
+function ensureTesseraFtsSynced(db: DatabaseSync): void {
+  try {
+    const contentCount = (db.prepare("SELECT COUNT(*) AS n FROM tesserae").get() as Row).n;
+    const docsizeCount = (db.prepare("SELECT COUNT(*) AS n FROM tesserae_fts_docsize").get() as Row)
+      .n;
+    if (contentCount !== docsizeCount) {
+      db.exec("INSERT INTO tesserae_fts(tesserae_fts) VALUES('rebuild')");
+    }
+  } catch {
+    // A diverged index degrades search to the memory source only; never block
+    // store open on index maintenance.
+  }
 }
 
 export function ensureDeltaColumns(db: DatabaseSync): void {

src/integration/agent-surface.ts

@@ -137,6 +137,9 @@ export function renderCompactSearchSurface(
   });
   return [
     options.preamble,
+    context.retrieval?.degraded
+      ? `[degraded retrieval: mode=${context.retrieval.mode}; reason=${context.retrieval.reason ?? "unknown"}] — results are lexical-only; vector ranking is unavailable until the embedding provider recovers.`
+      : "",
     ...lines,
     ...tesseraLines,
     context.logicalChainCount > 0

src/integration/search-projection.ts

@@ -37,6 +37,10 @@ export interface CompactSearchContext {
   /** Tessera (bookmark) hits — file locations attached to memories, resolved to
    *  lines (or marked stale) by the service layer. */
   tesserae?: TesseraHit[];
+  /** Retrieval health: present and `degraded: true` when results came back
+   *  lexically despite a configured provider (cooldown, missing key, index
+   *  not ready). Absent for callers that predate the field. */
+  retrieval?: MemoryContext["retrieval"];
 }
 
 /** Agent-facing search projection. Exact records and evidence remain behind `nmg get`. */
@@ -58,6 +62,9 @@ export function compactSearchContext(context: MemoryContext): CompactSearchConte
     activeGraphId: context.activeGraph?.id ?? null,
     deferredMemoryIds: context.progressiveDisclosure?.deferredMemoryIds ?? [],
     ...(context.tesserae && context.tesserae.length > 0 ? { tesserae: context.tesserae } : {}),
+    // Degraded retrieval must survive projection — a silently healthy-looking
+    // lexical surface is how a dead provider went unnoticed for weeks.
+    ...(context.retrieval?.degraded ? { retrieval: context.retrieval } : {}),
   };
 }
 

tests/cli/process.test.ts

@@ -72,13 +72,17 @@ test("search compact JSON exposes bounded headers without exact evidence", () =>
       "--data-dir",
       directory,
     ]);
+    // A clean environment (no embedding provider configured) keeps plain
+    // lexical the normal mode: no retrieval metadata is projected. A configured
+    // provider that fails to construct is the explicit-degradation case and is
+    // covered separately in service.test.ts.
     const compact = runLauncher([
       "search",
       "Durable detail",
       "--compact-json",
       "--data-dir",
       directory,
-    ]) as {
+    ], { NMG_EMBED_PROVIDER: "", NMG_EMBED_BASE_URL: "" } ) as {
       candidates: Array<{ id: string; preview: string; chains: string[] }>;
       logicalChainCount: number;
       activeGraphId: string | null;

tests/cli/service.test.ts

@@ -1823,6 +1823,36 @@ test("a configured-but-unreachable embedding provider degrades search to lexical
   }
 });
 
+test("a configured provider whose client cannot construct (missing key) degrades search explicitly", async () => {
+  const directory = mkdtempSync(join(tmpdir(), "nmg-cli-embed-nokey-"));
+  const service = new NmgService({
+    databasePath: join(directory, "nmg.sqlite"),
+    environment: { NMG_EMBED_PROVIDER: "gemini" }, // no NMG_EMBED_API_KEY
+  });
+  try {
+    await service.invoke("remember", {
+      statement: "User prefers Chinese explanations.",
+      nodeName: "Language preference",
+      memoryType: "preference",
+    });
+    // Construction fails persistently (no client object, no cooldown), so the
+    // old behavior was a silent healthy-looking lexical mode. It must now be
+    // an explicit degradation carrying the construction error.
+    const searched = await service.invoke("search", { query: "Chinese explanations" });
+    assert.equal(searched.results.length, 1);
+    assert.equal(searched.retrieval?.mode, "lexical");
+    assert.equal(searched.retrieval?.degraded, true);
+    assert.match(searched.retrieval?.reason ?? "", /api key/iu);
+    const status = await service.invoke("status");
+    assert.equal(status.embedding.configured, true);
+    assert.equal(status.embedding.indexId, null);
+    assert.match(String(status.embedding.reason ?? ""), /api key/iu);
+  } finally {
+    service.close();
+    removeTempDirectory(directory);
+  }
+});
+
 test("opt-in embedding auto-sync makes remembered records available to hybrid search", async () => {
   const directory = mkdtempSync(join(tmpdir(), "nmg-cli-embedding-auto-sync-"));
   const server = createServer((request, response) => {