diff --git a/src/cli/debug/refinements.test.ts b/src/cli/debug/refinements.test.ts index 7a34c5c8d95..9d1d7a43ef1 100644 --- a/src/cli/debug/refinements.test.ts +++ b/src/cli/debug/refinements.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, spyOn } from "bun:test"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; +import { Config } from "@/node/config"; import { appendRefinementEvent } from "@/node/services/refinement/refinementJournal"; import { TestTempDir } from "@/node/services/tools/testHelpers"; import { refinementsCommand } from "./refinements"; @@ -11,8 +12,14 @@ import { refinementsCommand } from "./refinements"; * created, inside a `/sessions/` layout so the confinement roots * resolve like a real mux home. */ -async function seedFixture(root: string): Promise<{ sessionDir: string; skillFile: string }> { +async function seedFixture( + root: string +): Promise<{ sessionDir: string; skillFile: string; config: Config }> { const sessionDir = path.join(root, "sessions", "ws-cli"); + // Rollback resolves shared-memory ownership from a config.json that must + // exist (an absent one reads as mid-rewrite): persist the empty default. + const config = new Config(root); + await config.editConfig((cfg) => cfg); const skillFile = path.join(root, "checkout", ".mux", "skills", "cli-skill", "SKILL.md"); await fsPromises.mkdir(path.dirname(skillFile), { recursive: true }); await fsPromises.writeFile(skillFile, "---\nname: cli-skill\n---\n", "utf-8"); @@ -24,7 +31,7 @@ async function seedFixture(root: string): Promise<{ sessionDir: string; skillFil inverse: { op: "delete-files", paths: [skillFile] }, evidence: { toolName: "agent_skill_write" }, }); - return { sessionDir, skillFile }; + return { sessionDir, skillFile, config }; } describe("debug refinements command", () => { @@ -37,7 +44,7 @@ describe("debug refinements command", () => { it("lists rows and performs a rollback with lineage output", async () => { using tempDir = new TestTempDir("test-debug-refinements"); - const { sessionDir, skillFile } = await seedFixture(tempDir.path); + const { sessionDir, skillFile, config } = await seedFixture(tempDir.path); const lines: string[] = []; const logSpy = spyOn(console, "log").mockImplementation((line: string) => { lines.push(line); @@ -50,7 +57,7 @@ describe("debug refinements command", () => { const rowId = lines[0].split(" ")[0]; lines.length = 0; - await refinementsCommand("ws-cli", { sessionDir, rollback: rowId }); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: rowId }); // Earlier test files in the same process may have reset exitCode to 0, // so assert "not failing" rather than "never touched". expect(process.exitCode ?? 0).toBe(0); @@ -74,14 +81,14 @@ describe("debug refinements command", () => { it("reports refusals on stderr and sets a failing exit code", async () => { using tempDir = new TestTempDir("test-debug-refinements-refuse"); - const { sessionDir } = await seedFixture(tempDir.path); + const { sessionDir, config } = await seedFixture(tempDir.path); const logSpy = spyOn(console, "log").mockImplementation(() => undefined); const errors: string[] = []; const errorSpy = spyOn(console, "error").mockImplementation((line: string) => { errors.push(line); }); try { - await refinementsCommand("ws-cli", { sessionDir, rollback: "missing-id" }); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: "missing-id" }); expect(process.exitCode).toBe(1); expect(errors.join("\n")).toContain("No refinement row"); } finally { @@ -89,4 +96,34 @@ describe("debug refinements command", () => { errorSpy.mockRestore(); } }); + + it("refuses a rollback while config.json is absent instead of assuming self-ownership", async () => { + using tempDir = new TestTempDir("test-debug-refinements-noconfig"); + const { sessionDir, skillFile, config } = await seedFixture(tempDir.path); + const lines: string[] = []; + const logSpy = spyOn(console, "log").mockImplementation((line: string) => { + lines.push(line); + }); + const errors: string[] = []; + const errorSpy = spyOn(console, "error").mockImplementation((line: string) => { + errors.push(line); + }); + try { + await refinementsCommand("ws-cli", { sessionDir }); + const rowId = lines[0].split(" ")[0]; + await fsPromises.rm(path.join(tempDir.path, "config.json")); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: rowId }); + expect(process.exitCode).toBe(1); + expect(errors.join("\n")).toContain("shared-memory ownership could not be resolved"); + expect( + await fsPromises.access(skillFile).then( + () => true, + () => false + ) + ).toBe(true); + } finally { + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); }); diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index d90ca13f4f4..4a20e7136c3 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -1,5 +1,10 @@ import * as path from "path"; -import { defaultConfig } from "@/node/config"; +import { defaultConfig, type Config } from "@/node/config"; +import { + resolveSharedWorkspaceMemoryTopology, + type SharedWorkspaceMemoryTopology, +} from "@/node/services/memoryWorkspaceOwner"; +import { getErrorMessage } from "@/common/utils/errors"; import { MemoryRefinementActionSchema, RollbackRefinementActionSchema, @@ -37,6 +42,8 @@ export interface RefinementsCommandOptions { force?: boolean; /** Test seam: bypass ~/.mux session resolution for fixture sessions. */ sessionDir?: string; + /** Test seam: the config whose task tree resolves shared-memory ownership. */ + config?: Pick; } /** @@ -50,8 +57,32 @@ export async function refinementsCommand( const sessionDir = opts.sessionDir ?? path.join(defaultConfig.sessionsDir, workspaceId); if (opts.rollback !== undefined) { + // Sub-agents journal workspace-scope rows that point into the owner's + // session dir; admit that root the same way the in-app tool does, from a + // config that must EXIST and read: a tolerant (or fresh-install) view + // would resolve a sub-agent to ITSELF, and the rollback would then mutate + // its hidden legacy notebook (no owner root, no adoption remap) and + // report success. Throws → the command fails before touching anything. + const config = opts.config ?? defaultConfig; + let topology: SharedWorkspaceMemoryTopology; + try { + topology = resolveSharedWorkspaceMemoryTopology(config, workspaceId); + } catch (error) { + console.error( + `Refusing rollback of '${opts.rollback}': shared-memory ownership could not be resolved (${getErrorMessage(error)})` + ); + process.exitCode = 1; + return; + } const result = await rollbackRefinement({ sessionDir, + sharedWorkspaceMemorySessionDir: topology.ownerSessionDir, + // Reloaded per check (plan-time and in-lock), not from the snapshot + // above: a live backend may register a new tree member while this + // process waits for the shared-store lock, and its rows must count. + // Same existence-requiring load: an unproven tree refuses the rollback. + listSharedWorkspaceMemoryPeerSessionDirs: () => + resolveSharedWorkspaceMemoryTopology(config, workspaceId).peerSessionDirs, id: opts.rollback, force: opts.force, evidence: { toolName: "debug-cli", actor: "user" }, diff --git a/src/common/constants/memory.ts b/src/common/constants/memory.ts index bf650683d65..f65757b305d 100644 --- a/src/common/constants/memory.ts +++ b/src/common/constants/memory.ts @@ -9,7 +9,8 @@ * per-project notes; never committed to the repo, survives * workspaces; carried by the settings backup only when the * user opts into the project bundle) - * - workspace -> /memory/ (host-local, deleted with the workspace) + * - workspace -> /memory/ of the task-tree OWNER (host-local, deleted + * with that workspace; sub-agents share their parent's store) */ /** Virtual root prefix all memory paths are expressed under. */ @@ -18,6 +19,14 @@ export const MEMORY_VIRTUAL_ROOT = "/memories"; export const MEMORY_SCOPES = ["global", "project", "workspace"] as const; export type MemoryScope = (typeof MEMORY_SCOPES)[number]; +/** + * `//memory.revision`: opaque token rewritten on every + * mutation of that owner's shared `/memories/workspace` store. Sessions and + * Memory tabs in OTHER backend processes (multi-instance) compare it before + * reusing a cached index/hot set — in-process consumers get change events. + */ +export const WORKSPACE_MEMORY_REVISION_FILE_NAME = "memory.revision"; + export type MemoryAccessLevel = "read" | "readwrite"; /** diff --git a/src/common/orpc/schemas/memory.ts b/src/common/orpc/schemas/memory.ts index 0bea6f09b40..09e2f571fa0 100644 --- a/src/common/orpc/schemas/memory.ts +++ b/src/common/orpc/schemas/memory.ts @@ -96,10 +96,12 @@ export type MemoryConsolidationRecordPayload = z.infer:`) when this row was + * copied from a removed sub-agent's journal into its memory owner's + * (sharedMemoryRowMigration.ts); lets a retried migration skip it. + */ + migratedFrom: z.string().optional(), + /** + * Where a migrated row was ORIGINALLY appended: the journal (workspace id) + * whose append sequence positions it, and its `seq` there. Two rows of one + * origin were serialized by that store's mutation lock (clock + append run + * inside it), so their origin sequence IS their mutation order — even when + * neither carries a usable clock value (pre-sharing history, a failed clock + * write). Copies are appended to the owner journal later than they happened + * and in migration order, so their own `seq` is no order evidence; a + * copy-of-copy keeps the first origin. Absent on a copy (older builds, + * corruption) = order unknown, never the copying journal's position. + * Native rows need no fields: their origin is (`workspaceId`, `seq`). + */ + originJournal: z.string().optional(), + originSeq: z.number().optional(), + /** + * Cross-session order key for rollback conflict detection (`sourceTs ?? ts`): + * a shared workspace store's monotonic clock, advanced under the store's + * mutation lock by every mutation (workspaceMemoryRevision.ts), so rows in + * an owner's and its sub-agents' journals — whose `ts`/`seq` are not + * comparable — still order totally; migrated rows keep their source value. + * Persisted rows are raw JSON: only a value isValidSourceClock accepts is + * order evidence; a present value outside that domain reads as order + * unknown (refinementRollback.ts), never as "earlier than everything". + */ + sourceTs: z.number().optional(), + /** + * The mutation landed but the shared store's clock write failed, so this + * row has NO defensible position relative to other rows (its `ts`/`seq` + * are journal-local). Rollback conflict detection treats such a row as + * conflicting with every overlapping row in either direction (force + * overrides), instead of ordering it by an incomparable timestamp. + */ + orderUnknown: z.literal(true).optional(), /** Expected post-action file hashes (RefinementPostStateSchema in refinement.ts). */ postState: JsonValueSchema.optional(), /** @@ -176,3 +215,52 @@ type DistributiveOmit = T extends unknown ? Omit export type DurableEventDraft = DistributiveOmit & { id?: string; }; + +/** + * A usable shared-store clock value: the clock is `max(Date.now(), prev + 1)` + * (workspaceMemoryRevision.ts), so a genuine value is a positive safe integer. + * Zero, negatives, fractions, unsafe integers and non-numbers are corruption. + */ +export function isValidSourceClock(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0; +} + +/** Where a refinement row was originally appended (see `originJournal`). */ +export interface RefinementRowOrigin { + journal: string; + seq: number; +} + +/** + * The journal position that orders a refinement row against rows of the same + * origin (RefinementDataSchema.originJournal). A native row is positioned by + * the journal it was READ from (`journalWorkspaceId`, the session's workspace + * id — the caller's knowledge, never the row's own `workspaceId`, which is + * persisted data a corrupt row could carry equal to another journal's; r79) + * and only while the row agrees with it. A migrated copy is positioned only + * by a carried, well-formed origin. Anything else is no order evidence (null). + */ +export function refinementRowOrigin( + row: { + workspaceId: string; + seq: number; + data: { migratedFrom?: string; originJournal?: unknown; originSeq?: unknown }; + }, + journalWorkspaceId: string | undefined +): RefinementRowOrigin | null { + if (row.data.migratedFrom === undefined) { + if (journalWorkspaceId === undefined || row.workspaceId !== journalWorkspaceId) return null; + return { journal: journalWorkspaceId, seq: row.seq }; + } + const { originJournal, originSeq } = row.data; + if ( + typeof originJournal !== "string" || + originJournal === "" || + typeof originSeq !== "number" || + !Number.isSafeInteger(originSeq) || + originSeq < 0 + ) { + return null; + } + return { journal: originJournal, seq: originSeq }; +} diff --git a/src/common/types/message.ts b/src/common/types/message.ts index a64cf5f2f8b..9c985081709 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -963,7 +963,31 @@ export interface ContextBudgetRejectedMessage { export interface MuxMetadata { /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; + /** + * The compaction epoch (opening boundary's history sequence, -1 before any + * boundary) under which the turn that produced this assistant row recorded + * its workspace-memory write policy, before the row was appended + * (TurnRequestBuilder start()). The post-compaction harvest accepts a turn + * only when this matches the epoch being harvested: a turn started before + * a destructive reset but appended after the new boundary carries the old + * epoch, whose deny the reset discarded. Builds that do not maintain the + * policy (older ones, after a downgrade) leave it unset, so their turns' + * user rows are never taken as accounted for (memoryConsolidationService.ts). + */ + workspaceMemoryPolicyEpoch?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) + /** + * Start sequence of the history segment this row was appended in + * (HistoryService, `history-segment.json`). A full clear opens a new + * segment whose sequences continue above every sequence the cleared + * history ever used, so a boundary's history sequence — and the + * boundary-less epoch identity `-(segmentStart + 1)` derived from this + * stamp (workspaceMemoryPolicyEpochOf) — never recurs across destructive + * clears: a turn started before a clear cannot be mistaken for one of the + * segment that replaced it. Omitted in the first segment (start 0), which + * legacy rows without the stamp belong to as well. + */ + historySegment?: number; /** Provider step boundaries in parts, persisted so continuous compaction can keep complete steps. */ stepStartPartIndices?: number[]; duration?: number; @@ -1081,6 +1105,18 @@ export interface MuxMetadata { */ rlmPreservedTailCopy?: boolean; + /** + * Compaction epoch (opening boundary's history sequence, -1 before any) the + * copied row was ORIGINALLY produced under — carried unchanged through + * repeated copies, so a copy of a copy still names the first epoch. The + * workspace-memory write policy of that epoch applies to the copy + * (TurnRequestBuilder → WorkspaceService.recordWorkspaceMemoryWritable): + * derived from history it would be wrong whenever the active-epoch read + * holds only the newest boundary. Absent on copies written before the + * field existed, whose source epochs never had a policy record either. + */ + rlmPreservedTailSourcePolicyEpoch?: number; + /** * @file mention snapshot token(s) this message provides content for. * Marks send-time materialized snapshot rows (the only @mention expansion diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index dc0230a8825..bf24d88bda1 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -3,13 +3,101 @@ import { describe, expect, it } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import { + compactionClosingPolicyEpoch, + duplicateUserMessageIds, + epochHasPriorTurnRows, findLatestCompactionBoundaryIndex, findLatestContextBoundaryIndex, hasProviderEligibleMessages, sliceMessagesForProviderFromLatestContextBoundary, sliceMessagesFromLatestCompactionBoundary, + workspaceMemoryPolicyEpochOf, } from "./compactionBoundary"; +describe("workspaceMemoryPolicyEpochOf", () => { + const boundary = (id: string, historySequence: number) => + createMuxMessage(id, "assistant", "summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + historySequence, + }); + + it("is -1 before any boundary for legacy and first-segment rows", () => { + expect(workspaceMemoryPolicyEpochOf([])).toBe(-1); + expect( + workspaceMemoryPolicyEpochOf([ + createMuxMessage("u1", "user", "a", { historySequence: 0 }), + createMuxMessage("u2", "user", "b", { historySequence: 1, historySegment: 0 }), + ]) + ).toBe(-1); + }); + + it("derives a segment-unique boundary-less identity from the segment stamp", () => { + // Rows of a later segment (after a full clear) never share the cleared + // segment's -1; any subset of the segment yields the same identity. + const rows = [ + createMuxMessage("u1", "user", "a", { historySequence: 7, historySegment: 7 }), + createMuxMessage("a1", "assistant", "b", { historySequence: 8, historySegment: 7 }), + ]; + expect(workspaceMemoryPolicyEpochOf(rows)).toBe(-8); + expect(workspaceMemoryPolicyEpochOf(rows.slice(1))).toBe(-8); + // A row that lost its stamp (rewritten in place by an older build) + // cannot pull the segment back to the legacy identity. + expect( + workspaceMemoryPolicyEpochOf([ + ...rows, + createMuxMessage("a2", "assistant", "c", { historySequence: 9 }), + ]) + ).toBe(-8); + }); + + it("ignores malformed stamps and prefers the latest boundary's sequence", () => { + expect( + workspaceMemoryPolicyEpochOf([ + createMuxMessage("u1", "user", "a", { + historySequence: 3, + historySegment: -4 as unknown as number, + }), + createMuxMessage("u2", "user", "b", { historySequence: 4, historySegment: 2.5 }), + ]) + ).toBe(-1); + expect( + workspaceMemoryPolicyEpochOf([ + boundary("s1", 5), + createMuxMessage("u1", "user", "a", { historySequence: 6, historySegment: 5 }), + ]) + ).toBe(5); + }); +}); + +describe("duplicateUserMessageIds", () => { + it("names ids shared by several user rows and makes them prior turns", () => { + const rows = [ + createMuxMessage("u1", "user", "first", { historySequence: 0 }), + createMuxMessage("a1", "assistant", "reply", { historySequence: 1 }), + createMuxMessage("u1", "user", "same id again", { historySequence: 2 }), + ]; + expect([...duplicateUserMessageIds(rows)]).toEqual(["u1"]); + expect(duplicateUserMessageIds(rows.slice(0, 2)).size).toBe(0); + // The current batch is matched by id: the duplicated id would hide the + // earlier row from the prior-turn check, so it counts as one. + const current = { userMessageId: "u1", preludeMessageIds: new Set() }; + expect(epochHasPriorTurnRows(rows, current)).toBe(true); + expect(epochHasPriorTurnRows(rows.slice(0, 2), current)).toBe(false); + }); +}); + +describe("compactionClosingPolicyEpoch", () => { + it("prefers the recorded closing epoch and falls back to the legacy identity", () => { + expect( + compactionClosingPolicyEpoch({ closingPolicyEpoch: -8, previousBoundaryHistorySequence: 3 }) + ).toBe(-8); + expect(compactionClosingPolicyEpoch({ previousBoundaryHistorySequence: 3 })).toBe(3); + expect(compactionClosingPolicyEpoch({})).toBe(-1); + }); +}); + describe("findLatestCompactionBoundaryIndex", () => { it("returns the newest compaction boundary via reverse scan", () => { const messages = [ @@ -378,3 +466,117 @@ describe("sliceMessagesFromLatestCompactionBoundary", () => { expect(sliced.map((msg) => msg.id)).toEqual(["u0", "summary-malformed", "u1"]); }); }); + +describe("epochHasPriorTurnRows", () => { + const current = { userMessageId: "u-now", preludeMessageIds: new Set(["p-now"]) }; + const withRows = (...rows: Array>) => + epochHasPriorTurnRows( + rows.map((args) => createMuxMessage(...args)), + current + ); + + it("counts an earlier user turn but not the batch being started", () => { + expect( + withRows( + ["u-now", "user", "now"], + ["p-now", "user", "prelude", { synthetic: true, fileAtMentionSnapshot: ["@a.md"] }] + ) + ).toBe(false); + expect(withRows(["u-old", "user", "earlier"], ["u-now", "user", "now"])).toBe(true); + expect(withRows(["a-old", "assistant", "answer"], ["u-now", "user", "now"])).toBe(false); + }); + + it("exempts a listed prelude id only for a row of prelude shape", () => { + // A prelude listing naming an ordinary user turn (raw history) must not + // hide that turn from the unknown-history rule. + expect(withRows(["u-now", "user", "now"], ["p-now", "user", "a real earlier turn"])).toBe(true); + // Nor a synthetic user TURN (auto-resume, CLI goal continuation): no snapshot. + expect( + withRows(["u-now", "user", "now"], ["p-now", "user", "continue", { synthetic: true }]) + ).toBe(true); + for (const marker of [ + { agentSkillSnapshot: { skillName: "s", scope: "project" as const, sha256: "x" } }, + { + mcpPromptSnapshot: { + serverName: "srv", + promptName: "p", + commandKey: "srv:p", + invokingMessageId: "u-now", + }, + }, + ]) { + expect( + withRows( + ["u-now", "user", "now"], + ["p-now", "user", "snapshot", { synthetic: true, ...marker }] + ) + ).toBe(false); + } + // Raw history: a snapshot field holding `null` or a value without the + // backend's shape is no snapshot — the row stays a turn of its own. + for (const malformed of [ + { fileAtMentionSnapshot: null }, + { fileAtMentionSnapshot: "@a.md" }, + { fileAtMentionSnapshot: [1] }, + { agentSkillSnapshot: null }, + { agentSkillSnapshot: { skillName: "s" } }, + { mcpPromptSnapshot: {} }, + { mcpPromptSnapshot: { serverName: "srv", promptName: "p" } }, + ]) { + const row = createMuxMessage("p-now", "user", "looks like a prelude", { synthetic: true }); + expect( + epochHasPriorTurnRows( + [ + createMuxMessage("u-now", "user", "now"), + { ...row, metadata: { ...row.metadata, ...(malformed as object) } }, + ], + current + ) + ).toBe(true); + } + }); + + it("ignores rows that are no turn of the epoch: compaction requests, tail copies, token-budget internals", () => { + expect( + withRows( + [ + "req", + "user", + "/compact", + { muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} } }, + ], + ["u-now", "user", "now"] + ) + ).toBe(false); + expect( + withRows( + ["copy", "user", "earlier", { rlmPreservedTailCopy: true }], + ["u-now", "user", "now"] + ) + ).toBe(false); + expect( + withRows( + [ + "lead", + "user", + "lead-in", + { muxMetadata: { type: "context-window-lead-in", rolloverId: "r1" } }, + ], + [ + "warn", + "user", + "warning", + { + muxMetadata: { + type: "context-budget-warning", + contextTokens: 1, + maxTokens: 2, + budgetTokens: 2, + }, + }, + ], + ["u-now", "user", "now"] + ) + ).toBe(false); + }); +}); diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 69aa23e0d3a..c83a816a2fb 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -6,7 +6,8 @@ import { import { isPositiveInteger } from "@/common/utils/numbers"; import { hasProviderReplayableContent } from "@/common/utils/messages/providerEligibility"; -import type { MuxMessage } from "@/common/types/message"; +import type { MuxMessage, MuxMetadata } from "@/common/types/message"; +import { isTokenBudgetInternalMessage } from "@/common/types/message"; export { CONTEXT_BOUNDARY_KINDS }; @@ -71,6 +72,85 @@ export function isDurableContextBoundaryMarker(message: MuxMessage | undefined): return getContextBoundaryKind(message) !== null; } +/** + * History sequence of the latest durable context boundary among `messages` + * (any kind), or undefined when there is none. Identifies the compaction + * epoch the rows after it belong to: compaction completion metadata carries it + * as `previousBoundaryHistorySequence`, and the workspace-memory policy + * accumulator is bound to it (WorkspaceService.recordWorkspaceMemoryWritable). + */ +export function latestContextBoundaryHistorySequence( + messages: readonly MuxMessage[] +): number | undefined { + let latest: number | undefined; + for (const message of messages) { + if (!isDurableContextBoundaryMarker(message)) continue; + const sequence = message.metadata?.historySequence; + if (typeof sequence !== "number" || !Number.isInteger(sequence) || sequence < 0) continue; + if (latest === undefined || sequence > latest) latest = sequence; + } + return latest; +} + +/** + * A persisted history sequence (stamp, request bound, segment start) in the + * clock's domain: a nonnegative safe integer. History rows are raw JSON, so + * every policy check reading one — the harvest gate, the tail-copy stamps, + * the segment start — must apply this same predicate (r90): a fractional, + * negative or unsafe value covers nothing anywhere, or one check would grant + * what another refused. + */ +export function isPersistedHistorySequence(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +/** + * Start sequence of the history segment `messages` belong to (the largest + * `historySegment` stamp among them; 0 for legacy rows and the first + * segment). Every row of a segment carries the same stamp, so any non-empty + * subset of the segment yields the same value; a full clear opens a segment + * with a strictly larger start (HistoryService). + */ +export function historySegmentStart(messages: readonly MuxMessage[]): number { + let start = 0; + for (const message of messages) { + const segment = message.metadata?.historySegment; + if (!isPersistedHistorySequence(segment)) continue; + if (segment > start) start = segment; + } + return start; +} + +/** + * The compaction epoch `messages` (an active-context read) belong to, as the + * workspace-memory write policy keys it: the latest durable boundary's + * history sequence, or `-(segmentStart + 1)` before any boundary of the + * segment (-1 in the first segment, as before the stamp existed). Sequences + * never recur across full clears (each clear opens a segment above every + * sequence used so far), so neither identity is ever reused for a different + * conversation: a turn that recorded its policy under the pre-clear identity + * cannot pass as one of the post-clear epoch, however the clear and its + * appends interleave across backends. Compaction completion reports the same + * value as `closingPolicyEpoch`, so the completion-side observation and every + * backend's turn records agree on which epoch a value belongs to. + */ +export function workspaceMemoryPolicyEpochOf(messages: readonly MuxMessage[]): number { + return latestContextBoundaryHistorySequence(messages) ?? -(historySegmentStart(messages) + 1); +} + +/** + * The policy epoch a compaction closed: `closingPolicyEpoch` when the + * completion recorded it, else the identity older builds used (the previous + * boundary's sequence, -1 before any) so persisted legacy records still key + * their turns' stamps. + */ +export function compactionClosingPolicyEpoch(metadata: { + closingPolicyEpoch?: number; + previousBoundaryHistorySequence?: number; +}): number { + return metadata.closingPolicyEpoch ?? metadata.previousBoundaryHistorySequence ?? -1; +} + /** * Locate the latest durable context boundary in reverse chronological order. * @@ -174,3 +254,113 @@ export function sliceMessagesForProviderFromLatestContextBoundary( ? messages.slice(boundaryIndex + 1) : messages.slice(boundaryIndex); } + +/** + * Whether the active epoch already holds a turn other than the one being + * started (`currentBatch`: the request's user row plus its prelude snapshot + * ids). Feeds the unknown-history rule of the workspace-memory write policy + * (WorkspaceService.recordWorkspaceMemoryWritable): an epoch with prior turns + * this process never recorded a policy for cannot be vouched for. Not turns: + * compaction request rows (they open an epoch), RLM keep-recent copies (the + * previous epoch's turns re-appended after the boundary; the harvest gate + * skips them too, and counting them would make another backend's first turn + * of the new epoch — racing the compacting backend's asynchronous policy + * carry — record an unknown-history deny for an all-writable epoch), and + * token-budget control rows (rollover lead-in, budget warning: backend + * template text prepended to the turn they precede; the harvest gate exempts + * them for the same reason, and counting one would record a false + * unknown-history deny for a fresh, otherwise writable epoch). + */ +export function epochHasPriorTurnRows( + activeContextMessages: readonly MuxMessage[], + currentTurn: { userMessageId: string | undefined; preludeMessageIds: ReadonlySet } +): boolean { + // Batches are matched by row id: two user rows sharing one id (persisted + // history is raw JSON) would both read as the current batch, hiding the + // earlier one from this check and from the harvest gate's coverage — so a + // duplicated id is itself an unaccounted prior turn (fail closed). A + // prelude listing exempts only rows of prelude shape (isRequestPreludeRow): + // an ordinary user turn named there stays a prior turn. + const duplicated = duplicateUserMessageIds(activeContextMessages); + return activeContextMessages.some( + (message) => + message.role === "user" && + (duplicated.has(message.id) || + !( + message.id === currentTurn.userMessageId || + (currentTurn.preludeMessageIds.has(message.id) && isRequestPreludeRow(message)) + )) && + message.metadata?.muxMetadata?.type !== "compaction-request" && + message.metadata?.rlmPreservedTailCopy !== true && + !isTokenBudgetInternalMessage(message) + ); +} + +/** + * Whether a row has the shape of a request prelude row — one the backend + * appends with a turn and lists in the user row's `requestPreludeMessageIds`: + * a synthetic USER row carrying the snapshot it materializes (@mention file + * snapshot, agent skill snapshot, MCP prompt snapshot), or a synthetic + * ASSISTANT row (family payloads). `synthetic` alone is not enough: the + * backend also persists synthetic user TURNS (auto-resume, CLI goal + * continuations), which are turns of their own. The id-keyed policy + * accounting (prior-turn check, harvest coverage, tail-copy stamps) honors a + * prelude listing only for prelude-shaped rows, so a listing that names any + * other user row (persisted history is raw JSON) cannot make that turn read + * as accounted for. The snapshot must have the shape the backend writes + * (r89): a `null` or arbitrary value under the field — corruption, or a + * turn hand-edited to look like a prelude — grants nothing. + */ +export function isRequestPreludeRow(message: MuxMessage): boolean { + const metadata = message.metadata; + if (metadata?.synthetic !== true) return false; + if (message.role === "assistant") return true; + return ( + message.role === "user" && + (isFileAtMentionSnapshotShape(metadata.fileAtMentionSnapshot) || + isAgentSkillSnapshotShape(metadata.agentSkillSnapshot) || + isMcpPromptSnapshotShape(metadata.mcpPromptSnapshot)) + ); +} + +function isFileAtMentionSnapshotShape(value: unknown): boolean { + return Array.isArray(value) && value.every((token) => typeof token === "string"); +} + +function isAgentSkillSnapshotShape(value: unknown): boolean { + if (value === null || typeof value !== "object") return false; + const snapshot = value as Partial>; + return ( + typeof snapshot.skillName === "string" && + typeof snapshot.scope === "string" && + typeof snapshot.sha256 === "string" + ); +} + +function isMcpPromptSnapshotShape(value: unknown): boolean { + if (value === null || typeof value !== "object") return false; + const snapshot = value as Partial>; + return ( + typeof snapshot.serverName === "string" && + typeof snapshot.promptName === "string" && + typeof snapshot.commandKey === "string" + ); +} + +/** + * Ids carried by more than one user row of `messages`. The policy checks + * account for user rows by id (a turn's batch is its user row plus the + * prelude ids that row lists), so an id shared by two rows would let the + * accounting of one vouch for the other; callers treat such ids as + * unaccounted for. + */ +export function duplicateUserMessageIds(messages: readonly MuxMessage[]): ReadonlySet { + const seen = new Set(); + const duplicated = new Set(); + for (const message of messages) { + if (message.role !== "user") continue; + if (seen.has(message.id)) duplicated.add(message.id); + else seen.add(message.id); + } + return duplicated; +} diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index c30c56e2245..16c2d7b07fc 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -2511,7 +2511,7 @@ export const TOOL_DEFINITIONS = { "Scopes (all paths are virtual):\n" + "- /memories/global/... — personal, permanent, shared across all projects\n" + "- /memories/project/... — private notes about this project; host-local, never committed to the repo (included in the settings backup only when the user opts in), survives workspaces\n" + - "- /memories/workspace/... — scratch state for this workspace; deleted with the workspace\n" + + "- /memories/workspace/... — scratch state for this workspace, shared with its sub-agents (a sub-agent reads and writes its parent's workspace notes); deleted with the owning workspace\n" + "Commands:\n" + "- view: list a directory (up to 2 levels, dotfiles excluded) or show a file with line numbers (offset/limit supported)\n" + "- create: create a new file; ERRORS if the file already exists (to overwrite: delete first, then create)\n" + diff --git a/src/common/utils/tools/toolPolicy.ts b/src/common/utils/tools/toolPolicy.ts index b282c3818c2..143af7c31a7 100644 --- a/src/common/utils/tools/toolPolicy.ts +++ b/src/common/utils/tools/toolPolicy.ts @@ -87,6 +87,16 @@ export function isSessionHistoryDisabled(policy?: ToolPolicy): boolean { return applyToolPolicyToNames(["session_history"], policy).length === 0; } +/** + * Whether the effective policy strips the `memory` tool. The persisted + * post-compaction harvest permission must reflect the FINAL toolset, not just + * the agent class: an editing-capable sub-agent whose policy denies memory + * must not have its transcript harvested into the (shared) workspace notebook. + */ +export function isMemoryToolDisabled(policy?: ToolPolicy): boolean { + return applyToolPolicyToNames(["memory"], policy).length === 0; +} + /** * Caller policy for a context-budget final-flush turn: the memory-only ceiling is appended * last so it wins regardless of the caller's or agent's own rules, while `memory` keeps diff --git a/src/node/config.test.ts b/src/node/config.test.ts index ea7e250736c..593ade5351c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -923,6 +923,35 @@ describe("Config", () => { expect(loaded.projects.size).toBe(0); }); + describe("loadExistingConfigOrThrow", () => { + it("throws for an absent config.json even though strict loads read it as empty", () => { + const config = new Config(tempDir); + expect(() => config.loadConfigOrDefault({ throwOnError: true })).not.toThrow(); + expect(() => config.loadExistingConfigOrThrow()).toThrow(/absent/); + }); + + it("rejects a config.json replaced during the read instead of returning a torn view", () => { + const configFile = path.join(tempDir, "config.json"); + fs.writeFileSync(configFile, JSON.stringify({ defaultProjectDir: "/tmp" })); + const config = new Config(tempDir); + expect(config.loadExistingConfigOrThrow().projects.size).toBe(0); + // Another backend's atomic rewrite lands between the pre-read stamp + // and the post-read one (a different size guarantees a new stamp). + const original = config.loadConfigOrDefault.bind(config); + const load = spyOn(config, "loadConfigOrDefault").mockImplementation((options) => { + const loaded = original(options); + fs.writeFileSync(configFile, JSON.stringify({ defaultProjectDir: "/tmp/replaced" })); + return loaded; + }); + try { + expect(() => config.loadExistingConfigOrThrow()).toThrow(/replaced during the read/); + } finally { + load.mockRestore(); + } + expect(() => config.loadExistingConfigOrThrow()).not.toThrow(); + }); + }); + it("keeps the canonical legacy identity when a secondary alias file is unreadable in lenient loads", async () => { // An id-less legacy entry with a HEALTHY canonical (generated-legacy) // metadata file and an unreadable basename-backed second candidate: diff --git a/src/node/config/index.ts b/src/node/config/index.ts index 4ecc7e34193..5b4a0e598cf 100644 --- a/src/node/config/index.ts +++ b/src/node/config/index.ts @@ -884,6 +884,11 @@ interface ConfigLoadFailureState { // re-log the same corrupt-config error once per instance. const configLoadFailureStates = new Map(); +/** Location of config.json under a Xum root (also used by callers that must stat it without a Config). */ +export function configFilePath(rootDir: string): string { + return path.join(rootDir, "config.json"); +} + function configLoadFailureState(configFile: string): ConfigLoadFailureState { let state = configLoadFailureStates.get(configFile); if (!state) { @@ -934,6 +939,21 @@ export class Config { readonly sessionsDir: string; readonly srcDir: string; private readonly configFile: string; + /** + * Cheap durable change signal for config.json: one stat, no parse. Any + * backend's rewrite changes size/mtime (atomic replace also changes the + * inode), so a memo built against a previous stamp knows to rebuild — + * unlike onConfigChanged, which only fires for THIS process's edits. A + * missing/unreadable file yields a distinct stamp. + */ + configFileStamp(): string { + try { + const st = fs.statSync(this.configFile, { bigint: true }); + return `${st.dev}:${st.ino}:${st.size}:${st.mtimeNs}`; + } catch { + return "missing"; + } + } private readonly providersConfigStore: ProvidersConfigStore; private readonly emitter = new EventEmitter(); /** @@ -969,7 +989,7 @@ export class Config { this.rootDir = sessionLocator.rootDir; this.sessionsDir = sessionLocator.sessionsDir; this.srcDir = sessionLocator.srcDir; - this.configFile = path.join(this.rootDir, "config.json"); + this.configFile = configFilePath(this.rootDir); this.providersConfigStore = providersConfigStore ?? new ProvidersConfigStore(this.rootDir); } @@ -1310,6 +1330,32 @@ export class Config { return { ids, hasWorkspaceEntriesWithoutIds }; } + /** + * Strict load that additionally REQUIRES config.json to exist. The strict + * mode of loadConfigOrDefault still treats ENOENT as a fresh install (an + * empty, valid config), which fail-closed callers — workspace removal's + * shared-memory handover, the workspace-memory policy accumulator — must + * not mistake for "this workspace is not registered" while the file is + * merely mid-rewrite. Stat before and after the read, and require the SAME + * stamp at both: a file present at both with one stamp is taken as present + * and unchanged during the read, while a replacement landing in between + * (another backend's atomic rewrite) means the bytes read may belong to + * neither snapshot's topology — the caller retries from one stable + * snapshot rather than act on a torn view. + */ + loadExistingConfigOrThrow(): ProjectsConfig { + const before = this.configFileStamp(); + const config = this.loadConfigOrDefault({ throwOnError: true }); + const after = this.configFileStamp(); + if (before === "missing" || after === "missing") { + throw new Error(`config.json is absent at ${this.configFile}`); + } + if (before !== after) { + throw new Error(`config.json at ${this.configFile} was replaced during the read`); + } + return config; + } + loadConfigOrDefault(options?: { throwOnError?: boolean }): ProjectsConfig { // Read as a Buffer and hand the same snapshot to the failure handler: backing up via a // second read could preserve a concurrent writer's replacement instead of the bytes that diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 2e454232851..dd4011b1e4b 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -13,7 +13,11 @@ import { TestClock } from "effect/testing"; import { SUBSCRIPTION_HEARTBEAT_INTERVAL_MS } from "@/common/utils/withQueueHeartbeat"; import { disposeAppRuntime, makeAppRuntime } from "@/node/services/di/appRuntime"; import type { ORPCContext } from "./context"; -import { subscribeWorkspaceActivity, subscribeDesignExperiment } from "./routerSubscriptions"; +import { + subscribeWorkspaceActivity, + subscribeDesignExperiment, + subscribeMemoryChanges, +} from "./routerSubscriptions"; test("subscription handlers forward the oRPC runtime Clock", async () => { const app = makeAppRuntime(TestClock.layer()); @@ -37,6 +41,92 @@ test("subscription handlers forward the oRPC runtime Clock", async () => { expect(workspaceService.listenerCount("activity")).toBe(0); }); +test("memory subscriptions match workspace-scope events on the shared memory owner", async () => { + // Workspace-scope change events carry the memory OWNER (task-tree root); + // a sub-agent's subscription must see edits to the notebook it shares. + const app = makeAppRuntime(TestClock.layer()); + const memoryService = new EventEmitter(); + const memoryConsolidationService = new EventEmitter(); + const controller = new AbortController(); + const ownerOf = new Map([["ws-child", "ws-owner"]]); + const context = { + "effect/context": app.context, + workspaceService: { getInfo: () => Promise.resolve(null) }, + memoryService: Object.assign(memoryService, { + resolveWorkspaceMemoryOwnerId: (workspaceId: string) => + ownerOf.get(workspaceId) ?? workspaceId, + workspaceMemoryRevision: () => Promise.resolve("rev-1"), + }), + memoryConsolidationService, + } as unknown as ORPCContext; + const stream = subscribeMemoryChanges(context, "ws-child", controller.signal); + try { + // Baseline handshake: the store revision is announced once as a + // root-addressed refresh so the client's listing catches up with a write + // that landed between its initial fetch and this subscription. + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: "", + }); + const first = stream.next(); + // The listener attaches once the generator has started running. + while (memoryService.listenerCount("change") === 0) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + const base = { + scope: "workspace", + path: "/memories/workspace/n.md", + actor: "agent", + projectPath: "", + }; + memoryService.emit("change", { ...base, workspaceId: "ws-other" }); + memoryService.emit("change", { ...base, workspaceId: "ws-child" }); + memoryService.emit("change", { ...base, workspaceId: "ws-owner" }); + // Global scope is never filtered: it marks the end of the batch, so + // receiving it second proves the other/child events were dropped. + const marker = { ...base, scope: "global", path: "/memories/global/g.md", workspaceId: "" }; + memoryService.emit("change", marker); + expect((await first).value).toEqual({ ...base, workspaceId: "ws-owner" }); + expect((await stream.next()).value).toEqual(marker); + // A workspace-scope event re-reads the store token asynchronously; the + // token is announced once read so a foreign write it may have absorbed is + // refetched too (see refreshStoreRevision in subscribeMemoryChanges). + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: "", + }); + + // Ownership change for THIS workspace (owner removed): synthesized + // root-addressed refresh + status refresh, now addressed to the new owner. + ownerOf.set("ws-child", "ws-child"); + memoryService.emit("ownersInvalidated", ["ws-unrelated"]); + memoryService.emit("ownersInvalidated", ["ws-child"]); + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "user", + workspaceId: "ws-child", + projectPath: "", + }); + expect((await stream.next()).value).toEqual({ + kind: "consolidation_status", + workspaceId: "ws-child", + projectPath: "", + }); + } finally { + controller.abort(); + await stream.return(undefined); + await disposeAppRuntime(app.managed); + } + expect(memoryService.listenerCount("change")).toBe(0); +}); + test("Design subscriptions publish sibling changes only after client shutdown", async () => { using temp = new DisposableTempDir("design-subscription"); const flags = path.join(temp.path, EXPERIMENT_OVERRIDES_FILE_NAME); diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 8c0a374f894..c212bb3720f 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -24,6 +24,7 @@ import type { LogEntry } from "@/node/services/logBuffer"; import { subscribeLogFeed } from "@/node/services/logBuffer"; import { resolveMemoryProjectIdentity, + toVirtualPath, type MemoryChangeEvent, } from "@/node/services/memoryService"; @@ -197,6 +198,9 @@ export function subscribeLogs( }); } +/** How often an open memory subscription re-checks its workspace's memory owner. */ +const MEMORY_OWNERSHIP_PROBE_INTERVAL_MS = 30_000; + export function subscribeMemoryChanges( context: ORPCContext, workspaceId: string | null, @@ -207,20 +211,107 @@ export function subscribeMemoryChanges( validate?.(); const metadata = workspaceId ? await context.workspaceService.getInfo(workspaceId) : null; const projectPath = metadata ? resolveMemoryProjectIdentity(metadata) : null; + // Workspace-scope events carry the memory OWNER (task-tree root), which is + // also the store this subscriber's workspace displays. Resolved per event + // (memoized, cheap): the owner can change while the tab stays open — a + // removed owner makes the child fall back to its own store. yield* runtimeSubscription(context, { signal, subscribe: (emit) => { + // Revision token of the displayed workspace store (see + // MemoryService.workspaceMemoryRevision), refreshed by every + // workspace event this subscription forwards so the probe below only + // fires for mutations this process never saw. Reads are async; a + // failed refresh leaves the old token, costing at most one redundant + // refresh on the next probe. + let storeRevision: string | null = null; + const rootRefresh = (): MemoryChangeEventPayload => ({ + scope: "workspace", + path: toVirtualPath("workspace", ""), + actor: "agent", + workspaceId: workspaceId + ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) + : "", + projectPath: projectPath ?? "", + }); + const refreshStoreRevision = (options?: { announce: boolean }) => { + if (!workspaceId) return; + context.memoryService.workspaceMemoryRevision(workspaceId).then( + (revision) => { + storeRevision = revision; + if (options?.announce) emit.push(rootRefresh()); + }, + () => undefined + ); + }; + // Baseline handshake: the client's initial listing and this + // subscription start independently, so a foreign write landing between + // them would be adopted here as the baseline while the listing shows + // the old files. Announce the baseline once it is read: the client + // refetches, and the listing is then at least as new as the token. + refreshStoreRevision({ announce: true }); const onChange = (event: MemoryChangeEvent) => { - if (event.scope === "workspace" && event.workspaceId !== workspaceId) return; + if ( + event.scope === "workspace" && + event.workspaceId !== + (workspaceId + ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) + : null) + ) + return; if (event.scope === "project" && event.projectPath !== projectPath) return; + // The token read below is asynchronous: it may absorb a foreign + // backend's write that landed after the client already refetched for + // this event, and the interval probe would then never announce it. + // Announcing the token once read (as the baseline handshake does) + // orders one more client refresh behind whatever the token saw; the + // cost is a redundant listing refetch per local mutation. + if (event.scope === "workspace") refreshStoreRevision({ announce: true }); emit.push(event); }; const onStatusChange = (event: MemoryConsolidationStatusChangeEventPayload) => emit.push(event); + // Ownership changed for this workspace (its owner was removed; it + // now reads its own store). No mutation event accompanies a removal, + // so synthesize a root-addressed refresh for the file list and the + // consolidation status, both of which described the old store. + const onOwnersInvalidated = (workspaceIds: string[]) => { + if (!workspaceId || !workspaceIds.includes(workspaceId)) return; + emit.push({ + scope: "workspace", + path: toVirtualPath("workspace", ""), + actor: "user", + workspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), + projectPath: projectPath ?? "", + }); + emit.push({ kind: "consolidation_status", workspaceId, projectPath: projectPath ?? "" }); + }; + // ownersInvalidated is emitted lazily, when something probes ownership. + // Another backend (multi-instance) removing the owner — or writing the + // shared store — leaves an idle tab with nothing to trigger that probe + // and no in-process change event, so probe here: one stat of + // config.json plus one small revision read per interval. A foreign + // write shows up as a changed token and synthesizes the same + // root-addressed refresh an ownership change does. + const ownershipProbe = workspaceId + ? setInterval(() => { + context.memoryService.workspaceMemoryRevision(workspaceId).then( + (revision) => { + if (revision === storeRevision) return; + storeRevision = revision; + emit.push(rootRefresh()); + }, + () => undefined + ); + }, MEMORY_OWNERSHIP_PROBE_INTERVAL_MS).unref() + : undefined; context.memoryService.on("change", onChange); + context.memoryService.on("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.on("statusChange", onStatusChange); return () => { + if (ownershipProbe !== undefined) clearInterval(ownershipProbe); context.memoryService.off("change", onChange); + context.memoryService.off("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.off("statusChange", onStatusChange); }; }, diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 0e9d2077634..641432c29ab 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -32,6 +32,7 @@ function createSession(args: { historyService: HistoryService; sessionDir: string; buildMemorySessionContext: AIService["buildMemorySessionContext"]; + probeMemoryStore?: AIService["probeMemoryStore"]; isExperimentEnabled?: AIService["isExperimentEnabled"]; }): AgentSession { const aiEmitter = new EventEmitter(); @@ -50,6 +51,7 @@ function createSession(args: { ), stopStream: mock(() => Promise.resolve({ success: true as const, data: undefined })), buildMemorySessionContext: args.buildMemorySessionContext, + probeMemoryStore: args.probeMemoryStore, isExperimentEnabled: args.isExperimentEnabled ?? (() => false), } as unknown as AIService; @@ -127,6 +129,152 @@ describe("AgentSession memory context", () => { expect(await priv.resolveMemoryContext("test-model")).toEqual(context); expect(await priv.resolveMemoryContext("test-model")).toEqual(context); expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + + // A write by another task-tree member to the shared workspace notebook + // invalidates from outside; the next resolve rebuilds from disk. + session.invalidateMemoryContext(); + expect(await priv.resolveMemoryContext("test-model")).toEqual(context); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + + // Invalidation DURING a build: the pre-write snapshot is served once but + // must not be cached, so the following resolve rebuilds again. + let finishBuild!: () => void; + buildMemorySessionContext.mockImplementationOnce( + () => new Promise((resolve) => (finishBuild = () => resolve(context))) + ); + session.invalidateMemoryContext(); + const inFlight = priv.resolveMemoryContext("test-model"); + await Promise.resolve(); + session.invalidateMemoryContext(); + finishBuild(); + expect(await inFlight).toEqual(context); + expect(await priv.resolveMemoryContext("test-model")).toEqual(context); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(4); + } finally { + await session.dispose(); + } + }); + + test("probes memory ownership before consulting the cache so a removed owner rebuilds this request", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-probe"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + const context: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => Promise.resolve(context)); + // Simulates MemoryService's stamp check finding a removed owner: the + // synchronous ownersInvalidated → core.ts → invalidateMemoryContext chain. + let ownerRemoved = false; + const sessionRef: { current?: AgentSession } = {}; + const probeMemoryStore = mock(() => { + // One-shot like the real memo invalidation: the stamp check fires once. + if (ownerRemoved) { + ownerRemoved = false; + sessionRef.current?.invalidateMemoryContext(); + } + return Promise.resolve(undefined); + }); + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryStore, + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + sessionRef.current = session; + const priv = session as unknown as PrivateSessionAccess; + try { + await priv.resolveMemoryContext("test-model"); + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + // A build is bracketed by two probes (before the cache read, after the + // build); a cache hit costs one. + expect(probeMemoryStore).toHaveBeenCalledTimes(3); + + ownerRemoved = true; + // The probe runs before the cache read, so THIS request rebuilds. + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + } finally { + await session.dispose(); + } + }); + + test("rebuilds when the owner store's revision advanced without an in-process change event", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-revision"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + const context: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => Promise.resolve(context)); + // Another backend process writing the shared notebook: no change event + // reaches this session, only the durable revision token differs. + let revision = "rev-1"; + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryStore: () => Promise.resolve(revision), + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + const priv = session as unknown as PrivateSessionAccess; + try { + await priv.resolveMemoryContext("test-model"); + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + + revision = "rev-2"; + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + // Stable again: the rebuilt context is cached under the new token. + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + } finally { + await session.dispose(); + } + }); + + test("discards a snapshot whose store changed during the build instead of serving it", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-midbuild"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + // The owner is removed by ANOTHER backend while the build reads hot files: + // no in-process event, the probe flips to "revoked" only after the build + // started, and the snapshot built from the former owner's notes must + // never reach the provider. + let revision = "rev-1"; + let flipDuringBuild = true; + const stale: MemorySessionContext = { + indexEntries: [{ path: "/memories/workspace/x.md", description: "" }], + hotMemoriesBlock: null, + }; + const fresh: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => { + if (flipDuringBuild) { + flipDuringBuild = false; + revision = "revoked"; + return Promise.resolve(stale); + } + return Promise.resolve(fresh); + }); + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryStore: () => Promise.resolve(revision), + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + const priv = session as unknown as PrivateSessionAccess; + try { + expect(await priv.resolveMemoryContext("test-model")).toBe(fresh); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + + // A store that will not hold still yields no context at all (bounded + // retries) rather than a snapshot of unknown provenance. + let tick = 0; + buildMemorySessionContext.mockImplementation(() => { + revision = `rev-${++tick}`; + return Promise.resolve(stale); + }); + expect(await priv.resolveMemoryContext("other-model")).toBeUndefined(); } finally { await session.dispose(); } diff --git a/src/node/services/agentSession.postCompactionAttachments.test.ts b/src/node/services/agentSession.postCompactionAttachments.test.ts index d7a119a3d6d..3dd9046b86e 100644 --- a/src/node/services/agentSession.postCompactionAttachments.test.ts +++ b/src/node/services/agentSession.postCompactionAttachments.test.ts @@ -140,7 +140,13 @@ function createSessionForHistory(historyService: HistoryService, sessionDir: str rootDir: path.dirname(sessionDir), sessionsDir: path.dirname(sessionDir), srcDir: "/tmp", - loadConfigOrDefault: mock(() => ({})), + // A context boundary resets the durable memory-policy accumulator, which + // looks this workspace up in the config snapshot (existence-requiring + // load; the mock delegates so the strict-mode override below propagates). + loadConfigOrDefault: mock(() => ({ projects: new Map() })), + loadExistingConfigOrThrow(this: Config) { + return this.loadConfigOrDefault({ throwOnError: true }); + }, } as unknown as Config; return new AgentSession({ @@ -248,8 +254,27 @@ describe("AgentSession post-compaction attachments", () => { expect(injected).not.toBeNull(); expect(getReadFilePaths(injected ?? [])).toEqual(["/tmp/pre-boundary-read.ts"]); - // A new context segment starts (context reset / full history clear): - // the reset was meant to discard that context, so... + // A new context segment starts (context reset / full history clear). + // Its durable policy-epoch reset reads config strictly: an unreadable + // config.json must surface as a retryable failure rather than a + // "successful" reset that leaves stale per-epoch records behind. + const sessionConfig = (session as unknown as { config: Config }).config; + const readable = sessionConfig.loadConfigOrDefault.bind(sessionConfig); + sessionConfig.loadConfigOrDefault = ((options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("config.json: unexpected token"); + return readable(options); + }) as Config["loadConfigOrDefault"]; + try { + expect( + await session.clearPostCompactionState().then( + () => null, + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + ).toContain("unexpected token"); + } finally { + sessionConfig.loadConfigOrDefault = readable; + } + // The reset was meant to discard that context, so... await session.clearPostCompactionState(); // ...no later turn may re-inject pre-boundary paths — neither diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index 54ef8c49644..209625ae1eb 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -205,6 +205,18 @@ export async function createAgentSessionHarness( const historyService = options.historyService ?? testHistory!.historyService; const config = options.config ?? testHistory?.config ?? createAgentSessionTestConfig(); const cleanup = testHistory?.cleanup ?? (() => Promise.resolve()); + // A registered workspace always has a config.json in production, and the + // session's compaction/reset boundaries require one to exist (an absent + // file reads as mid-rewrite, not as a fresh install). Persist the empty + // default so harness sessions do not fail those boundaries spuriously. + // Some suites pass a partial mock config (cast) without these methods. + if ( + typeof config.configFileStamp === "function" && + typeof config.editConfig === "function" && + config.configFileStamp() === "missing" + ) { + await config.editConfig((cfg) => cfg); + } const { aiEmitter, aiService } = options.aiService ? { aiEmitter: options.aiEmitter ?? new EventEmitter(), aiService: options.aiService } : createMockAiService({ diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 0061622535b..2e6dc115870 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -9,7 +9,10 @@ import { estimateFreshRequestTokensForModel } from "./contextBudgetCounting"; import type { RequestAssemblySnapshot } from "./events/eventSpine"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; -import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; +import { + compactionClosingPolicyEpoch, + sliceMessagesForProviderFromLatestContextBoundary, +} from "@/common/utils/messages/compactionBoundary"; import { randomUUID } from "crypto"; import { sandboxHostService } from "./sandbox/sandboxHostService"; import { applyToolPolicyToNames, isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; @@ -111,6 +114,17 @@ import { } from "@/common/utils/agentIds"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry, resolveWorkspaceModelFallbackChain } from "@/node/services/taskUtils"; +import { + setWorkspaceMemoryWritableForEpoch, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; +import { + carryWorkspaceMemoryDenyMarker, + clearWorkspaceMemoryDenyMarker, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; +import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import { buildStreamErrorEventData, createStreamErrorMessage, @@ -697,6 +711,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { replayStream?(workspaceId: string, options?: { afterTimestamp?: number }): Promise; getProvidersConfig(): ProvidersConfigMap | null; isExperimentEnabled(experimentId: ExperimentId): boolean; + probeMemoryStore?(workspaceId: string): Promise; buildMemorySessionContext?( workspaceId: string, modelString: string, @@ -749,7 +764,7 @@ interface AgentSessionOptions { runtimeConfig: RuntimeConfig | undefined; }) => Promise; /** Called when compaction completes (e.g., to clear idle compaction pending state) */ - onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void | Promise; /** Called with the terminal outcome of an idle compaction (persisted success / post-stream failure) */ onIdleCompactionOutcome?: (success: boolean) => void; /** Called when post-compaction context state may have changed (plan/file edits) */ @@ -771,6 +786,8 @@ interface CachedMemoryContext { tokenBudgetActive: boolean; memoryEnabled: boolean; hotSetEnabled: boolean; + /** Owner store revision (AIService.probeMemoryStore) the context was built from. */ + storeRevision: string | undefined; } interface SendMessageInternalOptions { @@ -1064,6 +1081,205 @@ export class AgentSession { * prompt-cache-stable bytes without preserving stale files forever. */ private memoryContextByModelString = new Map(); + + /** + * Drop the cached memory context so the next stream rebuilds the index and + * hot set from disk. Own memory tool calls clear it on tool-call-end; this + * entry point is for writes by OTHER sessions to a store this session also + * reads (a sub-agent editing the task tree's shared workspace notes). + */ + invalidateMemoryContext(): void { + this.memoryContextByModelString.clear(); + // A build already awaiting buildMemorySessionContext read the pre-write + // files; bumping the generation stops it from repopulating the cache. + this.memoryContextGeneration++; + } + + private memoryContextGeneration = 0; + + /** + * Workspace-memory write policy accumulated over the current compaction + * epoch, mirrored from the DURABLE accumulator in config.json + * (WorkspaceService.recordWorkspaceMemoryWritable performs the fail-closed + * AND against the persisted value, so backends sharing one chat.jsonl — + * XUM_ALLOW_MULTIPLE_INSTANCES — and a restarted process all contribute to + * one conjunction). Attached to compaction completions so the memory + * harvest — which writes to the (possibly shared) workspace store on this + * agent's behalf — honors a read-only turn anywhere in the epoch: the + * harvest reads EVERY message of the epoch. Both copies restart at a + * context boundary (compaction without a preserved tail, /clear, context + * reset, destructive history replace) via resetWorkspaceMemoryWritable. + */ + private workspaceMemoryWritable: { epoch: number; writable: boolean } | undefined; + + /** + * Record the mirror for `epoch` (the policy epoch the turn was recorded + * under). Keyed like the durable records: with several backends over one + * chat.jsonl, another backend's no-tail compaction or destructive clear + * opens a new epoch without any callback here, so an unkeyed mirror would + * carry the previous epoch's value into the new one — a stale deny would + * pin its first writable turn, a stale grant would keep the unknown-history + * rule from failing closed. + */ + recordWorkspaceMemoryWritable(effective: boolean, epoch: number): void { + assert(Number.isInteger(epoch), "workspace memory policy mirror epoch must be an integer"); + this.workspaceMemoryWritable = { epoch, writable: effective }; + } + + /** + * The mirror for `epoch`, for WorkspaceService's conjunction and the + * completion observation: undefined until a turn of THAT epoch recorded + * (a value recorded for another epoch says nothing about this one). + */ + workspaceMemoryWritableMirror(epoch: number): boolean | undefined { + return this.workspaceMemoryWritable?.epoch === epoch + ? this.workspaceMemoryWritable.writable + : undefined; + } + + /** In-flight durable epoch reset started by a no-tail compaction (see the completion callback). */ + private workspaceMemoryEpochReset: Promise | undefined; + + /** + * Resolves once the durable epoch reset of the last no-tail compaction (if + * any is still running) has settled, so a policy record for the new epoch + * never reads the closing epoch's accumulator or deny marker. + */ + async settleWorkspaceMemoryPolicyEpoch(): Promise { + await this.workspaceMemoryEpochReset; + } + + /** + * Destructive boundary (/clear, context reset, history replace): the + * in-memory mirror and every durable epoch record forget the discarded + * transcript. The new segment's epoch identities never recur (a full clear + * continues history sequences above the cleared segment, so neither a + * boundary's sequence nor the boundary-less `-(segmentStart + 1)` is + * reused; workspaceMemoryPolicyEpochOf), so the discarded records are + * inert to the new segment's readers — dropping them is hygiene, kept + * durable-or-throw like the other boundary invalidations so a reported + * success never leaves stale state behind. Nothing to do when the field is + * already absent. + * + * A COMPACTION boundary clears nothing durable (r77): its closing epoch's + * record and deny marker stay until the next destructive boundary. Two + * backends can each commit a boundary closing the same epoch, and each + * completion observes that epoch's verdict — the first observer consuming + * the record would leave the second reading only its own stale mirror and + * harvesting a read-only turn. Readers key by epoch and epoch keys never + * recur before a destructive boundary, so retained records are inert; the + * cost is one boolean per compaction epoch in config.json until then. + */ + private async resetWorkspaceMemoryWritable(): Promise { + // The mirror is forgotten only once the durable clear below completed — + // a reset that reports success while the persisted `-1` deny survives + // would pin the whole new segment to the stored-false fast path. + await clearWorkspaceMemoryDenyMarker( + this.config.rootDir, + path.join(this.config.sessionsDir, this.workspaceId) + ); + // Strict load: an unreadable — or transiently ABSENT — config.json would + // read as the empty default, in which this workspace has no records to + // clear; the reset would report success and leave the stale records + // behind. Throwing makes the boundary a retryable partial failure + // instead; a registered workspace always has a config. + const entry = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); + if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) { + this.workspaceMemoryWritable = undefined; + return; + } + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, this.workspaceId); + if (current === null) return cfg; + delete current.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + // Verified read-back: Config.saveConfig swallows write failures, so the + // awaited edit alone does not prove the clear landed. + const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); + if (after?.workspace.workspaceMemoryWritableByEpoch !== undefined) { + throw new Error( + `Workspace memory policy reset did not persist for ${this.workspaceId} (config write swallowed?)` + ); + } + this.workspaceMemoryWritable = undefined; + } + + /** + * Preserved-tail compaction: the tail copies were produced under the + * closing epoch's policy, so its accumulator carries into the new epoch — + * durably, by re-binding the config value and the deny marker recorded for + * `closingEpoch` to `nextEpoch` (another backend's first turn of the new + * epoch reads by epoch and would otherwise see nothing). The closing + * epoch's value is copied, never consumed (see resetWorkspaceMemoryWritable): + * another backend's boundary closing the same epoch observes it too. + */ + private async carryWorkspaceMemoryWritable( + closingEpoch: number, + nextEpoch: number + ): Promise { + const sessionDir = path.join(this.config.sessionsDir, this.workspaceId); + await carryWorkspaceMemoryDenyMarker(this.config.rootDir, sessionDir, closingEpoch, nextEpoch); + // Fail closed: the carry must be PROVEN, not assumed. A tolerant load + // would read a transiently unreadable config.json as "no closing record" + // and skip the carry; Config.saveConfig swallows write failures, so the + // awaited edit does not prove the new-epoch record landed either. Tail + // copies are excluded from the prior-turn check by design, so a fresh or + // foreign backend's first turn of the new epoch would then grant, and + // output conditioned on the read-only tail would become harvestable. + // When a DENY cannot be shown to have reached the new epoch, the + // session-dir marker (the same fallback recordWorkspaceMemoryWritable + // takes for an unwritable config) denies the new epoch instead. + let carried: boolean | undefined; + let failure: string | undefined; + try { + const entry = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); + const closingBefore = + entry === null ? undefined : workspaceMemoryWritableForEpoch(entry.workspace, closingEpoch); + if (closingBefore === undefined) return; + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, this.workspaceId); + if (current === null) return cfg; + const closing = workspaceMemoryWritableForEpoch(current.workspace, closingEpoch); + if (closing === undefined) return cfg; + // ANDed into a record another backend's first turn of the new epoch + // may already have made (its own conjunction could not see the closing + // value under the new key), never overwriting it. + const next = workspaceMemoryWritableForEpoch(current.workspace, nextEpoch); + carried = next === undefined ? closing : next && closing; + setWorkspaceMemoryWritableForEpoch(current.workspace, nextEpoch, carried); + return cfg; + }); + // Gone meanwhile (a destructive boundary): nothing to carry. + if (carried === undefined) return; + const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); + if ( + after === null || + workspaceMemoryWritableForEpoch(after.workspace, nextEpoch) !== carried + ) { + failure = "config write swallowed"; + } + } catch (error: unknown) { + failure = getErrorMessage(error); + } + // A grant that failed to persist leaves the new epoch without a record, + // which every reader treats as "grants normally" — nothing to fail + // closed on. A deny (or an unknown value) must be made durable somewhere. + if (failure === undefined || carried === true) return; + log.warn("Workspace memory policy carry not durable in config; recording a deny marker", { + workspaceId: this.workspaceId, + closingEpoch, + nextEpoch, + failure, + }); + // Same gate as WorkspaceService.denyDurableFallback: the marker's mkdir + // must not recreate a session dir a concurrent remover already deleted. + await withTargetMutationLock(this.config.rootDir, sessionDir, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, this.workspaceId)) return; + await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); + }); + } + /** * Cache the last-known experiment state so we don't spam metadata refresh * when post-compaction context is disabled. @@ -1228,7 +1444,52 @@ export class AgentSession { this.coordinator.recordCompactionSummary( (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null ); - onCompactionComplete?.(metadata); + const closingEpoch = compactionClosingPolicyEpoch(metadata); + const closing = this.workspaceMemoryWritableMirror(closingEpoch); + const observed = Promise.resolve( + onCompactionComplete?.({ + ...metadata, + ...(closing !== undefined ? { workspaceMemoryWritable: closing } : {}), + }) + ); + // New epoch. A preserved tail copies messages produced under this + // epoch's policy into the next one, so the fail-closed accumulator + // carries over with them (copied to the new epoch key, durably); + // otherwise the next normal turn restarts it. The mirror forgets the + // closing epoch right here, synchronously; the durable carry runs + // AFTER the completion observation settled (it reads the closing + // epoch's marker/config) and is awaited by this session's next turn + // (settleWorkspaceMemoryPolicyEpoch). The closing epoch's durable + // records are left in place (resetWorkspaceMemoryWritable explains + // why); every durable value is bound to its epoch, so they are + // invisible to the new epoch's turns on any backend. + const preservedTail = (metadata.preservedTailMessageCount ?? 0) > 0; + // The mirror follows the durable carry: a preserved tail re-binds the + // closing epoch's value to the new epoch (the copies were produced + // under it); otherwise the new epoch starts without one. + this.workspaceMemoryWritable = + preservedTail && closing !== undefined + ? { epoch: metadata.summaryHistorySequence, writable: closing } + : undefined; + const reset = observed + .catch(() => undefined) + .then(() => + preservedTail + ? this.carryWorkspaceMemoryWritable(closingEpoch, metadata.summaryHistorySequence) + : undefined + ) + .catch((error: unknown) => { + log.warn("Failed to reset the workspace memory policy epoch", { + workspaceId: this.workspaceId, + error, + }); + }) + .finally(() => { + if (this.workspaceMemoryEpochReset === reset) { + this.workspaceMemoryEpochReset = undefined; + } + }); + this.workspaceMemoryEpochReset = reset; }, onIdleCompactionOutcome, }); @@ -4381,32 +4642,49 @@ export class AgentSession { ); } if (await cancelBeforeAcceptance()) return Ok(undefined); - } else if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { - const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ - ...internal.preTurnMessages, - userMessage, - ]); - if (!batchAppendResult.success) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(batchAppendResult.error)); - } - persistedCancelableMessageIds.push( - ...internal.preTurnMessages.map((message) => message.id), - userMessage.id - ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } else if (!autoCompactionMessage) { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist - // normally. - const appendResult = await this.historyService.appendToHistory(this.workspaceId, userMessage); - if (!appendResult.success) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(appendResult.error)); + // normally. The snapshot rows appended above and the pre-turn payloads + // are this turn's request prelude; recorded on the user row exactly as + // the token-budget path does, so the post-compaction harvest gate can + // tell the turn's own batch from a row another backend interleaved + // (epochHasUncoveredUserRows matches prelude rows by id, never by + // adjacency) and the builder's unknown-history rule does not count the + // turn's own snapshots as turns nobody recorded. + const requestPreludeMessageIds = [ + ...(snapshotResult?.snapshotMessage ? [snapshotResult.snapshotMessage] : []), + ...skillSnapshotMessages, + ...mcpPromptSnapshotMessages, + ...(internal?.preTurnMessages ?? []), + ].map((row) => row.id); + if (requestPreludeMessageIds.length > 0) { + userMessage.metadata = { ...userMessage.metadata, requestPreludeMessageIds }; + } + if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { + const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ + ...internal.preTurnMessages, + userMessage, + ]); + if (!batchAppendResult.success) { + await rollbackPersistedTurnRows(); + return Err(createUnknownSendMessageError(batchAppendResult.error)); + } + persistedCancelableMessageIds.push( + ...internal.preTurnMessages.map((message) => message.id), + userMessage.id + ); + } else { + const appendResult = await this.historyService.appendToHistory( + this.workspaceId, + userMessage + ); + if (!appendResult.success) { + await rollbackPersistedTurnRows(); + return Err(createUnknownSendMessageError(appendResult.error)); + } + persistedCancelableMessageIds.push(userMessage.id); } - persistedCancelableMessageIds.push(userMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); } @@ -10306,6 +10584,10 @@ export class AgentSession { this.postCompactionLoadedSkills = []; this.postCompactionReadFilePaths = []; this.pendingPostCompactionStateToAcknowledge = null; + // A destructive context boundary (/clear, reset, history replace) starts + // a new harvest epoch: the fail-closed policy of the discarded transcript + // must not keep denying the new one. + await this.resetWorkspaceMemoryWritable(); // Durable-or-throw: a swallowed unlink failure would leave the stale // post-compaction.json to re-inject pre-boundary carryover after a // restart while the boundary caller reports success — the same @@ -10357,33 +10639,64 @@ export class AgentSession { this.aiService.isExperimentEnabled(id); const memoryEnabled = enabled(EXPERIMENT_IDS.MEMORY); const hotSetEnabled = enabled(EXPERIMENT_IDS.MEMORY_HOT_SET); - const cached = cache.get(modelString); - // Policy changes must not retain a previously injected extra (including index-only lookups). - if ( - cached?.tokenBudgetActive === tokenBudgetActive && - cached.memoryEnabled === memoryEnabled && - cached.hotSetEnabled === hotSetEnabled && - (cached.includesHotMemories || !includeHotMemories) - ) { - return cached.context ?? undefined; - } + const probe = async (): Promise => + memoryEnabled && typeof this.aiService.probeMemoryStore === "function" + ? await this.aiService.probeMemoryStore(this.workspaceId) + : undefined; + // Bounded: each retry means the store changed underneath the build; a + // store that will not hold still yields no memory context for this + // request rather than a snapshot of unknown provenance. + for (let attempt = 0; attempt < 3; attempt++) { + // Store probe first: a removed owner invalidates this cache synchronously + // (see AIService.probeMemoryStore), so the lookup below never serves an + // index built from a store this workspace no longer reads; and a store + // revision advanced by ANOTHER backend process (no in-process change + // event) fails the comparison below. + const storeRevision = await probe(); + const cached = cache.get(modelString); + // Policy changes must not retain a previously injected extra (including index-only lookups). + if ( + cached?.tokenBudgetActive === tokenBudgetActive && + cached.memoryEnabled === memoryEnabled && + cached.hotSetEnabled === hotSetEnabled && + cached.storeRevision === storeRevision && + (cached.includesHotMemories || !includeHotMemories) + ) { + return cached.context ?? undefined; + } - // Guard for test mocks that may not implement buildMemorySessionContext. - const context = - typeof this.aiService.buildMemorySessionContext === "function" - ? await this.aiService.buildMemorySessionContext(this.workspaceId, modelString, { - includeHotMemories, - tokenBudgetActive, - }) - : null; - cache.set(modelString, { - context, - includesHotMemories: includeHotMemories, - tokenBudgetActive, - memoryEnabled, - hotSetEnabled, + const generation = this.memoryContextGeneration; + // Guard for test mocks that may not implement buildMemorySessionContext. + const context = + typeof this.aiService.buildMemorySessionContext === "function" + ? await this.aiService.buildMemorySessionContext(this.workspaceId, modelString, { + includeHotMemories, + tokenBudgetActive, + }) + : null; + // Invalidated mid-build — by an in-process change (generation) or by a + // store the build cannot observe changing under it: a removal in + // ANOTHER backend revokes access ("revoked") without any local event, + // and the readability check ran before the hot-file reads. Such a + // snapshot is never served: the next attempt re-probes and rebuilds + // (a revoked store then lists nothing). + if (generation !== this.memoryContextGeneration || (await probe()) !== storeRevision) { + continue; + } + cache.set(modelString, { + context, + includesHotMemories: includeHotMemories, + tokenBudgetActive, + memoryEnabled, + hotSetEnabled, + storeRevision, + }); + return context ?? undefined; + } + log.debug("[AgentSession] memory context kept changing during its build; omitting it", { + workspaceId: this.workspaceId, }); - return context ?? undefined; + return undefined; } /** diff --git a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts new file mode 100644 index 00000000000..35841442c7c --- /dev/null +++ b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts @@ -0,0 +1,237 @@ +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; +import * as fsPromises from "fs/promises"; +import * as path from "path"; +import type { Config } from "@/node/config"; +import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; +import { getErrorMessage } from "@/common/utils/errors"; +import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + readWorkspaceMemoryDenyMarker, + workspaceMemoryDenyMarkerPath, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; +import { AgentSession } from "./agentSession"; +import { createStreamLifecycleMocks } from "./agentSession.testHarness"; +import type { AIService } from "./aiService"; +import type { BackgroundProcessManager } from "./backgroundProcessManager"; +import type { InitStateManager } from "./initStateManager"; +import { createTestHistoryService } from "./testHistoryService"; + +/** + * Durable epoch boundary of the workspace-memory write policy + * (AgentSession.resetWorkspaceMemoryWritable / carryWorkspaceMemoryWritable): + * Config.saveConfig swallows write failures, so both must prove their effect + * by reading back, and a deny the carry cannot prove reached the new epoch + * falls back to the session-dir marker. + */ +interface SessionInternals { + resetWorkspaceMemoryWritable(): Promise; + carryWorkspaceMemoryWritable(closingEpoch: number, nextEpoch: number): Promise; +} + +const WORKSPACE_ID = "policy-epoch-ws"; + +describe("AgentSession workspace memory policy epoch boundary", () => { + let cleanup: (() => Promise) | undefined; + const sessions: AgentSession[] = []; + + afterEach(async () => { + for (const session of sessions.splice(0)) await session.dispose(); + await cleanup?.(); + cleanup = undefined; + mock.restore(); + }); + + const createSession = async () => { + const harness = await createTestHistoryService(); + cleanup = harness.cleanup; + const config: Config = harness.config; + await fsPromises.mkdir(path.join(config.sessionsDir, WORKSPACE_ID), { recursive: true }); + await config.editConfig((cfg) => { + cfg.projects.set(SCRATCH_PROJECT_CONFIG_KEY, { + workspaces: [ + { + kind: "scratch", + path: path.join(config.rootDir, "scratch", WORKSPACE_ID), + id: WORKSPACE_ID, + name: WORKSPACE_ID, + runtimeConfig: { type: "local" }, + }, + ], + projectKind: "system", + trusted: true, + }); + return cfg; + }); + const session = new AgentSession({ + workspaceId: WORKSPACE_ID, + config, + historyService: harness.historyService, + aiService: { + on() { + return this; + }, + off() { + return this; + }, + ...createStreamLifecycleMocks(), + isStreaming: () => false, + } as unknown as AIService, + initStateManager: { + on() { + return this; + }, + off() { + return this; + }, + } as unknown as InitStateManager, + backgroundProcessManager: { + cleanup: mock(() => Promise.resolve()), + setMessageQueued: mock(() => undefined), + } as unknown as BackgroundProcessManager, + }); + sessions.push(session); + const records = () => + findWorkspaceEntry(config.loadConfigOrDefault(), WORKSPACE_ID)?.workspace + .workspaceMemoryWritableByEpoch; + const setRecords = (value: Record) => + config.editConfig((cfg) => { + findWorkspaceEntry(cfg, WORKSPACE_ID)!.workspace.workspaceMemoryWritableByEpoch = value; + return cfg; + }); + // A config write the Config layer swallowed: the edit runs on a loaded + // copy and resolves, nothing lands on disk. + const swallowNextWrite = () => + spyOn(config, "editConfig").mockImplementationOnce((edit) => { + edit(config.loadConfigOrDefault()); + return Promise.resolve(); + }); + return { + session, + config, + internals: session as unknown as SessionInternals, + sessionDir: path.join(config.sessionsDir, WORKSPACE_ID), + records, + setRecords, + swallowNextWrite, + }; + }; + + test("carry re-binds the closing epoch's value and proves it, falling back to the deny marker", async () => { + const { internals, sessionDir, records, setRecords, swallowNextWrite, config } = + await createSession(); + // Proven carry: the closing deny is copied to the new epoch key — and + // kept under its own, for another backend's boundary closing the same + // epoch (its completion observes the closing verdict too). + await setRecords({ "-1": false }); + await internals.carryWorkspaceMemoryWritable(-1, 7); + expect(records()).toEqual({ "-1": false, "7": false }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(false); + await internals.carryWorkspaceMemoryWritable(-1, 8); + expect(records()).toEqual({ "-1": false, "7": false, "8": false }); + + // Swallowed write: the deny never reached epoch 9 in config, so the + // session-dir marker denies epoch 9 instead of nothing. + await setRecords({ "-1": false }); + swallowNextWrite(); + await internals.carryWorkspaceMemoryWritable(-1, 9); + expect(records()).toEqual({ "-1": false }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(false); + + // Unreadable config: the closing value is unknown, which also fails closed. + const real = config.loadConfigOrDefault.bind(config); + const unreadable = spyOn(config, "loadConfigOrDefault").mockImplementation((options) => { + if (options?.throwOnError) throw new Error("EIO"); + return { ...real(), projects: new Map() }; + }); + try { + await internals.carryWorkspaceMemoryWritable(-1, 11); + } finally { + unreadable.mockRestore(); + } + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 11)).toBe(true); + + // A GRANT that failed to persist leaves the new epoch without a record, + // which readers treat as "grants normally": no marker. + await setRecords({ "-1": true }); + swallowNextWrite(); + await internals.carryWorkspaceMemoryWritable(-1, 13); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 13)).toBe(false); + }); + + test("the mirror answers only for the epoch it was recorded under", async () => { + const { session } = await createSession(); + session.recordWorkspaceMemoryWritable(false, 7); + expect(session.workspaceMemoryWritableMirror(7)).toBe(false); + // Another backend's boundary opened epoch 12 without any callback here: + // the stale value must neither deny the new epoch nor mask its unknown + // history. + expect(session.workspaceMemoryWritableMirror(12)).toBeUndefined(); + expect(session.workspaceMemoryWritableMirror(-1)).toBeUndefined(); + }); + + test("reset clears the mirror only once the durable clear is proven", async () => { + const { session, internals, records, setRecords, swallowNextWrite, config } = + await createSession(); + // Destructive boundary: every record goes, then the mirror. + session.recordWorkspaceMemoryWritable(false, -1); + await setRecords({ "-1": false, "5": true }); + await internals.resetWorkspaceMemoryWritable(); + expect(records()).toBeUndefined(); + expect(session.workspaceMemoryWritableMirror(-1)).toBeUndefined(); + + // Swallowed write: a surviving `-1: false` would pin the new segment to + // the stored-false fast path — the reset must fail (retryable) and keep + // the mirror rather than report success. + session.recordWorkspaceMemoryWritable(false, -1); + await setRecords({ "-1": false }); + swallowNextWrite(); + expect( + await internals.resetWorkspaceMemoryWritable().then(() => null, getErrorMessage) + ).toMatch(/did not persist/); + expect(records()).toEqual({ "-1": false }); + expect(session.workspaceMemoryWritableMirror(-1)).toBe(false); + + // An ABSENT config.json is not the empty default: a registered workspace + // always has one, so its absence is transient — the reset must fail + // (retryable) rather than clear the mirror over records it never saw. + session.recordWorkspaceMemoryWritable(false, -1); + await setRecords({ "-1": false }); + const configPath = path.join(config.rootDir, "config.json"); + const savedConfig = await fsPromises.readFile(configPath); + await fsPromises.rm(configPath); + try { + expect( + await internals.resetWorkspaceMemoryWritable().then(() => null, getErrorMessage) + ).toMatch(/absent/); + } finally { + await fsPromises.writeFile(configPath, savedConfig); + } + expect(session.workspaceMemoryWritableMirror(-1)).toBe(false); + expect(records()).toEqual({ "-1": false }); + }); + + test("a compaction boundary consumes neither the closing epoch's marker nor its wildcard", async () => { + const { internals, sessionDir } = await createSession(); + // Two backends' boundaries close epoch 3: each carry finds the entry. + await writeWorkspaceMemoryDenyMarker(sessionDir, 3); + await internals.carryWorkspaceMemoryWritable(3, 8); + await internals.carryWorkspaceMemoryWritable(3, 10); + for (const epoch of [3, 8, 10]) { + expect(await readWorkspaceMemoryDenyMarker(sessionDir, epoch)).toBe(true); + } + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(false); + // A wildcard (inherited from a malformed marker) is a deny of unknown + // epoch: the carry copies it to the new epoch and keeps it as-is. + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); + await writeWorkspaceMemoryDenyMarker(sessionDir, 12); + await internals.carryWorkspaceMemoryWritable(3, 14); + for (const epoch of [3, 14, 99]) { + expect(await readWorkspaceMemoryDenyMarker(sessionDir, epoch)).toBe(true); + } + // The destructive boundary is the only clear. + await internals.resetWorkspaceMemoryWritable(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + }); +}); diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index 6897bc32523..bbe50245152 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -8,8 +8,10 @@ import * as path from "node:path"; import { describe, it, expect, beforeEach, afterEach, mock, spyOn } from "bun:test"; +import { Err } from "@/common/types/result"; import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries"; import { AIService, resolveMuxProjectRootForHostFs } from "./aiService"; +import { WORKSPACE_MEMORY_POLICY_PERSIST_ERROR } from "./turnRequestBuilder"; import { discoverAvailableSubagentsForToolContext } from "./turnContextAssembler"; import { normalizeAnthropicBaseURL, @@ -2503,6 +2505,102 @@ describe("AIService.streamMessage compaction boundary slicing", () => { expect(initialMetadata.routeProvider).toBe("openrouter"); }); + it("removes the assistant placeholder when stream startup fails", async () => { + using xumHome = new DisposableTempDir("ai-service-startup-failure-placeholder"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-startup-failure-placeholder"; + const metadata = createLocalWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(xumHome.path, metadata); + const internals = harness.service as unknown as { + historyService: HistoryService; + streamManager: StreamManager; + }; + const deleted: string[] = []; + spyOn(internals.historyService, "deleteMessage").mockImplementation((_workspaceId, id) => { + deleted.push(id); + return Promise.resolve({ success: true, data: undefined }); + }); + spyOn(internals.streamManager, "startStream").mockResolvedValue( + Err({ type: "unknown", raw: "temp dir creation failed" }) + ); + + const result = await harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "continue")], + workspaceId, + modelString: "openai:gpt-5.2", + thinkingLevel: "medium", + }); + + expect(result.success).toBe(false); + // The placeholder carried the request bound and policy epoch stamp of a + // turn that never ran; left behind it would vouch for the user batch at + // the harvest gate (epochHarvestRefusal). + const appended = (internals.historyService.appendToHistory as ReturnType).mock + .calls as unknown as Array<[string, { id: string; role: string }]>; + const placeholder = appended.find(([, message]) => message.role === "assistant")?.[1]; + if (!placeholder) throw new Error("Expected an appended assistant placeholder"); + expect(deleted).toEqual([placeholder.id]); + }); + + it("denies the epoch when a failed turn's placeholder cannot be removed, and surfaces a lost deny", async () => { + using xumHome = new DisposableTempDir("ai-service-startup-failure-undeletable"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-startup-failure-undeletable"; + const metadata = createLocalWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(xumHome.path, metadata); + const internals = harness.service as unknown as { + historyService: HistoryService; + streamManager: StreamManager; + }; + // The harness runs without a memory tool, so start() already records a + // deny for the turn; the discard adds a second one. `outcomes` scripts + // the sink's durability per call. + const recorded: boolean[] = []; + let outcomes: boolean[] = []; + harness.service.turnRequestBuilderBindings.workspaceMemoryPolicySink = { + recordWorkspaceMemoryWritable: (_workspaceId, writable) => { + recorded.push(writable); + return Promise.resolve(outcomes.shift() ?? true); + }, + }; + spyOn(internals.historyService, "deleteMessage").mockResolvedValue(Err("concurrent rewrite")); + spyOn(internals.streamManager, "startStream").mockResolvedValue( + Err({ type: "unknown", raw: "temp dir creation failed" }) + ); + const stream = () => + harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "continue")], + workspaceId, + modelString: "openai:gpt-5.2", + thinkingLevel: "medium", + }); + + // The stamped placeholder stays behind: the epoch is denied instead, and + // the startup error is still the result. + const denied = await stream(); + expect(denied.success).toBe(false); + if (!denied.success) + expect(denied.error).toEqual({ type: "unknown", raw: "temp dir creation failed" }); + expect(recorded).toEqual([false, false]); + + // Neither removal nor deny durable: that failure is the result, not the + // startup error — the row is vouching for a batch no model saw. + outcomes = [true, false]; + const lost = await stream(); + expect(recorded).toEqual([false, false, false, false]); + expect(lost.success).toBe(false); + if (!lost.success) { + expect(lost.error.type).toBe("unknown"); + expect(lost.error.type === "unknown" ? lost.error.raw : "").toContain( + WORKSPACE_MEMORY_POLICY_PERSIST_ERROR + ); + } + }); + it("passes muxMetadata into initial stream metadata", async () => { using xumHome = new DisposableTempDir("ai-service-mux-metadata"); const projectPath = path.join(xumHome.path, "project"); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 66796bd7fac..99056d95075 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -23,6 +23,7 @@ import { type PreparedStreamMessage, type PreparedTurnRequest, type TurnRequestBuildContext, + WORKSPACE_MEMORY_POLICY_PERSIST_ERROR, } from "./turnRequestBuilder"; export { replaceOrAppendMessageById } from "./turnRequestBuilder"; export type { StreamMessageOptions } from "./turnRequestBuilder"; @@ -247,6 +248,23 @@ export class AIService extends EventEmitter { return this.experimentsService?.isExperimentEnabled(experimentId) === true; } + /** + * Re-check who owns this workspace's `/memories/workspace` store and return + * that store's revision token. Ownership is memoized and stamp-validated in + * MemoryService (one stat), the token is one small read; when another + * backend removed the owner since the last turn, the resulting + * `ownersInvalidated` event clears the affected sessions' cached context + * synchronously, and when another backend WROTE the shared store (no + * in-process change event) the token differs from the one recorded with the + * cached context. AgentSession calls this BEFORE consulting its cache so the + * current request, not the next one, rebuilds from the right, current store. + */ + async probeMemoryStore(workspaceId: string): Promise { + return await this.turnRequestBuilderBindings.memoryService?.workspaceMemoryRevision( + workspaceId + ); + } + /** * Build the session-segment memory context: the index snapshot advertised * in the memory tool description, plus the hot-memories block (pinned + @@ -1009,6 +1027,20 @@ export class AIService extends EventEmitter { startupState.pendingRunMetadataId = null; } buildOutcome.logStartOutcome("stream_start_failed", streamResult.error.type); + // No stream ran for this turn: discard its placeholder like an + // aborted startup's (TurnRequestBuilder denies the epoch when the row + // cannot be removed), so the never-started turn stays excluded from + // the harvest until a retry actually runs it. When neither could be + // made durable, that failure — not the startup error — is the result: + // the stamped row is still there, vouching for a batch no model saw. + if (!(await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId))) { + return Err({ + type: "unknown", + raw: `${WORKSPACE_MEMORY_POLICY_PERSIST_ERROR} (stream startup failed first: ${ + streamResult.error.type + })`, + }); + } return Err(streamResult.error); } @@ -1017,7 +1049,13 @@ export class AIService extends EventEmitter { this.clearTrackedPendingDevToolsRunMetadata(buildOutcome.assistantMessageId); startupState.pendingRunMetadataId = null; } - await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); + if (!(await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId))) { + return Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }); + } + } else { + // The stream is live: durable effects gated on "actually started" + // (memory harvest grant) may land now. + await buildOutcome.onStreamStarted?.(); } buildOutcome.logStartOutcome("started"); diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 2f23490aaf3..9a1b8b6e2c3 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1839,10 +1839,14 @@ describe("CompactionHandler", () => { onCompactionComplete, }); + // A real turn row: it carries its request bound and the epoch it + // recorded the workspace-memory policy under (none before any boundary). const tailAssistant = createMuxMessage("a1", "assistant", "tail answer", { model: "claude-x", usage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 }, contextUsage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 }, + requestHistorySequence: 2, + workspaceMemoryPolicyEpoch: -1, }); await seedHistory( createMuxMessage("u0", "user", "old head question"), @@ -1880,12 +1884,117 @@ describe("CompactionHandler", () => { expect(copy.metadata?.contextUsage).toBeUndefined(); // Copies must never masquerade as boundaries. expect(copy.metadata?.compactionBoundary).toBeUndefined(); + // The epoch the turn that covered the row recorded its policy under + // (none before this boundary): that policy governs the copy. + expect(copy.metadata?.rlmPreservedTailSourcePolicyEpoch).toBe(-1); } // Informational metadata survives. expect(epoch[2].metadata?.model).toBe("claude-x"); const metadata = onCompactionComplete.mock.calls[0]?.[0]; expect(metadata?.preservedTailMessageCount).toBe(2); + + // A second tail compaction re-copies the copies: they keep their + // ORIGINAL epoch (-1) while the new epoch's own rows carry this + // boundary's epoch — the chain stays visible to the policy conjunction. + const boundarySequence = epoch[0].metadata?.historySequence; + if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); + // A user row is covered by the assistant TURN row whose request bound + // anchors on it (the LAST user row at or below the bound) or that lists + // it as a prelude snapshot, and carries THAT turn's recorded epoch: an + // OLDER one for a turn that straddled a boundary (p2/u2/a2); none for a + // turn row WITHOUT a recorded policy (an older build's, u4/a4) — + // unknown, never vouched for; none for an accepted batch no assistant + // ever answered (u5: stream never started or crashed first); and none + // for another backend's row that landed between a turn's anchor and its + // assistant row (ux: the turn never consumed it, so "nearest later + // assistant" would vouch for repo-controlled content nobody vetted). + // Sequences: the boundary sits at boundarySequence, the two first-epoch + // copies at +1/+2, so the rows seeded here start at +3. + await seedHistory( + createMuxMessage("p2", "user", "prelude snapshot", { + synthetic: true, + fileAtMentionSnapshot: ["@notes.md"], + }), + // Listed as a prelude row but of ordinary shape: never stamped through + // the listing (r85). + createMuxMessage("px", "user", "a real turn's question listed as prelude"), + createMuxMessage("u2", "user", "second question", { + requestPreludeMessageIds: ["p2", "px"], + }), + createMuxMessage("a2", "assistant", "second answer", { + requestHistorySequence: boundarySequence + 5, // anchors on u2 + workspaceMemoryPolicyEpoch: -1, + }), + createMuxMessage("u3", "user", "third question"), + createMuxMessage("ux", "user", "foreign backend's batch"), + createMuxMessage("a3", "assistant", "third answer", { + requestHistorySequence: boundarySequence + 7, // anchors on u3, not ux + workspaceMemoryPolicyEpoch: boundarySequence, + }), + createMuxMessage("u4", "user", "old-build question"), + createMuxMessage("a4", "assistant", "old-build answer", { + requestHistorySequence: boundarySequence + 10, // anchors on u4 + }), + createMuxMessage("u5", "user", "unanswered question"), + // A row that recorded its policy under a foreign epoch but lost its + // request bound keeps that stamp (never the closing epoch); a present + // but malformed stamp stays unstamped; a synthetic payload row with + // neither belongs to the closing epoch. + createMuxMessage("a6", "assistant", "foreign-epoch answer without a bound", { + workspaceMemoryPolicyEpoch: -1, + }), + createMuxMessage("a7", "assistant", "corrupted stamp", { + workspaceMemoryPolicyEpoch: null as unknown as number, + }), + createMuxMessage("a8", "assistant", "synthetic payload"), + createMuxMessage("a9", "assistant", "old-build turn with a corrupted bound", { + requestHistorySequence: null as unknown as number, + }), + // A stamped turn whose bound is outside the sequence domain covers + // nothing (the harvest gate applies the same rule, r79/r80): its + // user row stays unstamped, the row itself keeps its stamp. + createMuxMessage("u10", "user", "question behind a fractional bound"), + createMuxMessage("a10", "assistant", "fractional bound", { + requestHistorySequence: boundarySequence + 18.5, + workspaceMemoryPolicyEpoch: boundarySequence, + }), + // An unsafe integer is outside the domain too (r90): the gate refuses + // it, so the copy must not read as covered either. + createMuxMessage("u11", "user", "question behind an unsafe bound"), + createMuxMessage("a11", "assistant", "unsafe bound", { + requestHistorySequence: Number.MAX_SAFE_INTEGER + 1, + workspaceMemoryPolicyEpoch: boundarySequence, + }), + createStampedCompactionRequest("compact-req-2", boundarySequence + 1) + ); + expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); + const secondEpoch = await historyService.getHistoryFromLatestBoundary(workspaceId); + if (!secondEpoch.success) throw new Error(secondEpoch.error); + expect( + secondEpoch.data.slice(1).map((copy) => copy.metadata?.rlmPreservedTailSourcePolicyEpoch) + ).toEqual([ + -1, // copy(u1) + -1, // copy(a1) + -1, // p2 (prelude of u2) + undefined, // px (listed, but no prelude shape) + -1, // u2 + -1, // a2 + boundarySequence, // u3 + undefined, // ux + boundarySequence, // a3 + undefined, // u4 + undefined, // a4 + undefined, // u5 + -1, // a6 (recorded stamp kept despite the missing bound) + undefined, // a7 (malformed stamp) + boundarySequence, // a8 (no stamp, no bound: synthetic payload row) + undefined, // a9 (no stamp, malformed bound: not a synthetic row) + undefined, // u10 (its turn's bound is fractional: never covered) + boundarySequence, // a10 (recorded stamp kept) + undefined, // u11 (its turn's bound is an unsafe integer: never covered) + boundarySequence, // a11 (recorded stamp kept) + ]); }); it("rewrites MCP snapshot invoking IDs to the copy IDs of LATER tail rows", async () => { diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 012db8bf17e..94d1bc5fdd7 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -21,7 +21,9 @@ import { type CompactionFollowUpRequest, type CompactionSummaryMetadata, type MuxMessage, + isTokenBudgetInternalMessage, } from "@/common/types/message"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { createCompactionSummaryMessageId } from "@/node/services/utils/messageIds"; import type { TelemetryService } from "@/node/services/telemetryService"; import { @@ -39,7 +41,12 @@ import { import { isDurableCompactedMarker, isDurableContextBoundaryMarker, + latestContextBoundaryHistorySequence, + duplicateUserMessageIds, + isPersistedHistorySequence, + isRequestPreludeRow, sliceMessagesFromLatestCompactionBoundary, + workspaceMemoryPolicyEpochOf, } from "@/common/utils/messages/compactionBoundary"; import { extractReadFilePaths, mergeReadFilePaths } from "@/common/utils/messages/extractReadFiles"; import { @@ -302,17 +309,6 @@ function isCompactedSummaryMessage(message: MuxMessage): boolean { return isDurableCompactedMarker(message.metadata?.compacted); } -function getLatestBoundaryHistorySequence(messages: readonly MuxMessage[]): number | undefined { - let latest: number | undefined; - for (const message of messages) { - if (!isDurableContextBoundaryMarker(message)) continue; - const sequence = message.metadata?.historySequence; - if (!isNonNegativeInteger(sequence)) continue; - if (latest === undefined || sequence > latest) latest = sequence; - } - return latest; -} - function getNextCompactionEpoch(messages: MuxMessage[]): number { let epochCursor = 0; @@ -1277,16 +1273,19 @@ export class CompactionHandler { } ); const idMap = new Map(params.tail.map((row) => [row.id, createPreservedTailCopyMessageId()])); - const copies = params.tail.map((row) => { - const copy = this.buildPreservedTailCopy(row, idMap); - // Continuous compaction prunes the just-finished answer too. Keep recent pages - // visible below the boundary while retaining RLM's usage/snapshot sanitizer. - copy.metadata = { ...copy.metadata, uiVisible: true }; - // User Esc keeps its interrupted marker and next-send continuation sentinel. - // Only our internal stop has an explicit durable Continue replacing it. - if (params.pendingFollowUp) delete copy.metadata.partial; - return copy; - }); + // Same closing epoch the completion metadata reports below. + const closingPolicyEpoch = workspaceMemoryPolicyEpochOf(params.messages); + const copies = this.buildCoveredTailCopies(params.tail, idMap, closingPolicyEpoch).map( + (copy) => { + // Continuous compaction prunes the just-finished answer too. Keep recent pages + // visible below the boundary while retaining RLM's usage/snapshot sanitizer. + copy.metadata = { ...copy.metadata, uiVisible: true }; + // User Esc keeps its interrupted marker and next-send continuation sentinel. + // Only our internal stop has an explicit durable Continue replacing it. + if (params.pendingFollowUp) delete copy.metadata.partial; + return copy; + } + ); return { boundary, copies }; } @@ -1343,7 +1342,8 @@ export class CompactionHandler { summaryMessageId: boundary.id, summaryHistorySequence: sequence, compactionEpoch: epoch, - previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), + previousBoundaryHistorySequence: latestContextBoundaryHistorySequence(params.messages), + closingPolicyEpoch: workspaceMemoryPolicyEpochOf(params.messages), compactionRequestMessageId: boundary.id, preservedTailMessageCount: copies.length, }); @@ -1406,7 +1406,10 @@ export class CompactionHandler { const nextCompactionEpoch = getNextCompactionEpoch(messages); assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); - const previousBoundaryHistorySequence = getLatestBoundaryHistorySequence(messages); + const previousBoundaryHistorySequence = latestContextBoundaryHistorySequence(messages); + // The policy epoch the compacted rows belong to: the key their turns + // recorded under (TurnRequestBuilder) and the one the harvest reads. + const closingPolicyEpoch = workspaceMemoryPolicyEpochOf(messages); const maxExistingHistorySequence = this.getMaxExistingHistorySequence(messages); // For idle compaction, preserve the original recency timestamp so the workspace @@ -1513,7 +1516,8 @@ export class CompactionHandler { const preservedTailCopies = this.buildPreservedTailCopies( messages, compactionRequestMessageId, - summaryMessage.id + summaryMessage.id, + closingPolicyEpoch ); const persistenceResult = @@ -1582,6 +1586,7 @@ export class CompactionHandler { summaryHistorySequence: persistedSequence, compactionEpoch: nextCompactionEpoch, previousBoundaryHistorySequence, + closingPolicyEpoch, compactionRequestMessageId, preservedTailMessageCount: preservedTailCopies.length, }); @@ -1599,7 +1604,8 @@ export class CompactionHandler { private buildPreservedTailCopies( messages: MuxMessage[], compactionRequestMessageId: string, - summaryMessageId: string + summaryMessageId: string, + closingPolicyEpoch: number ): MuxMessage[] { const requestIndex = messages.findIndex((message) => message.id === compactionRequestMessageId); if (requestIndex === -1) { @@ -1639,7 +1645,109 @@ export class CompactionHandler { for (const row of tailRows) { idMap.set(row.id, createPreservedTailCopyMessageId()); } - return tailRows.map((row) => this.buildPreservedTailCopy(row, idMap)); + return this.buildCoveredTailCopies(tailRows, idMap, closingPolicyEpoch); + } + + /** + * Copies of a whole tail with their policy epochs assigned by request + * COVERAGE — the same batch rule as the harvest gate + * (memoryConsolidationService epochHarvestRefusal). An assistant TURN row + * (it carries `requestHistorySequence`, the last history sequence its + * request was built from) consumed exactly one batch under the policy it + * recorded: the LAST user row at or below that bound plus the rows that + * user row lists in `requestPreludeMessageIds`, matched by exact id. Only + * those user rows carry the turn's recorded epoch; the turn row carries it + * itself. Not "the nearest later assistant": with several backends on one + * chat.jsonl — or a sub-agent report row appended while the parent's + * request was preparing — a user row can land between a turn's anchor and + * its assistant row without that turn ever having seen it, and stamping it + * would present unvetted (model- or repo-controlled) content to the next + * epoch as covered. Everything else stays unstamped — a user row no turn + * covered (an accepted batch whose stream never started or crashed first), + * one answered by a turn without a recorded policy (an older build's) — + * which the policy sink reads as unknown (deny). Token-budget control rows + * (backend template text, no agent or repository content) need no turn and + * belong to the closing epoch, as do assistant rows that carry no policy + * stamp and no bound (synthetic payloads, summaries). An assistant row + * WITH a recorded stamp keeps it even when its bound is missing or + * malformed. Copies of copies keep their original epoch regardless. + */ + private buildCoveredTailCopies( + tailRows: MuxMessage[], + idMap: Map, + closingPolicyEpoch: number + ): MuxMessage[] { + const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + const userRowById = new Map(); + for (const message of tailRows) { + const sequence = message.metadata?.historySequence; + if (message.role !== "user") continue; + userRowById.set(message.id, message); + if (typeof sequence === "number") userRows.push({ message, sequence }); + } + const coveredEpochById = new Map(); + // An id shared by two user rows names no batch: neither copy is stamped + // (the harvest gate refuses such an epoch; the copies must not present + // either row as covered to the next one). + const duplicated = duplicateUserMessageIds(tailRows); + for (const message of tailRows) { + if (message.role !== "assistant") continue; + const bound = message.metadata?.requestHistorySequence; + const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; + // Same domain check as the harvest gate (epochHarvestRefusal, r79 — + // the shared predicate, so an unsafe integer is refused alike, r90): a + // bound outside the clock's domain covers nothing there, so it must not + // stamp a batch here either — the copies would then carry an epoch + // without ever having been covered, and the association would grant + // in a later epoch what the gate refused in this one. + if (typeof policyEpoch !== "number" || !isPersistedHistorySequence(bound)) continue; + const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; + if (anchor === undefined) continue; + // Same prelude rule as the harvest gate: a listed id stamps a user row + // only when that row has prelude shape. + const prelude = getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds).filter( + (id) => { + const listed = userRowById.get(id); + return listed === undefined || isRequestPreludeRow(listed); + } + ); + for (const id of [anchor.id, ...prelude]) { + if (!coveredEpochById.has(id) && !duplicated.has(id)) { + coveredEpochById.set(id, policyEpoch); + } + } + } + return tailRows.map((row) => { + let epoch: number | undefined; + if (row.role === "assistant") { + // A recorded stamp is kept whatever the bound: a row that recorded + // its policy (possibly a read-only one, under a foreign epoch whose + // deny a destructive reset since discarded) but lost or corrupted its + // bound must not be reclassified as a synthetic non-turn row and + // handed the closing epoch — that would present it to the next epoch + // as current-policy content. History is raw JSON: a stamp that is + // present but not a number stays unstamped (unknown). Only a row + // with no stamp AND no bound at all is a synthetic payload/summary + // row (closing epoch); a row with a bound — present in any form, + // malformed included: it may be an older build's turn whose policy + // was never recorded — stays unstamped (unknown). + const stamp: unknown = row.metadata?.workspaceMemoryPolicyEpoch; + const bound: unknown = row.metadata?.requestHistorySequence; + epoch = + stamp !== undefined + ? typeof stamp === "number" + ? stamp + : undefined + : bound === undefined + ? closingPolicyEpoch + : undefined; + } else if (row.role !== "user" || isTokenBudgetInternalMessage(row)) { + epoch = closingPolicyEpoch; + } else { + epoch = coveredEpochById.get(row.id); + } + return this.buildPreservedTailCopy(row, idMap, epoch); + }); } /** @@ -1652,7 +1760,11 @@ export class CompactionHandler { * original rows remain visible above the boundary; fresh IDs keep UI * aggregation from collapsing a hidden copy over its visible original. */ - private buildPreservedTailCopy(row: MuxMessage, idMap: Map): MuxMessage { + private buildPreservedTailCopy( + row: MuxMessage, + idMap: Map, + coveringPolicyEpoch: number | undefined + ): MuxMessage { // IDs are preassigned for the whole tail (see caller) so forward-pointing // references (snapshot row → later invoking user row) rewrite correctly. const copyId = idMap.get(row.id); @@ -1671,12 +1783,28 @@ export class CompactionHandler { } : source?.mcpPromptSnapshot; + // The epoch whose workspace-memory write policy governs this row: a copy + // of a copy keeps its ORIGINAL epoch (the chain must stay visible to the + // policy conjunction; copies from before the field existed carry nothing + // forward — no policy record ever existed for their epochs). A first-time + // copy carries the epoch buildCoveredTailCopies assigned it: for a turn + // row its own recorded epoch — a turn that started before a destructive + // reset and landed after the new boundary belongs to the OLD epoch, whose + // policy the new one cannot vouch for — and a turn row without a record + // (an older build's) stays unstamped, read as unknown (deny). + const sourcePolicyEpoch = + source?.rlmPreservedTailCopy === true + ? source.rlmPreservedTailSourcePolicyEpoch + : coveringPolicyEpoch; return { ...row, id: copyId, metadata: { synthetic: true, rlmPreservedTailCopy: true, + ...(sourcePolicyEpoch !== undefined + ? { rlmPreservedTailSourcePolicyEpoch: sourcePolicyEpoch } + : {}), ...(source?.timestamp !== undefined ? { timestamp: source.timestamp } : {}), ...(source?.model !== undefined ? { model: source.model } : {}), ...(source?.thinkingLevel !== undefined ? { thinkingLevel: source.thinkingLevel } : {}), diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index 6abe54f1dec..309dcec62d9 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -57,7 +57,7 @@ import { MCPConfigService } from "@/node/services/mcpConfigService"; import { MCPServerManager } from "@/node/services/mcpServerManager"; import { MemoryConsolidationService } from "@/node/services/memoryConsolidationService"; import { MemoryMetaService } from "@/node/services/memoryMeta"; -import { MemoryService } from "@/node/services/memoryService"; +import { MemoryService, type MemoryChangeEvent } from "@/node/services/memoryService"; import { ProviderService } from "@/node/services/providerService"; import { SessionUsageService } from "@/node/services/sessionUsageService"; import { StreamManager } from "@/node/services/streamManager"; @@ -567,12 +567,33 @@ export const CoreWiringLive: Layer.Layer< }); turnRequestBuilderBindings.workspaceHeartbeatService = workspaceService; + turnRequestBuilderBindings.workspaceMemoryPolicySink = workspaceService; // Tool-started workflows share the same sidebar activity cache as ORPC-started workflows, // so terminal updates must prune active run counts regardless of launch path. turnRequestBuilderBindings.onWorkflowRunStatusChanged = (event) => workspaceService.emitWorkflowRunActivity(event); turnRequestBuilderBindings.workflowResultContinuationSender = workspaceService; workspaceService.setMemoryConsolidationService(memoryConsolidationService); + workspaceService.setSharedWorkspaceMemoryStore(memoryService); + // Workspace-scope change events carry the memory OWNER (task-tree root); + // every live session resolving to that owner reads the same notebook. + memoryService.on("change", (event: MemoryChangeEvent) => { + if (event.scope !== "workspace" || event.workspaceId === "") return; + // One config snapshot for the whole pass: cold owner lookups would + // otherwise parse the config once per live session, synchronously. + let cfg: ReturnType | undefined; + const snapshot = () => (cfg ??= config.loadConfigOrDefault()); + workspaceService.invalidateMemoryContextWhere( + (workspaceId) => + memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, snapshot) === event.workspaceId + ); + }); + // Ownership itself changed (an owner was removed while its sub-agents + // live on): those sessions' cached contexts still describe the old store. + memoryService.on("ownersInvalidated", (workspaceIds: string[]) => { + const affected = new Set(workspaceIds); + workspaceService.invalidateMemoryContextWhere((workspaceId) => affected.has(workspaceId)); + }); if (opts.devToolsService) { // DevTools debug-log cleanup when workspaces are archived/removed. workspaceService.setDevToolsService(opts.devToolsService); diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 4d4ef681bf3..8ccf9902a52 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -1409,13 +1409,15 @@ describe("HistoryService", () => { const cachedNext = counters.sequenceCounters.get(nextWorkspace); const next = row("next"); expect((await restarted.appendToHistory(nextWorkspace, next)).success).toBe(true); - expect(next.metadata?.historySequence).toBe(method === "clear" ? 0 : 101); + // A clear opens a new history segment above every cleared sequence + // (history-segment.json) rather than restarting at 0. + expect(next.metadata?.historySequence).toBe(101); const reloaded = new HistoryService(config); const later = row("later"); expect((await reloaded.appendToHistory(nextWorkspace, later)).success).toBe(true); - expect(later.metadata?.historySequence).toBe(method === "clear" ? 1 : 102); + expect(later.metadata?.historySequence).toBe(102); if (method !== "archive delete") { - expect(cachedNext).toBe(method === "clear" ? 0 : 101); + expect(cachedNext).toBe(101); } } ); @@ -2548,18 +2550,49 @@ describe("HistoryService", () => { expect(exists).toBe(false); }); - it("should reset sequence counter", async () => { + it("continues sequences above the cleared history in a new segment", async () => { const workspaceId = "workspace1"; - const msg1 = createMuxMessage("msg1", "user", "Hello"); - - await service.appendToHistory(workspaceId, msg1); + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.appendToHistory(workspaceId, createMuxMessage("msg2", "user", "Second")); await service.clearHistory(workspaceId); - const msg2 = createMuxMessage("msg2", "user", "New message"); - await service.appendToHistory(workspaceId, msg2); + // A sequence — or a policy epoch identity derived from one — must never + // name rows of two different conversations, so the cleared segment's + // sequences are retired for good and the new rows carry their segment. + const msg3 = createMuxMessage("msg3", "user", "New message"); + await service.appendToHistory(workspaceId, msg3); + expect(msg3.metadata?.historySequence).toBe(2); + expect(msg3.metadata?.historySegment).toBe(2); - const messages = await collectFullHistory(service, workspaceId); - expect(messages[0].metadata?.historySequence).toBe(0); + // Durable across a restart: a fresh service scanning the (short) new + // history must not fall back to the cleared sequences. + const restarted = new HistoryService(config); + const msg4 = createMuxMessage("msg4", "user", "After restart"); + await restarted.appendToHistory(workspaceId, msg4); + expect(msg4.metadata?.historySequence).toBe(3); + expect(msg4.metadata?.historySegment).toBe(2); + + // A second clear opens a segment strictly above the previous one even + // though only the new segment's rows were cleared. + await restarted.clearHistory(workspaceId); + const msg5 = createMuxMessage("msg5", "user", "Third conversation"); + await restarted.appendToHistory(workspaceId, msg5); + expect(msg5.metadata?.historySequence).toBe(4); + expect(msg5.metadata?.historySegment).toBe(4); + }); + + it("stamps rows appended by a foreign backend after a clear with the new segment", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + // A second backend with a stale cached counter (multi-instance). + const foreign = new HistoryService(config); + await foreign.appendToHistory(workspaceId, createMuxMessage("msg2", "user", "Foreign")); + await service.clearHistory(workspaceId); + + const late = createMuxMessage("msg3", "assistant", "in flight before the clear"); + expect((await foreign.appendToHistory(workspaceId, late)).success).toBe(true); + expect(late.metadata?.historySequence).toBe(2); + expect(late.metadata?.historySegment).toBe(2); }); it("should succeed when clearing non-existent history", async () => { @@ -2570,16 +2603,191 @@ describe("HistoryService", () => { expect(result.success).toBe(true); }); - it("should reset sequence counter even when file doesn't exist", async () => { + it("opens a new segment even when clearing an empty history", async () => { const workspaceId = "workspace-no-history"; + // A turn admitted before the clear may still persist its policy under + // the boundary-less identity of the segment being cleared; the new + // segment must not share it, so the segment moves on regardless. await service.clearHistory(workspaceId); const msg = createMuxMessage("msg1", "user", "First"); await service.appendToHistory(workspaceId, msg); const messages = await collectFullHistory(service, workspaceId); - expect(messages[0].metadata?.historySequence).toBe(0); + expect(messages[0].metadata?.historySequence).toBe(1); + expect(messages[0].metadata?.historySegment).toBe(1); + }); + + it("quarantines a malformed segment file and reseeds above the visible rows", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory(workspaceId); + const msg2 = createMuxMessage("msg2", "user", "Second"); + await service.appendToHistory(workspaceId, msg2); + expect(msg2.metadata?.historySegment).toBe(1); + const segmentPath = path.join(config.sessionsDir, workspaceId, "history-segment.json"); + await fs.writeFile(segmentPath, "not json"); + + // Self-healing rather than refusing every append until the file is + // deleted by hand: the corrupt file is quarantined and the segment + // reseeded above every sequence still visible, so the row lands and + // carries a fresh stamp (the epoch identity moves on — fail closed for + // the policy recorded under the old one, never a reused sequence). + // The reseeded start is floored by the wall clock: an empty history + // after a restart has no row, counter or cached start to prove a + // segment identity was never used before, and a small reseed could + // repeat one a late backend still names. + const before = Date.now(); + const restarted = new HistoryService(config); + const msg3 = createMuxMessage("msg3", "user", "Third"); + expect((await restarted.appendToHistory(workspaceId, msg3)).success).toBe(true); + const reseeded = msg3.metadata?.historySegment; + if (reseeded === undefined) throw new Error("expected a reseeded stamp"); + expect(reseeded).toBeGreaterThan(before); + expect(msg3.metadata?.historySequence).toBe(reseeded); + expect(JSON.parse(await fs.readFile(segmentPath, "utf-8"))).toEqual({ start: reseeded }); + const quarantined = (await fs.readdir(path.dirname(segmentPath))).filter((name) => + name.startsWith("history-segment.json.corrupt-") + ); + expect(quarantined).toHaveLength(1); + // A clear after the repair opens the next segment as usual. + await restarted.clearHistory(workspaceId); + const msg4 = createMuxMessage("msg4", "user", "Fourth"); + await restarted.appendToHistory(workspaceId, msg4); + expect(msg4.metadata?.historySegment).toBe(reseeded + 1); + }); + + it("reseeds an empty workspace's corrupt segment above any identity used before", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory(workspaceId); + // Segment 1 (identity -2) was used; the file is corrupted with nothing + // else surviving: no rows, and a fresh process has no cached state. + await fs.writeFile( + path.join(config.sessionsDir, workspaceId, "history-segment.json"), + "{broken" + ); + const restarted = new HistoryService(config); + const msg = createMuxMessage("msg2", "user", "After repair"); + expect((await restarted.appendToHistory(workspaceId, msg)).success).toBe(true); + expect(msg.metadata?.historySegment).toBeGreaterThan(1); + }); + + it("treats a segment start at the safe-integer boundary as malformed", async () => { + const workspaceId = "workspace1"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + // `start` is safe but `start + 1` is not: the file cannot seed a + // counter that moves, so it is quarantined and reseeded like any other + // malformed file. + const segmentPath = path.join(workspaceDir, "history-segment.json"); + await fs.writeFile(segmentPath, JSON.stringify({ start: Number.MAX_SAFE_INTEGER })); + const msg = createMuxMessage("msg1", "user", "Hello"); + expect((await service.appendToHistory(workspaceId, msg)).success).toBe(true); + const reseeded = msg.metadata?.historySegment; + if (reseeded === undefined) throw new Error("expected a reseeded stamp"); + expect(Number.isSafeInteger(reseeded + 1)).toBe(true); + expect(reseeded).toBeLessThan(Number.MAX_SAFE_INTEGER); + }); + + it("refuses a reseed it cannot persist without quarantining the malformed file", async () => { + const workspaceId = "workspace1"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + const segmentPath = path.join(workspaceDir, "history-segment.json"); + // A readable start at the ceiling: the first append of that segment is + // refused (the sequence could not be retired), but the start is cached. + await fs.writeFile(segmentPath, JSON.stringify({ start: Number.MAX_SAFE_INTEGER - 1 })); + expect( + (await service.appendToHistory(workspaceId, createMuxMessage("m1", "user", "x"))).success + ).toBe(false); + // The file then turns malformed: the reseed floors at the cached start + // and would land past the ceiling. It refuses BEFORE renaming, so the + // malformed file stays in place and the next attempt does not read an + // absent file as segment 0 and reuse the retired identity. + await fs.writeFile(segmentPath, "garbage"); + for (let attempt = 0; attempt < 2; attempt++) { + const refused = await service.appendToHistory( + workspaceId, + createMuxMessage(`m${attempt + 2}`, "user", "x") + ); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("safe successor"); + expect(await fs.readFile(segmentPath, "utf-8")).toBe("garbage"); + } + expect((await fs.readdir(workspaceDir)).filter((name) => name.includes(".corrupt-"))).toEqual( + [] + ); + }); + + it("ignores persisted sequences without a safe successor for the append and clear floors", async () => { + const workspaceId = "workspace2"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + // Hand-edited rows at and past 2^53 - 2 cannot floor a counter that + // has to move, nor open the segment that retires them (2^53 - 2 would + // put the next start at 2^53 - 1, which the reader rejects); they are + // skipped like fractional sequences (never refused), so the user can + // still send and /clear removes them. + await fs.writeFile( + path.join(workspaceDir, "chat.jsonl"), + [ + { ...createMuxMessage("sane", "user", "sane", { historySequence: 4 }), workspaceId }, + { + ...createMuxMessage("below", "user", "below the edge", { + historySequence: Number.MAX_SAFE_INTEGER - 1, + }), + workspaceId, + }, + { + ...createMuxMessage("edge", "user", "at the edge", { + historySequence: Number.MAX_SAFE_INTEGER, + }), + workspaceId, + }, + { + ...createMuxMessage("huge", "user", "past the edge", { + historySequence: Number.MAX_SAFE_INTEGER + 1, + }), + workspaceId, + }, + ] + .map((row) => JSON.stringify(row)) + .join("\n") + "\n" + ); + const appended = createMuxMessage("m", "user", "x"); + expect((await service.appendToHistory(workspaceId, appended)).success).toBe(true); + expect(appended.metadata?.historySequence).toBe(5); + expect((await service.clearHistory(workspaceId)).success).toBe(true); + expect( + await fs.access(path.join(workspaceDir, "chat.jsonl")).then( + () => true, + () => false + ) + ).toBe(false); + const next = createMuxMessage("n", "user", "y"); + expect((await service.appendToHistory(workspaceId, next)).success).toBe(true); + expect(next.metadata?.historySequence).toBe(6); + expect(next.metadata?.historySegment).toBe(6); + }); + + it("forks the current segment along with the history snapshot", async () => { + await service.appendToHistory("source", createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory("source"); + const kept = createMuxMessage("msg2", "user", "Kept"); + await service.appendToHistory("source", kept); + expect(kept.metadata?.historySegment).toBe(1); + + // The copied rows carry the source's stamp: the fork continues that + // segment instead of appending unstamped rows (segment 0) beside them. + expect((await service.copyHistorySnapshotToNewWorkspace("source", "fork")).success).toBe( + true + ); + const forked = createMuxMessage("msg3", "user", "In the fork"); + await service.appendToHistory("fork", forked); + expect(forked.metadata?.historySequence).toBe(2); + expect(forked.metadata?.historySegment).toBe(1); }); }); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 0b02d1d53cb..ef5cc24a6f1 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -197,6 +197,31 @@ function stripContextUsage(message: MuxMessage): MuxMessage { }; } +/** + * A history segment start that can be persisted AND read back: a nonnegative + * integer whose successor is still a safe integer (`start + 1` past 2^53 no + * longer moves). One predicate for the writer and the reader, so the service + * never persists a start it would treat as malformed on the next read. + */ +function isUsableHistorySegmentStart(value: unknown): value is number { + return isNonNegativeInteger(value) && Number.isSafeInteger(value + 1); +} + +/** + * A history sequence the counter can be floored at and that a full clear can + * retire (see getNewestHistorySequence): a nonnegative integer whose successor + * is a usable segment start, so every floor derived from it stays writable. + */ +function isUsableHistorySequence(value: unknown): value is number { + return isNonNegativeInteger(value) && isUsableHistorySegmentStart(value + 1); +} + +/** The persisted row's segment stamp, kept across in-place replacement (see MuxMetadata.historySegment). */ +function preservedHistorySegment(existing: MuxMessage): { historySegment?: number } { + const segment = existing.metadata?.historySegment; + return segment === undefined ? {} : { historySegment: segment }; +} + function getCompactionMetadataToPreserve( workspaceId: string, existingMessage: MuxMessage, @@ -416,8 +441,22 @@ export class HistoryService { private readonly CHAT_FILE = CHAT_FILE_NAME; private readonly CHAT_ARCHIVE_FILE = CHAT_ARCHIVE_FILE_NAME; private readonly PARTIAL_FILE = "partial.json"; + /** + * `{ start }`: the first history sequence of the current history segment. + * A full clear does not restart sequences at 0 — it opens a new segment + * above every sequence the cleared history used (advanceHistorySegment), + * so no sequence, and no epoch identity derived from one (a boundary's + * sequence, or `-(start + 1)` before any boundary; see + * workspaceMemoryPolicyEpochOf), is ever reused for a different + * conversation. Rows appended in a later segment carry its start as + * `metadata.historySegment`. Absent = first segment (start 0). + */ + private readonly HISTORY_SEGMENT_FILE = "history-segment.json"; // Track next sequence number per workspace in memory private sequenceCounters = new Map(); + // Current segment start per workspace, loaded with the counter + // (getMaxHistorySequence) and advanced by full clears under the history lock. + private historySegmentStarts = new Map(); // Workspaces whose chat.jsonl was already checked for a sealed (pre-boundary) // prefix this process. Guards the lazy one-time migration of legacy files; // new boundaries rotate eagerly at write time. @@ -1782,6 +1821,10 @@ export class HistoryService { Ok({ archive: await this.readExistingFile(this.getChatArchivePath(sourceWorkspaceId)), chat: await this.readExistingFile(this.getChatHistoryPath(sourceWorkspaceId)), + // The copied rows carry the source's segment stamp; the target must + // continue that segment (same floor, same stamp) or its own appends + // would sit unstamped beside stamped rows. + segment: await this.readExistingFile(this.getHistorySegmentPath(sourceWorkspaceId)), }) ); if (!snapshot.success) { @@ -1796,6 +1839,7 @@ export class HistoryService { for (const [targetPath, contents] of [ [this.getChatArchivePath(targetWorkspaceId), snapshot.data.archive], [this.getChatHistoryPath(targetWorkspaceId), snapshot.data.chat], + [this.getHistorySegmentPath(targetWorkspaceId), snapshot.data.segment], ] as const) { if (contents === null) { await fs.rm(targetPath, { force: true }); @@ -1803,6 +1847,7 @@ export class HistoryService { await writeFileAtomic(targetPath, contents); } } + this.historySegmentStarts.delete(targetWorkspaceId); return Ok(undefined); } ); @@ -1857,7 +1902,12 @@ export class HistoryService { for (const message of messages) { const sequence = message.metadata?.historySequence; - if (!isNonNegativeInteger(sequence)) { + // A sequence whose successor is not a usable segment start (>= 2^53 - 2; + // a hand-edited row) is malformed like a fraction: it cannot floor a + // counter that has to move nor be retired by a clear, so it is skipped — + // appends continue from the sane rows and a clear removes it — rather + // than refusing every append and clear. + if (!isUsableHistorySequence(sequence)) { continue; } @@ -1869,8 +1919,152 @@ export class HistoryService { return newest; } + private getHistorySegmentPath(workspaceId: string): string { + return path.join(this.getSessionDir(workspaceId), this.HISTORY_SEGMENT_FILE); + } + + /** + * Current segment start from `history-segment.json` (0 when absent), or + * null when the file is present but malformed. Unreadable (EACCES, EIO) + * throws: a cleared workspace whose file cannot be read would otherwise + * restart at 0 and reuse the sequences the clear retired. Only a safe + * integer is a usable start — above 2^53, `start + 1` can equal `start` + * and a clear would fail to open a strictly newer segment. + */ + private async readHistorySegmentStart(workspaceId: string): Promise { + let raw: string; + try { + raw = await fs.readFile(this.getHistorySegmentPath(workspaceId), "utf-8"); + } catch (error) { + if (!isErrnoWithCode(error, "ENOENT")) throw error; + return 0; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + const start: unknown = + typeof parsed === "object" && parsed !== null + ? (parsed as { start?: unknown }).start + : undefined; + // Same predicate as the writer: a file at the boundary is malformed too + // and reseeds (which refuses an unusable reseed before touching the file). + return isUsableHistorySegmentStart(start) ? start : null; + } + + /** + * Self-healing for a malformed `history-segment.json` (under the history + * write lock): the file is quarantined next to itself and reseeded with a + * start above every sequence still visible (the rows, the cached counter + * and the cached start), so appends and clears keep working instead of + * failing on every attempt until someone deletes the file by hand. The + * retired range a corrupt file may have named is not recoverable; rows + * appended after the reseed carry the new stamp, so the segment's + * boundary-less epoch identity changes and any policy recorded under the + * old one is simply never consulted again (harvests of that epoch fail + * closed rather than reading a stale record). + */ + private async reseedHistorySegmentUnderHistoryLock( + workspaceId: string, + visibleMaxSequence: number + ): Promise { + // An empty history after a restart leaves no row, counter or cached start + // to floor at, and a small reseed (1) could repeat the identity of an + // earlier segment that a late backend still names in its policy records. + // The wall clock is the one monotone source that survives all of that: + // sequences only need to compare, so the jump is harmless, and every + // later clear continues above it. + const start = + Math.max( + visibleMaxSequence, + (this.sequenceCounters.get(workspaceId) ?? 0) - 1, + this.historySegmentStarts.get(workspaceId) ?? 0, + Date.now() + ) + 1; + // Refuse BEFORE quarantining: a rename followed by a refused write would + // leave no segment file, and the next read would restart the segment at + // 0 — the retired range's identity reused, which the file exists to prevent. + this.requireUsableHistorySegmentStart(workspaceId, start); + const segmentPath = this.getHistorySegmentPath(workspaceId); + const quarantinePath = `${segmentPath}.corrupt-${Date.now()}`; + log.warn("Quarantining malformed history segment file and reseeding the segment", { + workspaceId, + quarantinePath, + }); + await fs.rename(segmentPath, quarantinePath); + await this.writeHistorySegmentStart(workspaceId, start); + return start; + } + + /** + * Reachable from persisted data (the cached counter or start of a + * hand-edited history at the 2^53 boundary): refuse rather than persist a + * start the reader would reject. + */ + private requireUsableHistorySegmentStart(workspaceId: string, start: number): void { + if (!isUsableHistorySegmentStart(start)) { + throw new Error( + `Cannot open a new history segment for ${workspaceId}: start ${String(start)} has no safe successor` + ); + } + } + + private async writeHistorySegmentStart(workspaceId: string, start: number): Promise { + this.requireUsableHistorySegmentStart(workspaceId, start); + await ensurePrivateDir(this.getSessionDir(workspaceId)); + await writeFileAtomic(this.getHistorySegmentPath(workspaceId), JSON.stringify({ start })); + this.historySegmentStarts.set(workspaceId, start); + } + + /** + * Open the next history segment (full clear, under the history write + * lock, BEFORE the files are rewritten): its start is above every sequence + * the history being cleared used, the cached counter, AND the previous + * start — an already-empty history still moves on, because a turn admitted + * before the clear can otherwise persist its policy under the boundary-less + * identity the cleared segment and the new one would share. Persisted + * before the rows are removed so a crash in between leaves the history + * intact rather than an empty history whose counter restarts at 0. + */ + private async advanceHistorySegmentUnderHistoryLock( + workspaceId: string, + clearedSequences: readonly number[] + ): Promise { + // Loads the segment cache as a side effect and floors at previousStart - 1. + const persistedMax = await this.getMaxHistorySequence(workspaceId); + const previousStart = this.historySegmentStarts.get(workspaceId) ?? 0; + const start = + clearedSequences.reduce( + (max, sequence) => (isUsableHistorySequence(sequence) && sequence > max ? sequence : max), + Math.max(persistedMax, (this.sequenceCounters.get(workspaceId) ?? 0) - 1, previousStart) + ) + 1; + await this.writeHistorySegmentStart(workspaceId, start); + this.sequenceCounters.set(workspaceId, start); + } + + /** + * Stamp a freshly sequenced row with the segment it is appended in (first + * segment rows stay unstamped, byte-identical to legacy rows). The cache is + * loaded by the in-lock counter refresh every append path runs first. + */ + private stampHistorySegment(workspaceId: string, metadata: MuxMetadata): MuxMetadata { + const start = this.historySegmentStarts.get(workspaceId); + assert(start !== undefined, "history segment start must be loaded before rows are stamped"); + return start === 0 ? metadata : { ...metadata, historySegment: start }; + } + + /** + * Called under the history write lock only (every caller assigns + * sequences): a malformed segment file is repaired in place here. + */ private async getMaxHistorySequence(workspaceId: string): Promise { - let maxSequence = -1; + // Floor: a cleared history has no rows, but its sequences continue above + // the retired segment (see HISTORY_SEGMENT_FILE). + const segmentStart = await this.readHistorySegmentStart(workspaceId); + if (segmentStart !== null) this.historySegmentStarts.set(workspaceId, segmentStart); + let maxSequence = (segmentStart ?? this.historySegmentStarts.get(workspaceId) ?? 0) - 1; // Full scan of the active file (cheap post-rotation; see getNextHistorySequence // for why we don't trust the tail alone). @@ -1884,8 +2078,11 @@ export class HistoryService { // The archive holds strictly-older sequences than chat.jsonl, so it only // decides the counter when chat.jsonl is missing/hand-edited. const archiveMax = await this.getArchiveTailMaxSequence(workspaceId); - - return Math.max(maxSequence, archiveMax); + const max = Math.max(maxSequence, archiveMax); + if (segmentStart === null) { + return (await this.reseedHistorySegmentUnderHistoryLock(workspaceId, max)) - 1; + } + return max; } /** @@ -2711,15 +2908,29 @@ export class HistoryService { // User rationale: a stale partial or hand-edited chat.jsonl can leave an old // historySequence at the tail. Initializing from the tail would make the next // live message look like an edit/truncation to the renderer, so scan for max. - const nextSeqNum = (await this.getMaxHistorySequence(workspaceId)) + 1; - assert( - isNonNegativeInteger(nextSeqNum), - "next history sequence counter must be a non-negative integer" - ); + const nextSeqNum = await this.getNextPersistedHistorySequence(workspaceId); this.sequenceCounters.set(workspaceId, nextSeqNum); return nextSeqNum; } + /** + * `max persisted sequence + 1`, refused when that is not itself a usable + * sequence. Reachable from persisted data (a hand-edited segment start at + * the 2^53 boundary): a row assigned there could not floor the counter on + * the next load (its duplicate would let a later finalization replace an + * unrelated row) nor be retired by a clear, so appends fail instead of + * writing it. + */ + private async getNextPersistedHistorySequence(workspaceId: string): Promise { + const nextSeqNum = (await this.getMaxHistorySequence(workspaceId)) + 1; + if (!isUsableHistorySequence(nextSeqNum)) { + throw new Error( + `History sequences of ${workspaceId} are exhausted: next sequence ${String(nextSeqNum)} has no safe successor` + ); + } + return nextSeqNum; + } + /** * Internal helper for appending to history without acquiring lock. */ @@ -2746,9 +2957,9 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { + message.metadata = this.stampHistorySegment(workspaceId, { historySequence: nextSeqNum, - }; + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } else { // Message already has metadata, but may need historySequence assigned @@ -2773,6 +2984,9 @@ export class HistoryService { ); } this.sequenceCounters.set(workspaceId, existingSeqNum + 1); + // A pre-sequenced row (recovered partial) is appended in the + // current segment like any other. + message.metadata = this.stampHistorySegment(workspaceId, message.metadata); } else { // Has metadata but no historySequence, assign one const nextSeqNum = await this.getNextHistorySequence(workspaceId); @@ -2780,10 +2994,10 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { + message.metadata = this.stampHistorySegment(workspaceId, { ...message.metadata, historySequence: nextSeqNum, - }; + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } } @@ -3033,7 +3247,7 @@ export class HistoryService { * precedes every operation (active file is bounded by rotation). */ private async refreshSequenceCounterUnderWriteLock(workspaceId: string): Promise { - const persistedNext = (await this.getMaxHistorySequence(workspaceId)) + 1; + const persistedNext = await this.getNextPersistedHistorySequence(workspaceId); const cached = this.sequenceCounters.get(workspaceId); if (cached === undefined || persistedNext > cached) { this.sequenceCounters.set(workspaceId, persistedNext); @@ -3123,7 +3337,10 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { ...message.metadata, historySequence: nextSeqNum }; + message.metadata = this.stampHistorySegment(workspaceId, { + ...message.metadata, + historySequence: nextSeqNum, + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } // Atomic all-or-nothing commit (r48): fs.appendFile is not @@ -3445,12 +3662,15 @@ export class HistoryService { message ); - // Preserve the historySequence, update everything else. + // Preserve the historySequence (and the segment the row was appended + // in: the replacement was built from an in-memory copy that may + // predate the stamp), update everything else. messages[i] = { ...message, metadata: { ...message.metadata, ...(preservedCompactionMetadata ?? {}), + ...preservedHistorySegment(existingMessage), historySequence: targetSequence, }, }; @@ -3601,6 +3821,7 @@ export class HistoryService { metadata: { ...summaryMessage.metadata, ...(preservedCompactionMetadata ?? {}), + ...preservedHistorySegment(messages[i]), historySequence: targetSequence, }, }; @@ -3619,7 +3840,10 @@ export class HistoryService { const nextSeqNum = await this.getNextHistorySequence(workspaceId); persistedSummary = { ...summaryMessage, - metadata: { ...summaryMessage.metadata, historySequence: nextSeqNum }, + metadata: this.stampHistorySegment(workspaceId, { + ...summaryMessage.metadata, + historySequence: nextSeqNum, + }), }; this.sequenceCounters.set(workspaceId, nextSeqNum + 1); appended.set(summaryMessage, persistedSummary); @@ -3632,7 +3856,13 @@ export class HistoryService { "persistBoundaryWithTailCopies expects unsequenced tail copies" ); const seq = await this.getNextHistorySequence(workspaceId); - const persistedCopy = { ...copy, metadata: { ...copy.metadata, historySequence: seq } }; + const persistedCopy = { + ...copy, + metadata: this.stampHistorySegment(workspaceId, { + ...copy.metadata, + historySequence: seq, + }), + }; this.sequenceCounters.set(workspaceId, seq + 1); appended.set(copy, persistedCopy); messages.push(persistedCopy); @@ -4187,8 +4417,8 @@ export class HistoryService { workspaceId ).advanceGenerationUnderHistoryLock(); } + await this.advanceHistorySegmentUnderHistoryLock(workspaceId, allSequences); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); - this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); } @@ -4246,8 +4476,8 @@ export class HistoryService { await this.getContinuousCompactionJournal( workspaceId ).advanceGenerationUnderHistoryLock(); + await this.advanceHistorySegmentUnderHistoryLock(workspaceId, allSequences); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); - this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); } @@ -4409,6 +4639,7 @@ export class HistoryService { const archiveFloor = (await this.getArchiveTailMaxSequence(newWorkspaceId)) + 1; this.sequenceCounters.set(newWorkspaceId, Math.max(oldCounter, archiveFloor)); this.sequenceCounters.delete(oldWorkspaceId); + this.historySegmentStarts.delete(oldWorkspaceId); return Ok(undefined); } @@ -4422,6 +4653,9 @@ export class HistoryService { // Transfer sequence counter to new workspace ID this.sequenceCounters.set(newWorkspaceId, oldCounter); this.sequenceCounters.delete(oldWorkspaceId); + // The segment file moved with the session directory; the new id's + // cache is loaded on its next append. + this.historySegmentStarts.delete(oldWorkspaceId); log.debug( `Migrated ${messages.length} messages from ${oldWorkspaceId} to ${newWorkspaceId}` diff --git a/src/node/services/memoryConsolidation.test.ts b/src/node/services/memoryConsolidation.test.ts index 2acadf5230d..9f1a60d3645 100644 --- a/src/node/services/memoryConsolidation.test.ts +++ b/src/node/services/memoryConsolidation.test.ts @@ -313,6 +313,48 @@ describe("consolidation memory tool rails", () => { ).toContain("polished"); }); + it("enforces pin protection inside the mutation against the store the command binds to", async () => { + using fixture = await createFixture(); + const sessionMemory = path.join(fixture.xumHome, "sessions", fixture.ctx.workspaceId, "memory"); + await fsPromises.mkdir(sessionMemory, { recursive: true }); + await fsPromises.writeFile(path.join(sessionMemory, "pinned.md"), "keep me\n"); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "pinned.md", { + projectPath: fixture.ctx.projectPath, + workspaceId: fixture.ctx.workspaceId, + }), + true + ); + // The tool's pre-check resolves the shared-store owner on its own; make + // that resolution disagree with the command's (as a transiently unreadable + // config.json can, falling back to a different store) so the pre-check + // looks at the wrong sidecar key and passes. The command itself must + // still refuse: its check runs in-lock against the owner its store is + // bound to. + const resolve = spyOn(fixture.memoryService, "resolveWorkspaceMemoryOwnerId"); + resolve.mockImplementationOnce(() => "ws-elsewhere"); + try { + const deletion = await execute(fixture.tool, { + command: "delete", + path: "/memories/workspace/pinned.md", + }); + expect(deletion.success).toBe(false); + if (!deletion.success) expect(deletion.error).toContain("pinned"); + expect(await pathExists(path.join(sessionMemory, "pinned.md"))).toBe(true); + resolve.mockImplementationOnce(() => "ws-elsewhere"); + const rename = await execute(fixture.tool, { + command: "rename", + old_path: "/memories/workspace/pinned.md", + new_path: "/memories/workspace/moved.md", + }); + expect(rename.success).toBe(false); + if (!rename.success) expect(rename.error).toContain("pinned"); + expect(await pathExists(path.join(sessionMemory, "pinned.md"))).toBe(true); + } finally { + resolve.mockRestore(); + } + }); + it("rejects deleting or renaming a directory that contains a pinned file", async () => { using fixture = await createFixture(); const nestedDir = path.join(fixture.globalMemoryDir, "nested"); diff --git a/src/node/services/memoryConsolidation.ts b/src/node/services/memoryConsolidation.ts index 509da74270c..cf84b904989 100644 --- a/src/node/services/memoryConsolidation.ts +++ b/src/node/services/memoryConsolidation.ts @@ -244,6 +244,11 @@ export function createConsolidationMemoryTool(args: { // Deletes/renames may target a directory (MemoryService removes // recursively), so reject when the path itself OR anything under it is // pinned — otherwise `delete dir/` would silently destroy dir/pinned.md. + // This pre-check gives the model (and dry-run staging) early feedback; + // the AUTHORITATIVE check runs inside MemoryService's mutation lock + // against the owner the command's store is actually bound to + // (`rejectPinned` below): the owner resolved here can differ from the + // command's when config.json is transiently unreadable in between. if (target.command === "delete" || target.command === "rename") { const { scope, relPath } = parseMemoryPath(target.path); assert( @@ -253,7 +258,11 @@ export function createConsolidationMemoryTool(args: { const entries = await metaService.getEntries(); const key = memoryLogicalKey(scope, relPath, { projectPath: ctx.projectPath, - workspaceId: ctx.workspaceId, + // Sidecar keys follow the shared store's owner (MemoryService.logicalKeyFor). + workspaceId: + ctx.workspaceId === "" + ? "" + : memoryService.resolveWorkspaceMemoryOwnerId(ctx.workspaceId), }); const subtreePrefix = `${key}/`; for (const [entryKey, entry] of entries) { @@ -327,6 +336,7 @@ export function createConsolidationMemoryTool(args: { const result = await executeMemoryCommand(memoryService, ctx, input, () => null, toolCallId, { expectedTargetFingerprint: args.expectedTargetFingerprints?.get(toolCallId), abortSignal: args.abortSignal, + rejectPinned: true, }); journal.push({ ...target, diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 4f9fdaca800..db92fc5eb74 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { Effect } from "effect"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -7,7 +8,10 @@ import type { LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai- import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import { createMuxMessage } from "@/common/types/message"; -import type { MemoryConsolidationStatusChangeEventPayload } from "@/common/orpc/schemas/memory"; +import type { + MemoryConsolidationStatusChangeEventPayload, + MemoryHarvestRecordPayload, +} from "@/common/orpc/schemas/memory"; import { MULTI_PROJECT_CONFIG_KEY } from "@/common/constants/multiProject"; import { EXPERIMENT_IDS } from "@/common/constants/experiments"; import { @@ -17,6 +21,7 @@ import { import { Ok } from "@/common/types/result"; import { Config } from "@/node/config"; import { + HARVEST_MAX_ATTEMPTS, MemoryConsolidationService, resolveDreamAgentBody, resolveDreamModelString, @@ -31,6 +36,7 @@ import { MemoryService } from "./memoryService"; import { SessionUsageService } from "./sessionUsageService"; import { TestTempDir } from "./tools/testHelpers"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; /** * Behavior under test: the orchestration rails around the runner — @@ -204,7 +210,10 @@ interface Fixture extends Disposable { setEnabled: (enabled: boolean) => void; /** When true, scripted runs emit a fatal stream error instead of finishing. */ setStreamFailing: (failing: boolean) => void; - addWorkspace: (id: string, opts?: { archivedAt?: string }) => Promise; + addWorkspace: ( + id: string, + opts?: { archivedAt?: string; parentWorkspaceId?: string } + ) => Promise; addMultiProjectWorkspace: (id: string, opts?: { bucket?: string }) => Promise; } @@ -283,6 +292,7 @@ async function createFixture(options?: { name: id, path: `/projects/demo/${id}`, archivedAt: opts?.archivedAt, + parentWorkspaceId: opts?.parentWorkspaceId, }); return cfg; }); @@ -317,9 +327,17 @@ async function seedCompactionEpoch( fixture: Fixture, workspaceId = "ws-dream" ): Promise { + const prompt = createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests."); + await fixture.historyService.appendToHistory(workspaceId, prompt); + // The turn ran: its policy was recorded before this reply was appended, and + // the reply's request snapshot covers the prompt (uncovered user rows make + // the harvest refuse; see below). await fixture.historyService.appendToHistory( workspaceId, - createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests.") + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) ); await fixture.historyService.appendToHistory( workspaceId, @@ -338,6 +356,9 @@ async function seedCompactionEpoch( expect(typeof summaryHistorySequence).toBe("number"); return { workspaceId, + // A normal editing-capable agent; the read-only / unknown gates are + // exercised explicitly where they matter. + workspaceMemoryWritable: true, summaryMessageId: "summary-1", summaryHistorySequence: summaryHistorySequence ?? -1, compactionEpoch: 1, @@ -395,6 +416,30 @@ describe("MemoryConsolidationService", () => { }, }), }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const childMetadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [childMetadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: childMetadata.summaryMessageId, + compactionEpoch: childMetadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: childMetadata, + }, + }, + }, + }) + ); const run = fixture.service.maybeRun("ws-dream", "manual"); await started; await fixture.service.cancelInFlightConsolidation("ws-dream"); @@ -405,10 +450,58 @@ describe("MemoryConsolidationService", () => { if (!result.success) { expect(result.error).toContain("stream failed"); } + // The cancelled owner's continuation must not start recovery of a + // sub-agent's retryable harvest (work outside the drained registry). + const childRecords = ( + JSON.parse( + await fsPromises.readFile(path.join(fixture.xumHome, "memory-consolidation.json"), "utf-8") + ) as { harvestsByWorkspace: Record> } + ).harvestsByWorkspace["ws-sub"]; + expect(Object.values(childRecords).map((record) => record.attemptCount)).toEqual([1]); // Idempotent with nothing in flight (the phantom-metadata removal path). await fixture.service.cancelInFlightConsolidation("ws-dream"); }); + it("a sub-agent's removal drain aborts the owner-keyed run made on its behalf", async () => { + // A child's trigger (and the post-harvest sweep) runs the OWNER's + // consolidation; a cancelled child harvest deliberately falls through to + // that sweep. The child's removal drain must still reach the owner-keyed + // run, or it would keep mutating the shared notebook after removal. + let streamStarted!: () => void; + const started = new Promise((resolve) => (streamStarted = resolve)); + using fixture = await createFixture({ + modelFactory: () => + new MockLanguageModelV3({ + doStream: (options) => { + streamStarted(); + return Promise.resolve({ + stream: new ReadableStream({ + start(controller) { + options.abortSignal?.addEventListener("abort", () => { + controller.error(new Error("request aborted")); + }); + }, + }), + }); + }, + }), + }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const run = fixture.service.maybeRun("ws-sub", "manual"); + await started; + await fixture.service.cancelInFlightConsolidation("ws-sub"); + const result = await run; + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("stream failed"); + // Locally cancelled child: neither its own trigger nor an owner run made + // on its behalf may start while teardown is under way. + const refused = await fixture.service.maybeRun("ws-dream", "manual", { + actingWorkspaceId: "ws-sub", + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("being removed"); + }); + it("runs, persists the journal record, and reports it via getRecord", async () => { using fixture = await createFixture(); const result = await fixture.service.maybeRun("ws-dream", "compaction"); @@ -1079,6 +1172,766 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(3); }); + it("recovers a sub-agent's failed harvest through the owner's run", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + + // The child's manual run redirects to the owner; the owner's recovery + // must still retry the CHILD's bucket (the launch sweep never visits it). + expect((await fixture.service.maybeRun("ws-sub", "manual")).success).toBe(true); + const status = await fixture.service.getStatus("ws-sub"); + expect(status.latestHarvestRecord?.status).toBe("completed"); + expect(status.latestHarvestRecord?.attemptCount).toBe(2); + }); + + it("refuses to harvest (and sweep) for an agent whose workspace memory is read-only", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + const metadata = await seedCompactionEpoch(fixture); + const refused = await fixture.service.maybeHarvestThenSweep({ + ...metadata, + summaryMessageId: "summary-readonly", + workspaceMemoryWritable: false, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("read-only"); + expect(fixture.modelCalls).toHaveLength(0); + // Recorded as terminal so recovery never retries it, with the reason. + const refusedRecord = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(refusedRecord?.status).toBe("failed"); + expect(refusedRecord?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect(refusedRecord?.error).toContain("read-only"); + + // Unknown policy (legacy record / no persisted value) fails closed too. + const unknown = await fixture.service.maybeHarvestThenSweep({ + ...metadata, + summaryMessageId: "summary-unknown", + workspaceMemoryWritable: undefined, + }); + expect(unknown.success).toBe(false); + if (!unknown.success) expect(unknown.error).toContain("unknown"); + expect(fixture.modelCalls).toHaveLength(0); + + // Explicitly writable harvests as before. + const allowed = await fixture.service.maybeHarvestThenSweep(metadata); + expect(allowed.success).toBe(true); + const harvested = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(harvested?.boundaryKey).toBe(metadata.summaryMessageId); + expect(harvested?.status).toBe("completed"); + }); + + it("keys the harvest by the recorded closing epoch, refusing a turn stamped before a clear", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-stale"); + // A full clear opens a new history segment: the boundary-less identity + // becomes -(segmentStart + 1) (workspaceMemoryPolicyEpochOf) and a turn + // admitted before the clear, stamped -1, can no longer pass as one of the + // new segment even though its row landed after the clear. + const seed = async (workspaceId: string, staleStamp: boolean) => { + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-1", "user", "before the clear") + ); + await fixture.historyService.clearHistory(workspaceId); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory(workspaceId, prompt); + const segmentStart = prompt.metadata?.historySegment; + if (segmentStart === undefined || segmentStart <= 0) + throw new Error("expected a new segment"); + const closingPolicyEpoch = -(segmentStart + 1); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: staleStamp ? -1 : closingPolicyEpoch, + }) + ); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory(workspaceId, summary); + return fixture.service.maybeHarvestThenSweep({ + workspaceId, + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + closingPolicyEpoch, + }); + }; + const stale = await seed("ws-stale", true); + expect(stale.success).toBe(false); + if (!stale.success) expect(stale.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); + const current = await seed("ws-dream", false); + expect(current.success).toBe(true); + expect(fixture.modelCalls.length).toBeGreaterThan(0); + }); + + it("refuses to harvest when a prelude listing names an ordinary user turn", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // A read-only backend's prompt was never answered; the next turn's user + // row lists it as a "prelude" (raw history). The listed row is not a + // synthetic prelude row, so it stays uncovered and the harvest refuses. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") + ); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests.", + { + requestPreludeMessageIds: ["late-1"], + } + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("never recorded"); + expect(fixture.modelCalls).toHaveLength(0); + }); + + it("refuses to harvest an epoch whose user rows share an id", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Coverage is keyed by id: covering the later row would mark the earlier + // (never-covered) row covered too, so a duplicated id refuses outright. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("pref-1", "user", "Read-only agent's prompt, never answered") + ); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("sharing one id"); + expect(fixture.modelCalls).toHaveLength(0); + }); + + it("refuses to harvest an epoch holding user rows no turn's request snapshot covers", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Backend A snapshots its request through pref-1; backend B appends a + // read-only turn's user row BEFORE that turn records its policy; A's + // assistant row lands above it carrying its snapshot bound; A compacts + // with a grant. The later assistant is no proof for B's row. + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + // Refused end to end: no sweep either (the Dream pass writes the shared + // notebook too), and the harvest record is terminal (never completed, + // never retried), so recovery cannot replay the grant. + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("never recorded"); + expect(fixture.modelCalls).toHaveLength(0); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect(record?.error).toContain("never recorded"); + expect(record?.refused).toBe(true); + // A later trigger for the same boundary (recovery, duplicate completion) + // finds the terminal refusal and does not fall through to the sweep. + const again = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(again.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + }); + + it("covers only a turn's own batch: a foreign row inside the request snapshot leaves the turn's own row uncovered", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Backend B (read-only) appends late-1 and pauses before its deny lands; + // backend A then snapshots THROUGH late-1 (its request's latest user row + // is now B's), records writable and replies. A's bound covers late-1 as + // A's batch — so pref-1, A's own row, belongs to no turn at all. + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + const foreign = createMuxMessage( + "late-1", + "user", + "Read-only agent's prompt, deny not recorded" + ); + await fixture.historyService.appendToHistory("ws-dream", foreign); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: foreign.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + + it("refuses a turn whose policy was recorded for another epoch, and ignores preserved-tail copies", async () => { + using fixture = await createFixture(); + await fixture.addWorkspace("ws-clean"); + await fixture.addWorkspace("ws-corrupt"); + await fixture.addWorkspace("ws-unbounded"); + await fixture.addWorkspace("ws-fractional"); + const seed = async ( + workspaceId: string, + foreignTurn: number | null | undefined, + options?: { withoutBound?: boolean; fractionalBound?: boolean } + ) => { + const reset = createMuxMessage("reset-1", "assistant", "", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory(workspaceId, reset); + const closingEpoch = reset.metadata?.historySequence ?? -1; + // RLM keep-recent copies of the previous epoch's turns: no stamps of + // their own, no coverage needed (their originals were judged already), + // and not harvested again. + for (const [id, role] of [ + ["copy-user", "user"], + ["copy-reply", "assistant"], + ] as const) { + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage(id, role, "Copied tail row.", { + synthetic: true, + rlmPreservedTailCopy: true, + }) + ); + } + const prompt = createMuxMessage("pref-1", "user", "Remember I prefer concise tests."); + await fixture.historyService.appendToHistory(workspaceId, prompt); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: + options?.fractionalBound === true + ? (prompt.metadata?.historySequence ?? 0) + 0.5 + : prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: closingEpoch, + }) + ); + if (foreignTurn !== undefined) { + // Backend B started a read-only turn under the previous epoch (-1) and + // recorded its deny there; backend A then reset the context (the deny + // went with the epoch) and B's assistant landed after the new boundary + // — without its user row, which the reset removed. Only the epoch + // stamp can surface it. A corrupted stamp (raw JSON row) refuses too. + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("b-reply", "assistant", "Read-only output.", { + ...(options?.withoutBound === true ? {} : { requestHistorySequence: closingEpoch - 1 }), + // `null` models a corrupted raw-JSON row. + workspaceMemoryPolicyEpoch: foreignTurn as unknown as number, + }) + ); + } + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 2, + }); + await fixture.historyService.appendToHistory(workspaceId, summary); + return fixture.service.maybeHarvestThenSweep({ + workspaceId, + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 2, + compactionRequestMessageId: "compact-request", + previousBoundaryHistorySequence: closingEpoch, + }); + }; + const refused = await seed("ws-dream", -1); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); + expect((await fixture.service.getStatus("ws-dream")).latestHarvestRecord?.refused).toBe(true); + // `null` is neither "no stamp" nor this epoch's: fail closed. + const corrupt = await seed("ws-corrupt", null); + expect(corrupt.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + // A foreign stamp refuses even when the row's request bound is missing + // or corrupt: its user row may be gone, so nothing else would surface it. + const unbounded = await seed("ws-unbounded", -1, { withoutBound: true }); + expect(unbounded.success).toBe(false); + // A bound outside the sequence domain (fractional) covers no user row: + // the turn stays uncovered and refuses (r79). + const fractional = await seed("ws-fractional", undefined, { fractionalBound: true }); + expect(fractional.success).toBe(false); + if (!fractional.success) expect(fractional.error).toContain("harvest refused"); + expect(fixture.modelCalls).toHaveLength(0); + expect(unbounded.success).toBe(false); + if (!unbounded.success) expect(unbounded.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); + + // Without the foreign turn the copies alone refuse nothing, and the + // harvest transcript leaves them out. + const granted = await seed("ws-clean", undefined); + expect(granted.success).toBe(true); + expect(fixture.modelPrompts.length).toBeGreaterThan(0); + expect(fixture.modelPrompts[0]).toContain("concise tests"); + expect(fixture.modelPrompts[0]).not.toContain("Copied tail row"); + }); + + it("takes no coverage from assistant rows of a build that did not record the policy", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // A downgraded build ran a (read-only) turn mid-epoch: its assistant row + // carries the request snapshot bound but no policy record, and it left + // the durable accumulator's stale grant untouched. The compaction then + // completes with that grant. + const prompt = createMuxMessage("pref-1", "user", "Read-only turn on the old build."); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + + it("refuses an unstamped turn row even when no user row of its own survives", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // An older/downgraded backend started a turn before a destructive reset + // and its assistant landed afterwards: the row keeps its request bound + // but has no policy stamp, and its user row is gone with the reset — so + // no uncovered user row would surface it. The row itself must refuse. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("orphan-reply", "assistant", "Produced under an unknown policy.", { + requestHistorySequence: 0, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + + it("covers a turn's request prelude rows by id, not by adjacency", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + let previousBoundaryHistorySequence: number | undefined; + const harvest = async (ids: { listed: string[]; summary: string; leadIn?: boolean }) => { + // Two synthetic snapshot rows precede the user row; only the listed + // ones are the turn's own prelude. The reply snapshots through the + // user row, so every user row lies below its bound. + if (ids.leadIn) { + // Token-budget control row (backend template text, same durable batch + // as the turn): needs no listing. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-lead-in`, "user", "A context window rollover started.", { + synthetic: true, + muxMetadata: { type: "context-window-lead-in", rolloverId: "r1" }, + }) + ); + } + const snapshots = [`${ids.summary}-snap-a`, `${ids.summary}-snap-b`].map((id) => + createMuxMessage(id, "user", "Snapshot content", { + synthetic: true, + fileAtMentionSnapshot: ["@notes.md"], + }) + ); + for (const snapshot of snapshots) { + await fixture.historyService.appendToHistory("ws-dream", snapshot); + } + const prompt = createMuxMessage( + `${ids.summary}-pref`, + "user", + "Remember I prefer concise tests.", + { + requestPreludeMessageIds: ids.listed, + } + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-reply`, "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: previousBoundaryHistorySequence ?? -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-compact`, "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage(ids.summary, "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: ids.summary, + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: `${ids.summary}-compact`, + ...(previousBoundaryHistorySequence !== undefined + ? { previousBoundaryHistorySequence } + : {}), + }); + previousBoundaryHistorySequence = summary.metadata?.historySequence; + return { + result, + record: (await fixture.service.getStatus("ws-dream")).latestHarvestRecord, + }; + }; + // Only one of the two adjacent snapshot rows is listed: the other is a + // foreign row that merely sits next to the batch. + const partial = await harvest({ listed: ["s1-snap-a"], summary: "s1" }); + expect(partial.result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + expect(partial.record?.status).toBe("failed"); + expect(partial.record?.error).toContain("never recorded"); + // Both listed: the whole batch is the turn's own and harvests. + const complete = await harvest({ + listed: ["s2-snap-a", "s2-snap-b"], + summary: "s2", + leadIn: true, + }); + expect(complete.result.success).toBe(true); + expect(complete.record?.status).toBe("completed"); + }); + + it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + const record = (await fixture.service.getStatus("ws-sub")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + // The owner's run no longer sees a retryable child bucket. + expect((await fixture.service.maybeRun("ws-dream", "manual")).success).toBe(true); + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe( + HARVEST_MAX_ATTEMPTS + ); + }); + + it("keeps a removal-finalized harvest record terminal against residual retryable writes", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + const boundaryKey = metadata.summaryMessageId; + const base = { + startedAt: Date.now() - 10_000, + attemptCount: 1, + boundaryKey, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + completionMetadata: metadata, + }; + // Residual runs of the bounded cancellation drain record through the same + // path as the live harvest; reach it directly to interleave with finalization. + const save = (record: MemoryHarvestRecordPayload) => + Effect.runPromise( + ( + fixture.service as unknown as { + saveHarvestRecordEffect: ( + workspaceId: string, + boundaryKey: string, + record: MemoryHarvestRecordPayload, + projectPath: string + ) => Effect.Effect; + } + ).saveHarvestRecordEffect("ws-sub", boundaryKey, record, "") + ); + await save({ ...base, status: "pending" }); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + const latest = async () => (await fixture.service.getStatus("ws-sub")).latestHarvestRecord; + expect((await latest())?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + + // A residual retryable failure landing after finalization must not reopen the bucket... + await save({ ...base, status: "failed", completedAt: Date.now(), error: "residual failure" }); + expect((await latest())?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect((await latest())?.error).toContain("workspace removed"); + // ...while a residual completion (its writes really landed) is kept as the truth, + // and finalization never demotes a completed record. + await save({ ...base, status: "completed", completedAt: Date.now(), acceptedCandidates: 1 }); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + expect((await latest())?.status).toBe("completed"); + }); + + it("serializes sidecar harvest writes with other backends through the cross-process lock", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + const sidecarPath = path.join(fixture.xumHome, "memory-consolidation.json"); + await fsPromises.writeFile( + sidecarPath, + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + // Another backend mid read-modify-write: it holds the sidecar's file lock + // (the in-process MutexMap cannot see it), so this finalization must wait. + const foreignHold = await acquireProcessFileLock({ + lockPath: `${sidecarPath}.lock`, + timeoutMs: 1_000, + label: "test foreign backend", + }); + let finalized = false; + const finalizing = fixture.service.finalizeHarvestsForRemoval("ws-sub").then(() => { + finalized = true; + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(finalized).toBe(false); + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe(1); + await foreignHold[Symbol.asyncDispose](); + await finalizing; + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe( + HARVEST_MAX_ATTEMPTS + ); + }); + it("normalizes stale max-attempt pending harvest records to failed", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); const metadata = await seedCompactionEpoch(fixture); @@ -1280,6 +2133,85 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(1); }); + it("redirects a sub-agent's Dream runs and status to the memory owner", async () => { + using fixture = await createFixture(); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + + // Launch sweep: a child's workspace write is keyed under the owner + // (MemoryService resolution), so the idle owner qualifies while the idle + // child is skipped even though it is listed. + await fixture.memoryService.create( + { runtime: null, checkoutCwd: "", workspaceId: "ws-sub", projectPath: "" }, + "/memories/workspace/from-child.md", + "shared lesson", + "agent" + ); + const dayAgo = Date.now() - 25 * 60 * 60 * 1000; + // A recently-used child keeps the whole tree's notebook "not idle": the + // owner must not be swept on its own stale recency. + await fixture.service.runLaunchSweep( + new Map([ + ["ws-sub", Date.now()], + ["ws-dream", dayAgo], + ]) + ); + expect(fixture.modelCalls).toHaveLength(0); + await fixture.service.runLaunchSweep( + new Map([ + ["ws-sub", dayAgo], + ["ws-dream", dayAgo], + ]) + ); + expect(fixture.modelCalls).toHaveLength(1); + expect(await fixture.service.getRecord("ws-sub")).toBeNull(); + expect(await fixture.service.getRecord("ws-dream")).not.toBeNull(); + + // Manual/compaction runs from the child consolidate the OWNER's store + // under the owner's lock; the record lands on the owner and the child's + // status view reports it (the tab shows the shared store). + const manual = await fixture.service.maybeRun("ws-sub", "manual"); + expect(manual.success).toBe(true); + expect(fixture.modelCalls).toHaveLength(2); + expect(await fixture.service.getRecord("ws-sub")).toBeNull(); + expect((await fixture.service.getStatus("ws-sub")).workspaceRecord).toEqual( + await fixture.service.getRecord("ws-dream") + ); + + // Archive is the owner's own one-shot promotion pass: a child archive is + // refused instead of running it. + const archive = await fixture.service.maybeRun("ws-sub", "archive"); + expect(archive.success).toBe(false); + if (!archive.success) expect(archive.error).toContain("owner"); + expect(fixture.modelCalls).toHaveLength(2); + + // A child mid-teardown must not start an owner run: locally cancelled + // (the child's removal drain could never cancel an owner-keyed run) or + // tombstoned by another backend. + await fixture.addWorkspace("ws-sub-gone", { parentWorkspaceId: "ws-dream" }); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-sub-gone"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile( + tombstonePath, + JSON.stringify({ workspaceId: "ws-sub-gone", removedAt: Date.now() }) + ); + const tombstoned = await fixture.service.maybeRun("ws-sub-gone", "manual"); + expect(tombstoned.success).toBe(false); + if (!tombstoned.success) expect(tombstoned.error).toContain("being removed"); + await fixture.service.cancelInFlightConsolidation("ws-sub"); + const cancelled = await fixture.service.maybeRun("ws-sub", "manual"); + expect(cancelled.success).toBe(false); + if (!cancelled.success) expect(cancelled.error).toContain("being removed"); + expect(fixture.modelCalls).toHaveLength(2); + + // A dangling parent chain resolves to a PRIVATE store (owner == self), so + // that workspace consolidates itself rather than being orphaned forever. + await fixture.addWorkspace("ws-orphan", { parentWorkspaceId: "ws-gone" }); + const orphan = await fixture.service.maybeRun("ws-orphan", "manual"); + expect(orphan.success).toBe(true); + expect(await fixture.service.getRecord("ws-orphan")).not.toBeNull(); + expect(fixture.modelCalls).toHaveLength(3); + }); + it("launch sweep skips archived workspaces and caps runs per launch", async () => { using fixture = await createFixture(); const dayAgo = Date.now() - 25 * 60 * 60 * 1000; diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 9e830b3ccbf..888c9ae9237 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -58,6 +58,7 @@ import { } from "@/node/services/branchSummary"; import { USAGE_WRITE_DRAIN_WINDOW_MS } from "@/constants/streamDrain"; import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { getErrorMessage } from "@/common/utils/errors"; import { Err, Ok } from "@/common/types/result"; @@ -66,6 +67,14 @@ import { resolveHeadlessAgentDefinition } from "@/node/services/agentDefinitions import type { AgentDefinitionPackage } from "@/common/types/agentDefinition"; import { log } from "@/node/services/log"; import type { HistoryService } from "@/node/services/historyService"; +import { isTokenBudgetInternalMessage, type MuxMessage } from "@/common/types/message"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; +import { + compactionClosingPolicyEpoch, + duplicateUserMessageIds, + isPersistedHistorySequence, + isRequestPreludeRow, +} from "@/common/utils/messages/compactionBoundary"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; @@ -93,6 +102,12 @@ interface MemoryConsolidationRunOptions { */ skipWorkspaceDebounce?: boolean; skipHarvestRecovery?: boolean; + /** + * Set by maybeRun when a sub-agent's trigger is redirected to its owner: the + * child's own durable removal tombstone then gates the owner run too, since + * a child mid-teardown must not mutate the shared notebook. + */ + actingWorkspaceId?: string; } interface ExperimentsCheck { @@ -285,12 +300,160 @@ function pruneHarvestRecords(records: Record): void } } -const HARVEST_MAX_ATTEMPTS = 3; +export const HARVEST_MAX_ATTEMPTS = 3; + +/** + * Bound on waiting for the cross-process sidecar lock; holders only do one + * small read-modify-write, so hitting it means another backend is wedged. + */ +const MEMORY_CONSOLIDATION_SIDECAR_LOCK_TIMEOUT_MS = 5_000; + +/** Completed, or failed with retries exhausted: nothing may retry it. */ +function isTerminalHarvestRecord(record: MemoryHarvestRecord): boolean { + return ( + record.status === "completed" || + (record.status === "failed" && record.attemptCount >= HARVEST_MAX_ATTEMPTS) + ); +} + +/** Terminal marker for a bucket whose transcript is being deleted (see finalizeHarvestsForRemoval). */ +function finalizeHarvestRecordForRemoval(record: MemoryHarvestRecord): MemoryHarvestRecord { + return { + ...record, + status: "failed", + completedAt: record.completedAt ?? Date.now(), + attemptCount: HARVEST_MAX_ATTEMPTS, + error: "workspace removed before the harvest could be retried; transcript no longer available", + }; +} + +/** Harvest refused with a terminal record already journaled (see runHarvestAttemptEffect). */ +class HarvestRefusedError extends Error { + constructor(reason: string) { + super(reason); + this.name = "HarvestRefusedError"; + } +} + +/** + * Why the compacted epoch's transcript cannot be harvested under the policy + * observed at completion, or null when every turn of it is accounted for. + * + * A turn records its write policy in start(), before its assistant row is + * appended, and the assistant row carries `requestHistorySequence` — the + * last history sequence its request was built from — plus + * `workspaceMemoryPolicyEpoch`, the epoch that policy was recorded under. A + * turn accounts for its own batch only when that epoch is the one being + * closed: a turn started in another backend before a destructive reset and + * appended after the new boundary recorded its (possibly read-only) policy + * for the epoch the reset discarded, and the reset may have removed its user + * row too, so nothing else would surface it — such a turn is refused + * outright. The turn's own batch is the LAST user row at or below the bound + * (the request's latest user message) plus the snapshot/payload rows that + * row lists in `requestPreludeMessageIds`. Only that batch is covered — not + * every user row below the bound: with several backends on one chat.jsonl, + * another backend's read-only batch can land between this turn's user row + * and its request snapshot while that backend's deny has not been recorded + * yet; the snapshot would then include the foreign row (making it this + * turn's latest user message) and this turn's own row would be left without + * a turn of its own — which is exactly what surfaces here as uncovered. Rows + * are matched by exact id, never by adjacency, so no interleaved foreign row + * can ride along. Assistant rows without the bound and without the epoch + * stamp (synthetic payload/summary rows that are no turn) cover nothing; a + * stamp that is present but malformed refuses like a foreign + * epoch's; a row carrying a bound but no stamp is such a turn with no record + * at all and refuses too (its user row may be gone with a reset). Token-budget + * control rows (rollover lead-in, budget warning) need no turn: backend + * template text appended in the same durable batch as the turn they precede, + * carrying neither agent nor repository content. + */ +function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: number): string | null { + // Coverage is keyed by row id: a duplicated user id (raw-JSON history) + // would let one row's turn vouch for the other, so it refuses outright. + if (duplicateUserMessageIds(messages).size > 0) { + return "the compacted epoch holds user rows sharing one id; harvest refused (fail closed)"; + } + const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + const userRowById = new Map(); + for (const message of messages) { + const sequence = message.metadata?.historySequence; + if (message.role !== "user") continue; + userRowById.set(message.id, message); + if (typeof sequence === "number") userRows.push({ message, sequence }); + } + const covered = new Set(); + for (const message of messages) { + if (message.role !== "assistant") continue; + const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; + if (policyEpoch === undefined) { + // No stamp: a synthetic payload/summary row (no request bound either) + // covers nothing. A row WITH a bound — present in any form — is a turn + // whose policy was never recorded (an older or downgraded build's, + // possibly started before a destructive reset that removed its user + // row, so no uncovered row would surface it below): refuse. + if (message.metadata?.requestHistorySequence !== undefined) { + return "the compacted epoch holds a turn whose memory policy was never recorded; harvest refused (fail closed)"; + } + continue; + } + // History rows are raw JSON: a stamp that is present but not the integer + // equal to the closing epoch — another epoch's, or a corrupted value such + // as null — proves no policy for this epoch and refuses the harvest. + // Checked BEFORE the bound: a foreign-epoch turn whose bound is missing + // or corrupt must still refuse (its user row may be gone with the reset + // that made it foreign, so nothing else would surface it). + if (!Number.isInteger(policyEpoch) || policyEpoch !== closingEpoch) { + return "the compacted epoch holds a turn whose memory policy was recorded for another epoch; harvest refused (fail closed)"; + } + const bound = message.metadata?.requestHistorySequence; + // Persisted rows are raw JSON: only a history sequence in the clock's + // domain covers anything (r79); a fractional or negative value leaves the + // turn uncovered and the refusal below fails closed. + if (!isPersistedHistorySequence(bound)) continue; + const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; + if (anchor === undefined) continue; + covered.add(anchor.id); + // A listed id covers a row only when that row has prelude shape: a + // listing naming an ordinary user turn (raw history) would otherwise + // let this turn vouch for content it never consumed. + for (const id of getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds)) { + const listed = userRowById.get(id); + if (listed === undefined || isRequestPreludeRow(listed)) covered.add(id); + } + } + const uncovered = messages.some( + (message) => + message.role === "user" && !covered.has(message.id) && !isTokenBudgetInternalMessage(message) + ); + return uncovered + ? "the compacted epoch holds user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)" + : null; +} export class MemoryConsolidationService extends EventEmitter { private readonly sidecarPath: string; /** Serializes sidecar read-modify-write cycles (journal persistence only). */ private readonly locks = new MutexMap(); + + /** + * Sidecar read-modify-write section. Two legs, like the durable journal's + * locks: the in-process MutexMap orders callers on this instance cheaply, + * and a cross-process lockfile (`.lock`) excludes OTHER backends + * over the same Xum root (multi-instance) — a residual harvest recording in + * one process must not read a stale record around another process's + * removal finalization, or the terminal-record guard in + * saveHarvestRecordEffect would be checked against a stale read. + */ + private withSidecarLock(fn: () => Promise): Promise { + return this.locks.withLock(this.sidecarPath, async () => { + await using _fileLock = await acquireProcessFileLock({ + lockPath: `${this.sidecarPath}.lock`, + timeoutMs: MEMORY_CONSOLIDATION_SIDECAR_LOCK_TIMEOUT_MS, + label: "memory consolidation sidecar", + }); + return await fn(); + }); + } /** * Per-workspace run lock holding the active run's promise. Reserved * SYNCHRONOUSLY in maybeRun before any await so two near-simultaneous @@ -318,10 +481,9 @@ export class MemoryConsolidationService extends EventEmitter { * post-harvest sweep, and a cancelled run still starts retryable-harvest * recovery, each with a fresh un-aborted signal. Entry points refuse and * new controllers start pre-aborted while a workspace is in this set. - * Entries are never cleared: removal is terminal, and if a force=false - * removal fails after the drain, losing background consolidation for the - * surviving workspace (until restart) matches the documented drained- - * producers tradeoff in WorkspaceService.removeWorkspace. Cross-PROCESS + * Entries are cleared only when removal aborts before its point of no + * return (releaseRemovalCancellation); once the tombstone is published, + * removal is terminal. Cross-PROCESS * teardown is covered by the durable removal tombstone instead (see * workspaceRemoval.ts), checked at memory mutation commit points. */ @@ -414,8 +576,12 @@ export class MemoryConsolidationService extends EventEmitter { const workspace = self.config.findWorkspace(workspaceId); const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); const globalRecord = findNewestWorkspaceRecord(file.workspaces); + // The workspace record describes the STORE the tab shows: for a + // sub-agent that is the owner's (runs are redirected there, see + // maybeRun); harvests stay per acting workspace. + const ownerWorkspaceId = self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); return { - workspaceRecord: file.workspaces[workspaceId] ?? null, + workspaceRecord: file.workspaces[ownerWorkspaceId] ?? null, projectRecord: projectPath === "" ? null : (file.projects[projectPath] ?? null), globalRecord, latestHarvestRecord: findNewestHarvestRecord(file.harvestsByWorkspace[workspaceId]), @@ -449,7 +615,7 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.uninterruptible( Effect.gen(function* () { yield* Effect.promise(() => - self.locks.withLock(self.sidecarPath, async () => { + self.withSidecarLock(async () => { const file = await self.load(); file.workspaces[workspaceId] = record; if (projectPath !== "") { @@ -475,16 +641,27 @@ export class MemoryConsolidationService extends EventEmitter { const self = this; return Effect.uninterruptible( Effect.gen(function* () { - yield* Effect.promise(() => - self.locks.withLock(self.sidecarPath, async () => { + const saved = yield* Effect.promise(() => + self.withSidecarLock(async () => { const file = await self.load(); file.harvestsByWorkspace[workspaceId] ??= {}; + const existing = file.harvestsByWorkspace[workspaceId][boundaryKey]; + // A terminal record is never reopened: removal finalization + // (finalizeHarvestsForRemoval) races the bounded cancellation + // drain's residual harvest runs on this file, and a residual + // pending/retryable-failure write landing afterwards would turn + // a bucket whose transcript is gone back into a retry candidate. + // Only a genuine completion may replace it (the writes happened). + if (existing !== undefined && isTerminalHarvestRecord(existing)) { + if (record.status !== "completed") return false; + } file.harvestsByWorkspace[workspaceId][boundaryKey] = record; pruneHarvestRecords(file.harvestsByWorkspace[workspaceId]); await writeFileAtomic(self.sidecarPath, JSON.stringify(file, null, 2)); + return true; }) ); - self.emitStatusChange(workspaceId, projectPath); + if (saved) self.emitStatusChange(workspaceId, projectPath); }) ); } @@ -494,10 +671,22 @@ export class MemoryConsolidationService extends EventEmitter { const self = this; return Effect.gen(function* () { const sidecar = yield* self.loadEffect(); - const records = sidecar.harvestsByWorkspace[workspaceId]; - if (records === undefined) return; + // Harvest buckets are keyed by the ACTING workspace, but a sub-agent's + // runs redirect to its owner (see maybeRun) and the launch sweep skips + // children, so an owner run must retry every tree member's bucket or a + // child's failed/stale harvest would never be recovered. + const cfg = self.config.loadConfigOrDefault(); + const records = Object.entries(sidecar.harvestsByWorkspace) + .filter( + ([bucketId]) => + !self.removalCancelled.has(bucketId) && + (bucketId === workspaceId || + self.memoryService.resolveWorkspaceMemoryOwnerId(bucketId, () => cfg) === workspaceId) + ) + .flatMap(([, bucket]) => Object.values(bucket)); + if (records.length === 0) return; - const retryable = Object.values(records) + const retryable = records .filter((record) => { if (record.completionMetadata === undefined) return false; if (record.attemptCount >= HARVEST_MAX_ATTEMPTS) return false; @@ -542,26 +731,48 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.promise(() => isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)); } - /** Register one run's removal controller; disposed when the run settles. */ - private trackRunController(workspaceId: string): { + /** + * Register one run's removal controller; disposed when the run settles. + * A run made on a sub-agent's behalf over the OWNER's store (a redirected + * trigger, the post-harvest sweep) is registered under the acting child's + * id too: the child's removal drain (cancelInFlightConsolidation) must abort + * it — a cancelled child harvest deliberately falls through to this sweep, + * which could otherwise keep mutating the shared notebook for its whole + * run after the removal's bounded drain returned. + */ + private trackRunController( + workspaceId: string, + actingWorkspaceId?: string + ): { controller: AbortController; dispose: () => void; } { const controller = new AbortController(); + const ids = [ + workspaceId, + ...(actingWorkspaceId !== undefined && actingWorkspaceId !== workspaceId + ? [actingWorkspaceId] + : []), + ]; // r61: a run registered after teardown began (follow-on sweep/recovery // racing the entry checks) must start already aborted. - if (this.removalCancelled.has(workspaceId)) { + if (ids.some((id) => this.removalCancelled.has(id))) { controller.abort(); } - const set = this.runControllers.get(workspaceId) ?? new Set(); - set.add(controller); - this.runControllers.set(workspaceId, set); + const sets = ids.map((id) => { + const set = this.runControllers.get(id) ?? new Set(); + set.add(controller); + this.runControllers.set(id, set); + return [id, set] as const; + }); return { controller, dispose: () => { - set.delete(controller); - if (set.size === 0 && this.runControllers.get(workspaceId) === set) { - this.runControllers.delete(workspaceId); + for (const [id, set] of sets) { + set.delete(controller); + if (set.size === 0 && this.runControllers.get(id) === set) { + this.runControllers.delete(id); + } } }, }; @@ -582,6 +793,84 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.runPromise(this.cancelInFlightConsolidationEffect(workspaceId)); } + /** + * Removal aborted BEFORE its point of no return (no tombstone published, the + * workspace stays registered and intact — e.g. a non-forced removal whose + * shared-memory handover found the owner notebook full): lift the teardown + * gate again, or the surviving workspace would refuse every Dream run and + * post-compaction harvest until restart. The drained in-flight runs are + * gone regardless (retryable harvests recover on the next trigger). + */ + releaseRemovalCancellation(workspaceId: string): void { + this.removalCancelled.delete(workspaceId); + } + + /** Terminal failed record for a policy-refused harvest (see maybeHarvestThenSweep). */ + private async recordRefusedHarvest( + metadata: CompactionCompletionMetadata, + reason: string + ): Promise { + const sidecar = await this.load(); + const existing = sidecar.harvestsByWorkspace[metadata.workspaceId]?.[metadata.summaryMessageId]; + if (existing?.status === "completed") return; + const workspace = this.config.findWorkspace(metadata.workspaceId); + const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); + const now = Date.now(); + await Effect.runPromise( + this.saveHarvestRecordEffect( + metadata.workspaceId, + metadata.summaryMessageId, + { + status: "failed", + startedAt: existing?.startedAt ?? now, + completedAt: now, + attemptCount: HARVEST_MAX_ATTEMPTS, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + completionMetadata: metadata, + acceptedCandidates: 0, + skippedCandidates: 0, + error: reason, + refused: true, + }, + projectPath + ) + ); + } + + /** + * Removal teardown for harvest state: the workspace's transcript is about + * to be deleted, so its failed/stale-pending harvest records can never be + * retried (recovery needs the compaction epoch's messages) — and once the + * config entry is gone they could not even be associated with the memory + * owner. Mark them terminal now so nothing lingers as "retryable". + */ + async finalizeHarvestsForRemoval(workspaceId: string): Promise { + // One read-check-write under the sidecar lock: residual harvest runs + // (cancelInFlightConsolidation's drain is bounded) may still be recording + // outcomes, and a completion landing between an unlocked read and this + // write must not be overwritten with a failure. + const finalized = await this.withSidecarLock(async () => { + const file = await this.load(); + const records = file.harvestsByWorkspace[workspaceId]; + if (records === undefined) return false; + let changed = false; + for (const [boundaryKey, record] of Object.entries(records)) { + if (isTerminalHarvestRecord(record)) continue; + records[boundaryKey] = finalizeHarvestRecordForRemoval(record); + changed = true; + } + if (changed) await writeFileAtomic(this.sidecarPath, JSON.stringify(file, null, 2)); + return changed; + }); + if (!finalized) return; + const workspace = this.config.findWorkspace(workspaceId); + this.emitStatusChange( + workspaceId, + workspace == null ? "" : resolveConsolidationProjectPath(workspace) + ); + } + /** * Teardown pipeline: uninterruptible end-to-end so the r61 mark, the abort * loop, and the residual-run handoff can never be separated, with the @@ -667,9 +956,41 @@ export class MemoryConsolidationService extends EventEmitter { options: MemoryConsolidationRunOptions = {} ): Promise> { if (!this.enabled()) return Err("memory-consolidation experiment is disabled"); - if (this.removalCancelled.has(workspaceId)) { + // Sub-agents share their task-tree owner's /memories/workspace store + // (MemoryService.resolveWorkspaceMemoryOwnerId): a child's manual or + // post-compaction run consolidates the OWNER's notebook under the owner's + // in-flight lock, so it never races the owner's own runs and the child's + // harvested candidates are actually swept. Archive is the owner's own + // one-shot promotion pass; a child archive must not trigger it. Compared + // by resolved owner, not parentWorkspaceId: a dangling/cyclic chain falls + // back to a private store that must stay consolidatable. + // + // The acting child's own teardown gate comes BEFORE the redirect: + // cancelInFlightConsolidation(child) marks only the child id, and an + // owner-keyed run reserved after that would neither be refused by the + // owner's check below nor be cancellable by the child's removal drain. + if ( + this.removalCancelled.has(workspaceId) || + (options.actingWorkspaceId !== undefined && + this.removalCancelled.has(options.actingWorkspaceId)) + ) { return Err("workspace is being removed; consolidation refused"); } + const ownerWorkspaceId = this.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); + if (ownerWorkspaceId !== workspaceId) { + if (trigger === "archive") { + return Err( + "sub-agent workspaces share their owner's workspace memory; the owner's archive pass promotes it" + ); + } + // The owner run's recovery covers this child's harvest bucket too. The + // child's durable tombstone is probed behind the reservation, like the + // owner's (actingWorkspaceId). + return this.maybeRun(ownerWorkspaceId, trigger, { + ...options, + actingWorkspaceId: options.actingWorkspaceId ?? workspaceId, + }); + } const active = this.inFlight.get(workspaceId); if (active !== undefined) { // Archive is the workspace's one-shot final pass (workspace→global @@ -686,7 +1007,7 @@ export class MemoryConsolidationService extends EventEmitter { // would otherwise also pass the check and start a second concurrent run // over the same directories. runPromise starts the fiber synchronously, // and the map is populated before this frame yields either way. - const removal = this.trackRunController(workspaceId); + const removal = this.trackRunController(workspaceId, options.actingWorkspaceId); const run = Effect.runPromise( this.runLockedEffect(workspaceId, trigger, options, removal.controller.signal) ); @@ -698,7 +1019,10 @@ export class MemoryConsolidationService extends EventEmitter { this.inFlight.delete(workspaceId); removal.dispose(); } - if (options.skipHarvestRecovery !== true) { + // Removal cancelled this run: recovery would spawn child harvests outside + // the cancellation registry being drained (their controllers were never + // registered), mutating the shared inbox during destructive teardown. + if (options.skipHarvestRecovery !== true && !this.removalCancelled.has(workspaceId)) { await Effect.runPromise(this.recoverRetryableHarvestsEffect(workspaceId)); } return result; @@ -719,6 +1043,12 @@ export class MemoryConsolidationService extends EventEmitter { if (yield* self.isRemovalTombstonedEffect(workspaceId)) { return Err("workspace is being removed; consolidation refused"); } + if ( + options.actingWorkspaceId !== undefined && + (yield* self.isRemovalTombstonedEffect(options.actingWorkspaceId)) + ) { + return Err("workspace is being removed; consolidation refused"); + } // Manual runs bypass debounce (an explicit /dream is explicit intent). // Archive too: it is the workspace's one-shot final pass — the only @@ -733,6 +1063,11 @@ export class MemoryConsolidationService extends EventEmitter { const workspace = self.config.findWorkspace(workspaceId); if (!workspace) return Err(`workspace not found: ${workspaceId}`); + // Sub-agents share their task-tree owner's /memories/workspace store + // (MemoryService.resolveWorkspaceMemoryOwnerId), so a Dream run from a + // child would consolidate the owner's notebook concurrently with the + // owner's own runs. Children still harvest into the shared inbox; only + // the owner sweeps it. const agentBody = yield* Effect.promise(() => resolveDreamAgentBody(self.config.rootDir)); if (agentBody === null) return Err("dream agent definition is missing"); @@ -749,11 +1084,18 @@ export class MemoryConsolidationService extends EventEmitter { } const projectPath = resolveConsolidationProjectPath(workspace); + // A child's redirected run sweeps under the owner's identity; the + // child's own removal tombstone still refuses every read and commit of + // the run (MemoryScopeContext.guardedWorkspaceId) — a remover in another + // backend cannot abort this controller. const ctx: MemoryScopeContext = { runtime: null, checkoutCwd: "", workspaceId, projectPath, + ...(options.actingWorkspaceId !== undefined && options.actingWorkspaceId !== workspaceId + ? { guardedWorkspaceId: options.actingWorkspaceId } + : {}), }; const result = yield* Effect.promise(async () => @@ -837,6 +1179,20 @@ export class MemoryConsolidationService extends EventEmitter { if (this.removalCancelled.has(metadata.workspaceId)) { return Err("workspace is being removed; harvest refused"); } + // The harvest writes /memories/workspace on the agent's behalf — for a + // sub-agent, into the OWNER's shared notebook — and then sweeps it. A + // read-only (explore-like) agent's transcript must not reach either, and + // an UNKNOWN policy (legacy record, no persisted value) fails closed. The + // refusal is recorded as a terminal harvest record so recovery does not + // retry it forever and the Memory tab shows why the epoch was skipped. + if (metadata.workspaceMemoryWritable !== true) { + const reason = + metadata.workspaceMemoryWritable === false + ? "workspace memory is read-only for this agent; harvest and sweep refused" + : "workspace memory write policy is unknown for this epoch; harvest refused (fail closed)"; + await this.recordRefusedHarvest(metadata, reason); + return Err(reason); + } const boundaryRunKey = `${metadata.workspaceId}:${metadata.summaryMessageId}`; const active = this.harvestInFlight.get(boundaryRunKey); @@ -878,6 +1234,11 @@ export class MemoryConsolidationService extends EventEmitter { const boundaryKey = metadata.summaryMessageId; const sidecar = yield* self.loadEffect(); const existing = sidecar.harvestsByWorkspace[metadata.workspaceId]?.[boundaryKey]; + // An epoch refused earlier (terminal record) stays refused on every + // later trigger — recovery, a duplicate completion — sweep included. + if (existing?.refused === true) { + return Err(existing.error ?? "harvest of this epoch was refused (fail closed)"); + } const existingAttemptCount = existing?.attemptCount ?? 0; const stalePending = existing === undefined ? false : isStalePendingHarvestRecord(existing); @@ -922,8 +1283,19 @@ export class MemoryConsolidationService extends EventEmitter { // completed-record save rejecting — journals a failed record and // still falls through to the sweep, so the fold handles the error // channel AND defects identically. + // A refusal (policy never recorded for some turn of the epoch) + // already journaled its terminal record; re-journaling would make it + // retryable again. It also ends the run here, WITHOUT the owner sweep: + // the reasons the harvest is refused (an unaccounted turn, a stale + // grant) are exactly the reasons a model-driven Dream pass over the + // owner's shared notebook must not run on this epoch's behalf either. + const refused = { reason: null as string | null }; const journalHarvestFailure = (error: unknown): Effect.Effect => Effect.gen(function* () { + if (error instanceof HarvestRefusedError) { + refused.reason = error.message; + return; + } yield* self.saveHarvestRecordEffect( metadata.workspaceId, boundaryKey, @@ -955,9 +1327,19 @@ export class MemoryConsolidationService extends EventEmitter { removalSignal, }) .pipe(Effect.catch(journalHarvestFailure), Effect.catchDefect(journalHarvestFailure)); + if (refused.reason !== null) return Err(refused.reason); } - return yield* Effect.promise(() => self.runCompactionSweepAfterHarvest(metadata.workspaceId)); + // A sub-agent's inbox lives in the owner's store: wait on and run the + // OWNER's consolidation (see maybeRun) so the harvest is actually swept — + // still as the acting CHILD, so the child's in-process cancellation, + // durable tombstone and removal drain bind the owner-keyed run too. + return yield* Effect.promise(() => + self.runCompactionSweepAfterHarvest( + self.memoryService.resolveWorkspaceMemoryOwnerId(metadata.workspaceId), + metadata.workspaceId + ) + ); }); } @@ -982,6 +1364,22 @@ export class MemoryConsolidationService extends EventEmitter { catch: (error) => error, }); if (!epoch.success) return yield* Effect.fail(new Error(epoch.error)); + // RLM keep-recent copies (compactionHandler) duplicate the previous + // epoch's last turns after its boundary: that epoch's harvest already + // judged the originals, and the copies carry no turn stamps of their + // own, so they neither need covering nor get harvested twice. + const messages = epoch.data.messages.filter( + (message) => message.metadata?.rlmPreservedTailCopy !== true + ); + // Every scanned user row must be covered by a turn whose write policy + // was recorded for THIS epoch (see epochHarvestRefusal). Uncovered rows + // have an unknown policy the grant evaluated at completion could not + // have accounted for. Terminal refusal: a retry would replay that grant. + const refusal = epochHarvestRefusal(messages, compactionClosingPolicyEpoch(metadata)); + if (refusal !== null) { + yield* Effect.promise(() => self.recordRefusedHarvest(metadata, refusal)); + return yield* Effect.fail(new HarvestRefusedError(refusal)); + } const modelString = resolveDreamModelString(self.config, metadata.workspaceId); const modelResult = yield* Effect.tryPromise({ @@ -1007,7 +1405,7 @@ export class MemoryConsolidationService extends EventEmitter { memoryService: self.memoryService, ctx, completionMetadata: metadata, - messages: epoch.data.messages, + messages, summary: epoch.data.summary, // Timeout + removal (r60); see the runLockedEffect signal for rationale. abortSignal: AbortSignal.any([ @@ -1069,7 +1467,8 @@ export class MemoryConsolidationService extends EventEmitter { } private async runCompactionSweepAfterHarvest( - workspaceId: string + workspaceId: string, + actingWorkspaceId: string ): Promise> { for (;;) { const active = this.inFlight.get(workspaceId); @@ -1082,6 +1481,7 @@ export class MemoryConsolidationService extends EventEmitter { const result = await this.maybeRun(workspaceId, trigger, { skipWorkspaceDebounce: true, skipHarvestRecovery: true, + ...(actingWorkspaceId !== workspaceId ? { actingWorkspaceId } : {}), }); if (!result.success && result.error === "a consolidation run is already in flight") { continue; @@ -1193,7 +1593,8 @@ export class MemoryConsolidationService extends EventEmitter { let globalLastRunAt = findNewestWorkspaceRecord(sidecar.workspaces)?.lastRunAt ?? 0; const archivedById = new Map(); const projectPathByWorkspace = new Map(); - for (const [configProjectPath, project] of self.config.loadConfigOrDefault().projects) { + const cfg = self.config.loadConfigOrDefault(); + for (const [configProjectPath, project] of cfg.projects) { for (const workspace of project.workspaces) { if (workspace.id === undefined) continue; archivedById.set( @@ -1216,8 +1617,18 @@ export class MemoryConsolidationService extends EventEmitter { ]) ); - let started = 0; + // Shared stores: a child's writes are keyed under its owner, so the + // owner's row absorbs every tree member's recency (a notebook is idle + // only when the WHOLE tree is) and child rows are then dropped — + // running a child would just redirect to the owner anyway. + const treeRecency = new Map(); for (const [workspaceId, recency] of recencyByWorkspace) { + const ownerId = self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, () => cfg); + treeRecency.set(ownerId, Math.max(treeRecency.get(ownerId) ?? 0, recency)); + } + + let started = 0; + for (const [workspaceId, recency] of treeRecency) { if (started >= MEMORY_CONSOLIDATION_LAUNCH_SWEEP_CAP) break; if (now - recency < MEMORY_CONSOLIDATION_IDLE_MS) continue; if (archivedById.get(workspaceId) === true) continue; diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts new file mode 100644 index 00000000000..857bfd4ecea --- /dev/null +++ b/src/node/services/memoryLegacyAdoption.ts @@ -0,0 +1,529 @@ +/** + * Legacy-notebook adoption manifest: the durable record of which files of a + * sub-agent's PRE-SHARING private notebook (`/memory`, written by + * builds that kept `/memories/workspace` per workspace) were folded into the + * task-tree owner's shared store, and where each landed + * (MemoryService.adoptLegacyPrivateStore). Shared with the refinement + * rollback engine: refinement rows journaled before the upgrade address the + * legacy files, while the note the user sees since is the owner copy. + */ +import type { Dirent } from "node:fs"; +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; +import writeFileAtomic from "write-file-atomic"; +import type { RefinementInverse } from "@/common/types/refinement"; + +/** + * File in the sub-agent's SESSION dir (beside its legacy `memory` dir, never + * inside it) recording, per relPath, the sha256 of the content already copied + * into the shared store (adoptLegacyPrivateStore). Outside the legacy root on + * purpose: everything under `/memory` is the model-writable + * `/memories/workspace` namespace of a downgraded build (the path grammar + * admits dotfiles), and this manifest's `created`/`target`/hash fields are + * trusted as provenance — a fabricated settled record with an absent source + * would make deletion reconciliation remove a matching owner note. The + * session dir itself is not addressable through any memory path. + */ +export const LEGACY_ADOPTION_MANIFEST_FILE_NAME = "memory-adoption-manifest.json"; + +export function legacyAdoptionManifestPath(childSessionDir: string): string { + return path.join(childSessionDir, LEGACY_ADOPTION_MANIFEST_FILE_NAME); +} + +/** + * One adopted legacy file: content hash, child sidecar fingerprint, owner-store + * relPath, and whether the adoption CREATED that owner file (provenance: only + * such a copy may be removed again when the legacy source disappears; a + * pre-existing identical owner note is the owner's own). `pending`: written + * BEFORE the copy lands (provenance must not depend on the copy's existence: a + * retry finding the bytes already at the target could not tell an interrupted + * adoption from an owner note); cleared once the sidecar fold completed. + */ +export interface LegacyAdoptionRecord { + content: string; + sidecar: string; + target: string; + created?: boolean; + pending?: boolean; + /** + * Identity of the owner file this adoption wrote (`ino:size:mtimeNs` right + * after the write). Deletion reconciliation requires the copy to be THIS + * generation of the file, not merely to hold the adopted bytes: an owner + * who deleted and recreated (or edited and restored) the note to identical + * bytes owns the new file, and a byte match alone would let a downgraded + * child's source deletion remove it. Absent (write before stamping, or the + * stamp could not be taken): never unchanged — the copy is preserved. + */ + targetStamp?: string; + /** + * The copy this adoption created was since replaced outside it (rewritten, + * or deleted and recreated to identical bytes: `targetStamp` no longer + * matches), so the file is the owner's own. Kept apart from a note the + * owner already had when it was first adopted (`created` never set): that + * one still folds the child's pin toggles, a replaced copy never does — + * `created` alone cannot tell the two apart once provenance is lost. + */ + replaced?: boolean; + /** + * Hash of the bytes an in-place replacement is about to write (set on the + * pending prior record, cleared once the pass completes). With `content` + * (the pre-write bytes) this lets a retry recognize the copy as this + * adoption's on either side of an interrupted write. + */ + replacementContent?: string; + /** + * Identity (`ino:size:mtimeNs`) of the staged bytes an in-place replacement + * is about to install, taken on the staging entry before the install (a + * rename keeps it) and set together with `replacementContent`. A retry + * finds the installed copy by this stamp; a byte match alone never counts. + */ + replacementStamp?: string; + /** + * Reconciliation of a deleted source is under way: the copy is about to be + * (or was just) removed. Set before the removal so a crash between the + * removal and the tombstone write is recovered as "removed by us" rather + * than "changed by the owner". + */ + pendingDeletion?: boolean; + /** + * The legacy source was deleted (or renamed away) on a downgraded build and + * the copy reconciled. Kept rather than dropped: the child's pre-sharing + * refinement rows for this note (a delete's restore inverse, a rename's + * mirrored rename) still address the legacy path and need the mapping to + * be rolled back into the shared store; a reappearing source is adopted + * afresh (the record's other fields are stale then). + */ + deleted?: boolean; +} + +/** + * Parse one manifest record. Lifecycle flags are raw JSON: a value that is + * neither absent nor boolean fails CLOSED — `pending`/`pendingDeletion` read + * as set (the pass is redone), `created`/`deleted` as unset (no destructive + * provenance; the source is reconciled as a plain unlisted note), `replaced` + * as set (the child's pin no longer reaches the file) — so a corrupted flag + * can never make an interrupted pass look settled. + */ +function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null { + if (typeof value !== "object" || value === null) return null; + const record = value as Record; + if ( + typeof record.content !== "string" || + typeof record.sidecar !== "string" || + typeof record.target !== "string" + ) { + return null; + } + // A present but non-string replacement hash is a malformed RECORD (not a + // flag to fail closed on): without it, a replacement pass that crashed + // after writing the new owner bytes leaves a copy reconciliation cannot + // recognize as this adoption's — a later source deletion would tombstone + // it as owner-owned and removal would report a complete handover while the + // adoption-created note stays visible without provenance. + if (record.replacementContent !== undefined && typeof record.replacementContent !== "string") { + return null; + } + if (record.targetStamp !== undefined && typeof record.targetStamp !== "string") return null; + if (record.replacementStamp !== undefined && typeof record.replacementStamp !== "string") { + return null; + } + const flag = (raw: unknown, malformed: boolean): boolean | undefined => + raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; + return { + content: record.content, + sidecar: record.sidecar, + target: record.target, + created: flag(record.created, false), + pending: flag(record.pending, true), + pendingDeletion: flag(record.pendingDeletion, true), + deleted: flag(record.deleted, false), + replaced: flag(record.replaced, true), + replacementContent: record.replacementContent, + replacementStamp: record.replacementStamp, + targetStamp: record.targetStamp, + }; +} + +/** + * Read of the adoption manifest. A MISSING file reads as "nothing adopted" + * for every caller. Tolerant callers also read an unreadable (EACCES, EIO) + * or malformed file — bad JSON, a non-object, a record missing its string + * fields — as empty (self-healing: the next pass rewrites it). `strict` + * callers throw on all of those: the adoption pass and the removal handover + * decide what may be deleted on the manifest's authority, and an empty + * substitute would drop provenance — a malformed record whose downgraded + * source is already gone can no longer be reconciled (its target is in the + * bad record), and removal would delete the child session while the + * adoption-created owner copy stays visible for good. A Map, not a plain + * object: a legacy note may + * legitimately be named `__proto__` (any store-valid relPath), and assigning + * that key on an ordinary object hits the prototype setter instead of + * creating an entry the serialization would carry — the note would then be + * re-adopted (and the owner clock advanced) on every access. JSON.parse and + * Object.fromEntries create own properties, so the round-trip is exact. + */ +export async function readLegacyAdoptionManifest( + manifestPath: string, + options?: { strict?: boolean } +): Promise> { + let raw: string; + try { + raw = await fsPromises.readFile(manifestPath, "utf-8"); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (options?.strict === true && code !== "ENOENT" && code !== "ENOTDIR") throw error; + return new Map(); + } + const malformed = (detail: string): Map => { + if (options?.strict === true) { + throw new Error(`the legacy adoption manifest at ${manifestPath} is malformed (${detail})`); + } + return new Map(); + }; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return malformed("not JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return malformed("not an object"); + } + const entries: Array<[string, LegacyAdoptionRecord]> = []; + for (const [relPath, value] of Object.entries(parsed)) { + const record = parseLegacyAdoptionRecord(value); + if (record === null) return malformed(`record '${relPath}'`); + entries.push([relPath, record]); + } + return new Map(entries); +} + +/** + * The file identity a LegacyAdoptionRecord.targetStamp records, or why there + * is none: "absent" only when the stat PROVES the path is gone (ENOENT / + * ENOTDIR); any other failure (EACCES, EIO) is "unreadable" — it says + * nothing about the path, so callers deciding on absence must refuse. + */ +export async function adoptionTargetPresence( + absPath: string +): Promise<{ stamp: string } | "absent" | "unreadable"> { + try { + const stat = await fsPromises.lstat(absPath, { bigint: true }); + return { stamp: `${stat.ino}:${stat.size}:${stat.mtimeNs}` }; + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + return code === "ENOENT" || code === "ENOTDIR" ? "absent" : "unreadable"; + } +} + +/** + * The file identity a LegacyAdoptionRecord.targetStamp records; null when the + * file cannot be stat'ed (the record then carries no stamp: preserved). + */ +export async function adoptionTargetStamp(absPath: string): Promise { + const presence = await adoptionTargetPresence(absPath); + return typeof presence === "string" ? null : presence.stamp; +} + +/** + * Every non-directory entry (files, symlinks, anything) under `absDir`, + * recursively, as relPaths prefixed with `dirRel`; empty when the directory + * is absent. Throws on any other traversal failure (the caller then refuses + * rather than guess). + */ +async function listEntriesUnder(absDir: string, dirRel: string): Promise> { + const found = new Set(); + const walk = async (abs: string, rel: string): Promise => { + let entries: Dirent[]; + try { + entries = await fsPromises.readdir(abs, { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code === "ENOENT" || code === "ENOTDIR") return; + throw error; + } + for (const entry of entries) { + const childRel = `${rel}/${entry.name}`; + if (entry.isDirectory() && !entry.isSymbolicLink()) { + await walk(path.join(abs, entry.name), childRel); + } else { + found.add(childRel); + } + } + }; + await walk(absDir, dirRel); + return found; +} + +function setsEqual(a: ReadonlySet, b: ReadonlySet): boolean { + return a.size === b.size && [...a].every((value) => b.has(value)); +} + +/** Thrown for a legacy path the shared store does not represent (see below). */ +export class LegacyPathNotAdoptedError extends Error { + constructor(legacyPath: string, reason: "not-adopted" | "owner-owned" | "replaced") { + super( + reason === "owner-owned" + ? `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook; the shared workspace store holds an identical note the owner already had (adoption created nothing), so a rollback there would alter the owner's own note` + : reason === "replaced" + ? `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook; its adopted copy in the shared workspace store was since replaced (rewritten, or deleted and recreated) outside this sub-agent's rows, so the file there is the owner's own and a rollback would alter or remove it` + : `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` + ); + this.name = "LegacyPathNotAdoptedError"; + } +} + +/** + * Retargets refinement inverses (and any other recorded path) of a sub-agent + * whose legacy private notebook was adopted into the owner's store: a path + * under `/memory` becomes the owner-store path its note was + * folded into, so a rollback reverts the copy the shared notebook actually + * serves rather than the hidden legacy file (which the next adoption pass + * would re-import as a conflicting duplicate). Paths outside the legacy root + * pass through unchanged. A legacy path the manifest does not know throws + * LegacyPathNotAdoptedError — fail closed rather than mutate an invisible + * file. So does a record the adoption did NOT create (`created` unset: the + * owner already had an identical note of its own): the child's rows never + * touched that file, and applying their inverses there — a create row's + * delete-files in particular — would alter or remove the owner's own note. + * Likewise a created record whose target is no longer THIS adoption's + * generation of the file (`targetStamp`, the same rule deletion + * reconciliation applies): an owner save is unjournaled and may keep the + * bytes, so neither peer rows nor post-state hashes would notice — the + * replacement is the owner's, and the mapping is refused (r74). The stamps + * are read when the remapper is created; callers create it under the store's + * mutation lock (the rollback engine re-derives it there), and the engine + * re-stamps the targets its own retargeted applies rewrite + * (refreshLegacyAdoptionTargetStamps) so the child's remaining rows for the + * same note stay mappable. + * Only the manifest's `target` is trusted for the destination's relPath; + * callers re-run their confinement checks on the mapped result. `strict` + * (removal's row migration) throws on an unreadable manifest instead of + * treating every legacy path as unadopted. + */ +export async function createLegacyPathRemapper(args: { + childSessionDir: string; + ownerSessionDir: string; + strict?: boolean; +}): Promise<{ + path(filePath: string): string; + inverse(inverse: RefinementInverse): RefinementInverse; +}> { + const legacyRoot = path.join(path.resolve(args.childSessionDir), "memory"); + const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const adopted = await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.resolve(args.childSessionDir)), + { strict: args.strict === true } + ); + // Directory endpoints (pre-sharing directory renames) map only when the + // owner's on-disk subtree is EXACTLY the adopted descendants: a structural + // rename moves whatever is there, so an owner note added beside the + // adopted copies (no refinement row of its own) would travel along + // unnoticed. Precomputed for every directory prefix the manifest knows. + // Created records whose owner target is still this lineage's generation + // (see LegacyAdoptionRecord.targetStamp). A record reconciled as deleted + // is current while its target stays absent — or holds the generation a + // retargeted rollback recreated there (re-stamped below); anything else at + // that path is the owner's. + // Value: whether that generation is a file on disk (a tombstoned record + // whose copy a rollback restored counts as present). Absence must be + // PROVEN (ENOENT/ENOTDIR): a target that merely cannot be stat'ed right + // now (EACCES, EIO) may well hold an owner-created replacement, and + // calling it absent would let a restore or rename land on it — such a + // record is simply not current (the rollback is refused as owner-owned). + const currentGeneration = new Map(); + for (const [rel, record] of adopted) { + if (record.created !== true || record.pending === true) continue; + const presence = await adoptionTargetPresence( + path.join(ownerRoot, ...record.target.split("/")) + ); + if ( + record.targetStamp !== undefined && + typeof presence !== "string" && + presence.stamp === record.targetStamp + ) { + currentGeneration.set(rel, "present"); + } else if (record.deleted === true && presence === "absent") { + currentGeneration.set(rel, "absent"); + } + } + const ownerSubtreeExact = new Map(); + const directoryPrefixes = new Set(); + for (const rel of adopted.keys()) { + const parts = rel.split("/"); + for (let depth = 1; depth < parts.length; depth++) { + directoryPrefixes.add(parts.slice(0, depth).join("/")); + } + } + for (const dirRel of directoryPrefixes) { + const descendants = [...adopted].filter(([rel]) => rel.startsWith(`${dirRel}/`)); + const oneToOne = descendants.every( + ([rel, entry]) => entry.target === rel && currentGeneration.has(rel) + ); + const expected = new Set( + descendants.filter(([rel]) => currentGeneration.get(rel) === "present").map(([rel]) => rel) + ); + ownerSubtreeExact.set( + dirRel, + oneToOne && + setsEqual( + expected, + await listEntriesUnder(path.join(ownerRoot, ...dirRel.split("/")), dirRel) + ) + ); + } + const legacyRelPath = (filePath: string): string | null => { + const relative = path.relative(legacyRoot, path.resolve(filePath)); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return null; + return relative.split(path.sep).join("/"); + }; + const remapPath = (filePath: string): string => { + const relPath = legacyRelPath(filePath); + if (relPath === null) return filePath; + const record = adopted.get(relPath); + if (record === undefined) { + // A directory endpoint (a pre-sharing directory rename): the manifest + // records files only. Mappable when every adopted descendant landed at + // its own relPath in the owner store as this adoption's copy — the + // owner directory then IS the adopted directory. Descendants placed + // elsewhere (conflict imports) or owner-owned make the structural + // move ambiguous: refused. + if (ownerSubtreeExact.get(relPath) === true) { + return path.join(ownerRoot, ...relPath.split("/")); + } + throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); + } + if (record.pending === true) throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); + if (record.created !== true) throw new LegacyPathNotAdoptedError(filePath, "owner-owned"); + if (!currentGeneration.has(relPath)) throw new LegacyPathNotAdoptedError(filePath, "replaced"); + return path.join(ownerRoot, ...record.target.split("/")); + }; + // The destination of a rename INVERSE is the name the child's rename + // vacated. A rename made before the first upgrade leaves no record for it + // (adoption saw only the post-rename names), yet the row is still + // rollbackable once its `from` side maps (r75): the vacated name lands + // beside the adopted copies. The engine requires it absent before moving, + // so an owner note there refuses like for any rename. + const remapRenameDestination = (filePath: string): string => { + const relPath = legacyRelPath(filePath); + if (relPath === null || adopted.has(relPath) || directoryPrefixes.has(relPath)) { + return remapPath(filePath); + } + return path.join(ownerRoot, ...relPath.split("/")); + }; + return { + path: remapPath, + inverse: (inverse) => { + switch (inverse.op) { + case "delete-files": + return { ...inverse, paths: inverse.paths.map(remapPath) }; + case "rename": { + const from = remapPath(inverse.from); + return { ...inverse, from, to: remapRenameDestination(inverse.to) }; + } + case "restore-files": + return { + ...inverse, + files: inverse.files.map((file) => ({ ...file, path: remapPath(file.path) })), + ...(inverse.deletePaths === undefined + ? {} + : { deletePaths: inverse.deletePaths.map(remapPath) }), + }; + } + }, + }; +} + +/** + * Re-stamp adopted targets the rollback engine just rewrote or removed while + * applying an inverse on the child's behalf (createLegacyPathRemapper mapped + * the child's legacy paths onto them, or the row is the child's own rollback + * row over the shared store): the write is the child's own lineage acting, so + * the new generation stays mappable for the child's remaining rows over the + * same note (create + edit unwind LIFO). `paths` may be directories (a rename + * endpoint): every record whose target lies beneath is re-stamped, so after a + * retargeted rename the vacated side's records lose their stamp and the + * restored side's (tombstoned by the downgraded build's rename) take the + * moved files' generation (r75). A rename whose restored side has NO record + * (the child renamed before the first upgrade, so adoption only ever saw the + * new names) gets tombstoned records for the moved copies (r76): keyed by the + * legacy path the child's older rows address, carrying the moved file's + * generation — `deleted` because no legacy source exists there (reconciliation + * skips tombstones; a later source is a fresh note), yet mappable while the + * copy is that generation or absent again. Runs under the owner store's + * mutation lock the engine holds (the lock adoption passes take too). A target + * that is gone loses its stamp — nothing maps there until adoption places the + * note anew. Best-effort by contract: a failure here only leaves stale stamps, + * which refuse (never mutate) later. + */ +export async function refreshLegacyAdoptionTargetStamps(args: { + childSessionDir: string; + ownerSessionDir: string; + paths: readonly string[]; + renamed?: { from: string; to: string }; +}): Promise { + const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const ownerRel = (filePath: string): string | null => { + const relative = path.relative(ownerRoot, path.resolve(filePath)); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return null; + return relative.split(path.sep).join("/"); + }; + const touched = new Set(); + for (const filePath of args.paths) { + const rel = ownerRel(filePath); + if (rel !== null) touched.add(rel); + } + const renamed = + args.renamed === undefined + ? null + : { from: ownerRel(args.renamed.from), to: ownerRel(args.renamed.to) }; + if (renamed?.from != null) touched.add(renamed.from); + if (renamed?.to != null) touched.add(renamed.to); + if (touched.size === 0) return; + const beneath = (target: string, rel: string): boolean => + target === rel || target.startsWith(`${rel}/`); + const manifestPath = legacyAdoptionManifestPath(path.resolve(args.childSessionDir)); + const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); + let dirty = false; + const stampOf = async (target: string): Promise => + (await adoptionTargetStamp(path.join(ownerRoot, ...target.split("/")))) ?? undefined; + for (const record of adopted.values()) { + if (record.created !== true || record.pending === true) continue; + if (![...touched].some((rel) => beneath(record.target, rel))) continue; + const stamp = await stampOf(record.target); + if (stamp === record.targetStamp) continue; + record.targetStamp = stamp; + dirty = true; + } + if (renamed?.from != null && renamed.to != null) { + const targets = new Set([...adopted.values()].map((record) => record.target)); + for (const record of [...adopted.values()]) { + // Every copy beneath the vacated endpoint — at its own relPath (the + // directory proof requires one-to-one) or a lone file's conflict import + // under imported// (r77) — now sits at the same relative + // position under the restored name. The restored name IS the legacy + // path the child's older rows address: a restored side without a record + // mapped one-to-one (remapRenameDestination); one that had a record + // (any target) is re-stamped above and gets no second record. + if (record.created !== true || record.pending === true) continue; + if (!beneath(record.target, renamed.from)) continue; + const movedRel = renamed.to + record.target.slice(renamed.from.length); + if (adopted.has(movedRel) || targets.has(movedRel)) continue; + const stamp = await stampOf(movedRel); + if (stamp === undefined) continue; // not moved after all: nothing to vouch for + adopted.set(movedRel, { + content: record.content, + sidecar: record.sidecar, + target: movedRel, + created: true, + deleted: true, + targetStamp: stamp, + }); + dirty = true; + } + } + if (!dirty) return; + await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { + encoding: "utf-8", + }); +} diff --git a/src/node/services/memoryMeta.test.ts b/src/node/services/memoryMeta.test.ts index b39581fb4b8..daff2743e0d 100644 --- a/src/node/services/memoryMeta.test.ts +++ b/src/node/services/memoryMeta.test.ts @@ -1,9 +1,9 @@ -import { describe, it, expect } from "bun:test"; +import { describe, it, expect, spyOn } from "bun:test"; import { Effect } from "effect"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; -import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; +import { MemoryMetaService, MemoryMetaWriteError, memoryLogicalKey } from "./memoryMeta"; import { TestTempDir } from "./tools/testHelpers"; describe("memoryLogicalKey", () => { @@ -45,6 +45,82 @@ describe("memoryLogicalKey", () => { }); describe("MemoryMetaService", () => { + it("does not cache an empty view taken while the sidecar was unreadable", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + const service = new MemoryMetaService(tempDir.path); + await service.setPinned("global:prefs.md", true); + // Transient read failure (EACCES interval) with the file's stamp unchanged: + // this read heals to empty, but the next one must retry the file — not + // serve the empty view and then write it back over the real pins. + const reader = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))) as never); + const reloaded = new MemoryMetaService(tempDir.path); + expect(await reloaded.getPinnedKeys()).toEqual(new Set()); + reader.mockRestore(); + expect(await reloaded.getPinnedKeys()).toEqual(new Set(["global:prefs.md"])); + await reloaded.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + + it("refuses a mutation whose read of the sidecar failed instead of overwriting it", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + await new MemoryMetaService(tempDir.path).setPinned("global:prefs.md", true); + // The mutating call itself hits the transient failure: its healed empty + // view must not become the file, or every existing pin is erased. + const reader = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))) as never); + const fresh = new MemoryMetaService(tempDir.path); + try { + const failure = await fresh.setPinned("workspace:ws-1:scratch.md", true).then( + () => null, + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(MemoryMetaWriteError); + expect((failure as MemoryMetaWriteError).reason).toContain("could not be read"); + } finally { + reader.mockRestore(); + } + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md"]) + ); + // Once readable again the same instance mutates normally. + await fresh.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + + it("does not serve a cached first-run view when the sidecar stat itself fails", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + const service = new MemoryMetaService(tempDir.path); + // Cached "missing" (normal first run in this process)... + expect(await service.getPinnedKeys()).toEqual(new Set()); + // ...then another backend creates the sidecar. + await new MemoryMetaService(tempDir.path).setPinned("global:prefs.md", true); + // A transient stat failure must not read as "still missing": the stale + // empty cache would otherwise be written over the foreign pins. + const stat = spyOn(fsPromises, "stat").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + const failure = await service.setPinned("workspace:ws-1:scratch.md", true).then( + () => null, + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(MemoryMetaWriteError); + } finally { + stat.mockRestore(); + } + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md"]) + ); + await service.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + it("persists pins across instances via the sidecar file", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); @@ -64,6 +140,22 @@ describe("MemoryMetaService", () => { ); }); + it("sees another backend's sidecar writes and never overwrites them from a stale cache", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + // Two services over one Xum root stand in for two backend processes + // (XUM_ALLOW_MULTIPLE_INSTANCES): neither sees the other's in-memory cache. + const a = new MemoryMetaService(tempDir.path); + const b = new MemoryMetaService(tempDir.path); + await b.getEntries(); // B caches the (empty) sidecar + await a.setPinned("workspace:ws-owner:notes.md", true); + // B's stamp-validated load picks up A's write... + expect(await b.getPinnedKeys()).toEqual(new Set(["workspace:ws-owner:notes.md"])); + // ...and B's own mutation starts from the current file, keeping A's pin. + await b.recordAccess("global:other.md", { write: false }); + expect(await a.getPinnedKeys()).toEqual(new Set(["workspace:ws-owner:notes.md"])); + expect((await a.getEntries()).has("global:other.md")).toBe(true); + }); + it("unpinning removes the key", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index 137e650e6c3..26050d60e4f 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -13,6 +13,7 @@ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; import { Effect, Schema, Semaphore } from "effect"; import type { MemoryScope } from "@/common/constants/memory"; import { getErrorMessage } from "@/common/utils/errors"; @@ -65,6 +66,12 @@ export interface MemoryMetaEntry { lastWriteAt: number | null; } +function maxTimestamp(a: number | null, b: number | null): number | null { + if (a === null) return b; + if (b === null) return a; + return Math.max(a, b); +} + const EMPTY_ENTRY: MemoryMetaEntry = { pinned: false, accessCount: 0, @@ -136,6 +143,9 @@ export class MemoryMetaWriteError extends Schema.TaggedError => + this.mutate((entries) => { + for (const [key, source] of Object.entries(entries)) { + if (!keyInSubtree(key, sourceLogicalKey)) continue; + const targetKey = `${targetLogicalKey}${key.slice(sourceLogicalKey.length)}`; + const target = entries[targetKey]; + entries[targetKey] = + target === undefined + ? { ...source } + : { + pinned: options.pinned === "source" ? source.pinned : target.pinned, + accessCount: Math.max(target.accessCount, source.accessCount), + lastAccessedAt: maxTimestamp(target.lastAccessedAt, source.lastAccessedAt), + lastWriteAt: maxTimestamp(target.lastWriteAt, source.lastWriteAt), + }; + } + }), + /** * Drop all entries for a deleted file or directory subtree so a future file * at the same path never resurrects stale pins or stats. @@ -247,30 +301,76 @@ export class MemoryMetaService { * (logged for diagnosis) and only writes can fail. */ private load(): Effect.Effect { + return Effect.map(this.loadWithHealth(), (loaded) => loaded.meta); + } + + /** + * `load()` plus whether this view is a healed substitute for a sidecar that + * exists but could not be read. Reads may serve that substitute; a mutation + * must not (see mutate()). + */ + private loadWithHealth(): Effect.Effect<{ meta: MemoryMetaFile; readFailed: boolean }> { // eslint-disable-next-line @typescript-eslint/no-this-alias -- Effect.gen generator bodies do not inherit `this` const self = this; return Effect.gen(function* () { - if (self.cache !== null) return self.cache; - const parsed = yield* Effect.tryPromise({ - try: async (): Promise => - JSON.parse(await fsPromises.readFile(self.metaPath, "utf-8")), + const stamp = yield* Effect.promise(() => self.fileStamp()); + // A failed stat (null) never matches: another backend may have written + // the file since the cached "missing" was taken. + if (stamp !== null && self.cache !== null && stamp === self.cacheStamp) { + return { meta: self.cache, readFailed: false }; + } + let readFailed = false; + const raw = yield* Effect.tryPromise({ + try: (): Promise => fsPromises.readFile(self.metaPath, "utf-8"), catch: (error) => error, }).pipe( Effect.catch((error) => { // Missing file is the normal first-run case; anything else is healed to empty. if ((error as NodeJS.ErrnoException).code !== "ENOENT") { log.debug("[MemoryMetaService] healing unreadable sidecar", { error }); + readFailed = true; } - return Effect.succeed(null); + return Effect.succeed(null); }) ); + let parsed: unknown = null; + if (raw !== null) { + try { + parsed = JSON.parse(raw); + } catch (error) { + // Corrupt content (unlike a failed read) IS the file's state: healing + // it to empty and letting the next mutation rewrite it is the fix. + log.debug("[MemoryMetaService] healing corrupt sidecar", { error }); + } + } self.cache = sanitizeMetaFile(parsed); - return self.cache; + // The stamp is remembered only for a real read (or a genuinely absent + // file): a transiently unreadable sidecar (EACCES interval, a writer + // mid-swap) heals to empty for THIS call, but caching that empty view + // under the file's unchanged stamp would keep serving it once readable + // again — and the next mutation would write the pins and stats away. + self.cacheStamp = readFailed ? null : stamp; + return { meta: self.cache, readFailed: readFailed || stamp === null }; }); } /** - * Read-modify-write cycle under the sidecar semaphore. Persists before + * Cheap change signal for the sidecar (same scheme as Config.configFileStamp). + * "missing" only on a proven ENOENT (the normal first run); null when the + * stat itself failed — not cacheable, and a mutation must not proceed on it. + */ + private async fileStamp(): Promise { + try { + const st = await fsPromises.stat(this.metaPath, { bigint: true }); + return `${st.dev}:${st.ino}:${st.size}:${st.mtimeNs}`; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : null; + } + } + + /** + * Read-modify-write cycle under the sidecar semaphore (in-process) and the + * sidecar lockfile (other backends over the same Xum root). Persists before * updating the in-memory cache so observers never see state that didn't make * it to disk. Entries that end up entirely default are dropped. */ @@ -281,7 +381,31 @@ export class MemoryMetaService { const self = this; return this.writeLock.withPermit( Effect.gen(function* () { - const meta = yield* self.load(); + const fileLock = yield* Effect.tryPromise({ + try: () => + acquireProcessFileLock({ + lockPath: `${self.metaPath}.lock`, + timeoutMs: MEMORY_META_LOCK_TIMEOUT_MS, + label: "memory metadata sidecar", + }), + catch: (cause) => + new MemoryMetaWriteError({ metaPath: self.metaPath, reason: getErrorMessage(cause) }), + }); + yield* Effect.addFinalizer(() => Effect.promise(() => fileLock[Symbol.asyncDispose]())); + // Stamp-validated: sees a foreign backend's write that landed since. + const { meta, readFailed } = yield* self.loadWithHealth(); + // A read that healed to empty is fine to serve, but rewriting the + // sidecar from it would erase every existing pin and usage stat the + // moment the file becomes readable again. Fail the mutation instead; + // the caller retries on a later call, which re-reads. + if (readFailed) { + return yield* Effect.fail( + new MemoryMetaWriteError({ + metaPath: self.metaPath, + reason: "sidecar exists but could not be read; refusing to overwrite it", + }) + ); + } const entries = { ...meta.entries }; update(entries); for (const [key, entry] of Object.entries(entries)) { @@ -307,9 +431,10 @@ export class MemoryMetaService { }), }); self.cache = next; + self.cacheStamp = yield* Effect.promise(() => self.fileStamp()); }) ); - }) + }).pipe(Effect.scoped) ); } @@ -327,6 +452,21 @@ export class MemoryMetaService { return Effect.runPromise(this.effects.getEntries()); } + /** + * `getEntries()` that refuses a healed substitute: throws when the sidecar + * exists but could not be read. For decisions that consume the entries + * destructively — the legacy-notebook handover before a sub-agent's session + * is deleted folds child-keyed pins/usage into the owner key; an empty + * substitute would fold nothing, report success, and strand the entries. + */ + async getEntriesOrThrow(): Promise> { + const { meta, readFailed } = await Effect.runPromise(this.loadWithHealth()); + if (readFailed) { + throw new Error(`memory metadata sidecar could not be read at ${this.metaPath}`); + } + return new Map(Object.entries(meta.entries).map(([key, entry]) => [key, { ...entry }])); + } + async setPinned(logicalKey: string, pinned: boolean): Promise { await Effect.runPromise(this.effects.setPinned(logicalKey, pinned)); } @@ -344,6 +484,15 @@ export class MemoryMetaService { await Effect.runPromise(this.effects.renameKeys(oldLogicalKey, newLogicalKey)); } + /** Fold a subtree's entries into `targetLogicalKey`, keeping the source (see effects). */ + async mergeKeys( + sourceLogicalKey: string, + targetLogicalKey: string, + options: { pinned: "target" | "source" } + ): Promise { + await Effect.runPromise(this.effects.mergeKeys(sourceLogicalKey, targetLogicalKey, options)); + } + /** * Drop all entries for a deleted file or directory subtree so a future file * at the same path never resurrects stale pins or stats. diff --git a/src/node/services/memoryOperations.test.ts b/src/node/services/memoryOperations.test.ts index deee04bd986..cdc15b4ab33 100644 --- a/src/node/services/memoryOperations.test.ts +++ b/src/node/services/memoryOperations.test.ts @@ -198,6 +198,33 @@ describe("memory operations", () => { expect(await memoryMetaService.getPinnedKeys()).toEqual(new Set(["global:prefs.md"])); }); + test("setPinned broadcasts a change event so other tabs on the same store refetch", async () => { + const context = createContext({ enabled: true }); + await saveMemory(context, { + workspaceId: "ws-mem", + path: "/memories/workspace/pinned.md", + content: "x", + expectedSha256: null, + }); + const events: MemoryChangeEvent[] = []; + memoryService.on("change", (event: MemoryChangeEvent) => events.push(event)); + const result = await setMemoryPinned(context, { + workspaceId: "ws-mem", + path: "/memories/workspace/pinned.md", + pinned: true, + }); + expect(result).toEqual({ success: true, data: undefined }); + expect(events).toEqual([ + { + scope: "workspace", + path: "/memories/workspace/pinned.md", + actor: "user", + workspaceId: "ws-mem", + projectPath, + }, + ]); + }); + test("list exposes usage stats; UI reads do not count as uses", async () => { const client = createClient({ enabled: true }); await client.memory.save({ @@ -246,6 +273,15 @@ describe("memory operations", () => { expect(await memoryMetaService.getPinnedKeys()).toEqual(new Set()); }); + // The subscription announces its store-revision baseline once with a + // root-addressed workspace refresh (see subscribeMemoryChanges); tests + // about forwarded events skip it. + const isBaselineRefresh = (event: MemoryChangeEventPayload): boolean => + !("kind" in event) && + event.scope === "workspace" && + event.path === "/memories/workspace" && + event.actor === "agent"; + test("onChange streams change events from UI saves", async () => { const client = createClient({ enabled: true }); const controller = new AbortController(); @@ -256,6 +292,7 @@ describe("memory operations", () => { const firstEvent = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; return event; } return null; @@ -409,6 +446,7 @@ describe("memory operations", () => { const received: MemoryChangeEventPayload[] = []; const consumer = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; received.push(event); if (received.length >= 3) break; } @@ -481,6 +519,7 @@ describe("memory operations", () => { const received: MemoryChangeEventPayload[] = []; const consumer = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; received.push(event); break; } diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index 0837e5250ba..603ae34344d 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -23,7 +23,7 @@ import { resolveMemoryProjectIdentity, type MemoryScopeContext, } from "./memoryService"; -import { memoryLogicalKey } from "./memoryMeta"; +import { MemoryMetaWriteError, memoryLogicalKey } from "./memoryMeta"; type MemoryContext = Pick< ORPCContext, @@ -57,6 +57,8 @@ function workspaceNotFound(workspaceId: string | null | undefined): string { interface ResolvedMemoryScope { projectPath: string; + /** Task-tree root whose store backs workspace scope ("" without a workspace); keys sidecar pins/stats. */ + ownerWorkspaceId: string; scopeCtx: MemoryScopeContext; } @@ -73,19 +75,24 @@ function resolveMemoryScope( if (workspaceId == null) return { projectPath: "", + ownerWorkspaceId: "", scopeCtx: { runtime: null, checkoutCwd: "", workspaceId: "", projectPath: "" }, }; const metadata = yield* Effect.promise(() => context.workspaceService.getInfo(workspaceId)); if (!metadata) return yield* Effect.fail(new MemoryWorkspaceNotFoundError({ workspaceId })); const projectPath = resolveMemoryProjectIdentity(metadata); + const scopeCtx: MemoryScopeContext = { + runtime: createRuntimeForWorkspace(metadata), + checkoutCwd: "", + workspaceId, + projectPath, + }; return { projectPath, - scopeCtx: { - runtime: createRuntimeForWorkspace(metadata), - checkoutCwd: "", - workspaceId, - projectPath, - }, + // Same per-context resolution the store/notify paths use, so sidecar keys + // and the physical store never disagree about the owner within a request. + ownerWorkspaceId: context.memoryService.ownerWorkspaceIdFor(scopeCtx), + scopeCtx, }; }); } @@ -102,7 +109,7 @@ export function listMemoryEffect(context: MemoryContext, input: Input context.memoryService.setPinned(resolved.scopeCtx, input.path, input.pinned), + catch: (error: unknown) => error, + }).pipe( + Effect.map(() => ({ success: true as const, data: undefined })), + // Sidecar write failures (disk full, permissions) arrive as the typed + // MemoryMetaWriteError; lock timeouts and command errors map onto the + // same legacy string error channel instead of escaping as an untyped + // INTERNAL_SERVER_ERROR rejection. + Effect.catch((error) => + Effect.succeed({ + success: false as const, + error: + error instanceof MemoryMetaWriteError + ? `Failed to persist pin state: ${error.reason}` + : `Failed to update pin: ${getErrorMessage(error)}`, + }) ) - .pipe( - Effect.map(() => ({ success: true as const, data: undefined })), - // Sidecar write failures (disk full, permissions) arrive as the typed - // MemoryMetaWriteError and map onto the legacy string error channel - // instead of escaping as an untyped INTERNAL_SERVER_ERROR rejection. - Effect.catchTag("MemoryMetaWriteError", (error) => - Effect.succeed({ - success: false as const, - error: `Failed to persist pin state: ${error.reason}`, - }) - ) - ); + ); }).pipe(Effect.catchTag("MemoryWorkspaceNotFoundError", workspaceNotFoundAsStringError)); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index a71d2e2fe5c..c6f04c3e022 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "bun:test"; +import { describe, it, expect, spyOn } from "bun:test"; import { MEMORY_MAX_FILES_PER_SCOPE, MEMORY_MAX_FILE_BYTES } from "@/common/constants/memory"; @@ -6,6 +6,7 @@ import { createHash } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import { Config } from "@/node/config"; +import { getErrorMessage } from "@/common/utils/errors"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { extractMemoryDescription, @@ -13,10 +14,16 @@ import { MemoryService, projectMemoryDirName, resolveMemoryProjectIdentity, + type MemoryChangeEvent, type MemoryScopeContext, type PinnedFileMutation, } from "./memoryService"; -import { MemoryMetaService } from "./memoryMeta"; +import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, +} from "./memoryLegacyAdoption"; import { MemoryRefinementActionSchema, REFINEMENT_CAPTURE_MAX_FILES, @@ -24,7 +31,17 @@ import { RefinementInverseSchema, } from "@/common/types/refinement"; import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; -import { TestTempDir } from "./tools/testHelpers"; +import { rollbackRefinement } from "./refinement/refinementRollback"; +import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; +import { reclaimExcessRefinementInverseBlobs, sha256Hex } from "./refinement/refinementJournal"; +import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; +import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; +import { sharedWorkspaceMemoryPeerSessionDirs } from "./memoryWorkspaceOwner"; +import { createRefinementRollbackTool } from "./tools/refinement_rollback"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; +import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; +import { memoryMutationLockKey, withTargetMutationLock } from "./refinement/targetMutationLocks"; +import { TestTempDir, mockToolCallOptions } from "./tools/testHelpers"; function pathExists(target: string): Promise { return fsPromises.access(target).then( @@ -85,6 +102,9 @@ function projectMemoryRoot(fixture: MemoryFixture): string { ); } +/** Store clock segment of a workspaceMemoryRevision token (the rest are file/legacy stamps). */ +const clockOf = (token: string): number => Number(MemoryService.revisionClockOf(token)); + describe("MemoryService", () => { describe("create + view round-trip", () => { it("creates and views a global memory file at /memory/global", async () => { @@ -863,6 +883,4388 @@ describe("MemoryService", () => { }); }); + describe("sub-agent workspace memory sharing", () => { + /** Register owner → child → grandchild so parentWorkspaceId chains resolve. */ + async function registerTaskTree(fixture: MemoryFixture): Promise { + await fixture.config.editConfig((cfg) => { + cfg.projects.set(FIXTURE_PROJECT_PATH, { + workspaces: [ + { id: "ws-owner", name: "owner", path: "/checkouts/owner" }, + { + id: "ws-child", + name: "child", + path: "/checkouts/child", + parentWorkspaceId: "ws-owner", + }, + { + id: "ws-grandchild", + name: "grandchild", + path: "/checkouts/grandchild", + parentWorkspaceId: "ws-child", + }, + { id: "ws-solo", name: "solo", path: "/checkouts/solo" }, + ], + }); + return cfg; + }); + } + + it("resolves the task-tree root as the owner; unknown and parentless ids resolve to themselves", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-owner")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-solo")).toBe("ws-solo"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-unregistered")).toBe( + "ws-unregistered" + ); + }); + + it("resolves from a caller snapshot without touching the config file", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const cfg = fixture.config.loadConfigOrDefault(); + const stamp = spyOn(fixture.config, "configFileStamp"); + const load = spyOn(fixture.config, "loadConfigOrDefault"); + // Bulk passes (launch sweep over every recorded workspace) must not pay + // one synchronous stat per workspace on the main process. + for (const id of ["ws-owner", "ws-child", "ws-grandchild", "ws-solo"]) { + fixture.service.resolveWorkspaceMemoryOwnerId(id, () => cfg); + } + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild", () => cfg)).toBe( + "ws-owner" + ); + expect(stamp).not.toHaveBeenCalled(); + expect(load).not.toHaveBeenCalled(); + }); + + it("keeps a grandchild on the root store via the pinned owner after its parent is removed", async () => { + using fixture = await createFixture("ws-grandchild"); + await registerTaskTree(fixture); + // Removal of the intermediate "ws-child" pins memoryOwnerWorkspaceId on + // its children before deregistering it (WorkspaceService.remove). + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + for (const ws of project.workspaces) { + if (ws.parentWorkspaceId === "ws-child") ws.memoryOwnerWorkspaceId = "ws-owner"; + } + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-child"); + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-owner"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/still-shared.md", + "root store", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "still-shared.md") + ) + ).toBe(true); + // A pinned owner that is itself gone falls back to self. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-grandchild"); + }); + + it("stores a sub-agent's workspace notes in the owner's session dir, visible to the whole tree", async () => { + using fixture = await createFixture("ws-grandchild"); + await registerTaskTree(fixture); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/context-notes.md", + "found the bug in parser.ts", + "agent" + ); + expect(created.success).toBe(true); + + // Physically in the OWNER's session dir, not the grandchild's. + const ownerPhysical = path.join( + fixture.config.sessionsDir, + "ws-owner", + "memory", + "context-notes.md" + ); + expect(await fsPromises.readFile(ownerPhysical, "utf-8")).toBe("found the bug in parser.ts"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-grandchild", "memory")) + ).toBe(false); + + // Owner and sibling child read the same file through their own contexts. + for (const workspaceId of ["ws-owner", "ws-child"]) { + const viewed = await fixture.service.view( + { ...fixture.ctx, workspaceId }, + "/memories/workspace/context-notes.md" + ); + expect(viewed.success).toBe(true); + if (viewed.success) expect(viewed.output).toContain("found the bug in parser.ts"); + } + // An unrelated workspace does not see it. + const solo = await fixture.service.view( + { ...fixture.ctx, workspaceId: "ws-solo" }, + "/memories/workspace/context-notes.md" + ); + expect(solo.success).toBe(false); + + // Change events name the owner so the owner's Memory tab (and every + // tree member's) refreshes — the create and each shared read (the two + // views re-rank the shared hot set); sidecar stats are keyed by the + // owner too. + expect(events).toEqual( + Array.from({ length: 3 }, () => ({ + scope: "workspace", + path: "/memories/workspace/context-notes.md", + actor: "agent", + workspaceId: "ws-owner", + projectPath: FIXTURE_PROJECT_PATH, + })) + ); + const meta = await fixture.metaService.getEntries(); + expect( + meta.get( + memoryLogicalKey("workspace", "context-notes.md", { + projectPath: "", + workspaceId: "ws-owner", + }) + )?.lastWriteAt + ).not.toBeNull(); + expect( + meta.has( + memoryLogicalKey("workspace", "context-notes.md", { + projectPath: "", + workspaceId: "ws-grandchild", + }) + ) + ).toBe(false); + }); + + it("journals a sub-agent's workspace-scope mutation in its own session and rolls it back via the owner root", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/n.md", + "shared", + "agent" + ); + expect(created.success).toBe(true); + + // Attribution stays with the acting workspace: the row is in the + // child's journal but its inverse points into the owner's memory dir. + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const events = await readRefinementEvents(childSessionDir); + expect(events).toHaveLength(1); + expect(await readRefinementEvents(ownerSessionDir)).toHaveLength(0); + const physical = path.join(ownerSessionDir, "memory", "n.md"); + expect(events[0].data.inverse).toEqual({ op: "delete-files", paths: [physical] }); + + // Confinement: the child's own memory root does not admit the path... + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + id: events[0].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("outside every memory scope root"); + expect(await pathExists(physical)).toBe(true); + + // ...the caller-supplied owner session dir does. + const rolledBack = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: events[0].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await pathExists(physical)).toBe(false); + }); + it("re-resolves the owner after config changes so a removed owner's child falls back to its own store", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); + + // The owner is deregistered (removal with a live shared-checkout child); + // the dangling chain must not keep pointing at the tombstoned owner. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + return cfg; + }); + // Live sessions of formerly-shared children are told to drop their cache. + expect(invalidated).toEqual([["ws-child"]]); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/after.md", + "own store now", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "after.md")) + ).toBe(true); + }); + + it("ignores config edits that leave the memory topology unchanged", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); + + // Ordinary churn (a retitle) must not make every live child rebuild its + // memory context, and the unchanged mapping stays memoized. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces.find((ws) => ws.id === "ws-child")!.title = "renamed"; + return cfg; + }); + expect(invalidated).toEqual([]); + const load = spyOn(fixture.config, "loadConfigOrDefault"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(load).not.toHaveBeenCalled(); + }); + + it("re-resolves the owner after an EXTERNAL config rewrite (another backend removed it)", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + + // Rewrite config.json directly: no local onConfigChanged fires, only the + // file's stamp changes — as when a second backend deregisters the owner. + const configFile = path.join(fixture.xumHome, "config.json"); + // On disk, projects are [path, project] tuples. + const raw = JSON.parse(await fsPromises.readFile(configFile, "utf-8")) as { + projects: Array<[string, { workspaces: Array<{ id: string }> }]>; + }; + const project = raw.projects.find(([projectPath]) => projectPath === FIXTURE_PROJECT_PATH); + expect(project).toBeDefined(); + project![1].workspaces = project![1].workspaces.filter((ws) => ws.id !== "ws-owner"); + await fsPromises.writeFile(configFile, JSON.stringify(raw, null, 2)); + // Same-tick same-size rewrites can leave mtime unchanged; force a distinct stamp. + await fsPromises.utimes( + configFile, + new Date(Date.now() + 5_000), + new Date(Date.now() + 5_000) + ); + + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + }); + + it("announces the self→shared transition when config.json recovers after being unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); + + // config.json vanishes (another backend mid-rewrite): the child cannot + // resolve its tree and falls back to its private store... + const configFile = path.join(fixture.xumHome, "config.json"); + const parked = `${configFile}.parked`; + await fsPromises.rename(configFile, parked); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + expect(invalidated).toEqual([]); + + // ...and once it is back, sessions that built a context on the fallback + // store must be told, even though no shared mapping was ever memoized. + await fsPromises.rename(parked, configFile); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(invalidated).toEqual([["ws-child"]]); + }); + + it("does not memoize the self fallback taken while config.json is unreadable but unchanged", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // The file stats the same (no stamp change) but cannot be read/parsed + // for a moment (EACCES interval, non-atomic writer): the lenient load + // yields the empty default while the strict one throws. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + // Readability returns without the stamp moving: the next resolution + // must see the real tree instead of a pinned fallback. + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + }); + + it("keeps the owner memo retryable when a local config edit notifies while the file is unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // A local edit removes the owner. The change notification fires while + // the file cannot be read (a swallowed late write failure): the memo + // must not be stamped as current, or the stale mapping survives until + // an unrelated rewrite once readability returns without a stamp change. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + await fixture.config.editConfig((cfg) => { + for (const project of cfg.projects.values()) { + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + } + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + }); + + it("rescans the owner notebook for the revision token only on root changes or per interval", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + await fixture.service.create(fixture.ctx, "/memories/workspace/dir/nested.md", "v1", "agent"); + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const nestedStats = () => + lstatSpy.mock.calls.filter(([target]) => String(target).startsWith(ownerRoot)).length; + const lstatSpy = spyOn(fsPromises, "lstat"); + try { + const first = await foreign.workspaceMemoryRevision("ws-owner"); + expect(nestedStats()).toBeGreaterThan(0); + // A cached-context probe on every turn (and every Memory tab interval) + // must not walk the notebook again: the clock and sidecar segments + // carry this build's writes, the scan only exists for downgraded ones. + lstatSpy.mockClear(); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(first); + expect(nestedStats()).toBe(0); + // A note added at the root moves the root mtime (one stat): rescanned + // at once, token changed — a downgraded build's new note shows up. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(ownerRoot, "old-build.md"), "downgraded write"); + const afterRootWrite = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterRootWrite).not.toBe(first); + expect(nestedStats()).toBeGreaterThan(0); + // This build's own writes need no rescan to be seen: the clock moved. + lstatSpy.mockClear(); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dir/nested.md", + "v1", + "v2", + "agent" + ); + expect(await foreign.workspaceMemoryRevision("ws-owner")).not.toBe(afterRootWrite); + expect(nestedStats()).toBe(0); + // A redirected child's token also fingerprints its legacy notebook: + // that scan is memoized the same way (r84) — the first probe walks + // it, a repeated probe does not, a new legacy note is seen at once. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir", "old.md"), "legacy"); + const legacyStats = () => + lstatSpy.mock.calls.filter(([target]) => String(target).startsWith(legacyRoot + path.sep)) + .length; + lstatSpy.mockClear(); + const childToken = await foreign.workspaceMemoryRevision("ws-child"); + expect(legacyStats()).toBeGreaterThan(0); + lstatSpy.mockClear(); + expect(await foreign.workspaceMemoryRevision("ws-child")).toBe(childToken); + expect(legacyStats()).toBe(0); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "new.md"), "downgraded write"); + expect(await foreign.workspaceMemoryRevision("ws-child")).not.toBe(childToken); + expect(legacyStats()).toBeGreaterThan(0); + } finally { + lstatSpy.mockRestore(); + } + }); + + it("advances the owner store's revision token on shared writes, visible to another backend", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // A second MemoryService over the same Xum root stands in for another + // backend process: it receives none of this instance's change events. + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect(MemoryService.revisionClockOf(await foreign.workspaceMemoryRevision("ws-owner"))).toBe( + "missing" + ); + + await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "v1", "agent"); + const afterCreate = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterCreate).not.toBe("missing"); + // The child's token tracks the owner clock, plus its own legacy notebook + // state (see below); the child has none yet. + const childToken = await foreign.workspaceMemoryRevision("ws-child"); + expect(childToken.startsWith(`${afterCreate}\u0000`)).toBe(true); + // A downgraded backend writing the child's LEGACY notebook (or toggling a + // child-keyed pin) moves no owner clock, yet the child's cached context + // must miss so its next access adopts the change. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "legacy"); + const afterLegacyWrite = await foreign.workspaceMemoryRevision("ws-child"); + expect(afterLegacyWrite).not.toBe(childToken); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "old.md", { projectPath: "", workspaceId: "ws-child" }), + true + ); + expect(await foreign.workspaceMemoryRevision("ws-child")).not.toBe(afterLegacyWrite); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + await fsPromises.rm(legacyRoot, { recursive: true, force: true }); + + // Other scopes leave the workspace store's token alone... + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + // ...a downgraded build writing straight into the owner's canonical + // notebook moves no clock, but the token still changes (file stamps)... + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "old-build.md"), + "written by a downgraded build" + ); + const afterOldBuildWrite = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterOldBuildWrite).not.toBe(afterCreate); + expect(clockOf(afterOldBuildWrite)).toBe(clockOf(afterCreate)); + // ...a pin toggle (hot-set input, no store write) advances it... + await fixture.service.setPinned(fixture.ctx, "/memories/workspace/shared.md", true); + const afterPin = await foreign.workspaceMemoryRevision("ws-owner"); + expect(clockOf(afterPin)).toBeGreaterThan(clockOf(afterCreate)); + // ...and an owner-keyed sidecar change whose clock write was lost (the + // revision write is best-effort; a downgraded build toggling the pin + // moves no clock either) still changes the token. + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "shared.md", { projectPath: "", workspaceId: "ws-owner" }), + false + ); + const afterSidecarOnly = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterSidecarOnly).not.toBe(afterPin); + expect(clockOf(afterSidecarOnly)).toBe(clockOf(afterPin)); + // ...while every shared-store mutation advances it. + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/shared.md", + "v1", + "v2", + "agent" + ); + const afterEdit = await foreign.workspaceMemoryRevision("ws-owner"); + expect(clockOf(afterEdit)).toBeGreaterThan(clockOf(afterPin)); + // A read-side access (view / recall) re-ranks the shared hot set through + // the owner-keyed usage stats: it advances the clock and announces the + // owner's store like a pin does, so the rest of the tree (and other + // backends) drop their cached hot set too. + const events: MemoryChangeEvent[] = []; + fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); + expect( + (await fixture.service.view(fixture.ctx, "/memories/workspace/shared.md")).success + ).toBe(true); + const afterView = await foreign.workspaceMemoryRevision("ws-owner"); + expect(clockOf(afterView)).toBeGreaterThan(clockOf(afterEdit)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + scope: "workspace", + path: "/memories/workspace/shared.md", + workspaceId: "ws-owner", + }); + await fixture.service.recordRecall(fixture.ctx, "/memories/workspace/shared.md"); + expect(clockOf(await foreign.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + clockOf(afterView) + ); + expect(events).toHaveLength(2); + // Global reads have no store clock and stay silent. + expect((await fixture.service.view(fixture.ctx, "/memories/global/g.md")).success).toBe(true); + expect(events).toHaveLength(2); + }); + + it("refuses to commit into a self-fallback store once config.json has recovered", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // Mid-command race: the command resolves its store while config.json is + // unreadable (self-fallback) and the file recovers before the commit + // check inside the mutation lock. Only the command's FIRST resolution + // is faked; the pre-commit re-resolution sees the recovered tree. + spyOn(fixture.service, "resolveWorkspaceMemoryOwnerId").mockImplementationOnce( + () => "ws-child" + ); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/late.md", + "x", + "agent" + ); + expect(created.success).toBe(false); + if (!created.success) expect(created.error).toContain("Ownership of the workspace notebook"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "late.md")) + ).toBe(false); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-owner", "memory", "late.md")) + ).toBe(false); + }); + + it("re-resolves the owner per command and refuses reads once the owner is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + // One context serves a whole stream (createMemoryTool): a cached owner + // must not outlive the command that resolved it. + expect(fixture.service.ownerWorkspaceIdFor(fixture.ctx)).toBe("ws-owner"); + const resolve = spyOn(fixture.service, "resolveWorkspaceMemoryOwnerId"); + expect((await fixture.service.view(fixture.ctx, "/memories/workspace/n.md")).success).toBe( + true + ); + expect(resolve).toHaveBeenCalled(); + + // Another backend removed the owner: its durable tombstone (no local + // event) must stop the child's reads of the shared notebook. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + const refused = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + const root = await fixture.service.view(fixture.ctx, "/memories"); + expect(root.success).toBe(true); + if (root.success) expect(root.output).toContain("unavailable"); + // The prompt-context path is guarded too: the probe reports revocation + // (invalidating a cached context) and the index no longer lists the store. + expect(await fixture.service.workspaceMemoryRevision("ws-child")).toBe("revoked"); + expect( + (await fixture.service.listIndexEntries(fixture.ctx)).some( + (entry) => entry.scope === "workspace" + ) + ).toBe(false); + }); + + it("reports revocation for a tombstone published while the revision token was being built", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const before = await fixture.service.workspaceMemoryRevision("ws-child"); + expect(before).not.toBe("revoked"); + // Removal lands between the entry check and the token's reads: the + // unchanged pre-removal token would let a cached context keep serving + // the owner's notes to the removed child's next request. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + const service = fixture.service as unknown as { + buildWorkspaceMemoryRevisionToken: (...args: unknown[]) => Promise; + }; + const original = service.buildWorkspaceMemoryRevisionToken.bind(fixture.service); + const build = spyOn(service, "buildWorkspaceMemoryRevisionToken").mockImplementationOnce( + async (...args) => { + const token = await original(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + return token; + } + ); + try { + expect(await fixture.service.workspaceMemoryRevision("ws-child")).toBe("revoked"); + expect(build).toHaveBeenCalledTimes(1); + } finally { + build.mockRestore(); + } + }); + + it("guards a context acting on a removed child's behalf like the child itself", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // A child's consolidation run sweeps under the OWNER's identity; the + // child's removal by another backend (tombstone, no local signal) must + // still refuse that run's reads and commits in every scope. + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner", guardedWorkspaceId: "ws-child" }; + await fixture.service.create(ownerCtx, "/memories/workspace/n.md", "shared", "agent"); + await fixture.service.create(ownerCtx, "/memories/global/g.md", "global", "agent"); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + for (const attempt of [ + () => fixture.service.view(ownerCtx, "/memories/workspace/n.md"), + () => + fixture.service.strReplace(ownerCtx, "/memories/workspace/n.md", "shared", "x", "agent"), + () => fixture.service.strReplace(ownerCtx, "/memories/global/g.md", "global", "x", "agent"), + () => fixture.service.create(ownerCtx, "/memories/project/p.md", "p", "agent"), + ]) { + const result = await attempt(); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("ws-child was removed"); + } + expect( + await fsPromises.readFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "n.md"), + "utf-8" + ) + ).toBe("shared"); + // The owner's own contexts are unaffected. + const plainOwner = { ...fixture.ctx, workspaceId: "ws-owner" }; + expect((await fixture.service.view(plainOwner, "/memories/workspace/n.md")).success).toBe( + true + ); + }); + + it("refuses a read whose workspace was tombstoned while the legacy adoption pass ran", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + // The adoption pass (owner-store lock) is the window: another backend's + // removal of ws-child publishes its tombstone after the readability + // check that opened the store, and the pass swallows its own refusal. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + spyOn( + fixture.service as unknown as { adoptLegacyPrivateStore: () => Promise }, + "adoptLegacyPrivateStore" + ).mockImplementationOnce(async () => { + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const refused = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + }); + + it("withholds a read whose workspace was tombstoned after the pre-read check", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "global", "agent"); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + // Another backend's removal lands between the check that opened the + // store and the read itself: every path that exposes the store's bytes + // or listing re-checks before returning them. + const service = fixture.service as unknown as { + openWorkspaceStore: (...args: unknown[]) => Promise; + }; + const original = service.openWorkspaceStore.bind(fixture.service); + const tombstoneAfterOpen = () => + spyOn(service, "openWorkspaceStore").mockImplementationOnce(async (...args) => { + await original(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const untombstone = () => fsPromises.rm(tombstonePath, { force: true }); + + tombstoneAfterOpen(); + const file = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(file.success).toBe(false); + if (!file.success) expect(file.error).toContain("was removed"); + await untombstone(); + + // The usage record waits for the owner-store lock, which a removal + // holds while it publishes the tombstone: landing there, after the + // bytes were read, must still withhold them. + const usageService = fixture.service as unknown as { + recordUsage: (...args: unknown[]) => Promise; + }; + const originalUsage = usageService.recordUsage.bind(fixture.service); + const usage = spyOn(usageService, "recordUsage").mockImplementationOnce(async (...args) => { + await originalUsage(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const lateFile = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(usage).toHaveBeenCalledTimes(1); + expect(lateFile.success).toBe(false); + if (!lateFile.success) expect(lateFile.error).toContain("was removed"); + usage.mockRestore(); + await untombstone(); + + tombstoneAfterOpen(); + const dir = await fixture.service.view(fixture.ctx, "/memories/workspace"); + expect(dir.success).toBe(false); + if (!dir.success) expect(dir.error).toContain("was removed"); + await untombstone(); + + tombstoneAfterOpen(); + const root = await fixture.service.view(fixture.ctx, "/memories"); + expect(root.success).toBe(true); + if (root.success) { + expect(root.output).toContain("unavailable"); + expect(root.output).not.toContain("n.md"); + } + await untombstone(); + + tombstoneAfterOpen(); + const entries = await fixture.service.listIndexEntries(fixture.ctx); + expect(entries.map((entry) => entry.scope)).toEqual(["global"]); + await untombstone(); + + tombstoneAfterOpen(); + const ui = await fixture.service.readFileWithSha(fixture.ctx, "/memories/workspace/n.md"); + expect(ui.success).toBe(false); + await untombstone(); + + // Hot-set reads happen after the index enumeration passed: the + // tombstone landing before the file read drops the item. + const hotBefore = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: (text) => Promise.resolve(text.length), + }); + expect(hotBefore.some((item) => item.path === "/memories/workspace/n.md")).toBe(true); + // Interleaving: the tombstone lands after listIndexEntries built the + // candidate list and before the hot-set file reads. + const originalList = fixture.service.listIndexEntries.bind(fixture.service); + const listIndex = spyOn(fixture.service, "listIndexEntries").mockImplementationOnce( + async (ctx) => { + const result = await originalList(ctx); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + return result; + } + ); + try { + const hot = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: (text) => Promise.resolve(text.length), + }); + expect(hot.some((item) => item.path === "/memories/workspace/n.md")).toBe(false); + expect(hot.some((item) => item.path === "/memories/global/g.md")).toBe(true); + } finally { + listIndex.mockRestore(); + await untombstone(); + } + // Selection keeps awaiting token counts after the file reads: a + // tombstone landing there still withholds the workspace items. + let counted = 0; + const hotAfterCount = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: async (text) => { + if (counted++ === 0) { + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + } + return text.length; + }, + }); + expect(counted).toBeGreaterThan(0); + expect(hotAfterCount.some((item) => item.path === "/memories/workspace/n.md")).toBe(false); + expect(hotAfterCount.some((item) => item.path === "/memories/global/g.md")).toBe(true); + await untombstone(); + }); + + it("refuses a pin toggle once the owner it was bound to is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + // Owner removed by another backend between the tab's owner resolution + // and the pin's lock acquisition: the pin must not be committed under + // the dead owner's logical key while the route reports success. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + const refused = await fixture.service + .setPinned(fixture.ctx, "/memories/workspace/n.md", true) + .then( + () => null, + (error: unknown) => error + ); + expect(refused).toBeInstanceOf(Error); + expect(getErrorMessage(refused)).toContain("was removed"); + expect((await fixture.metaService.getPinnedKeys()).size).toBe(0); + expect(events).toEqual([]); + }); + + it("adopts a sub-agent's pre-sharing private notebook into the shared store, keeping the legacy copy downgrade-readable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // Notes written by a build that kept the child's workspace scope in its + // own session dir, plus a pin recorded under the child's logical key. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "only-child.md"), "child notes"); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "same.md"), "identical"); + await fsPromises.writeFile(path.join(legacyRoot, "clash.md"), "child version"); + await fixture.metaService.setPinned("workspace:ws-child:only-child.md", true); + // The owner already holds one identical and one conflicting file. + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + await fixture.service.create( + ownerCtx, + "/memories/workspace/sub/same.md", + "identical", + "agent" + ); + await fixture.service.create( + ownerCtx, + "/memories/workspace/clash.md", + "owner version", + "agent" + ); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + + const listed = await fixture.service.listIndexEntries(fixture.ctx); + expect(listed.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "clash.md", + "imported/ws-child/clash.md", + "only-child.md", + "sub/same.md", + ]); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "child notes" + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "clash.md"), "utf-8")).toBe( + "owner version" + ); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "clash.md"), "utf-8") + ).toBe("child version"); + // The pin is copied to the owner-keyed logical key; the child-keyed + // entry stays for a downgraded build, which keys by the child id. + expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ + "workspace:ws-child:only-child.md", + "workspace:ws-owner:only-child.md", + ]); + // The legacy copy stays where a downgraded build reads it; the tree's + // tabs were told once and a second access is a no-op. + expect(await fsPromises.readFile(path.join(legacyRoot, "only-child.md"), "utf-8")).toBe( + "child notes" + ); + expect(events).toEqual([ + { + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: FIXTURE_PROJECT_PATH, + }, + ]); + await fixture.service.listIndexEntries(fixture.ctx); + expect(events).toHaveLength(1); + + // Edited through the shared store, then a backend restart: the legacy + // copy is known to be folded in already and must not resurface as a + // stale duplicate. + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/only-child.md", + "child notes", + "shared edit", + "agent" + ); + // A downgraded build wrote a new note into the legacy dir meanwhile. + await fsPromises.writeFile(path.join(legacyRoot, "downgrade.md"), "written on old build"); + // An earlier adoption was interrupted right after writing this file's + // bytes: identical bytes in the owner store, pin only under the child + // key, nothing in the manifest. The retry must still copy the pin. + await fsPromises.writeFile(path.join(legacyRoot, "half.md"), "half adopted"); + await fsPromises.writeFile(path.join(ownerRoot, "half.md"), "half adopted"); + await fixture.metaService.setPinned("workspace:ws-child:half.md", true); + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const restartedEvents: unknown[] = []; + restarted.on("change", (event) => restartedEvents.push(event)); + const revisionBefore = clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")); + const relisted = await restarted.listIndexEntries(fixture.ctx); + // The pass wrote one file and copied one pin: both change what other + // backends derive from the store, so the clock moved and the tabs heard. + expect(clockOf(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + revisionBefore + ); + expect(restartedEvents).toHaveLength(1); + // Metadata-only pass (nothing to write, one pin to copy): same signals. + await fsPromises.writeFile(path.join(legacyRoot, "meta-only.md"), "same bytes"); + await fsPromises.writeFile(path.join(ownerRoot, "meta-only.md"), "same bytes"); + await fixture.metaService.setPinned("workspace:ws-child:meta-only.md", true); + const metaOnly = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const metaOnlyEvents: unknown[] = []; + metaOnly.on("change", (event) => metaOnlyEvents.push(event)); + const revisionMid = clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")); + await metaOnly.listIndexEntries(fixture.ctx); + expect(clockOf(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + revisionMid + ); + expect(metaOnlyEvents).toHaveLength(1); + expect(await fixture.metaService.getPinnedKeys()).toContain( + "workspace:ws-owner:meta-only.md" + ); + + // Sidecar-only changes made on a downgraded build (bytes untouched) are + // folded in on the next upgrade: an unpin of half.md under the child key + // reaches the owner key (its recorded target still holds the bytes)... + const freshService = () => + new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await fixture.metaService.setPinned("workspace:ws-child:half.md", false); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).not.toContain("workspace:ws-owner:half.md"); + // ...while the owner's OWN later choice is not undone by an unchanged + // child entry on every restart. + await fixture.metaService.setPinned("workspace:ws-owner:half.md", true); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).toContain("workspace:ws-owner:half.md"); + // only-child.md's recorded target was replaced by the shared edit: the + // child's pin change must not land on the owner's new content. The + // legacy note is placed anew (imported/) and carries the child's state. + await fixture.metaService.setPinned("workspace:ws-child:only-child.md", false); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).toContain( + "workspace:ws-owner:only-child.md" + ); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "only-child.md"), + "utf-8" + ) + ).toBe("child notes"); + expect(await fixture.metaService.getPinnedKeys()).not.toContain( + "workspace:ws-owner:imported/ws-child/only-child.md" + ); + expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "clash.md", + "downgrade.md", + "half.md", + "imported/ws-child/clash.md", + "only-child.md", + "sub/same.md", + ]); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "shared edit" + ); + expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ + "workspace:ws-child:meta-only.md", + "workspace:ws-owner:half.md", + "workspace:ws-owner:meta-only.md", + "workspace:ws-owner:only-child.md", + ]); + + // A workspace that is its own owner keeps its private store untouched. + const solo = { ...fixture.ctx, workspaceId: "ws-solo" }; + await fixture.service.create(solo, "/memories/workspace/mine.md", "solo", "agent"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-solo", "memory", "mine.md")) + ).toBe(true); + }); + + it("stops adopting legacy notes at the shared store's remaining file capacity", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner two below the cap; child brings five (one identical to an owner + // file, which needs no slot). + await Promise.all( + Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE - 2 }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ) + ); + await fsPromises.writeFile(path.join(ownerRoot, "shared.md"), "same"); + for (const name of ["a.md", "b.md", "c.md", "d.md"]) { + await fsPromises.writeFile(path.join(legacyRoot, name), `child ${name}`); + } + await fsPromises.writeFile(path.join(legacyRoot, "shared.md"), "same"); + + const listed = await fixture.service.listIndexEntries(fixture.ctx); + const workspaceFiles = listed.filter((e) => e.scope === "workspace").map((e) => e.relPath); + // Exactly at the cap, never above: one slot was already taken by + // shared.md's owner copy, so only one of the four new notes fit. + expect(workspaceFiles).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); + expect(workspaceFiles.filter((f) => ["a.md", "b.md", "c.md", "d.md"].includes(f))).toEqual([ + "a.md", + ]); + // A create into the full scope is refused like before, so the invariant holds. + const full = await fixture.service.create( + fixture.ctx, + "/memories/workspace/new.md", + "x", + "agent" + ); + expect(full.success).toBe(false); + // Freed capacity lets a later pass of the SAME process fold in the rest: + // an incomplete pass is not memoized, since its retry depends on owner + // state the legacy check key does not observe. + await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0000.md", "agent"); + await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0001.md", "agent"); + const relisted = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath); + expect(relisted).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); + expect(relisted.filter((f) => ["a.md", "b.md", "c.md", "d.md"].includes(f))).toEqual([ + "a.md", + "b.md", + "c.md", + ]); + }); + + it("adopts addressable dot-entry notes and refuses the handover over unrepresentable ones", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + // The path grammar admits dotfiles, so a downgraded child may hold a + // real note at `.note` that no listing ever showed — including one + // whose text `create` accepted but the lossy-decode gate cannot vouch + // for. Such an entry must hold up removal like a listed note would + // (r73), not be written off as a stray `.DS_Store`. + await fsPromises.mkdir(path.join(legacyRoot, ".hidden"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, ".note"), "dot note"); + await fsPromises.writeFile(path.join(legacyRoot, ".hidden", "n.md"), "nested dot note"); + // Every in-namespace path is a note's, including one named like the + // pass's own staging area (which lives OUTSIDE the memory root, r91): + // adoption must never mistake it for staged bytes and remove it. + await fsPromises.mkdir(path.join(legacyRoot, "memory-adoption-staging"), { + recursive: true, + }); + await fsPromises.writeFile( + path.join(legacyRoot, "memory-adoption-staging", "n.md"), + "staging-named note" + ); + await fsPromises.writeFile( + path.join(legacyRoot, ".DS_Store"), + Buffer.from([0, 0, 1, 255, 254]) + ); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/1 legacy workspace memory note\(s\)/); + // The representable dot-entries were folded in by that same pass. + expect(await fsPromises.readFile(path.join(ownerRoot, ".note"), "utf-8")).toBe("dot note"); + expect(await fsPromises.readFile(path.join(ownerRoot, ".hidden", "n.md"), "utf-8")).toBe( + "nested dot note" + ); + expect(await pathExists(path.join(ownerRoot, ".DS_Store"))).toBe(false); + expect( + await fsPromises.readFile(path.join(ownerRoot, "memory-adoption-staging", "n.md"), "utf-8") + ).toBe("staging-named note"); + // Removing the stray entry lets a retried (non-forced) handover complete. + await fsPromises.rm(path.join(legacyRoot, ".DS_Store")); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + // Still addressable through the shared store, like on the old build. + const viewed = await fixture.service.view(fixture.ctx, "/memories/workspace/.note"); + expect(viewed.success).toBe(true); + if (viewed.success) expect(viewed.output).toContain("dot note"); + }); + + it("adopts the legacy notebook for removal without any prior access, and throws instead of deferring", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "only-child.md"), "child notes"); + // No workspace-memory entry point ever served ws-child in this process: + // removal's handover must fold the notes in by itself. + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "child notes" + ); + // Idempotent: a retried removal re-runs the pass (the per-process memo + // is bypassed) and finds nothing new. + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + // A removal that verified a different owner than the store now resolves + // to must not adopt into the wrong notebook. + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-other") + .then(() => null, getErrorMessage) + ).toMatch(/resolved to ws-owner/); + // Failures surface (the access-time pass only logs and retries later). + // A sidecar that exists but cannot be read is no "no metadata": the + // handover would copy the note without its pin and report success. + await fsPromises.writeFile(path.join(legacyRoot, "late.md"), "written later"); + const metaPath = path.join(fixture.xumHome, "memory-meta.json"); + const savedMeta = await fsPromises.readFile(metaPath).catch(() => null); + await fsPromises.rm(metaPath, { force: true }); + await fsPromises.mkdir(metaPath); // EISDIR on read + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/sidecar could not be read/); + } finally { + await fsPromises.rmdir(metaPath); + if (savedMeta !== null) await fsPromises.writeFile(metaPath, savedMeta); + } + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + // Same for the adoption manifest: unreadable (not missing) aborts. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const savedManifest = await fsPromises.readFile(manifestPath); + await fsPromises.rm(manifestPath); + await fsPromises.mkdir(manifestPath); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EISDIR/); + } finally { + await fsPromises.rmdir(manifestPath); + await fsPromises.writeFile(manifestPath, savedManifest); + } + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + // Malformed (not missing) aborts too: a bad record whose source is + // gone could not be reconciled, and an empty substitute would drop the + // provenance for its owner copy while removal deletes the child. + for (const [label, body] of [ + ["not JSON", "{nope"], + ["not an object", "[]"], + ["record 'note.md'", JSON.stringify({ "note.md": { content: 1 } })], + [ + "record 'late.md'", + JSON.stringify({ + "late.md": { content: "x", sidecar: "", target: "late.md", replacementContent: 5 }, + }), + ], + ] as const) { + await fsPromises.writeFile(manifestPath, body); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toContain(`malformed (${label})`); + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + } + await fsPromises.writeFile(manifestPath, savedManifest); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(true); + }); + + it("transfers provenance when a renamed legacy note lands on its own conflict copy", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Conflict: the owner has a different a.md, so the child's is adopted + // under imported//a.md. + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + // The downgraded build renames the source to exactly that imported + // path: the new record reuses the identical target; the old record's + // reconciliation must hand the copy over, not delete it. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + // The old record stays as a tombstone (rollbacks of the child's + // pre-sharing rows for a.md still need its mapping). + expect(Object.keys(manifest).sort()).toEqual(["a.md", "imported/ws-child/a.md"]); + expect(manifest["a.md"]).toMatchObject({ deleted: true }); + expect(manifest["imported/ws-child/a.md"]).toMatchObject({ + target: "imported/ws-child/a.md", + created: true, + }); + // With provenance transferred, deleting the renamed source removes the copy. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); + }); + + it("transfers the installed generation to the successor across an interrupted replacement", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const settled = (await readLegacyAdoptionManifest(manifestPath)).get("a.md")!; + // The downgraded build edits a.md; the replacement pass installed the + // new bytes (a new generation) but crashed before settling: the record + // is pending with the OVERWRITTEN generation's targetStamp and the + // installed one's replacementStamp. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a2"); + await fsPromises.rm(importedCopy); + await fsPromises.writeFile(importedCopy, "child's a2"); + const installed = (await adoptionTargetStamp(importedCopy))!; + expect(installed).not.toBe(settled.targetStamp); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "a.md": { + ...settled, + pending: true, + replacementContent: sha256Hex("child's a2"), + replacementStamp: installed, + }, + }) + ); + // Before the retry, the source is renamed onto the conflict-copy path: + // the successor record reuses the installed file, and must inherit the + // generation actually on disk — not the overwritten one, which would + // make the copy read as replaced by the owner at once. + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + const successor = (await readLegacyAdoptionManifest(manifestPath)).get( + "imported/ws-child/a.md" + )!; + expect(successor).toMatchObject({ target: "imported/ws-child/a.md", created: true }); + expect(successor.targetStamp).toBe(installed); + // With the right generation, deleting the renamed source removes the copy. + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); + }); + + it("keeps an owner-edited conflict copy the owner's when a renamed legacy note lands on it", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner edits the conflict copy: it is the owner's now. + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/imported/ws-child/a.md", + "child's a", + "owner's edit", + "agent" + ); + // The downgraded build renames the source onto that path with the + // owner's bytes: the new record reuses the file, but no provenance + // transfers — the old copy no longer holds the adopted bytes. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rm(path.join(legacyRoot, "a.md")); + await fsPromises.writeFile( + path.join(legacyRoot, "imported", "ws-child", "a.md"), + "owner's edit" + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(manifest["imported/ws-child/a.md"].created).not.toBe(true); + // ...and the obsolete record's tombstone drops its destructive + // provenance: the child's old rows may not map onto the owner's note. + expect(manifest["a.md"]).toMatchObject({ deleted: true }); + expect(manifest["a.md"].created).not.toBe(true); + // Deleting the renamed source leaves the owner's note in place. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "a.md"), "utf-8") + ).toBe("owner's edit"); + }); + + it("removal lists an oversized legacy notebook completely, counting every unplaceable note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner store with one free slot; legacy notebook two notes past the + // per-scope cap in one flat directory. A capped walk would list cap+1 + // notes and report cap skipped; every note beyond the cap must be + // listed and reported so removal cannot delete an unlisted one. + await Promise.all([ + ...Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE - 1 }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ), + ...Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE + 2 }, (_, i) => + fsPromises.writeFile(path.join(legacyRoot, `n${String(i).padStart(4, "0")}.md`), "n") + ), + ]); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(new RegExp(`^${MEMORY_MAX_FILES_PER_SCOPE + 1} legacy workspace memory note`)); + }); + + it("re-adopts when a downgraded build edits a nested legacy note in place or only its pin", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v1"); + // In-place edit of an existing nested file on the old build: neither the + // legacy root's mtime nor the (unknown to it) store clock moves. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v2 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The copy this adoption created was untouched by the owner: the new + // bytes replace it in place (an imported/ duplicate would strand the + // old copy, provenance lost, in the shared notebook). + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "v2 (downgrade)" + ); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"))).toBe( + false + ); + // Sidecar-only change (a pin toggled on the old build under the child + // key): no file stat changes at all, yet the owner key must follow. + const childKey = memoryLogicalKey("workspace", "sub/note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "sub/note.md", { + projectPath: "", + workspaceId: "ws-owner", + }) + ) + ).toBe(true); + // Once the OWNER edited the copy it is the owner's: a further legacy + // edit is placed anew under imported/. + await fixture.service.strReplace( + { ...fixture.ctx, workspaceId: "ws-owner" }, + "/memories/workspace/sub/note.md", + "v2", + "owner's v3", + "agent" + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v4 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"), + "utf-8" + ) + ).toBe("v4 (downgrade)"); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "owner's v3 (downgrade)" + ); + }); + + it("follows legacy deletions and renames for copies the adoption created, never owner notes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // `same.md` pre-exists identically on the owner side (reused, not created); + // `mine.md` and `moved.md` are created by the adoption; `edited.md` too, + // but the owner edits it afterwards. + await fsPromises.writeFile(path.join(ownerRoot, "same.md"), "identical"); + for (const [name, body] of [ + ["same.md", "identical"], + ["mine.md", "child note"], + ["moved.md", "to be renamed"], + ["edited.md", "child draft"], + ]) { + await fsPromises.writeFile(path.join(legacyRoot, name), body); + } + await fixture.service.listIndexEntries({ ...fixture.ctx }); + for (const name of ["same.md", "mine.md", "moved.md", "edited.md"]) { + expect( + await fsPromises.stat(path.join(ownerRoot, name)).then( + () => true, + () => false + ) + ).toBe(true); + } + await fixture.service.setPinned({ ...fixture.ctx }, "/memories/workspace/mine.md", true); + await fixture.service.strReplace( + { ...fixture.ctx }, + "/memories/workspace/edited.md", + "draft", + "final", + "agent" + ); + // The downgraded build deletes same.md and mine.md, renames moved.md, and + // deletes edited.md. + await new Promise((resolve) => setTimeout(resolve, 5)); + for (const name of ["same.md", "mine.md", "edited.md"]) { + await fsPromises.rm(path.join(legacyRoot, name)); + } + await fsPromises.rename( + path.join(legacyRoot, "moved.md"), + path.join(legacyRoot, "renamed.md") + ); + const relisted = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + // Created + unchanged copies are gone (mine.md, moved.md); the reused + // owner note and the owner-edited copy stay; the rename's new name is + // adopted. + expect(relisted).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "edited.md"), "utf-8")).toBe( + "child final" + ); + // The removed copy's owner-side pin went with it. + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "mine.md", { projectPath: "", workspaceId: "ws-owner" }) + ) + ).toBe(false); + // Idempotent: a further pass changes nothing. + const again = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + expect(again).toEqual(relisted); + // A lossy legacy listing (readdir failure tolerated by listFiles) is not + // proof of deletion: the copies stay while the sources provably exist. + await fsPromises.writeFile(path.join(legacyRoot, "renamed.md"), "to be renamed (v2)"); + const lossy = spyOn(fsPromises, "readdir").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + lossy.mockRestore(); + } + // ...and the edited source replaces its untouched adopted copy in place. + expect( + (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort() + ).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "renamed.md"), "utf-8")).toBe( + "to be renamed (v2)" + ); + }); + + it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + // Adopted before the child ever had a sidecar entry (no view, no pin). + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner pins the shared copy... + await fixture.metaService.setPinned(ownerKey, true); + // ...then the downgraded build merely views the legacy note: the child + // sidecar gains a usage-only entry — the default unpinned state, not a + // pin transition — so the owner's pin stands. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // Another view once an entry exists: usage changes, the pin bit does not. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // A pin the child actually toggles on the old build is the newer intent. + await fixture.metaService.setPinned(ownerKey, false); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // ...but not once the copy is the OWNER's generation (deleted and + // recreated with identical bytes): a later child toggle no longer folds + // in (r79), and the record stops claiming the copy — the same rule + // deletion reconciliation and the rollback remapper apply. + const ownerCopy = path.join(fixture.config.sessionsDir, "ws-owner", "memory", "note.md"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fixture.metaService.setPinned(childKey, false); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + expect(record.replaced).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The record now persists without `created`, like a note the owner had + // all along — but that one folds child toggles, this one must not: the + // next toggle (a fresh process, so nothing is remembered in memory) + // leaves the owner-owned replacement alone too (r80). + await fixture.metaService.setPinned(childKey, true); + await fixture.metaService.setPinned(ownerKey, false); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); + }); + + it("imports a legacy edit beside an adopted copy the owner recreated with the same bytes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerRoot, "note.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The owner deletes the copy and recreates it with the adopted bytes — + // the owner's generation now — and, before any pass re-inspects it, + // the downgraded build edits the legacy source. The bytes still hash + // to the record's, but the stamp no longer matches: the edit must not + // replace the owner's note in place; it lands in the import directory. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "note.md"), "utf-8") + ).toBe("v2"); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.target).toBe("imported/ws-child/note.md"); + expect(record.created).toBe(true); + expect(record.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "imported", "ws-child", "note.md"))) ?? + undefined + ); + // The staged bytes were installed by rename; nothing lingers (the + // staging dir sits beside the memory root, outside the namespace). + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); + }); + + it("never claims a copy by byte match alone: a pending record without its receipt's generation", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "same bytes"); + // A fresh adoption crashed after recording its pending manifest but + // BEFORE installing the copy — the receipt names staged bytes that + // never reached the target. Another backend (or the owner) then created + // an owner note with the very same bytes at the planned target. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + await fsPromises.mkdir(path.dirname(manifestPath), { recursive: true }); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + targetStamp: "1:10:1", + }, + }) + ); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "same bytes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const record = (await readLegacyAdoptionManifest(manifestPath)).get("note.md")!; + expect(record.pending).toBeUndefined(); + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + // The same for an older build's stamp-less pending record: ambiguous, + // so it claims nothing. + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + expect((await readLegacyAdoptionManifest(manifestPath)).get("note.md")!.created).toBe(false); + // Deleting the legacy source therefore leaves the owner's note alone. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "same bytes" + ); + // No staged bytes linger beside the owner store. + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); + }); + + it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "note.md", { projectPath: "", workspaceId: "ws-child" }), + true + ); + // The copy lands, then the sidecar fold fails before the manifest + // records the adoption as complete. + const failing = spyOn(fixture.metaService, "mergeKeys").mockImplementationOnce(() => + Promise.reject(new Error("sidecar unavailable")) + ); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + failing.mockRestore(); + } + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "child notes" + ); + // The retry finds identical bytes at the target (no-write path) and must + // still know this adoption created the copy... + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(manifest["note.md"]).toMatchObject({ created: true }); + expect(manifest["note.md"].pending).toBeUndefined(); + // ...so a deletion on the downgraded build still follows it out. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + }); + + it("retains adoption provenance while the copy of a deleted legacy note cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + expect(await pathExists(target)).toBe(true); + // The downgraded build deletes the source while the copy's stat fails + // transiently: neither the copy nor its provenance may go. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "stat").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realStat as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + // Recovered: the retained provenance lets the copy follow its source out. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + + it("treats a legacy directory replaced by a note as deleting its adopted descendants", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir", "note.md"), "nested"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(true); + // The downgraded build replaces dir/ with a regular note: the old + // descendant's probe fails ENOTDIR — proof of deletion, like ENOENT. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir"), "now a note"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(false); + // The new note itself lands under imported/ (the owner still has a + // directory at that path). + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "dir"), "utf-8") + ).toBe("now a note"); + }); + + it("retains adoption provenance while the copy of a deleted legacy note cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + // stat succeeds, the content read fails: not "changed" — keep the entry. + const realOpen = fsPromises.open.bind(fsPromises); + const unreadable = spyOn(fsPromises, "open").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realOpen as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + expect( + Object.keys( + JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record + ) + ).toEqual(["note.md"]); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + + it("retries instead of duplicating when an adopted note's prior copy cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + // Sidecar-only change (downgraded build pinned the note) while the + // prior copy cannot be read: no imported/ duplicate, record untouched. + await fixture.metaService.setPinned(childKey, true); + const realOpen = fsPromises.open.bind(fsPromises); + const unreadable = spyOn(fsPromises, "open").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realOpen as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); + // Readable again: the pin folds into the same copy. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + }); + + it("never represents a legacy note through a symlinked owner path", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner path is a link to an unlisted dotfile with identical bytes: + // following it would call the note "already represented" while the + // shared notebook never lists it. + await fsPromises.writeFile(path.join(ownerRoot, ".hidden"), "child notes"); + await fsPromises.symlink(".hidden", path.join(ownerRoot, "note.md")); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "note.md"), "utf-8") + ).toBe("child notes"); + expect((await fsPromises.lstat(path.join(ownerRoot, "note.md"))).isSymbolicLink()).toBe(true); + }); + + it("keeps a tombstoned record for a deleted legacy source and re-adopts a reappearing one", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const tombstoned = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; target: string } + >; + expect(tombstoned["note.md"]).toMatchObject({ target: "note.md", deleted: true }); + // A copy restored into the shared store (a rollback of the deletion) + // is not reconciled away again: the tombstone is final. + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v1"); + await fixture.service.create(fixture.ctx, "/memories/workspace/other.md", "o", "agent"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(true); + // The source reappears on the old build: adopted as a fresh note. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + const readopted = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + >; + expect(readopted["note.md"]).toMatchObject({ created: true }); + expect(readopted["note.md"].deleted).toBeUndefined(); + }); + + it("refuses to restore a deleted legacy note while its owner target cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The pre-sharing delete: its inverse restores the legacy path. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "delete", path: "/memories/workspace/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "note.md"), text: "v1" }], + }, + }, + }); + const deleteRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const target = path.join(ownerRoot, "note.md"); + const rollback = () => + rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: path.dirname(ownerRoot), + id: deleteRow.id, + evidence: { toolName: "test", actor: "user" }, + }); + // The tombstoned record is current only while the target is PROVEN + // absent: a stat failure that proves nothing (EACCES) may hide an + // owner-created replacement, so the restore is refused rather than + // landing on it. + const realLstat = fsPromises.lstat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "lstat").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realLstat as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + const refused = await rollback(); + expect(refused.success).toBe(false); + expect(await pathExists(target)).toBe(false); + } finally { + unreadable.mockRestore(); + } + // Proven absent: the restore lands in the shared store. + const restored = await rollback(); + expect(restored.success).toBe(true); + expect(await fsPromises.readFile(target, "utf-8")).toBe("v1"); + }); + + it("ignores a manifest a downgraded child wrote into its model-writable legacy root", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fixture.service.create( + fixture.ctx, + "/memories/workspace/note.md", + "owner note", + "agent" + ); + // The downgraded build's memory tool serves `/memory` as + // /memories/workspace and its path grammar admits dotfiles: a model + // there can plant a settled record claiming the owner's note as this + // adoption's creation whose source is already gone. Read as provenance, + // deletion reconciliation would remove the owner's note. + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile( + path.join(legacyRoot, ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { + content: sha256Hex("owner note"), + sidecar: "", + target: "note.md", + created: true, + }, + }) + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "owner note" + ); + // The planted file is just an (addressable) dot-entry note of the child: + // adopted as such, never read as provenance — the real manifest records + // that note and knows nothing of `note.md`. + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual([".adopted-into-shared-store.json"]); + expect(manifest[".adopted-into-shared-store.json"]).toMatchObject({ created: true }); + }); + + it("preserves an owner note recreated with the adopted bytes when the legacy source is deleted", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // ABA on the owner side: the owner deletes the adopted copy and later + // writes a note of its own at the same path with the same bytes (or + // edits and restores it). The bytes match the record; the file is not + // this adoption's copy any more. + await fixture.service.deletePath(ownerCtx, "/memories/workspace/note.md", "agent"); + await fixture.service.create(ownerCtx, "/memories/workspace/note.md", "v1", "agent"); + // The downgraded child then deletes its source. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + // Tombstoned as owner-owned: no destructive provenance survives. + expect(manifest["note.md"]).toMatchObject({ deleted: true, created: false }); + }); + + it("recovers an interrupted in-place replacement without duplicating the note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The downgraded build edits the note; the replacement pass crashed + // after recording its pending state but before writing the bytes — + // the on-disk state that leaves: the PRIOR record marked pending, the + // owner copy still holding the old bytes. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + expect(prior).toMatchObject({ target: "note.md", created: true }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: true } }) + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // The retry recognizes the surviving old bytes as this adoption's copy + // and replaces them in place — no imported/ duplicate, provenance kept. + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await restarted.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); + expect(manifest["note.md"].pending).toBeUndefined(); + // A pin the child toggled together with an edit survives an interrupted + // replacement: the pending record keeps the PRIOR sidecar state, so the + // retry still sees the transition and applies it over the owner's pin. + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + await fixture.metaService.setPinned(ownerKey, false); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3"); + await fixture.metaService.setPinned(childKey, true); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...settled, pending: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v3"); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // A crash after the replacement write but before the settled manifest: + // the pending record still carries the OVERWRITTEN generation's stamp. + // The retry binds the replacement on disk (r75) — settling the stale + // stamp would refuse the child's rollbacks as "replaced" and leave the + // copy behind when the source is deleted. + const settled2 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; targetStamp?: string } + > + )["note.md"]; + expect(settled2.targetStamp).toBeDefined(); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3-replaced"); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v3-replaced"); + // The receipt the pass took on the staged bytes (a rename keeps it). + const receipt = async () => (await adoptionTargetStamp(path.join(ownerRoot, "note.md")))!; + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + ...settled2, + pending: true, + replacementContent: sha256Hex("v3-replaced"), + replacementStamp: await receipt(), + }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const rebound = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { pending?: boolean; targetStamp?: string } + > + )["note.md"]; + expect(rebound.pending).toBeUndefined(); + expect(rebound.targetStamp).not.toBe(settled2.targetStamp); + expect(rebound.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "note.md"))) ?? undefined + ); + // The opposite crash window: the replacement bytes landed but the final + // manifest write did not, and the downgraded build deletes the source + // before the retry. The pending record names both hashes, so the copy + // is still recognized as this adoption's and follows the source out. + const settled3 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v4"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + ...settled3, + pending: true, + replacementContent: sha256Hex("v4"), + replacementStamp: await receipt(), + }, + }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + }); + + it("recovers a deletion interrupted between the copy's removal and the tombstone", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The crash state: source deleted, deletion recorded as pending, copy + // already removed, tombstone never written. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + // Removed by us, not changed by the owner: destructive provenance kept, + // so the child's delete row still maps onto the shared store. + expect(tombstone).toMatchObject({ deleted: true, created: true }); + expect(tombstone.pendingDeletion).toBeUndefined(); + }); + + it("does not read owner state at a pending-deletion target as removed by us", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The deletion was recorded pending but failed before the removal; the + // owner replaced the copy with a directory of its own in the meantime. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.mkdir(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile(path.join(ownerRoot, "note.md", "inner.md"), "owner's"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["note.md"]; + expect(tombstone.deleted).toBe(true); + expect(tombstone.created).not.toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md", "inner.md"), "utf-8")).toBe( + "owner's" + ); + // Same for a containment failure: an escaping symlink at the target is + // owner state, not proof of absence. + await fsPromises.writeFile(path.join(legacyRoot, "link.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const linkPrior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["link.md"] as Record; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "link.md")); + await fsPromises.rm(path.join(ownerRoot, "link.md")); + await fsPromises.symlink( + path.join(fixture.xumHome, "outside.md"), + path.join(ownerRoot, "link.md") + ); + await fsPromises.writeFile(path.join(fixture.xumHome, "outside.md"), "outside"); + const current = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + unknown + >; + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ ...current, "link.md": { ...linkPrior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const linkTombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["link.md"]; + expect(linkTombstone.deleted).toBe(true); + expect(linkTombstone.created).not.toBe(true); + expect((await fsPromises.lstat(path.join(ownerRoot, "link.md"))).isSymbolicLink()).toBe(true); + }); + + it("reads malformed manifest lifecycle flags fail-closed", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // An interrupted adoption's `pending` corrupted to a string: the copy + // was never written. The record must not read as settled — removal's + // handover reconstructs the copy instead of reporting completion. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: "true" } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pending?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pending).toBeUndefined(); + }); + + it("re-adopts a source that reappeared identically while its deletion was pending", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // Crash after the copy's removal, before the tombstone; the downgraded + // build then recreates the source with the same bytes. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.utimes(path.join(legacyRoot, "note.md"), new Date(), new Date()); + // Removal's handover must restore the copy, not report "nothing to do" + // and delete the only remaining note with the child session. + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pendingDeletion).toBeUndefined(); + }); + + it("keeps adopting a legacy note named __proto__ exactly once", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "__proto__"), "proto notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "__proto__"), "utf-8")).toBe( + "proto notes" + ); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["__proto__"]); + // A fresh process (empty memo) finds the record and leaves the clock alone. + const revision = await fixture.service.workspaceMemoryRevision("ws-owner"); + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await restarted.listIndexEntries({ ...fixture.ctx }); + expect(await restarted.workspaceMemoryRevision("ws-owner")).toBe(revision); + }); + + it("removal adoption refuses to leave a note behind and runs under held locks", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner notebook at the cap: the child's note has no slot. + await Promise.all( + Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ) + ); + await fsPromises.writeFile(path.join(legacyRoot, "stranded.md"), "only copy"); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/could not be folded/); + // A legacy listing that cannot be completed (readdir failure) is no + // "nothing to adopt": removal must abort rather than delete a note it + // never saw. + const lossy = spyOn(fsPromises, "readdir").mockImplementation((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EIO/); + } finally { + lossy.mockRestore(); + } + // A legacy root that cannot be inspected (EACCES) is not "nothing to + // adopt" either: removal must abort rather than delete it unseen. + const realLstat = fsPromises.lstat.bind(fsPromises); + const unreadableRoot = spyOn(fsPromises, "lstat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === legacyRoot + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realLstat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/could not be inspected/); + } finally { + unreadableRoot.mockRestore(); + } + // Space frees up; the in-lock delta pass (removal holds the owner-store + // lock already) folds the note in without re-acquiring the lock. + await fsPromises.rm(path.join(ownerRoot, "o0000.md")); + // A destination whose stat fails (not a proven absence) is not free: the + // pass aborts instead of overwriting whatever the owner keeps there. + await fsPromises.writeFile(path.join(ownerRoot, "stranded.md"), "owner's own"); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadableTarget = spyOn(fsPromises, "stat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === path.join(ownerRoot, "stranded.md") + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realStat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EIO/); + } finally { + unreadableTarget.mockRestore(); + } + expect(await fsPromises.readFile(path.join(ownerRoot, "stranded.md"), "utf-8")).toBe( + "owner's own" + ); + await fsPromises.rm(path.join(ownerRoot, "stranded.md")); + await withTargetMutationLock( + fixture.xumHome, + memoryMutationLockKey(fixture.xumHome, ownerRoot), + () => + fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner", { + locksHeld: true, + }) + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "stranded.md"), "utf-8")).toBe( + "only copy" + ); + }); + + it("keeps memoized owners when a changed config.json cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // The stamp moves (a rewrite) but the contents are unreadable for a + // moment: the memo must not be replaced by the empty default's self + // fallbacks, and the pass must be retried once readable. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + spyOn(fixture.config, "configFileStamp").mockReturnValue("rewritten"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + }); + + it("folds in a note written under a self-fallback once ownership resolves to the tree root again", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // Shared access first: the (absent) legacy store is checked against ws-owner. + expect((await fixture.service.listIndexEntries(fixture.ctx)).length).toBe(0); + // config.json goes missing: the child resolves to itself and writes a + // note into its private dir. + const configPath = path.join(fixture.xumHome, "config.json"); + const savedConfig = await fsPromises.readFile(configPath); + await fsPromises.rm(configPath); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/fallback.md", + "written while config was gone", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "fallback.md")) + ).toBe(true); + // Config recovers: the same process must fold that note into the + // shared store instead of trusting its earlier "nothing to adopt". + await fsPromises.writeFile(configPath, savedConfig); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // Index builds get their own context object in production (the owner + // cache is per context); mirror that instead of reusing the command's. + const listed = await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(listed.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "fallback.md", + ]); + expect( + await fsPromises.readFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "fallback.md"), + "utf-8" + ) + ).toBe("written while config was gone"); + + // ANOTHER backend hits the same fallback while this process's resolution + // never changes: its write advances the child's store clock, which this + // process notices on its next access and folds the note in. + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + spyOn(foreign, "resolveWorkspaceMemoryOwnerId").mockReturnValue("ws-child"); + const foreignWrite = await foreign.create( + { ...fixture.ctx }, + "/memories/workspace/foreign.md", + "written by another backend's fallback", + "agent" + ); + expect(foreignWrite.success).toBe(true); + const afterForeign = await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(afterForeign.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "fallback.md", + "foreign.md", + ]); + }); + + it("never imports through a symlinked legacy notebook root or escaped files", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const outside = path.join(fixture.xumHome, "outside"); + await fsPromises.mkdir(outside, { recursive: true }); + await fsPromises.writeFile(path.join(outside, "secret.md"), "host file"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + // Root itself is a symlink: refused outright (lstat, never followed). + await fsPromises.symlink(outside, path.join(childSessionDir, "memory")); + expect( + (await fixture.service.listIndexEntries(fixture.ctx)).filter((e) => e.scope === "workspace") + ).toEqual([]); + + // Destination side: the owner store's imported/ component is a + // symlink out of the root. A conflicting legacy note would land there; + // the write is refused (and nothing is written outside), the note stays + // in the legacy dir unrecorded. + await fsPromises.unlink(path.join(childSessionDir, "memory")); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot); + await fsPromises.writeFile(path.join(legacyRoot, "clash.md"), "child version"); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + await fsPromises.mkdir(path.join(ownerRoot, "imported"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "clash.md"), "owner version"); + await fsPromises.symlink(outside, path.join(ownerRoot, "imported", "ws-child")); + const escaped = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect( + (await escaped.listIndexEntries(fixture.ctx)) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath) + ).toEqual(["clash.md"]); + expect(await pathExists(path.join(outside, "clash.md"))).toBe(false); + expect(await pathExists(legacyAdoptionManifestPath(path.dirname(legacyRoot)))).toBe(false); + await fsPromises.unlink(path.join(ownerRoot, "imported", "ws-child")); + await fsPromises.rm(legacyRoot, { recursive: true }); + await fsPromises.rm(path.join(ownerRoot, "clash.md")); + + // Real root whose entries point outside: symlinked entries are not + // regular files to the walk, and a symlinked subdirectory is never + // descended into. + await fsPromises.mkdir(legacyRoot); + await fsPromises.symlink(path.join(outside, "secret.md"), path.join(legacyRoot, "link.md")); + await fsPromises.symlink(outside, path.join(legacyRoot, "linked-dir")); + await fsPromises.writeFile(path.join(legacyRoot, "real.md"), "real note"); + const fresh = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect( + (await fresh.listIndexEntries(fixture.ctx)) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath) + ).toEqual(["real.md"]); + }); + + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const [row] = await readRefinementEvents(childSessionDir); + + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: row.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + }); + + it("refuses a rollback from a tombstoned acting workspace (orphaned journal)", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const [row] = await readRefinementEvents(childSessionDir); + + // Removal took the orphan path: the child's journal stays on disk, but + // the child is tombstoned and must not mutate the owner's live notebook. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: row.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("this workspace was removed"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + expect(await readRefinementEvents(childSessionDir)).toHaveLength(1); + }); + + it("sees a live tree member's later shared-store edit as divergence when rolling back", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const peersOf = (workspaceId: string) => () => + sharedWorkspaceMemoryPeerSessionDirs( + fixture.config.loadConfigOrDefault(), + fixture.config.sessionsDir, + workspaceId + ); + expect(peersOf("ws-owner")()).toEqual([ + childSessionDir, + path.join(fixture.config.sessionsDir, "ws-grandchild"), + ]); + expect(peersOf("ws-solo")()).toEqual([]); + + // Owner renames a directory; the child then edits a file beneath the + // destination. That edit lives only in the child's journal. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "v1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "child edit", + "agent" + ); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const renameRow = ownerRows.find( + (row) => (row.data.action as { op: string }).op === "rename" + )!; + + // Membership that cannot be established (unreadable config) refuses the + // rollback instead of guessing an empty tree. + const unresolvable = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => { + throw new Error("config.json unreadable"); + }, + evidence: { toolName: "test", actor: "user" }, + }); + expect(unresolvable.success).toBe(false); + if (!unresolvable.success) expect(unresolvable.error).toContain("could not be resolved"); + + // Own journal only: the rename looks cleanly undoable and would move + // the child's newer content back without a word. + const blind = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [], + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeTargetLock: () => Promise.reject(new Error("would have applied")), + }); + expect(blind.success).toBe(false); + if (!blind.success) expect(blind.error).toContain("would have applied"); + + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("touched the same paths"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("child edit"); + + // Rolling the child's edit back first (its journal sees the owner's + // rename as EARLIER, not a conflict) unblocks the owner's rollback — + // unless another child edit lands between the owner's plan-time scan + // and its target lock: the journals are re-read under the lock. + const [childRow] = await readRefinementEvents(childSessionDir); + const childUndo = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-child"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(childUndo.success).toBe(true); + const raced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeTargetLock: async () => { + const late = await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "late child edit", + "agent" + ); + expect(late.success).toBe(true); + }, + }); + expect(raced.success).toBe(false); + if (!raced.success) expect(raced.error).toContain("a concurrent mutation landed"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("late child edit"); + // LIFO: undo the late edit (its own journal, netted out) and the + // owner's rollback goes through. + const childRows = await readRefinementEvents(childSessionDir); + const lateRow = childRows[childRows.length - 1]; + expect(lateRow.data.rollbackOf).toBeUndefined(); + const lateUndo = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: lateRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-child"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(lateUndo.success).toBe(true); + const ownerUndo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(ownerUndo.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "notes", "a.md"), "utf-8") + ).toBe("v1"); + }); + + it("migrates a removed sub-agent's live shared-memory rows into the owner's journal, rollbackable there", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // Shared-store edit (migrates), an edit already rolled back (skipped), + // and a global edit (not the owner's store: stays with the child). + await fixture.service.create(fixture.ctx, "/memories/workspace/keep.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/keep.md", + "v1", + "v2", + "agent" + ); + await fixture.service.create(fixture.ctx, "/memories/workspace/undone.md", "x", "agent"); + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); + const childRows = await readRefinementEvents(childSessionDir); + const undone = childRows.find( + (row) => (row.data.action as { path: string }).path === "/memories/workspace/undone.md" + )!; + const rolledBack = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: undone.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + + // A rollback of the rollback re-applies "redone.md": it is live again. + await fixture.service.create(fixture.ctx, "/memories/workspace/redone.md", "r", "agent"); + const redone = (await readRefinementEvents(childSessionDir)).find( + (row) => + (row.data.action as { path?: string }).path === "/memories/workspace/redone.md" && + row.data.rollbackOf === undefined + )!; + const undoRedone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: redone.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undoRedone.success).toBe(true); + if (!undoRedone.success) return; + expect( + ( + await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: undoRedone.data.rollbackRowId ?? "", + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + // Three live edits plus redone.md's full rollback lineage (rollback and + // its re-apply); undone.md's dead lineage stays behind. + expect(await migrate()).toBe(5); + // Idempotent: a retried removal migrates nothing twice. + expect(await migrate()).toBe(0); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect( + ownerRows.map((row) => [ + (row.data.action as { op: string }).op, + (row.data.action as { path?: string }).path, + (row.data.evidence as { workspaceId: string }).workspaceId, + ]) + ).toEqual([ + ["create", "/memories/workspace/keep.md", "ws-owner"], + ["str_replace", "/memories/workspace/keep.md", "ws-owner"], + ["create", "/memories/workspace/redone.md", "ws-owner"], + ["rollback", undefined, "ws-owner"], + ["rollback", undefined, "ws-owner"], + ]); + expect(ownerRows.every((row) => row.data.migratedFrom?.startsWith("ws-child:"))).toBe(true); + // The copied rollback rows point at the owner-side copies, not at + // child ids that no longer exist anywhere. + expect(ownerRows[3].data.rollbackOf).toBe(ownerRows[2].id); + expect((ownerRows[3].data.action as { of: string }).of).toBe(ownerRows[2].id); + expect(ownerRows[4].data.rollbackOf).toBe(ownerRows[3].id); + + // The child is gone; the owner rolls the edit back from its own journal + // (payload blobs were copied, postState hashes preserved). + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const keep = path.join(ownerSessionDir, "memory", "keep.md"); + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRows[1].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(true); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + }); + + it("migrates a pre-sharing row through the adoption manifest so the adopted copy stays rollbackable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Journaled before sharing: the inverse addresses the legacy notebook. + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + // A self-fallback write's clock value: the child's PRIVATE store's, + // not the owner's — meaningless once the row is retargeted. + sourceTs: 42, + }, + }); + // Also a legacy row for a note the shared store never took (unplaceable). + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/never.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "never.md")] }, + }, + }); + // Adoption folds old.md into the owner store. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom?.startsWith("ws-child:") === true + )!; + expect((copy.data.inverse as { files: Array<{ path: string }> }).files[0].path).toBe( + ownerCopy + ); + expect((copy.data.postState as { files: Array<{ path: string }> }).files[0].path).toBe( + ownerCopy + ); + // A retargeted row's clock value belonged to the private store: its + // order among the owner's rows is unknown, not that value. + expect(copy.data.sourceTs).toBeUndefined(); + expect(copy.data.orderUnknown).toBe(true); + const rolledBack = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + + it("an owner rollback of a migrated copy ignores the still-registered child's original row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + }, + }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + // Removal aborted after the pre-teardown pass: the child stays + // registered (a peer of the owner) with its original row in place. + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom?.startsWith("ws-child:") === true + )!; + const rolledBack = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + + it("a retried removal re-copies a row whose earlier owner-side copy is unusable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const childRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(1); + // The copy's inverse is corrupted on disk before the removal is retried. + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${childRow.id}"`)) return line; + const row = JSON.parse(line) as { data: { inverse: unknown } }; + row.data.inverse = { op: "bogus" }; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + // Not "already copied": the intact source is copied again, and the new + // copy is the one the owner can roll back. + expect(await migrate()).toBe(1); + const copies = (await readRefinementEvents(ownerSessionDir)).filter( + (row) => row.data.migratedFrom === `ws-child:${childRow.id}` + ); + expect(copies).toHaveLength(2); + const usable = copies.find((row) => (row.data.inverse as { op: string }).op !== "bogus")!; + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: usable.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(true); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(false); + // A third pass sees the usable copy: nothing more to do. + expect(await migrate()).toBe(0); + }); + + it("migrates a row whose only rollback record is malformed instead of dropping both", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A rollback row that names the create but whose action and inverse + // are unusable: it cannot be copied, so counting it as a completed + // rollback would delete the child journal with neither inverse kept. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { op: "bogus" }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + ); + expect(copy).toBeDefined(); + expect(copy!.data.rollbackOf).toBeUndefined(); + }); + + it("re-copies a rollback row whose earlier copy was corrupted instead of treating the target as rolled back", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + const rollback = async (id: string) => { + const result = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(result.success).toBe(true); + return (await readRefinementEvents(childSessionDir)).at(-1)!; + }; + // create → rolled back → re-applied: the create is live, both rollback + // rows are its lineage. + const undo = await rollback(createRow.id); + await rollback(undo.id); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(3); + // The first ROLLBACK's copy is corrupted before the removal is retried. + // Its bare `rollbackOf` must not block re-copying the intact source + // rollback: the owner would keep an unusable rollback record over a + // target the engine then reads as live. + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${undo.id}"`)) return line; + const row = JSON.parse(line) as { data: { action: unknown } }; + row.data.action = { op: "bogus" }; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + expect(await migrate()).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const createCopy = ownerRows.find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + )!; + const undoCopies = ownerRows.filter((row) => row.data.migratedFrom === `ws-child:${undo.id}`); + expect(undoCopies).toHaveLength(2); + expect( + undoCopies.filter((row) => (row.data.action as { op: string }).op === "rollback") + ).toHaveLength(1); + expect(undoCopies.every((row) => row.data.rollbackOf === createCopy.id)).toBe(true); + // A copy whose action still PARSES but names another target is corrupt + // by the engine's own rule (isUsableRollbackRow, r79): the migration + // must judge copies by that same predicate (r80), or a retried removal + // skips the intact source and the lineage is lost. + const disagreeing = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${undo.id}"`)) return line; + const row = JSON.parse(line) as { data: { action: { op: string; of?: string } } }; + if (row.data.action.op === "rollback") row.data.action.of = "someone-else"; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, disagreeing.join("\n")); + expect(await migrate()).toBe(1); + const recopied = (await readRefinementEvents(ownerSessionDir)).filter( + (row) => + row.data.migratedFrom === `ws-child:${undo.id}` && + (row.data.action as { op: string; of?: string }).of === createCopy.id + ); + expect(recopied).toHaveLength(1); + }); + + it("never turns a corrupt source rollback row into a usable owner rollback", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A rollback row whose parseable action names ANOTHER target than + // `rollbackOf`: the engine rejects it (isUsableRollbackRow); remapping + // `of` to the create's copy would make the owner believe the create was + // rolled back while its bytes are still on disk. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "rollback", of: "someone-else" }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect(ownerRows.filter((row) => row.data.rollbackOf !== undefined)).toHaveLength(0); + }); + + it("does not let a non-memory rollback row kill a memory row's liveness during migration", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A well-formed rollback row whose `kind` is not "memory": the copy loop + // skips it, so it must not count as a completed rollback either — or + // removal would delete the journal without copying the live create. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "skill", + action: { op: "rollback", of: createRow.id }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + ); + expect(copy).toBeDefined(); + }); + + it("refuses to hand over a live row whose action is malformed instead of dropping it", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // A live edit with a usable inverse but a corrupt action: its inverse + // paths are the evidence conflict detection needs, and the owner journal + // cannot carry it without an action — removal must not delete the only + // copy (a forced removal accepts the loss explicitly). + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + }, + }); + const attempt = migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await attempt.then(() => null, getErrorMessage)).toContain("action is malformed"); + }); + + it("a peer's corrupt rollback row does not hide the peer's live edit from conflict detection", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/dir/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/dir", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + // The child edits beneath the renamed destination (later by the store + // clock), then a rollback row naming that edit lands with a corrupt + // action: the edit is still on disk. + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + const childEdit = (await readRefinementEvents(childSessionDir)).at(-1)!; + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "moved", "a.md"), text: "c2" }], + }, + rollbackOf: childEdit.id, + }, + }); + const attempt = () => + rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + const expectRefused = async () => { + const refused = await attempt(); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + `later refinement row ${childEdit.id}` + ); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }; + await expectRefused(); + // Lineage fields that disagree (a well-formed rollback action naming + // another row) are corrupt too (r79). + const childJournalPath = path.join(childSessionDir, "durable-events.jsonl"); + const rewriteChildRows = async (edit: (row: Record) => void) => { + const rewritten = (await fsPromises.readFile(childJournalPath, "utf-8")) + .split("\n") + .map((line) => { + if (line.trim() === "") return line; + const row = JSON.parse(line) as Record; + edit(row); + return JSON.stringify(row); + }); + await fsPromises.writeFile(childJournalPath, rewritten.join("\n")); + }; + await rewriteChildRows((row) => { + const data = row.data as { rollbackOf?: string; action: unknown }; + if (data.rollbackOf === childEdit.id) data.action = { op: "rollback", of: "other-row" }; + }); + await expectRefused(); + // A peer row whose persisted workspaceId was corrupted to the OWNER's + // must not be ordered by the owner journal's sequence (r79): the origin + // binds to the journal the row was read from, so the row falls back to + // the store clock and still reads as the later mutation. + await rewriteChildRows((row) => { + row.workspaceId = "ws-owner"; + }); + expect( + (await readRefinementEvents(childSessionDir)).every((row) => row.workspaceId === "ws-owner") + ).toBe(true); + await expectRefused(); + }); + + it("treats a malformed store clock as order-unknown instead of 'earlier than everything'", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory; the child then edits a file beneath the + // destination. The child row's clock is corrupted to -1 on disk: trusted, + // it would sort BEFORE the rename and the rollback would move the + // child's newer content silently. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + const journalPath = path.join(childSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes('"sourceTs"')) return line; + const row = JSON.parse(line) as { data: { sourceTs: number } }; + row.data.sourceTs = -1; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: ownerRename.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("Refusing rollback"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }); + + it("keeps a still-registered child's original row when its migrated copy is unusable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory (a rename row carries no post-state hash, so + // a later edit beneath the destination is visible ONLY as a row), the + // child edits a file under the destination, then a removal of the child + // aborts after migrating the child's row (the child stays registered). + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + // The copy's inverse is corrupted on disk (the row survives the + // self-healing read; only its inverse no longer parses). + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const lines = (await fsPromises.readFile(journalPath, "utf-8")).split("\n"); + let corrupted = 0; + const rewritten = lines.map((line) => { + if (!line.includes('"migratedFrom":"ws-child:')) return line; + const row = JSON.parse(line) as { data: { inverse: unknown } }; + row.data.inverse = { op: "bogus" }; + corrupted++; + return JSON.stringify(row); + }); + expect(corrupted).toBe(1); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + // Rolling back the owner's rename would move the child's newer content + // along: the child's intact original must still surface as the conflict + // the unusable copy can no longer report. + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: ownerRename.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("Refusing rollback"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }); + + it("follows a row rolled back between the two handover passes with its rollback row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/keep.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/keep.md", + "v1", + "v2", + "agent" + ); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + // Pre-teardown pass copies the live edit... + expect(await migrate()).toBe(2); + // ...then another backend rolls it back in the child journal before the + // in-lock delta pass runs. + const edit = (await readRefinementEvents(childSessionDir)).find( + (row) => (row.data.action as { op: string }).op === "str_replace" + )!; + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: edit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + const keep = path.join(ownerSessionDir, "memory", "keep.md"); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + // The delta pass copies exactly the rollback row, remapped onto the + // owner-side copy of the edit. + expect(await migrate()).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const editCopy = ownerRows.find((row) => row.data.migratedFrom === `ws-child:${edit.id}`)!; + const rollbackCopy = ownerRows.find((row) => row.data.rollbackOf !== undefined)!; + expect(rollbackCopy.data.rollbackOf).toBe(editCopy.id); + // The owner journal knows the edit is no longer live: rolling it back + // again is refused instead of re-applying an inverse that already ran. + const again = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(again.success).toBe(false); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + // Rolling back the copied rollback (re-apply) works from the owner journal. + const redo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: rollbackCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(redo.success).toBe(true); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v2"); + }); + + it("migrates a row whose inverse payload was reclaimed as an audit-only record", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory (no post-state hash), then the child edits + // a file under the destination; the child's inverse payload is then + // reclaimed under its quota before the child is removed. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "v1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "child-v2", + "agent" + ); + const childJournal = sharedDurableEventJournal(childSessionDir); + await reclaimExcessRefinementInverseBlobs(childJournal, [ + { ref: `sha256:${"e".repeat(64)}`, size: REFINEMENT_INVERSE_BLOB_QUOTA_BYTES }, + ]); + const childEdit = (await readRefinementEvents(childSessionDir)).find( + (row) => (row.data.action as { op: string }).op === "str_replace" + )!; + const childBlobRef = (childEdit.data.inverse as { files: Array<{ blobRef: string }> }) + .files[0].blobRef; + expect(await childJournal.blobs.has(childBlobRef as never)).toBe(false); + + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const copy = ownerRows.find((row) => row.data.migratedFrom === `ws-child:${childEdit.id}`)!; + // Paths and (dangling) payload reference preserved; nothing published. + expect( + (copy.data.inverse as { files: Array<{ path: string; blobRef: string }> }).files + ).toEqual([ + { path: path.join(ownerSessionDir, "memory", "moved", "a.md"), blobRef: childBlobRef }, + ]); + // Unrollbackable, like any evicted payload... + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(false); + if (!undo.success) expect(undo.error).toContain("no longer available"); + // ...but still evidence: rolling the owner's rename back would move the + // child's newer content, so it is reported as a conflict instead. + const renameRow = ownerRows.find( + (row) => (row.data.action as { op: string }).op === "rename" + )!; + const renameUndo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(renameUndo.success).toBe(false); + if (!renameUndo.success) expect(renameUndo.error).toContain("diverges"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("child-v2"); + }); + + it("concurrent migrations of the same child copy each row exactly once", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/a.md", "a", "agent"); + await fixture.service.create(fixture.ctx, "/memories/workspace/b.md", "b", "agent"); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + // Two removals of one child racing (two backends): both unlocked + // pre-filters see an empty owner journal, so only the in-lock check can + // keep the second from appending duplicate rows. + const counts = await Promise.all([migrate(), migrate()]); + expect(counts[0] + counts[1]).toBe(2); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect(ownerRows.map((row) => row.data.migratedFrom).sort()).toEqual( + (await readRefinementEvents(childSessionDir)).map((row) => `ws-child:${row.id}`).sort() + ); + }); + + it("migrated rows keep their real order relative to the owner's own later edits", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Child edits first, owner edits the same file later, THEN the child is + // removed: the migrated (older) child row is appended after the owner's. + await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "c1", "agent"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/shared.md", + "c1", + "o2", + "agent" + ); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const ownerEdit = ownerRows.find((row) => row.data.migratedFrom === undefined)!; + const migrated = ownerRows.find((row) => row.data.migratedFrom !== undefined)!; + expect(migrated.seq).toBeGreaterThan(ownerEdit.seq); + + // LIFO unrolling works without force: the owner's edit is the newest + // mutation of the file, so it rolls back first... + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + const shared = path.join(ownerSessionDir, "memory", "shared.md"); + expect(await fsPromises.readFile(shared, "utf-8")).toBe("c1"); + // ...and then the migrated child create. + const second = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: migrated.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(second.success).toBe(true); + expect(await pathExists(shared)).toBe(false); + }); + + it("unwinds a removed child's own overlapping history LIFO through the owner journal", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Two pre-sharing rows over one note: a create, then an edit. Both are + // retargeted on migration (order-unknown against the OWNER's rows), but + // their order against EACH OTHER is the child journal's sequence. + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(2); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const copyOf = (source: { id: string }) => + ownerRows.find((row) => row.data.migratedFrom === `ws-child:${source.id}`)!; + const createCopy = copyOf(childCreate); + const editCopy = copyOf(childEdit); + for (const [copy, source] of [ + [createCopy, childCreate], + [editCopy, childEdit], + ] as const) { + expect(copy.data.orderUnknown).toBe(true); + expect(copy.data.originJournal).toBe("ws-child"); + expect(copy.data.originSeq).toBe(source.seq); + } + // The older copy is not the newest mutation of the note: refused. + const stale = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(stale.success).toBe(false); + expect(stale.success ? "" : stale.error).toContain("later refinement row"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + // Newest first, no force needed... + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // ...then the create. + const second = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(second.success).toBe(true); + expect(await pathExists(ownerCopy)).toBe(false); + }); + + it("orders re-copied and provenance-less migrated rows by source position, not migration order", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(2); + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewriteCopyOf = async ( + source: { id: string }, + edit: (data: Record) => void + ) => { + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${source.id}"`)) return line; + const row = JSON.parse(line) as { data: Record }; + edit(row.data); + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + }; + // The CREATE's copy is corrupted before the removal is retried: the + // retry re-copies it, so the OLDER mutation now has the higher + // owner-journal seq (and a later append time). + await rewriteCopyOf(childCreate, (data) => { + data.inverse = { op: "bogus" }; + }); + expect(await migrate()).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const editCopy = ownerRows.find( + (row) => row.data.migratedFrom === `ws-child:${childEdit.id}` + )!; + const createCopy = ownerRows.find( + (row) => + row.data.migratedFrom === `ws-child:${childCreate.id}` && + (row.data.inverse as { op: string }).op !== "bogus" + )!; + expect(createCopy.seq).toBeGreaterThan(editCopy.seq); + expect(createCopy.data.originSeq!).toBeLessThan(editCopy.data.originSeq!); + // Migration order is not mutation order: the edit is still the newest + // mutation of the note and rolls back without force. + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // A copy WITHOUT a carried origin (an older build's migration, or the + // fields lost to corruption) has no position: its order against every + // other row stays unknown, so the create's rollback needs force. + await rewriteCopyOf(childEdit, (data) => { + delete data.originJournal; + delete data.originSeq; + }); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "order relative to this row is unknown" + ); + const forced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + force: true, + evidence: { toolName: "test", actor: "user" }, + }); + expect(forced.success).toBe(true); + expect(await pathExists(ownerCopy)).toBe(false); + }); + + it("re-stamps an adopted copy a retargeted rollback rewrites, refusing one the owner replaced", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v1") }], + }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); + const recordStamp = async () => + (await readLegacyAdoptionManifest(manifestPath)).get("old.md")!.targetStamp; + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); + const rollback = (id: string) => + rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id, + evidence: { toolName: "test", actor: "user" }, + }); + // An apply that loses the rollback lock at the commit point is + // compensated; the compensated file is yet another generation, and the + // record follows that one too (r77) — otherwise the retry would refuse. + const lostLock = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeCommit: () => Promise.reject(new Error("lost the rollback lock")), + }); + expect(lostLock.success).toBe(false); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); + // Rolling the edit back rewrites the copy: a new generation, but this + // lineage's own — the record follows it, so the create still maps. + expect((await rollback(childEdit.id)).success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); + // An owner save meanwhile (Memory tab: unjournaled, bytes unchanged) + // makes the copy the owner's: the create's delete-files is refused, + // force or not, and the note stays. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fixture.service.saveFile( + { ...fixture.ctx, workspaceId: "ws-owner" }, + "/memories/workspace/old.md", + "v1", + sha256Hex("v1"), + "user" + ); + for (const force of [false, true]) { + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childCreate.id, + force, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain("since replaced"); + } + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + + it("orders owner and child rows of the shared store by one store clock, advanced by rollbacks too", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // Interleaved edits from two journals: `ts`/`seq` are not comparable + // across them (and can tie within a millisecond), the store clock is. + await fixture.service.create(fixture.ctx, "/memories/workspace/s.md", "c1", "agent"); + await fixture.service.strReplace(ownerCtx, "/memories/workspace/s.md", "c1", "o1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/s.md", + "o1", + "c2", + "agent" + ); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + const [ownerEdit] = await readRefinementEvents(ownerSessionDir); + const clocks = [childCreate, ownerEdit, childEdit].map((row) => row.data.sourceTs); + expect(clocks.every((clock) => typeof clock === "number")).toBe(true); + expect(clocks[0]!).toBeLessThan(clocks[1]!); + expect(clocks[1]!).toBeLessThan(clocks[2]!); + // The published token never lags a row's clock (change events tick it once more). + expect( + clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")) + ).toBeGreaterThanOrEqual(Math.max(...(clocks as number[]))); + + // The owner's edit is not the newest for that path: refused without force. + const stale = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(stale.success).toBe(false); + // A rollback (here through the engine directly, as the debug CLI does) + // is a store mutation too: it advances the clock other backends watch... + const before = await fixture.service.workspaceMemoryRevision("ws-owner"); + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + const after = await fixture.service.workspaceMemoryRevision("ws-owner"); + expect(clockOf(after)).toBeGreaterThan(clockOf(before)); + // ...and its row takes the next clock value, so the owner's edit is now + // the newest and rolls back cleanly. + const rollbackRow = (await readRefinementEvents(childSessionDir)).find( + (row) => row.data.rollbackOf === childEdit.id + )!; + expect(rollbackRow.data.sourceTs).toBe(clockOf(after)); + expect( + ( + await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + }); + + it("a shared-store row whose clock write failed conflicts with every overlapping row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/dir/s.md", "o1", "agent"); + // The child's edit lands, but the owner store's clock cannot be written: + // the row must not fall back to its journal-local `ts` (incomparable + // with the owner journal's rows) — it is journaled as order-unknown. + const revisionPath = path.join(ownerSessionDir, "memory.revision"); + await fsPromises.rm(revisionPath); + await fsPromises.mkdir(revisionPath); // a directory: the clock write fails + try { + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dir/s.md", + "o1", + "c1", + "agent" + ); + } finally { + await fsPromises.rmdir(revisionPath); + } + const [childEdit] = await readRefinementEvents(childSessionDir); + expect(childEdit.data.sourceTs).toBeUndefined(); + expect(childEdit.data.orderUnknown).toBe(true); + // The owner renames the directory afterwards (ordered by the clock). + await fixture.service.rename( + ownerCtx, + "/memories/workspace/dir", + "/memories/workspace/moved", + "agent" + ); + const [, ownerRename] = await readRefinementEvents(ownerSessionDir); + // Rolling the rename back would move the child's edit without seeing + // it if the row were ordered by `ts`; unknown order fails closed. + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "order relative to this row is unknown" + ); + const forced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + force: true, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(forced.success).toBe(true); + // A rollback whose own clock write fails is journaled order-unknown too. + await fsPromises.rm(revisionPath, { force: true }); + await fsPromises.mkdir(revisionPath); + try { + expect( + ( + await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + force: true, + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + } finally { + await fsPromises.rmdir(revisionPath); + } + const rollbackRow = (await readRefinementEvents(childSessionDir)).find( + (row) => row.data.rollbackOf === childEdit.id + )!; + expect(rollbackRow.data.sourceTs).toBeUndefined(); + expect(rollbackRow.data.orderUnknown).toBe(true); + // A clock that EXISTS but is unreadable/malformed must not be advanced + // from zero (a lower value would order this mutation before rows it + // followed): the row is order-unknown instead. + await fsPromises.writeFile(revisionPath, "garbage"); + await fixture.service.create(ownerCtx, "/memories/workspace/after.md", "x", "agent"); + const afterRow = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + expect(afterRow.data.sourceTs).toBeUndefined(); + expect(afterRow.data.orderUnknown).toBe(true); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe("garbage"); + // A numeric PREFIX is malformed too (parseInt would accept it). + await fsPromises.writeFile(revisionPath, "2000000000000000e1"); + await fixture.service.create(ownerCtx, "/memories/workspace/after2.md", "x", "agent"); + const after2 = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + expect(after2.data.orderUnknown).toBe(true); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe("2000000000000000e1"); + }); + + it("refuses a rollback while a peer's adoption manifest cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/o.md", "v1", "agent"); + const [ownerCreate] = await readRefinementEvents(ownerSessionDir); + // The child's (pre-sharing) manifest is unreadable: its adopted rows + // cannot be consulted, so the owner's rollback must not proceed blind. + const manifestPath = legacyAdoptionManifestPath(childSessionDir); + await fsPromises.mkdir(manifestPath, { recursive: true }); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerCreate.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain("adoption manifest could not be read"); + await fsPromises.rmdir(manifestPath); + expect( + ( + await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerCreate.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + }); + + it("ignores migrated copies of its own rows when a child rolls back after an aborted removal", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/dup.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dup.md", + "v1", + "v2", + "agent" + ); + // Pre-teardown migration ran, then the removal aborted: the owner + // journal holds copies of the child's rows while the child lives on. + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(2); + const [, childEdit] = await readRefinementEvents(childSessionDir); + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "dup.md"), "utf-8") + ).toBe("v1"); + }); + + it("aborts removal-time row migration when the adoption manifest is unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/m.md", "v1", "agent"); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); + await fsPromises.mkdir(manifestPath, { recursive: true }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }).then(() => null, getErrorMessage) + ).toMatch(/EISDIR/); + }); + + it("the refinement_rollback tool refuses memory rollbacks into a read-only scope", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const [row] = await readRefinementEvents(childSessionDir); + const physical = path.join(ownerSessionDir, "memory", "n.md"); + + const makeTool = (access: MemoryScopeAccess) => + createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemory: () => ({ ownerSessionDir, peerSessionDirs: [] }), + memory: { service: fixture.service, ctx: fixture.ctx, access }, + }); + const run = async (access: MemoryScopeAccess) => + (await makeTool(access).execute!({ id: row.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + + // Explore-like agent: workspace scope is read-only → the rollback (a + // write into the owner's shared notebook) is refused before the engine. + const refused = await run({ global: "read", project: "read", workspace: "read" }); + expect(refused.success).toBe(false); + expect(refused.error).toContain("read-only"); + expect(await pathExists(physical)).toBe(true); + + // A context whose scope roots do not contain the row's paths (here an + // unrelated workspace's) cannot evaluate the policy: fail closed even + // with read-write access. + const foreignTool = createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemory: () => ({ ownerSessionDir, peerSessionDirs: [] }), + memory: { + service: fixture.service, + ctx: { ...fixture.ctx, workspaceId: "ws-solo" }, + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" }, + }, + }); + const unclassifiable = (await foreignTool.execute!( + { id: row.id, reason: "test" }, + mockToolCallOptions + )) as { success: boolean; error?: string }; + expect(unclassifiable.success).toBe(false); + expect(unclassifiable.error).toContain("Cannot classify"); + expect(await pathExists(physical)).toBe(true); + + const allowed = await run({ + global: "readwrite", + project: "readwrite", + workspace: "readwrite", + }); + expect(allowed.success).toBe(true); + expect(await pathExists(physical)).toBe(false); + + // A pre-sharing row (journaled while the child owned its store, so its + // inverse addresses /memory) whose note was since adopted: the + // policy gate classifies the ADOPTED owner path — the one the engine + // will touch — instead of refusing the legacy path as unclassifiable. + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "legacy.md"), "v2"); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "legacy.md"), "v2"); + await fsPromises.writeFile( + legacyAdoptionManifestPath(path.dirname(legacyRoot)), + JSON.stringify({ + "legacy.md": { + content: "x", + sidecar: "", + target: "legacy.md", + created: true, + targetStamp: await adoptionTargetStamp( + path.join(ownerSessionDir, "memory", "legacy.md") + ), + }, + }) + ); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/legacy.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "legacy.md"), text: "v1" }], + }, + }, + }); + const legacyRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + const legacyRefused = (await makeTool({ + global: "read", + project: "read", + workspace: "read", + }).execute!({ id: legacyRow.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + expect(legacyRefused.success).toBe(false); + expect(legacyRefused.error).toContain("read-only"); + const legacyAllowed = (await makeTool({ + global: "readwrite", + project: "readwrite", + workspace: "readwrite", + }).execute!({ id: legacyRow.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + expect(legacyAllowed.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "legacy.md"), "utf-8") + ).toBe("v1"); + expect(await fsPromises.readFile(path.join(legacyRoot, "legacy.md"), "utf-8")).toBe("v2"); + }); + + it("the refinement_rollback tool refuses while shared-memory ownership cannot be proven", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const [row] = await readRefinementEvents(childSessionDir); + // config.json mid-rewrite at execution time: the topology resolver + // throws instead of degrading to "the child owns its notebook" (which + // would drop the owner root and the peer list from the rollback). + const tool = createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemory: () => { + throw new Error("config.json is absent"); + }, + memory: { + service: fixture.service, + ctx: fixture.ctx, + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" }, + }, + }); + const refused = (await tool.execute!( + { id: row.id, reason: "test" }, + mockToolCallOptions + )) as { + success: boolean; + error?: string; + }; + expect(refused.success).toBe(false); + expect(refused.error).toContain("config.json is absent"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + expect((await readRefinementEvents(childSessionDir)).length).toBe(1); + }); + + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const events: MemoryChangeEvent[] = []; + fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); + const ownerMemory = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + fixture.service.notifyExternalMutation(fixture.ctx, [ + path.join(ownerMemory, "a.md"), + path.join(ownerMemory, "dir", "b.md"), + path.join(fixture.xumHome, "memory", "global", "g.md"), + path.join(fixture.xumHome, "elsewhere", "x.md"), + ownerMemory, // the root itself is not a file inside the scope + ]); + expect(events.map((event) => [event.scope, event.path, event.workspaceId]).sort()).toEqual([ + ["global", "/memories/global", "ws-owner"], + ["workspace", "/memories/workspace", "ws-owner"], + ]); + }); + }); + describe("memory index entries", () => { it("lists files across scopes with sanitized frontmatter descriptions", async () => { using fixture = await createFixture(); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index f1078a54810..30e6e55b099 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -17,13 +17,14 @@ * documented limitation. */ import { EventEmitter } from "events"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; import YAML from "yaml"; import assert from "@/common/utils/assert"; import { CONTEXT_NOTES_MEMORY_PATH } from "@/common/constants/contextBudget"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; import { MEMORY_HOT_SET_MAX_ITEM_BYTES, MEMORY_INDEX_DESCRIPTION_MAX_CHARS, @@ -47,6 +48,20 @@ import { withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, + type LegacyAdoptionRecord, +} from "@/node/services/memoryLegacyAdoption"; +import { + resolveWorkspaceMemoryOwnerId, + workspaceMemoryOwnerResolver, +} from "@/node/services/memoryWorkspaceOwner"; +import { + advanceWorkspaceMemoryRevision, + readWorkspaceMemoryRevision, +} from "@/node/services/refinement/workspaceMemoryRevision"; import { REFINEMENT_CAPTURE_MAX_FILES, REFINEMENT_CAPTURE_MAX_TOTAL_BYTES, @@ -71,7 +86,12 @@ export interface MemoryScopeContext { runtime: Runtime | null; /** Workspace checkout cwd. Kept in the context shape for existing callers; storage ignores it. */ checkoutCwd: string; - /** Workspace ID; workspace scope root is /memory. */ + /** + * ACTING workspace ID. The workspace scope root is the memory OWNER's + * /memory, where the owner is the task-tree root: sub-agent + * children (parentWorkspaceId set) share their parent's workspace notes + * (see MemoryService.resolveWorkspaceMemoryOwnerId). + */ workspaceId: string; /** * Stable project identity from Xum config (the project root path, never the @@ -79,6 +99,15 @@ export interface MemoryScopeContext { * and sidecar logical keys; empty when no project identity is available. */ projectPath: string; + /** + * A further workspace on whose behalf this context acts, guarded like the + * acting one: a sub-agent's consolidation run sweeps the OWNER's notebook + * under the owner's identity (`workspaceId`), and the child's removal — + * possibly by another backend, which cannot abort this run — must refuse + * every read and commit of that run at the tombstone check, not only its + * start (r77). + */ + guardedWorkspaceId?: string; } export type MemoryActor = "agent" | "user"; @@ -331,7 +360,7 @@ export function projectMemoryDirName(projectPath: string): string { return `${base}-${hash}`; } -function toVirtualPath(scope: MemoryScope, relPath: string): string { +export function toVirtualPath(scope: MemoryScope, relPath: string): string { return relPath === "" ? `${MEMORY_VIRTUAL_ROOT}/${scope}` : `${MEMORY_VIRTUAL_ROOT}/${scope}/${relPath}`; @@ -360,8 +389,23 @@ interface MemoryStore { /** assertRootSafe + create the root if missing (write paths only). */ ensureRoot(): Promise; /** Relative paths of all non-dotfile files under the root, sorted. */ - listFiles(): Promise; - kind(relPath: string): Promise; + /** + * Files under the root. Tolerant and bounded by default (self-healing: an + * unreadable directory lists as empty; the walk stops past the per-scope + * cap); `strict` throws on any traversal failure and is unbounded, for + * callers whose decision must not rest on a possibly partial listing. + * Dot-entries are omitted unless `includeDotfiles`: listings and the index + * hide them, but the path grammar admits them, so a note such as `.note` + * is addressable — the legacy adoption pass must see it or removal would + * delete the only copy. + */ + listFiles(options?: { strict?: boolean; includeDotfiles?: boolean }): Promise; + /** + * Kind of an entry, null when absent. Tolerant by default (any stat failure + * reads as absent); `strict` throws unless the absence is proven (ENOENT / + * ENOTDIR), for callers about to overwrite whatever is there. + */ + kind(relPath: string, options?: { strict?: boolean }): Promise; /** * Read at most `maxBytes` from the head of the file. Index/hot-set builds * use this so files edited outside MemoryService cannot force unbounded reads @@ -382,6 +426,105 @@ interface MemoryStore { assertContained(relPath: string): Promise; } +/** + * Owner-store directory that receives a sub-agent's legacy private notes whose + * relPath the owner already uses with different content (adoptLegacyPrivateStore). + */ +const LEGACY_IMPORT_DIR = "imported"; +/** + * Directory beside the owner's memory root (in its session dir, OUTSIDE the + * model-writable memory namespace — a legacy note may legitimately live under + * any in-namespace path, dot-entries included) where the adoption pass stages + * a copy's bytes before installing them by rename + * (adoptLegacyPrivateStoreOrThrow); emptied at the start of every pass. + */ +const LEGACY_ADOPTION_STAGING_DIR_NAME = "memory-adoption-staging"; + +function legacyAdoptionStagingDir(store: MemoryStore): string { + return path.join(path.dirname(store.physicalRoot), LEGACY_ADOPTION_STAGING_DIR_NAME); +} + +/** + * Pin bit of a manifest record's child sidecar fingerprint. No child entry at + * that adoption is the default, unpinned state (a usage entry a downgraded + * build creates by merely viewing the note is not a pin transition); null + * only for an unparsable fingerprint. + */ +function legacySidecarPinned(sidecar: string): boolean | null { + if (sidecar === "") return false; + try { + const parsed: unknown = JSON.parse(sidecar); + return typeof parsed === "object" && parsed !== null + ? ((parsed as { pinned?: unknown }).pinned ?? false) === true + : null; + } catch { + return null; + } +} + +/** + * Change stamp of a sub-agent's legacy private store: its store clock (any + * MemoryService write there advances it, including a foreign backend's + * self-fallback write), the root directory's mtime, and every listed file's + * size + mtime — a DOWNGRADED build editing an existing nested note moves + * neither the clock (it does not know it) nor the root mtime. Bounded by the + * per-scope file cap and paid only while a legacy directory exists. Missing + * pieces read as fixed tokens. + */ +async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Promise { + const revision = await readWorkspaceMemoryRevision(childSessionDir).catch(() => null); + const rootMtime = await fsPromises + .stat(legacyRoot) + .then((stat) => String(stat.mtimeMs)) + .catch(() => "missing"); + const files = await new LocalMemoryStore(legacyRoot) + .listFiles({ includeDotfiles: true }) + .catch(() => []); + const fileStamps = await Promise.all( + files.map(async (relPath) => { + const stamp = await fsPromises + .lstat(path.join(legacyRoot, relPath), { bigint: true }) + .then((stat) => `${stat.size}:${stat.mtimeNs}`) + .catch(() => "missing"); + return `${relPath}=${stamp}`; + }) + ); + return `${revision ?? "none"}:${rootMtime}:${fileStamps.join("\u0001")}`; +} + +/** + * Minimum spacing of the owner-store file scan a workspaceMemoryRevision + * token carries (legacyStoreStamp over the shared notebook). The scan exists + * only for a DOWNGRADED build's in-place edits, which move no clock; this + * build's writers — local or another backend — advance the store clock, + * which every token reads fresh. Unthrottled, each cached-context probe (every + * turn) and every Memory tab interval would list and lstat the whole notebook. + */ +const OWNER_STORE_SCAN_INTERVAL_MS = 60_000; + +/** A stat failure that proves the path is absent (vs. one that says nothing about it). */ +function isMissingPathError(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException | null)?.code; + return code === "ENOENT" || code === "ENOTDIR"; +} + +/** + * Link-aware kind of a path: symlinks are reported as such, never followed. + * "missing" only when proven (ENOENT/ENOTDIR); any other failure (EACCES, + * EIO) is "unreadable" — a legacy notebook whose root cannot be inspected + * must not read as "nothing to adopt" to a removal about to delete it. + */ +async function lstatKind( + absPath: string +): Promise<"dir" | "symlink" | "other" | "missing" | "unreadable"> { + try { + const stat = await fsPromises.lstat(absPath); + return stat.isSymbolicLink() ? "symlink" : stat.isDirectory() ? "dir" : "other"; + } catch (error) { + return isMissingPathError(error) ? "missing" : "unreadable"; + } +} + function isPathWithinRoot( realRoot: string, candidate: string, @@ -424,16 +567,24 @@ class LocalMemoryStore implements MemoryStore { await fsPromises.mkdir(this.physicalRoot, { recursive: true }); } - async listFiles(): Promise { + async listFiles(options?: { strict?: boolean; includeDotfiles?: boolean }): Promise { const results: string[] = []; const walk = async (dirRel: string): Promise => { // Bounded walk: files may have been edited outside MemoryService. +1 lets - // callers detect overflow (e.g. the index logs its truncation). - if (results.length > MEMORY_MAX_FILES_PER_SCOPE) return; + // callers detect overflow (e.g. the index logs its truncation). Strict + // callers need the COMPLETE set (an omitted file would silently count + // as "nothing to adopt" and could lose its only copy), so the bound + // does not apply to them. + if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; let entries; try { entries = await fsPromises.readdir(this.abs(dirRel), { withFileTypes: true }); - } catch { + } catch (error) { + // Strict callers (removal's legacy handover) must not take a partial + // listing for the whole; a missing ROOT is the genuine empty case. + if (options?.strict === true && !(dirRel === "" && hasErrorCode(error, "ENOENT"))) { + throw error; + } return; // Self-healing: missing/unreadable dirs list as empty. } // Iterate in path-string order — directories key as "name/" so the DFS @@ -448,8 +599,8 @@ class LocalMemoryStore implements MemoryStore { }); for (const entry of entries) { // Per-entry cap: a single flat directory can exceed the cap on its own. - if (results.length > MEMORY_MAX_FILES_PER_SCOPE) return; - if (entry.name.startsWith(".")) continue; + if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; + if (options?.includeDotfiles !== true && entry.name.startsWith(".")) continue; const childRel = dirRel === "" ? entry.name : `${dirRel}/${entry.name}`; if (entry.isDirectory()) { await walk(childRel); @@ -462,11 +613,12 @@ class LocalMemoryStore implements MemoryStore { return results.sort(); } - async kind(relPath: string): Promise { + async kind(relPath: string, options?: { strict?: boolean }): Promise { try { const stat = await fsPromises.stat(this.abs(relPath)); return stat.isDirectory() ? "dir" : "file"; - } catch { + } catch (error) { + if (options?.strict === true && !isMissingPathError(error)) throw error; return null; } } @@ -599,6 +751,22 @@ export class MemoryService extends EventEmitter { private readonly metaService: MemoryMetaService ) { super(); + // Parent links are immutable, but an OWNER can be removed while a + // shared-checkout descendant keeps running; its deregistration lands as a + // config change, after which the child must re-resolve (and fall back to + // its own store) instead of writing into the tombstoned owner forever. + // Local edits notify here; edits by ANOTHER backend (multi-instance) are + // caught by the config-file stamp check in resolveWorkspaceMemoryOwnerId. + // The notification fires after the file write, so adopting the new stamp + // here keeps the next resolve from repeating the invalidation — but only + // once the memo was actually rebuilt from the file: an unreadable file at + // notification time (a swallowed late write failure, an EACCES interval) + // keeps the old stamp so the next resolve retries, exactly like the + // stamp check in resolveWorkspaceMemoryOwnerId. + this.config.onConfigChanged(() => { + const stamp = this.config.configFileStamp(); + if (this.invalidateWorkspaceMemoryOwnerMemo()) this.workspaceMemoryOwnerConfigStamp = stamp; + }); } // ------------------------------------------------------------------------- @@ -608,15 +776,186 @@ export class MemoryService extends EventEmitter { // Best-effort: stats failures must never break a memory command. // ------------------------------------------------------------------------- + /** + * Memo of every owner this process resolved (self-resolutions included), + * valid for one config-file stamp: a workspace's parentWorkspaceId is fixed + * at creation and IDs are never reused, so a mapping can only change through + * a config rewrite, which the stamp check / onConfigChanged catch. Fallback + * observations (unregistered ID, config missing or malformed → self) are + * memoized too, so their recovery to a shared owner is a visible transition + * (see invalidateWorkspaceMemoryOwnerMemo). + */ + private readonly workspaceMemoryOwnerById = new Map(); + /** Config-file stamp (Config.configFileStamp) the memo was built against. */ + private workspaceMemoryOwnerConfigStamp: string | null = null; + + /** + * Memoized resolveWorkspaceMemoryOwnerId (see memoryWorkspaceOwner.ts). The + * config is only loaded on a memo miss. Callers resolving many workspaces + * in one synchronous pass (launch sweep, recovery, config-change diffing) + * supply a shared `snapshot`, which bypasses the memo entirely: neither the + * per-call config stat (O(n) synchronous statSync on the main process for + * n workspaces) nor memoization apply — the snapshot can predate the current + * file stamp (another backend rewriting config.json mid-pass) and would + * otherwise be cached under the newer stamp. + */ + resolveWorkspaceMemoryOwnerId( + workspaceId: string, + snapshot?: () => ReturnType + ): string { + if (snapshot !== undefined) return resolveWorkspaceMemoryOwnerId(snapshot(), workspaceId); + const stamp = this.config.configFileStamp(); + if (stamp !== this.workspaceMemoryOwnerConfigStamp) { + // Adopt the new stamp only once its contents were actually read: a + // changed-but-unreadable file (see below) must be retried on the next + // call, not remembered as "seen". + if (this.invalidateWorkspaceMemoryOwnerMemo()) this.workspaceMemoryOwnerConfigStamp = stamp; + } + const cached = this.workspaceMemoryOwnerById.get(workspaceId); + if (cached !== undefined) return cached; + // Only a successful load is memoized. A config.json that stats fine but + // cannot be read or parsed right now (EACCES interval, half-written by a + // non-atomic writer) yields the fresh-install default — the self + // fallback — and the stamp will not move when readability returns, so a + // memo taken now would pin the child to its private notebook until an + // unrelated config rewrite. The fallback is still returned (callers + // degrade to the private store), just re-resolved on the next call. + let cfg: ReturnType; + try { + cfg = this.config.loadConfigOrDefault({ throwOnError: true }); + } catch (error) { + log.debug("[MemoryService] config unreadable; workspace memory owner not memoized", { + workspaceId, + error, + }); + return resolveWorkspaceMemoryOwnerId(this.config.loadConfigOrDefault(), workspaceId); + } + const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + this.workspaceMemoryOwnerById.set(workspaceId, owner); + return owner; + } + + /** + * Re-resolve every memoized workspace against the current config and tell + * listeners which ones now map to a DIFFERENT owner: their live sessions + * hold a memory context built from the previous store (an owner that was + * just removed — or the private fallback store used while config.json was + * missing/malformed and the tree could not be resolved), so core.ts + * invalidates those caches and the memory subscription refreshes. There is + * no memory-change event for either transition to ride on. + * + * Most config edits (titles, models, task status) leave the topology alone; + * emitting for those would make every live child rebuild its index and hot + * set from disk on ordinary churn, so only real owner changes are reported. + * One parse, only when something was memoized. Returns false when the + * config could not be read (readable stat, unreadable/unparseable content): + * the memoized mappings are RETAINED rather than replaced by the empty + * default's self fallbacks — those would be pinned until the stamp moved, + * which a restored permission bit never does — and the caller keeps the old + * stamp so the pass is retried on the next resolution. + */ + private invalidateWorkspaceMemoryOwnerMemo(): boolean { + if (this.workspaceMemoryOwnerById.size === 0) return true; + // One parse and one ID index for the whole pass (O(n), not O(n²)). + let cfg: ReturnType; + try { + cfg = this.config.loadConfigOrDefault({ throwOnError: true }); + } catch (error) { + log.debug("[MemoryService] config unreadable; keeping memoized workspace memory owners", { + error, + }); + return false; + } + const resolve = workspaceMemoryOwnerResolver(cfg); + const changed: string[] = []; + for (const [workspaceId, previousOwner] of this.workspaceMemoryOwnerById) { + const owner = resolve(workspaceId); + this.workspaceMemoryOwnerById.set(workspaceId, owner); + if (owner !== previousOwner) changed.push(workspaceId); + } + if (changed.length > 0) this.emit("ownersInvalidated", changed); + return true; + } + + /** + * Per-context owner cache: a context object is created per command / per + * index+hot-set build and reused for every entry within it, so the stamp + * stat behind resolveWorkspaceMemoryOwnerId runs once per operation instead + * of once per candidate file. Staleness is bounded to that one operation; + * writes are still gated by the store-bound tombstone check. + */ + private readonly ownerByContext = new WeakMap(); + + /** + * Sub-agents whose pre-sharing private notebook was found absent or already + * adopted during this process lifetime, keyed to the owner and legacy-store + * state observed at the time (see adoptLegacyPrivateStore). + */ + private readonly legacyStoreCheckedAgainst = new Map(); + + /** Last file scan per store (owner notebooks and redirected children's legacy notebooks; see throttledStoreStamp). */ + private readonly ownerStoreStampMemo = new Map< + string, + { rootMtime: string; stamp: string; scannedAt: number } + >(); + + /** + * Owner of the workspace scope for this context ("" when there is no + * workspace). Public so callers that key sidecar metadata for the same + * context (memoryOperations) bind to the exact owner the store resolved to. + */ + ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { + if (ctx.workspaceId === "") return ""; + const cached = this.ownerByContext.get(ctx); + if (cached !== undefined) return cached; + const owner = this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + this.ownerByContext.set(ctx, owner); + return owner; + } + /** Logical sidecar key, or null when the scope has no stable identity. */ private logicalKeyFor(ctx: MemoryScopeContext, scope: MemoryScope, relPath: string) { if (scope === "project" && ctx.projectPath === "") return null; return memoryLogicalKey(scope, relPath, { projectPath: ctx.projectPath, - workspaceId: ctx.workspaceId, + // Pins/usage stats follow the physical file, so a shared notebook has + // one ranking regardless of which tree member touched it. Only the + // workspace key embeds the id; skip the lookup for the other scopes. + workspaceId: scope === "workspace" ? this.ownerWorkspaceIdFor(ctx) : ctx.workspaceId, }); } + /** + * Consolidation's pin protection (pinned files are editable but never + * deleted/renamed; a directory counts when anything under it is pinned), + * evaluated INSIDE the mutation lock against the owner the command's store + * is bound to: logicalKeyFor and getStore share this command's owner + * resolution (ownerWorkspaceIdFor), so the key checked is the key of the + * file about to be removed. A guard run before the command against a + * separately resolved owner (the private-store fallback while config.json + * was unreadable) would check the wrong sidecar entries and let an + * owner-pinned note go. Strict sidecar read: an unreadable pin file must + * refuse, not read as "nothing pinned". + */ + private async assertNotPinnedForRemoval( + ctx: MemoryScopeContext, + scope: MemoryScope, + relPath: string, + virtualPath: string + ): Promise { + const key = this.logicalKeyFor(ctx, scope, relPath); + if (key === null) return; + const subtreePrefix = `${key}/`; + for (const [entryKey, entry] of await this.metaService.getEntriesOrThrow()) { + if (entry.pinned !== true) continue; + if (entryKey === key || entryKey.startsWith(subtreePrefix)) { + throw new MemoryCommandError( + `${virtualPath} is pinned by the user (directly or via a pinned file inside it); pinned files may be edited but never deleted or renamed.` + ); + } + } + } + private async recordUsage( ctx: MemoryScopeContext, scope: MemoryScope, @@ -626,6 +965,29 @@ export class MemoryService extends EventEmitter { try { const key = this.logicalKeyFor(ctx, scope, relPath); if (key === null) return; + if (scope === "workspace" && !options.write) { + // A read-side access (view, recall) re-ranks the shared hot set the + // whole task tree derives from the owner's sidecar entries — like a + // pin does — so it is published the same way: store clock under the + // store's mutation lock (other backends' probes), then the change + // event (this backend's other sessions and Memory tabs). Writes are + // already inside their mutation's lock and publish with it; a + // read holds no lock yet, hence the explicit one here, with the same + // commit guard so a removed owner's directory is never recreated. + const store = this.getStore(ctx, scope); + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + await this.assertMutationCommittable( + ctx, + store, + undefined, + toVirtualPath(scope, relPath) + ); + await this.metaService.recordAccess(key, options); + await this.advanceStoreRevision(store); + }); + this.emitChange(ctx, scope, relPath, "agent"); + return; + } await this.metaService.recordAccess(key, options); } catch (error) { log.debug("[MemoryService] failed to record memory usage", { scope, relPath, error }); @@ -717,15 +1079,22 @@ export class MemoryService extends EventEmitter { ); } return new LocalMemoryStore( - workspaceMemoryStorePath(this.config.sessionsDir, ctx.workspaceId) + workspaceMemoryStorePath(this.config.sessionsDir, this.ownerWorkspaceIdFor(ctx)) ); } } } private async runCommand( + ctx: MemoryScopeContext, operation: () => Promise ): Promise { + // The per-context owner cache is scoped to ONE command: createMemoryTool + // reuses a context for a whole stream, and a cached owner would otherwise + // let a child keep reading its parent's notebook after the tree changed + // (owner removed by another backend — no local event) for as long as the + // stream lives. Re-resolving costs one memoized, stamp-validated lookup. + this.ownerByContext.delete(ctx); try { return await operation(); } catch (error) { @@ -751,11 +1120,930 @@ export class MemoryService extends EventEmitter { relPath: string ): Promise { const store = this.getStore(ctx, scope); + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); await store.assertRootSafe(); await store.assertContained(relPath); return store; } + /** + * Every workspace-scope entry point (commands, root listing, index build) + * goes through here: refuse revoked access, then fold a sub-agent's + * pre-sharing private notebook into the shared store it now resolves to. + */ + private async openWorkspaceStore(ctx: MemoryScopeContext, store: MemoryStore): Promise { + await this.assertWorkspaceStoreReadable(ctx, store); + await this.adoptLegacyPrivateStore(ctx, store); + // The adoption pass waits for and holds the owner-store lock, a window in + // which another backend's removal can publish the acting workspace's (or + // the owner's) tombstone. The pass itself refuses on its commit guard + // and swallows that as a retryable adoption failure, so re-check here: + // the caller is about to read the owner's still-live notebook on behalf + // of a workspace that no longer exists. + await this.assertWorkspaceStoreReadable(ctx, store); + } + + /** + * Upgrade compatibility for the shared task-tree notebook. Sub-agents + * created by builds before sharing kept `/memories/workspace` in their OWN + * session dir (//memory). getStore now redirects them to + * the owner's root, which would make those notes invisible — and removal + * later deletes the child's session dir, discarding them for good. On the + * child's first shared-store access per process, copy every legacy file + * into the owner store (same relPath when free or identical; otherwise + * under imported//) and copy pins/stats to the owner key. + * + * The legacy directory is left in place, untouched: it is exactly where a + * DOWNGRADED build reads (and writes) this child's notebook, so the notes + * stay visible across upgrade↔downgrade (the child-keyed sidecar entries + * stay for the same reason) and files the import cannot carry + * (binary/oversize, doubly conflicting) are never moved anywhere. + * The copy is idempotent — identical files are skipped, differing ones land + * under imported// — so notes edited during a downgrade are folded in + * again on the next upgrade. Writes made through the shared store meanwhile + * live in the owner's notebook, which the downgraded build shows there. + * + * Security: the legacy root must be a real directory (a symlinked root + * would let an index build copy arbitrary host text into the shared + * notebook and the model's context), and every file passes the store's + * containment check before it is read. Runs under the owner store's + * mutation lock with the same commit guard as file mutations, and never + * throws: a failure (lock timeout, disk) is retried on the next access, + * while the caller proceeds with the shared store. Not journaled: this is + * a mechanical copy, not an agent edit; pre-upgrade child journal rows keep + * targeting the legacy physical paths. + */ + private async adoptLegacyPrivateStore( + ctx: MemoryScopeContext, + store: MemoryStore + ): Promise { + const childId = ctx.workspaceId; + if (childId === "") return; + const owner = this.storeOwnerWorkspaceId(store); + assert(owner !== null, "workspace-scope stores live under sessionsDir"); + if (owner === childId) { + // Not redirected: the private store IS the store. Recorded so a later + // redirect (config recovered) is seen as a change of the key below. + this.legacyStoreCheckedAgainst.set(childId, owner); + return; + } + try { + await this.adoptLegacyPrivateStoreOrThrow(ctx, store, owner); + } catch (error) { + log.warn( + "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", + { + childId, + owner, + error, + } + ); + } + } + + /** + * Removal handover: before a sub-agent's session directory is deleted, + * fold its legacy private notebook (if any) into the owner store. The + * access-time adoption above only runs when some workspace-memory entry + * point serves the child; a child removed right after an upgrade (an + * inactive-descendant deletion cascade, say) may never have had one, and + * the deletion would discard its notes for good. Runs BEFORE any teardown + * step (removal reuses the owner it verified with a strict config load), so + * a failure aborts the removal with the workspace intact: this variant + * THROWS instead of deferring to a next access that will never come — also + * when a listed note could not be represented in the owner store (shared + * notebook at its file cap, both destinations taken by different content, + * unreadable as text): the pass would count it as skipped and the deletion + * would take the only copy. Removal runs it twice: pre-teardown, and again + * inside the removal locks (`locksHeld`, the owner-store lock among them) + * right before the tombstone, catching a note a self-fallback backend + * committed into the legacy directory in between — the child's own store + * lock is held there too, so nothing can land after that pass. + */ + async adoptLegacyPrivateStoreForRemoval( + childWorkspaceId: string, + ownerWorkspaceId: string, + options?: { locksHeld: boolean } + ): Promise { + assert(childWorkspaceId.length > 0, "adoptLegacyPrivateStoreForRemoval requires a child id"); + assert( + ownerWorkspaceId.length > 0 && ownerWorkspaceId !== childWorkspaceId, + "adoptLegacyPrivateStoreForRemoval requires a distinct owner id" + ); + // Workspace-scope keys and roots embed only the workspace id (see + // logicalKeyFor / getStore), so no project identity is needed here. + const ctx: MemoryScopeContext = { + runtime: null, + checkoutCwd: "", + workspaceId: childWorkspaceId, + projectPath: "", + }; + const store = this.getStore(ctx, "workspace"); + // The store resolved from current config must be the owner removal + // verified; a disagreement means the topology changed under removal's + // feet, and adopting into the wrong notebook would be worse than aborting. + const resolvedOwner = this.storeOwnerWorkspaceId(store); + if (resolvedOwner !== ownerWorkspaceId) { + throw new Error( + `shared memory owner of ${childWorkspaceId} resolved to ${String(resolvedOwner)} while removal verified ${ownerWorkspaceId}` + ); + } + const { skipped } = await this.adoptLegacyPrivateStoreOrThrow(ctx, store, ownerWorkspaceId, { + force: true, + locksHeld: options?.locksHeld === true, + }); + if (skipped > 0) { + throw new Error( + `${skipped} legacy workspace memory note(s) of ${childWorkspaceId} could not be folded into ${ownerWorkspaceId}'s shared notebook (full, conflicting, or not text); removing the session directory would discard them` + ); + } + } + + /** + * Fingerprint of a sub-agent's legacy private notebook as seen by the + * adoption pass: owner, legacy root kind, the legacy store's stamp (child + * store clock, root entry, listed files' size/mtime) and the child-keyed + * sidecar entries. Also folded into the child's memory probe token + * (workspaceMemoryRevision): a downgraded backend's edit to the legacy note + * or its pin never moves the OWNER store's clock, so a cached session + * context keyed on that clock alone would keep serving the pre-edit index + * until some unrelated owner mutation. Cheap when no legacy root exists + * (one lstat). + */ + private async legacyAdoptionCheckKey( + childId: string, + owner: string, + options?: { + /** + * Probe-token use: the legacy file scan is memoized like the owner + * store's (throttledStoreStamp). The adoption pass itself must stay + * exact (it decides whether to re-run) and never passes this. + */ + throttled: boolean; + } + ): Promise<{ legacyRootKind: Awaited>; checkKey: string }> { + const childSessionDir = path.join(this.config.sessionsDir, childId); + const legacyRoot = path.join(childSessionDir, "memory"); + const legacyRootKind = await lstatKind(legacyRoot); + // Workspace-scope keys embed only the workspace id (see logicalKeyFor). + const childKeyPrefix = memoryLogicalKey("workspace", "", { + projectPath: "", + workspaceId: childId, + }); + const childSidecarFingerprint = + legacyRootKind === "dir" + ? JSON.stringify( + [...(await this.metaService.getEntries())] + .filter(([key]) => key.startsWith(childKeyPrefix)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ) + : ""; + const legacyStamp = + legacyRootKind !== "dir" + ? "" + : options?.throttled === true + ? await this.throttledStoreStamp(`legacy\u0000${childId}`, childSessionDir, legacyRoot) + : await legacyStoreStamp(childSessionDir, legacyRoot); + const checkKey = `${owner}\u0000${legacyRootKind}\u0000${legacyStamp}\u0000${childSidecarFingerprint}`; + return { legacyRootKind, checkKey }; + } + + /** + * The adoption pass (see adoptLegacyPrivateStore). `force` skips the + * per-process "already checked" memo: removal wants the pass to run against + * the current legacy directory regardless of what an earlier access saw. + * `locksHeld`: the caller already holds the owner store's mutation lock + * (removal's in-lock delta pass), so it is not re-acquired. Returns how many + * listed legacy notes could NOT be represented in the owner store this pass. + */ + private async adoptLegacyPrivateStoreOrThrow( + ctx: MemoryScopeContext, + store: MemoryStore, + owner: string, + options?: { force: boolean; locksHeld?: boolean } + ): Promise<{ skipped: number }> { + const childId = ctx.workspaceId; + // Checked once per (child, owner, legacy-store state) per process. The + // owner is part of the key because ownership can move: a command served + // while config.json was missing/malformed resolves the child to itself + // and writes into the legacy dir. The legacy store's own state is part of + // it because ANOTHER backend can do the same while this process's + // resolution never changes: its self-fallback write advances the child's + // store clock (memory.revision in the child's session dir) and replaces a + // root entry, so either signal re-runs the pass — as does any listed + // file's size/mtime. The child-keyed sidecar entries are the fourth + // input: a downgraded build can change only a pin or usage counter, which + // the manifest reconciles (sidecar fingerprint) but no file stat shows. + // The pass itself is idempotent. + const childSessionDir = path.join(this.config.sessionsDir, childId); + const legacyRoot = path.join(childSessionDir, "memory"); + const { legacyRootKind, checkKey } = await this.legacyAdoptionCheckKey(childId, owner); + if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) { + return { skipped: 0 }; + } + // Not "nothing to adopt": a root that could not be inspected may hold the + // only copy of downgrade-era notes. Access-time callers log and retry; + // removal aborts with the session intact. + const unreadableRoot = (): Error => + new Error(`the legacy workspace memory root of ${childId} could not be inspected`); + if (legacyRootKind === "unreadable") throw unreadableRoot(); + if (legacyRootKind !== "dir") { + if (legacyRootKind === "symlink") { + log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { + childId, + legacyRoot, + }); + } + this.legacyStoreCheckedAgainst.set(childId, checkKey); + return { skipped: 0 }; + } + // Files adopted this pass (bytes written OR only their sidecar entries + // folded in): either changes what the shared store's readers derive from it. + let adoptedCount = 0; + let skipped = 0; + const pass = async (): Promise => { + await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); + const rootKindUnderLock = await lstatKind(legacyRoot); + if (rootKindUnderLock === "unreadable") throw unreadableRoot(); + if (rootKindUnderLock !== "dir") return; // swapped while waiting for the lock + const legacy = new LocalMemoryStore(legacyRoot); + // Strict: a note omitted by a partial listing would count as "nothing + // to adopt" (skipped stays 0) and removal would then delete its only + // copy. A traversal failure fails the pass instead (access-time: + // retried on the next access; removal: aborted, session intact). + // Dot-entries included: no listing shows them, but the path grammar + // admits them, so `.note` may be a real note of the downgraded child. + const files = await legacy.listFiles({ strict: true, includeDotfiles: true }); + // What was already folded in, kept in the child's session dir OUTSIDE + // the legacy root (which is a downgraded build's model-writable + // namespace; see legacyAdoptionManifestPath): per relPath the content + // hash, the fingerprint of the child-keyed sidecar entry, and where the + // copy landed. Content: without it, a note later edited through the shared + // store would be re-imported as a stale duplicate on every backend + // start. Sidecar: a downgraded build can change only a pin or usage + // stats, which must reach the owner key without the bytes changing. + // Strict reads throughout: this pass decides what the handover may + // consider done (and removal then deletes the child session on that + // basis), so a transiently unreadable manifest, sidecar or owner + // listing must fail the pass rather than stand in as "empty". + const manifestPath = legacyAdoptionManifestPath(childSessionDir); + const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); + const sidecarEntries = await this.metaService.getEntriesOrThrow(); + // The per-scope file cap is a store invariant (create/rename enforce + // it): the copy stops at the owner store's remaining capacity so a + // combined notebook cannot exceed it — an over-full scope is silently + // truncated by the index and refuses every later create. Files left + // behind stay unrecorded and are retried once space frees up. Complete + // owner listing: an undercount would let the copy push the store past + // the cap and hide an adopted note's only copy once readable again. + let remainingCapacity = + MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles({ strict: true })).length; + // Staged bytes a crashed pass never installed: their records claim + // nothing (no file carries the receipt), so they are simply dropped. + const stagingDir = legacyAdoptionStagingDir(store); + await fsPromises.rm(stagingDir, { recursive: true, force: true }); + let capacityExhausted = false; + let manifestDirty = false; + let imported = 0; + const writeManifest = () => + writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { + encoding: "utf-8", + }); + for (const relPath of files) { + // Same read gates as a memory command: containment (no symlink + // escape), size cap, and text-only (a lossy utf-8 decode cannot be + // carried by a text write). + const content = await legacy + .assertContained(relPath) + .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) + .catch(() => null); + if (content === null || content.includes("\uFFFD")) { + // Dot-entries too (r73): `.note` is addressable, so a real note + // there may hold text `create` permitted (U+FFFD included) or be + // transiently unreadable — exempting dot-entries would report a + // complete handover and let removal take the only copy. A stray + // `.DS_Store` costs a forced removal, never a note. + skipped++; + continue; + } + const childKey = memoryLogicalKey("workspace", relPath, { + projectPath: ctx.projectPath, + workspaceId: childId, + }); + const childEntry = sidecarEntries.get(childKey); + const record: LegacyAdoptionRecord = { + content: sha256Hex(content), + sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), + target: "", + created: false, + }; + // A record whose source was reconciled as deleted is kept only for + // the path mapping (rollbacks); a reappearing source is a fresh note. + const priorRecord = adopted.get(relPath); + const previous = priorRecord?.deleted === true ? undefined : priorRecord; + if ( + previous?.content === record.content && + previous.sidecar === record.sidecar && + previous.pending !== true && + // A deletion under way removed (or is about to remove) the copy: a + // source reappearing with the same bytes must be adopted anew. + previous.pendingDeletion !== true + ) { + continue; // folded in earlier, nothing changed since + } + // `generation`: the stamp of the copy an in-place replacement installs + // over, re-checked right before the install. + let target: { + relPath: string; + write: boolean; + replaces?: boolean; + generation?: string; + } | null = null; + // A child's pin toggle folds into the copy only while the copy is + // this adoption's generation (see below) or the owner's identical + // note it was folded into at first adoption; a copy the owner + // replaced since keeps the owner's pin — recorded as `replaced`, so + // the next pass still knows (its `created` is gone either way). + let foldChildPin = true; + if (previous !== undefined) { + // The recorded target is reused only while it still holds bytes + // this adoption put there — the owner may have edited, replaced or + // deleted it since, and the child's note must not land on unrelated + // content or a missing file. Unchanged legacy bytes (only the + // sidecar moved): reuse without a write. Legacy bytes edited on the + // downgraded build while the copy THIS adoption created is still + // untouched: the copy is replaced in place — placing the new bytes + // elsewhere would strand the old copy, provenance lost, in the + // model-visible notebook. Otherwise the note is placed anew. + // Inspected strictly: a prior target that merely cannot be stat'ed + // or read right now is not "replaced" — retry on the next access + // instead (the pass stays incomplete). + let priorContent: string | null; + try { + priorContent = await this.inspectAdoptedCopy(store, previous.target); + } catch (error) { + log.warn( + "[MemoryService] cannot inspect an adopted note's prior copy; retrying later", + { childId, owner, relPath, target: previous.target, error } + ); + skipped++; + continue; + } + // Ours only while the copy is a generation this adoption installed + // (LegacyAdoptionRecord.targetStamp / replacementStamp — receipts + // taken on the staged bytes BEFORE they appear at the target, so + // even a pass interrupted between manifest and install left one). + // Identical bytes in another generation are the owner's (r79: the + // same rule deletion reconciliation and the rollback remapper + // apply): the owner may have deleted and recreated the note with + // the very same bytes, or — a pass interrupted before its install + // — another backend may have created it at the planned target + // (r89). `pending` alone is never provenance: a stamp-less pending + // record (an older build's) is ambiguous and claims nothing. + const currentStamp = + (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; + const ours = + previous.created === true && + currentStamp !== undefined && + (currentStamp === previous.targetStamp || currentStamp === previous.replacementStamp); + if (priorContent === content) { + target = { relPath: previous.target, write: false }; + record.created = ours; + record.targetStamp = ours ? currentStamp : undefined; + const replaced = previous.replaced === true || (previous.created === true && !ours); + if (replaced) record.replaced = true; + foldChildPin = !replaced; + } else if ( + ours && + priorContent !== null && + [previous.content, previous.replacementContent].includes(sha256Hex(priorContent)) + ) { + // Legacy bytes edited on the downgraded build while the copy is + // still this adoption's (either side of an interrupted + // replacement): replaced in place. + target = { + relPath: previous.target, + write: true, + replaces: true, + generation: currentStamp, + }; + } + } + if (target === null) { + target = await this.legacyImportTarget(store, childId, relPath, content); + if (target === null) { + skipped++; + continue; + } + } + if (target.write) { + if (target.replaces !== true && remainingCapacity <= 0) { + capacityExhausted = true; + skipped++; + continue; + } + // Destination containment immediately before the write (the + // same check a memory create runs): a symlinked component under + // the owner root — e.g. imported/ pointing elsewhere — + // must never let the copy land outside the store. + try { + await store.assertContained(target.relPath); + } catch (error) { + log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { + childId, + relPath, + target: target.relPath, + error, + }); + skipped++; + continue; + } + // Staged install: the bytes are written to a hidden staging entry + // of the owner store first and their identity taken there (a + // rename keeps ino, size and mtime), so the manifest can record the + // receipt of the copy BEFORE the copy appears at the target. A pass + // interrupted at any point then leaves a record that either names + // a file not yet there (nothing claimed) or names the installed + // generation by stamp; a plain byte match never has to stand in + // for provenance. Without the record, an installed copy would read + // as the owner's own note, and a legacy deletion could then never + // follow it out of the shared store. A replacement keeps the PRIOR + // record (old hash and stamp, same target) while pending: on either + // side of the install the retry recognizes the file by its stamp. + const stagingPath = path.join(stagingDir, randomUUID()); + try { + await fsPromises.mkdir(stagingDir, { recursive: true }); + await writeFileAtomic(stagingPath, content, { encoding: "utf-8" }); + } catch (error) { + log.warn("[MemoryService] cannot stage a legacy note for adoption; retrying later", { + childId, + relPath, + error, + }); + skipped++; + continue; + } + const stagedStamp = await adoptionTargetStamp(stagingPath); + if (stagedStamp === null) { + await fsPromises.rm(stagingPath, { force: true }); + skipped++; + continue; + } + adopted.set( + relPath, + target.replaces === true && previous !== undefined + ? { + ...previous, + pending: true, + replacementContent: record.content, + replacementStamp: stagedStamp, + } + : { + ...record, + target: target.relPath, + created: true, + pending: true, + targetStamp: stagedStamp, + } + ); + await writeManifest(); + // The destination as decided above, re-checked under the lock right + // before the install: a fresh placement must still be free, a + // replacement must still be the generation it was decided against. + // Anything else is owner state the rename must not clobber — the + // staged bytes are dropped, the record restored, and the note is + // placed on the next pass. + const installable = + target.replaces === true + ? (await adoptionTargetStamp(store.physicalPath(target.relPath))) === + target.generation + : (await store.kind(target.relPath, { strict: true })) === null; + const restoreRecord = async () => { + await fsPromises.rm(stagingPath, { force: true }); + if (previous === undefined) adopted.delete(relPath); + else adopted.set(relPath, previous); + await writeManifest(); + }; + if (!installable) { + await restoreRecord(); + log.warn( + "[MemoryService] adoption destination changed before install; retrying later", + { + childId, + relPath, + target: target.relPath, + } + ); + skipped++; + continue; + } + // The install: same session dir, so a plain rename (an EXDEV — the + // memory root mounted apart from its session dir — fails this note, + // not the pass). + try { + const destination = store.physicalPath(target.relPath); + await fsPromises.mkdir(path.dirname(destination), { recursive: true }); + await fsPromises.rename(stagingPath, destination); + } catch (error) { + await restoreRecord(); + log.warn("[MemoryService] cannot install a staged legacy note; retrying later", { + childId, + relPath, + target: target.relPath, + error, + }); + skipped++; + continue; + } + if (target.replaces !== true) remainingCapacity--; + imported++; + record.created = true; + // The generation of the file just installed (see targetStamp): the + // staged receipt, unless the filesystem re-stamped the rename. + record.targetStamp = + (await adoptionTargetStamp(store.physicalPath(target.relPath))) ?? undefined; + } + record.target = target.relPath; + // Pins/stats were keyed by the child: fold them into the owner key. + // The child-keyed entry stays — like the legacy file, it is what a + // downgraded build reads. Recorded in the manifest only once this + // succeeded, so an adoption interrupted after its writeFile (or a + // failing sidecar write) retries this step on the next access. A + // first adoption keeps the owner's own pin (a note the owner tracked + // independently); a PIN the CHILD toggled since its last adoption + // (downgrade-time pin/unpin) is the newer intent and wins. Only the + // pin bit counts for that: a downgraded build merely viewing the note + // changes its usage counters, which must not drag the owner's pin + // back to the child's unchanged value. + if (childEntry !== undefined) { + // A pending record still carries the sidecar state it was recorded + // with: a fresh adoption's is the child's current state (no + // transition → first-adoption semantics), a pending replacement's + // is the prior record's — the child's toggle since must not be + // lost to the interrupted pass. + const priorPinned = previous === undefined ? null : legacySidecarPinned(previous.sidecar); + // Only an actual boolean transition of the child's pin overrides + // the owner's; an unknown prior state never does. + const childPinChanged = priorPinned !== null && priorPinned !== childEntry.pinned; + try { + await this.metaService.mergeKeys( + childKey, + memoryLogicalKey("workspace", target.relPath, { + projectPath: ctx.projectPath, + workspaceId: owner, + }), + { pinned: childPinChanged && foldChildPin ? "source" : "target" } + ); + } catch (error) { + log.warn( + "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", + { relPath, error } + ); + // Counts as skipped: the note's pin/usage metadata is still + // stranded under the child key, and removal must not delete the + // child session (the only trigger for a retry) on that basis. + skipped++; + continue; + } + } + adopted.set(relPath, record); + manifestDirty = true; + adoptedCount++; + } + // Legacy notes deleted or renamed on the downgraded build: a copy THIS + // adoption created, still holding the adopted bytes, follows the source + // out of the shared notebook (a rename's new name is adopted above like + // a fresh note). Provenance and unchanged content are both required — + // an owner note that merely happened to be identical, or an adopted + // copy the owner has since edited, is the owner's and stays. Unlisted + // sources are only ever judged against the listing that succeeded + // above; a failed listing never reaches this point. + const listed = new Set(files); + for (const [relPath, previous] of adopted) { + if (listed.has(relPath) || previous.deleted === true) continue; + // Absence from the listing is not proof: LocalMemoryStore.listFiles + // tolerates readdir failures (a partial list). Only a provable ENOENT + // on the source itself counts; any other outcome keeps the entry + // (and the copy) for a later pass. + // ENOTDIR is proof too: the downgraded build replaced `dir/` with a + // regular note, deleting every descendant. + const sourceGone = await fsPromises.lstat(path.join(legacyRoot, relPath)).then( + () => false, + (error: unknown) => isMissingPathError(error) + ); + if (!sourceGone) continue; + let unchangedForTombstone = false; + if (previous.created === true) { + // Strict probe: a target that merely could not be stat'ed is not + // "changed" — dropping the entry on that basis would lose the + // provenance for good and leave the obsolete copy visible forever + // once the filesystem recovers. Keep the entry (and the pass + // incomplete) so the next access reconciles it. + let targetKind: MemoryEntryKind; + let targetContained = false; + try { + targetContained = await store.assertContained(previous.target).then( + () => true, + () => false + ); + targetKind = targetContained + ? await store.kind(previous.target, { strict: true }) + : null; + } catch (error) { + log.warn( + "[MemoryService] cannot inspect an adopted legacy note's copy; retrying later", + { + childId, + owner, + relPath, + target: previous.target, + error, + } + ); + skipped++; + continue; + } + // Same for the content read: a failure other than the cap check + // (MemoryCommandError: the owner grew the copy past the cap, which + // IS a change) says nothing about the content. + let current: string | null = null; + if (targetKind === "file") { + try { + current = await this.readBoundedTextFile(store, previous.target, previous.target); + } catch (error) { + if (!(error instanceof MemoryCommandError)) { + log.warn( + "[MemoryService] cannot read an adopted legacy note's copy; retrying later", + { + childId, + owner, + relPath, + target: previous.target, + error, + } + ); + skipped++; + continue; + } + } + } + // Ours only while it is a generation this adoption installed + // (targetStamp; replacementStamp on the far side of an interrupted + // in-place replacement — both receipts taken on the staged bytes, + // so a crash cannot have kept them from being recorded): identical + // bytes in a file the owner deleted and recreated, or edited and + // restored, are the owner's, and a record without a stamp preserves. + const currentHash = current === null ? null : sha256Hex(current); + const stamp = await adoptionTargetStamp(store.physicalPath(previous.target)); + const unchanged = + currentHash !== null && + stamp !== null && + ((currentHash === previous.content && stamp === previous.targetStamp) || + (previous.pending === true && + currentHash === previous.replacementContent && + stamp === previous.replacementStamp)); + // A listed note may now point at this very target (the downgraded + // build renamed `a.md` to the path its conflict copy was adopted + // under, and the new record reused the identical file): the target + // is that note's copy now. Provenance transfers to the successor + // record instead of the file being deleted from under it. + const successor = [...adopted].find( + ([rel, record]) => + rel !== relPath && listed.has(rel) && record.target === previous.target + ); + // A target PROVEN absent (contained path, strict probe ENOENT) while + // a deletion was pending was removed by the interrupted pass, not + // changed by the owner. A directory, symlink, escaping component or + // over-cap file there is owner state. + const removedByUs = + previous.pendingDeletion === true && targetContained && targetKind === null; + unchangedForTombstone = (unchanged || removedByUs) && successor === undefined; + if (successor !== undefined) { + // Only a copy still holding the adopted bytes is ours to hand + // over; one the owner edited since is the owner's, and the + // successor keeps its own (non-created) provenance. + if (unchanged && stamp !== null && successor[1].created !== true) { + successor[1].created = true; + // The generation observed on disk — the receipt `unchanged` + // matched (r90: on the far side of an interrupted replacement + // that is `replacementStamp`, not the overwritten generation's + // `targetStamp`, which would make the successor read as + // replaced by the owner at once). + successor[1].targetStamp = stamp; + manifestDirty = true; + } + } else if (unchanged) { + // Deletion provenance first: a crash after the removal but before + // the tombstone write must not make the retry read the missing + // copy as owner-changed (and drop the child's rollback mapping). + adopted.set(relPath, { ...previous, pendingDeletion: true }); + await writeManifest(); + // Metadata next: a sidecar failure then aborts the pass with the + // file and manifest entry intact, so the retry repeats both; + // the reverse order would strand the owner-key pin/usage once + // the file was gone and the entry dropped. + await this.metaService.removeKeys( + memoryLogicalKey("workspace", previous.target, { + projectPath: ctx.projectPath, + workspaceId: owner, + }) + ); + await store.remove(previous.target); + adoptedCount++; + log.info("[MemoryService] removed an adopted legacy note deleted on the old build", { + childId, + owner, + relPath, + target: previous.target, + }); + } + } + // Kept as a tombstone, not dropped: the child's pre-sharing rows for + // this note still need relPath → target to be rolled back into the + // shared store (a delete's restore lands at the reconciled target; + // the reconciliation above never runs again for it). + // Destructive provenance survives only while the target was still + // this adoption's copy and nobody took it over: a copy the owner + // edited (or one handed to a successor record) is not the old path's + // to delete or restore any more. + adopted.set(relPath, { + ...previous, + pendingDeletion: undefined, + deleted: true, + created: previous.created === true && unchangedForTombstone, + }); + manifestDirty = true; + } + if (manifestDirty) await writeManifest(); + await fsPromises.rm(stagingDir, { recursive: true, force: true }); + if (capacityExhausted) { + log.warn( + "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", + { childId, owner, cap: MEMORY_MAX_FILES_PER_SCOPE } + ); + } + if (adoptedCount > 0) { + // A metadata-only adoption (identical bytes, child pin folded in) + // still changes the hot set other backends derive, so the clock + // moves too. + await this.advanceStoreRevision(store); + log.info( + "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", + { childId, owner, imported, skipped } + ); + } + }; + if (options?.locksHeld === true) { + await pass(); + } else { + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), pass); + } + // Recorded against the state observed BEFORE the pass: a foreign write + // landing during it changes the stamp and re-runs the (idempotent) pass. + // Only a complete pass is memoized: a note left unrepresented (owner + // store full, both destinations taken, unreadable, sidecar fold failed) + // is retried on the next access, and that retry depends on OWNER-side + // state the check key does not observe. + if (skipped === 0) { + this.legacyStoreCheckedAgainst.set(childId, checkKey); + } else { + this.legacyStoreCheckedAgainst.delete(childId); + } + if (adoptedCount > 0) this.emitChange(ctx, "workspace", "", "agent"); + return { skipped }; + } + + /** + * Where a legacy file lands in the owner store: its own relPath when free + * (write) or already identical (no write); the per-child import directory + * when the owner has different content there; null when even that slot is + * taken by different content (the file stays only in the legacy directory). + */ + private async legacyImportTarget( + store: MemoryStore, + childId: string, + relPath: string, + content: string + ): Promise<{ relPath: string; write: boolean } | null> { + for (const candidate of [relPath, `${LEGACY_IMPORT_DIR}/${childId}/${relPath}`]) { + // Never even compare through an escaping path (the write site re-checks). + const contained = await store.assertContained(candidate).then( + () => true, + () => false + ); + if (!contained) continue; + // A symlink is never a destination: the store's listing excludes links + // (and dotfiles), so a note "represented" through one would be + // invisible to the shared notebook, and a write would land through it. + const linkKind = await lstatKind(store.physicalPath(candidate)); + if (linkKind === "unreadable") { + throw new Error(`cannot inspect adoption destination ${candidate}`); + } + if (linkKind === "symlink") continue; + // Strict: a destination that merely could not be stat'ed (EACCES, EIO) + // is not free — declaring it so would overwrite whatever the owner + // keeps there once the copy runs. The failure aborts the pass instead + // (access-time: retried; removal: session intact). + const kind = await store.kind(candidate, { strict: true }); + if (kind === null) return { relPath: candidate, write: true }; + if (kind === "file") { + const existing = await this.readBoundedTextFile(store, candidate, candidate).catch( + () => null + ); + if (existing === content) return { relPath: candidate, write: false }; + } + } + return null; + } + + /** + * Content of an adopted note's copy in the owner store, or null when no + * regular listed file is there (absent, a directory, a symlink, or grown + * past the cap — each a change the owner made). Throws when the copy + * cannot be inspected at all (EACCES, EIO): callers retry later. + */ + private async inspectAdoptedCopy(store: MemoryStore, relPath: string): Promise { + const contained = await store.assertContained(relPath).then( + () => true, + () => false + ); + if (!contained) return null; + const linkKind = await lstatKind(store.physicalPath(relPath)); + if (linkKind === "unreadable") throw new Error(`cannot inspect adopted copy ${relPath}`); + if (linkKind !== "other") return null; // missing, dir, or a symlink + try { + return await this.readBoundedTextFile(store, relPath, relPath); + } catch (error) { + if (error instanceof MemoryCommandError) return null; // over the cap + throw error; + } + } + + /** + * The workspace whose session dir physically holds `store` + * (//memory → owner), or null for global/project roots, + * which live elsewhere. + */ + private storeOwnerWorkspaceId(store: MemoryStore): string | null { + const rel = path.relative(this.config.sessionsDir, store.physicalRoot); + if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null; + return rel.split(path.sep)[0]; + } + + /** Acting workspace plus the store's owner: both must be alive to touch the store. */ + private guardedWorkspaceIds(ctx: MemoryScopeContext, store: MemoryStore): string[] { + const owner = this.storeOwnerWorkspaceId(store); + return [ + ...new Set([ + ctx.workspaceId, + ...(owner === null ? [] : [owner]), + ...(ctx.guardedWorkspaceId === undefined || ctx.guardedWorkspaceId === "" + ? [] + : [ctx.guardedWorkspaceId]), + ]), + ]; + } + + /** + * Reads have no commit guard, so a removed child's stream in ANOTHER backend + * (which the remover cannot cancel) could keep viewing its former owner's + * notebook — including notes written after the removal — through the + * shared store. Refuse workspace-scope reads once the acting workspace or + * the store's owner is tombstoned (the tombstone is durable and + * cross-process; see workspaceRemoval.ts). + */ + private async assertWorkspaceStoreReadable( + ctx: MemoryScopeContext, + store: MemoryStore + ): Promise { + if (ctx.workspaceId === "") return; + for (const workspaceId of this.guardedWorkspaceIds(ctx, store)) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + throw new MemoryCommandError( + `Workspace ${workspaceId} was removed; the workspace memory store is no longer available` + ); + } + } + } + + /** + * Post-read gate for workspace-scope reads. openWorkspaceStore checks the + * tombstones BEFORE the read; another backend can publish the acting + * workspace's (or the owner's) removal tombstone while the read is in + * flight, and the bytes would then be exposed on behalf of a workspace that + * no longer exists. Re-checked after every read whose result leaves the + * service (view, listings, index/hot-set builds, UI reads), before the + * result is returned. Other scopes are never shared and have no tombstone. + */ + private async assertWorkspaceReadExposable( + ctx: MemoryScopeContext, + scope: MemoryScope, + store: MemoryStore + ): Promise { + if (scope !== "workspace") return; + await this.assertWorkspaceStoreReadable(ctx, store); + } + private requireFilePath(parsed: ParsedMemoryPath, virtualPath: string): MemoryScope { if (parsed.scope === null || parsed.relPath === "") { throw new MemoryCommandError( @@ -769,14 +2057,18 @@ export class MemoryService extends EventEmitter { * Append the invertible `refinement` row for one memory mutation (RLM r2). * * Rows land in the ACTING workspace's session journal even though memory - * files can be global/project-scoped: the journal is per-session, so + * files can be global/project-scoped — or, for a sub-agent's workspace + * scope, live in the OWNER's session dir: the journal is per-session, so * cross-workspace edits to a shared file are attributed to (and invertible - * from) whichever workspace made them — the intended v1 scope. When the + * from) whichever workspace made them — the intended v1 scope. Rollback of + * a sub-agent's workspace-scope row admits the owner's memory root via + * RollbackRefinementOptions.sharedWorkspaceMemorySessionDir. When the * context has no workspace, there is no session journal; skip (log-only). * Never throws: journaling failures must not fail the memory command. */ private async journalRefinement( ctx: MemoryScopeContext, + store: MemoryStore, action: MemoryRefinementAction, inverse: RefinementInverseDraft, actor: MemoryActor, @@ -789,6 +2081,16 @@ export class MemoryService extends EventEmitter { }); return; } + // Workspace-scope rows carry the owner store's clock so rows from every + // tree member's journal order consistently (see workspaceMemoryRevision.ts). + // The mutation is already on disk, so a failed clock write cannot fail the + // command; the row is journaled as `orderUnknown` instead — the rollback + // engine then treats it as conflicting with every overlapping row rather + // than ordering it by its journal-local `ts`, which another journal's rows + // cannot be compared against (a child's later edit beneath a directory + // the owner renamed would otherwise read as older and be moved silently). + const sourceTs = await this.advanceStoreRevision(store); + const orderUnknown = sourceTs === undefined && this.storeOwnerWorkspaceId(store) !== null; await appendRefinementEvent({ sessionDir: path.join(this.config.sessionsDir, ctx.workspaceId), workspaceId: ctx.workspaceId, @@ -801,9 +2103,76 @@ export class MemoryService extends EventEmitter { ...(toolCallId !== undefined ? { toolCallId } : {}), }, ...(postFiles !== undefined ? { postFiles } : {}), + ...(sourceTs !== undefined ? { sourceTs } : {}), + ...(orderUnknown ? { orderUnknown: true } : {}), }); } + /** + * Refuse to COMMIT a mutation whose caller was torn down (r59/r61). Checked + * INSIDE the target mutation lock immediately before the first durable + * write; a mutation that already committed always journals (mutation → row + * → ack) so rollback lineage stays intact. Two teardown signals: + * + * - The caller's abort signal (r59): consolidation/refine passes receive no + * hard tool cancellation — an execution wedged in pre-commit I/O (e.g. a + * named pipe under a memory root) is detached by the caller's bounded + * drain, and once the I/O unblocks after workspace teardown it would + * still write durable memory AND append its refinement journal row into + * the deleted session directory, recreating it. + * - The durable removal tombstone (r61): with multiple backends over one + * Xum root, the remover cannot abort a dream/harvest run in ANOTHER + * process — that run's signal stays live after removal. The tombstone is + * published under the same memory target locks this check runs inside + * (see workspaceRemoval.ts), so a foreign backend's mutation observes + * removal here at commit time and refuses instead of recreating the + * deleted session directory via its write or journal append. + * + * Both the acting workspace and the workspace that physically owns the + * RESOLVED store are checked: a removed sub-agent must not keep writing + * into its parent's notebook, and a removed owner must not have its session + * directory recreated by a lingering child's write. The owner is derived + * from the store the command already bound to — not re-resolved — so an + * ownership change between resolution and lock acquisition cannot make the + * check pass for the new owner while the write lands in the old one. + * + * The bound owner is then compared with a fresh resolution: the + * per-context cache (ownerWorkspaceIdFor) may hold a self-fallback taken + * while config.json was missing or malformed, and if the file recovers + * before this command commits, the write would land in the child's private + * store although the tree is shared again. Refused as a recoverable error; + * the retried command resolves the owner anew. + */ + private async assertMutationCommittable( + ctx: MemoryScopeContext, + store: MemoryStore, + signal: AbortSignal | undefined, + virtualPath: string + ): Promise { + if (signal?.aborted === true) { + throw new MemoryCommandError( + `Mutation of ${virtualPath} was cancelled before commit (caller torn down)` + ); + } + if (ctx.workspaceId === "") return; + const boundOwner = this.storeOwnerWorkspaceId(store); + if (boundOwner !== null) { + const currentOwner = this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + if (currentOwner !== boundOwner) { + throw new MemoryCommandError( + `Ownership of the workspace notebook changed while mutating ${virtualPath} (now ${currentOwner}); retry the command` + ); + } + } + for (const workspaceId of this.guardedWorkspaceIds(ctx, store)) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + throw new MemoryCommandError( + `Workspace ${workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` + ); + } + } + } + /** * Capture the restore payload for a delete (file or recursive directory) * BEFORE it is removed. Returns null when capture fails or the subtree @@ -917,17 +2286,234 @@ export class MemoryService extends EventEmitter { scope: MemoryScope, relPath: string, actor: MemoryActor - ) { + ): void { const event: MemoryChangeEvent = { scope, path: toVirtualPath(scope, relPath), actor, - workspaceId: ctx.workspaceId, + // Owner, not actor: subscribers filter workspace-scope events by the + // store they display, and every tree member displays the owner's. + workspaceId: this.ownerWorkspaceIdFor(ctx), projectPath: ctx.projectPath, }; + // Pure in-process signal. The cross-process token (store clock) is NOT + // advanced here: it must move under the store's mutation lock, which + // mutations hold when they journal (journalRefinement / saveFile) and the + // rollback engine holds when it appends its row; notifyPinChange takes it + // explicitly. this.emit("change", event); } + /** + * Advance the owner store's clock (see workspaceMemoryRevision.ts) for a + * workspace-scope mutation and return it for the row's `sourceTs`. Callers + * MUST hold the store's target mutation lock (the clock's monotonicity is + * only as good as the lock around its read→write). Best-effort: a failure + * (removed owner directory — mutations are already refused pre-commit, see + * assertMutationCommittable; a pin toggle on a never-written store) must + * not fail the command, so the row then falls back to its journal `ts`. + */ + private async advanceStoreRevision(store: MemoryStore): Promise { + const owner = this.storeOwnerWorkspaceId(store); + if (owner === null) return undefined; + try { + return await advanceWorkspaceMemoryRevision(path.join(this.config.sessionsDir, owner)); + } catch (error) { + log.debug("[MemoryService] failed to advance workspace memory revision", { owner, error }); + return undefined; + } + } + + /** + * Current revision token of the store backing `workspaceId`'s + * `/memories/workspace` (owner-resolved; one memoized ownership check plus + * one small read). Compared by consumers that cache a derived view of the + * store — AgentSession's memory context, the Memory tab subscription — so + * a mutation from ANOTHER backend process (multi-instance), which emits no + * change event here, still invalidates them on their next probe. A store + * never written or unreadable yields a fixed sentinel. + */ + async workspaceMemoryRevision(workspaceId: string): Promise { + assert(workspaceId.length > 0, "workspaceMemoryRevision requires a workspaceId"); + const owner = this.resolveWorkspaceMemoryOwnerId(workspaceId); + // Access revoked (acting workspace or owner tombstoned by any backend): + // a distinct token so a cached context built from the owner's notes is + // invalidated and the rebuild (listIndexEntries) then excludes the store. + const revoked = async (): Promise => { + for (const guarded of new Set([workspaceId, owner])) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return true; + } + return false; + }; + if (await revoked()) return "revoked"; + const token = await this.buildWorkspaceMemoryRevisionToken(workspaceId, owner); + // Re-checked AFTER the token's reads: a tombstone published while they + // were in flight is not part of the token, so the unchanged pre-removal + // token would let a cached context keep serving the owner's notes to the + // removed workspace's next request. + return (await revoked()) ? "revoked" : token; + } + + private async buildWorkspaceMemoryRevisionToken( + workspaceId: string, + owner: string + ): Promise { + const ownerSessionDir = path.join(this.config.sessionsDir, owner); + const revision = await readWorkspaceMemoryRevision(ownerSessionDir); + // The clock is what this build's writers advance. A downgraded build + // sharing the root writes straight into the owner's canonical notebook + // without touching it, so the token also carries the store's own file + // stamps (root mtime, per-file size + mtime; bounded by the per-scope + // file cap): stat fingerprints are cache hints — enough to make a cached + // context miss — never proof that authorizes any mutation. + // The owner-keyed sidecar entries (pins, usage) rank the shared hot set + // and are written by every backend; a pin whose revision write then + // failed (advanceStoreRevision is best-effort) is still visible here. + const ownerKeyPrefix = memoryLogicalKey("workspace", "", { + projectPath: "", + workspaceId: owner, + }); + const ownerSidecar = JSON.stringify( + [...(await this.metaService.getEntries())] + .filter(([key]) => key.startsWith(ownerKeyPrefix)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ); + const token = `${revision === null ? "missing" : String(revision)}\u0000${await this.ownerStoreStamp( + owner, + ownerSessionDir + )}\u0000${ownerSidecar}`; + // A redirected sub-agent's token also tracks its legacy private notebook + // (see legacyAdoptionCheckKey): its next store access adopts the change, + // so the cached context must miss as soon as the legacy state moves. + if (owner === workspaceId) return token; + return `${token}\u0000${ + (await this.legacyAdoptionCheckKey(workspaceId, owner, { throttled: true })).checkKey + }`; + } + + /** + * The token's downgrade-compatibility segment. The per-file scan reruns + * when the notebook root's mtime moved (one stat: a note created, deleted + * or renamed at the top level by any build) and otherwise at most once per + * OWNER_STORE_SCAN_INTERVAL_MS per owner, shared by every session and tab + * probing that store. Only a downgraded build's in-place edit of an + * existing note waits for the interval; this build's writes advance the + * clock segment. + */ + private ownerStoreStamp(owner: string, ownerSessionDir: string): Promise { + return this.throttledStoreStamp( + `owner\u0000${owner}`, + ownerSessionDir, + workspaceMemoryStorePath(this.config.sessionsDir, owner) + ); + } + + /** + * legacyStoreStamp with the per-store memo described at ownerStoreStamp; + * also used for a redirected child's legacy notebook in its probe token, + * which otherwise walks that directory on every probe as well. + */ + private async throttledStoreStamp( + memoKey: string, + sessionDir: string, + root: string + ): Promise { + const rootMtime = await fsPromises + .stat(root) + .then((stat) => String(stat.mtimeMs)) + .catch(() => "missing"); + const memo = this.ownerStoreStampMemo.get(memoKey); + const now = Date.now(); + if (memo?.rootMtime === rootMtime && now - memo.scannedAt < OWNER_STORE_SCAN_INTERVAL_MS) { + return memo.stamp; + } + const stamp = await legacyStoreStamp(sessionDir, root); + this.ownerStoreStampMemo.set(memoKey, { rootMtime, stamp, scannedAt: now }); + return stamp; + } + + /** The store clock segment of a workspaceMemoryRevision token (tests, diagnostics). */ + static revisionClockOf(token: string): string { + return token.split("\u0000", 1)[0] ?? token; + } + + /** + * Announces memory files mutated outside this service by a refinement + * rollback (which applies inverses straight to disk). Physical paths are + * classified against this context's scope roots and one root-addressed + * event per touched scope is emitted, so Memory tabs refresh and — for the + * shared workspace store — every task-tree session drops its cached + * context (see the change listener wired in di/layers/core.ts). The store + * clock is not touched: the rollback engine advanced it under the target + * lock when it journaled its row (refinementRollback.ts). + */ + notifyExternalMutation(ctx: MemoryScopeContext, physicalPaths: readonly string[]): void { + const touched = new Set(); + for (const physicalPath of physicalPaths) { + const scope = this.scopeOfPhysicalPath(ctx, physicalPath); + if (scope !== null) touched.add(scope); + } + for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); + } + + /** + * The memory scope whose root (for this context) contains `physicalPath`, + * or null when the path lies outside every available scope root. Lets + * out-of-band writers (refinement rollback) apply the same per-scope write + * policy the memory tool enforces. + */ + scopeOfPhysicalPath(ctx: MemoryScopeContext, physicalPath: string): MemoryScope | null { + for (const scope of MEMORY_SCOPES) { + let root: string; + try { + root = this.getStore(ctx, scope).physicalRoot; + } catch (error) { + if (error instanceof MemoryCommandError) continue; // scope unavailable in this context + throw error; + } + const rel = path.relative(root, path.resolve(physicalPath)); + if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) return scope; + } + return null; + } + + /** + * Toggle a pin (Memory tab). Pins live in the sidecar, not the store, so + * nothing else emits a change or moves the store clock: for the workspace + * scope the sidecar write AND the clock advance happen under the store's + * mutation lock, so a lock timeout fails BEFORE anything is committed (no + * durable pin with a stale foreign hot set and a failed route), and the + * other tree members' tabs are told afterwards. Sidecar write failures + * surface as MemoryMetaWriteError. + */ + async setPinned(ctx: MemoryScopeContext, virtualPath: string, pinned: boolean): Promise { + const parsed = parseMemoryPath(virtualPath); + const scope = this.requireFilePath(parsed, virtualPath); + const key = this.logicalKeyFor(ctx, scope, parsed.relPath); + if (key === null) { + throw new MemoryCommandError( + "Project memory is unavailable: no project is associated with this session" + ); + } + if (scope === "workspace") { + const store = this.getStore(ctx, scope); + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + // Same commit guard as file mutations: `key` and `store` were bound to + // the owner the context resolved BEFORE the lock. If that owner was + // removed meanwhile (removal publishes its tombstone under this lock) + // or ownership moved, the pin would land under a dead logical key and + // the route would still report success. Refuse instead. + await this.assertMutationCommittable(ctx, store, undefined, virtualPath); + await this.metaService.setPinned(key, pinned); + // A pin changes the hot set other backends derive from this store. + await this.advanceStoreRevision(store); + }); + } else { + await this.metaService.setPinned(key, pinned); + } + this.emitChange(ctx, scope, parsed.relPath, "user"); + } + /** * Announces that a project's memory was mutated outside this service. The settings-backup * restore writes memory files directly (under the shared memory mutation lock), and @@ -958,7 +2544,7 @@ export class MemoryService extends EventEmitter { virtualPath: string, options?: { offset?: number; limit?: number } ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); if (parsed.scope === null) { // Virtual root: list every scope. @@ -967,9 +2553,11 @@ export class MemoryService extends EventEmitter { sections.push(`- ${scope}/`); try { const store = this.getStore(ctx, scope); + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); // Read-only: never create roots just to list (missing ⇒ empty). await store.assertRootSafe(); const files = await store.listFiles(); + await this.assertWorkspaceReadExposable(ctx, scope, store); sections.push(...renderTree(files, MEMORY_VIEW_MAX_DEPTH - 1, " ")); } catch (error) { // Self-healing: an unavailable scope must not break the whole view. @@ -986,6 +2574,7 @@ export class MemoryService extends EventEmitter { // write — but the scope itself always exists in the protocol. if (kind === "dir" || (kind === null && parsed.relPath === "")) { const files = await store.listFiles(); + await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); const prefix = parsed.relPath === "" ? "" : `${parsed.relPath}/`; const scopedFiles = files .filter((file) => file.startsWith(prefix)) @@ -1003,6 +2592,12 @@ export class MemoryService extends EventEmitter { const content = await this.readBoundedTextFile(store, parsed.relPath, virtualPath); const output = renderFileView(content, options); await this.recordUsage(ctx, parsed.scope, parsed.relPath, { write: false }); + // AFTER recordUsage — the last await before the content leaves: a + // read-side usage record waits for the owner-store lock, which a + // removal holds for its handover before publishing the tombstone and + // releasing; recordUsage's own refusal is swallowed (usage is + // best-effort), so it cannot stand in for this gate. + await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); return { success: true, output }; }); } @@ -1015,7 +2610,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); assertWithinFileSizeCap(fileText); @@ -1025,7 +2620,7 @@ export class MemoryService extends EventEmitter { // only INSIDE the lock and after the removal check (r62), so the // mkdir serializes with removal's locked deletion and cannot // recreate a removed session directory. - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.ensureRoot(); const existing = await store.kind(parsed.relPath); if (existing !== null) { @@ -1039,18 +2634,21 @@ export class MemoryService extends EventEmitter { `The ${scope} memory scope is full (${MEMORY_MAX_FILES_PER_SCOPE} files); delete unused files first` ); } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, fileText); - // Row is written before the create is acknowledged (mutation → row → ack). + // Usage stats land BEFORE the row advances the store clock: a foreign + // backend rebuilding its hot set on the new revision must already see + // them (see workspaceMemoryRevision.ts). Row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, + store, { op: "create", path: toVirtualPath(scope, parsed.relPath) }, { op: "delete-files", paths: [store.physicalPath(parsed.relPath)] }, actor, toolCallId, [{ path: store.physicalPath(parsed.relPath), content: fileText }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1069,7 +2667,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); if (oldStr.length === 0) { @@ -1080,11 +2678,13 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const updated = computeStrReplaceUpdate(content, oldStr, newStr, virtualPath); assertWithinFileSizeCap(updated); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); - // Row is written before the edit is acknowledged (mutation → row → ack). + // Usage stats before the row (store clock); row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, + store, { op: "str_replace", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1094,7 +2694,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: store.physicalPath(parsed.relPath), content: updated }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Edited ${toVirtualPath(scope, parsed.relPath)}` }; }); @@ -1111,7 +2710,7 @@ export class MemoryService extends EventEmitter { expectedFingerprint?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); @@ -1133,11 +2732,13 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const { updated, insertedLineCount } = computeInsertUpdate(content, insertLine, insertText); assertWithinFileSizeCap(updated); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); - // Row is written before the edit is acknowledged (mutation → row → ack). + // Usage stats before the row (store clock); row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, + store, { op: "insert", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1147,7 +2748,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: store.physicalPath(parsed.relPath), content: updated }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1185,7 +2785,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); if (mutation.command === "str_replace" && mutation.oldStr.length === 0) { @@ -1193,7 +2793,9 @@ export class MemoryService extends EventEmitter { } const store = await this.resolveStore(ctx, scope, parsed.relPath); return withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + // Root materialized only inside the lock and after the removal check + // (r62), like create(): for a sub-agent this is the OWNER's store. + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.ensureRoot(); const kind = await store.kind(parsed.relPath); if (kind === "dir") { @@ -1228,12 +2830,14 @@ export class MemoryService extends EventEmitter { mutation.insertText ).updated; assertWithinFileSizeCap(updated, maxFileBytes); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); const physicalPath = store.physicalPath(parsed.relPath); - // Row is written before the write is acknowledged (mutation → row → ack). + // Usage stats before the row (store clock); row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, + store, { op: mutation.command, path: toVirtualPath(scope, parsed.relPath) }, previous === null ? { op: "delete-files", paths: [physicalPath] } @@ -1242,7 +2846,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: physicalPath, content: updated }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1261,7 +2864,7 @@ export class MemoryService extends EventEmitter { | { command: "delete"; path: string } | { command: "rename"; path: string; new_path: string } ): Promise<{ ok: true } | { ok: false; error: string }> { - const result = await this.runCommand(async () => { + const result = await this.runCommand(ctx, async () => { const parsed = parseMemoryPath(command.path); const scope = this.requireFilePath(parsed, command.path); switch (command.command) { @@ -1374,9 +2977,10 @@ export class MemoryService extends EventEmitter { actor: MemoryActor, toolCallId?: string, expectedFingerprint?: string, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + options?: { rejectPinned?: boolean } ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); @@ -1385,6 +2989,9 @@ export class MemoryService extends EventEmitter { if (kind === null) { throw new MemoryCommandError(`No memory file or directory at ${virtualPath}`); } + if (options?.rejectPinned === true) { + await this.assertNotPinnedForRemoval(ctx, scope, parsed.relPath, virtualPath); + } // r55: staged refine deletes were approved against the target's // staging-time state — a target edited between staging and apply // must refuse rather than silently destroying the newer contents. @@ -1401,18 +3008,24 @@ export class MemoryService extends EventEmitter { // Prior contents must be captured before removal; the row itself is // written after the mutation succeeds and before it is acknowledged. const inverse = await this.captureDeleteInverse(store, parsed.relPath, kind); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.remove(parsed.relPath); + // Sidecar first, then the row advances the store clock (see create). + await this.recordDelete(ctx, scope, parsed.relPath); if (inverse !== null) { await this.journalRefinement( ctx, + store, { op: "delete", path: toVirtualPath(scope, parsed.relPath) }, inverse, actor, toolCallId ); + } else { + // Unjournaled delete (unrepresentable subtree): the store still + // changed, so other backends' cached views must still see it. + await this.advanceStoreRevision(store); } - await this.recordDelete(ctx, scope, parsed.relPath); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1428,9 +3041,10 @@ export class MemoryService extends EventEmitter { newVirtualPath: string, actor: MemoryActor, toolCallId?: string, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + options?: { rejectPinned?: boolean } ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const oldParsed = parseMemoryPath(oldVirtualPath); const newParsed = parseMemoryPath(newVirtualPath); const scope = this.requireFilePath(oldParsed, oldVirtualPath); @@ -1448,6 +3062,9 @@ export class MemoryService extends EventEmitter { if (oldKind === null) { throw new MemoryCommandError(`No memory file or directory at ${oldVirtualPath}`); } + if (options?.rejectPinned === true) { + await this.assertNotPinnedForRemoval(ctx, scope, oldParsed.relPath, oldVirtualPath); + } // Pre-flight (mirrored in validateMutation): store.rename would mkdir // the destination parent INSIDE the source before the filesystem // rejects the move — refuse cleanly instead of polluting the source. @@ -1463,11 +3080,14 @@ export class MemoryService extends EventEmitter { if (newKind !== null) { throw new MemoryCommandError(`Destination ${newVirtualPath} already exists`); } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, oldVirtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, oldVirtualPath); await store.rename(oldParsed.relPath, newParsed.relPath); + // Sidecar first, then the row advances the store clock (see create). + await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + store, { op: "rename", path: toVirtualPath(scope, oldParsed.relPath), @@ -1481,7 +3101,6 @@ export class MemoryService extends EventEmitter { actor, toolCallId ); - await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); this.emitChange(ctx, scope, oldParsed.relPath, actor); this.emitChange(ctx, scope, newParsed.relPath, actor); return { @@ -1546,6 +3165,7 @@ export class MemoryService extends EventEmitter { const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); + await this.assertWorkspaceReadExposable(ctx, scope, store); // Deliberately NOT recorded as a use: this is a human browsing the // Memory tab/settings, and usage stats must reflect agent reads only so // UI browsing never inflates hot-set ranking. (UI saves still count — @@ -1587,7 +3207,7 @@ export class MemoryService extends EventEmitter { async () => { // UI save can create new files: materialize the scope root on // first use — in-lock, after the removal check (r62; see create). - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.ensureRoot(); const kind = await store.kind(parsed.relPath); if (kind === "dir") { @@ -1614,9 +3234,12 @@ export class MemoryService extends EventEmitter { ); } } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); + // UI saves are not journaled, so advance the store clock here + // (in-lock, after the sidecar so foreign rebuilds see the stats). + await this.advanceStoreRevision(store); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, data: { sha256: sha256Hex(content) } }; } @@ -1642,8 +3265,17 @@ export class MemoryService extends EventEmitter { async listIndexEntries(ctx: MemoryScopeContext): Promise { const entries: MemoryIndexEntry[] = []; for (const scope of MEMORY_SCOPES) { + // Per-scope buffer: the scope's entries join the result only once the + // post-read gate below passed, so a tombstone published mid-enumeration + // drops the whole scope rather than a prefix of it. + const scopeEntries: MemoryIndexEntry[] = []; try { const store = this.getStore(ctx, scope); + // Prompt context is a read of the (possibly shared) store: a removed + // child's stream in another backend must not keep indexing / hot-set + // reading its former owner's notes. Refused here (skipped below) like + // any other scope failure. + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); // Read-only enumeration (stream startup, Memory tab) must not create // scope roots unnecessarily. Missing roots list as empty. await store.assertRootSafe(); @@ -1691,8 +3323,10 @@ export class MemoryService extends EventEmitter { } catch { // Unreadable file: list it without a description. } - entries.push({ path: toVirtualPath(scope, relPath), scope, relPath, description }); + scopeEntries.push({ path: toVirtualPath(scope, relPath), scope, relPath, description }); } + await this.assertWorkspaceReadExposable(ctx, scope, store); + entries.push(...scopeEntries); } catch (error) { log.debug("[MemoryService] skipping scope in memory index", { scope, error }); } @@ -1726,24 +3360,41 @@ export class MemoryService extends EventEmitter { lastAccessedAt: stats?.lastAccessedAt ?? null, }; }); - return selectHotMemories({ + const selected = await selectHotMemories({ candidates, countTokens: options.countTokens, tokenBudgetActive: options.tokenBudgetActive, onlyContextNotes: options.onlyContextNotes, - readFile: (virtualPath) => { + readFile: async (virtualPath) => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); // Paths come from listIndexEntries (already enumerated under the scope // roots), so no extra containment walk is needed for these reads. // Bounded prefix: selection truncates to MEMORY_HOT_SET_MAX_ITEM_BYTES // anyway; +1 byte preserves its over-budget (truncation marker) check. - return this.getStore(ctx, scope).readFilePrefix( + const store = this.getStore(ctx, scope); + const content = await store.readFilePrefix( parsed.relPath, MEMORY_HOT_SET_MAX_ITEM_BYTES + 1 ); + await this.assertWorkspaceReadExposable(ctx, scope, store); + return content; }, }); + // Selection keeps awaiting (token counting, repeatedly) after the last + // per-file gate: a tombstone published meanwhile must still withhold the + // buffered owner notes. Final check once selection is done; the workspace + // items are dropped (the scope reads as unavailable, like in the index). + const isWorkspaceItem = (item: MemoryHotSetItem): boolean => + parseMemoryPath(item.path).scope === "workspace"; + if (selected.some(isWorkspaceItem)) { + try { + await this.assertWorkspaceStoreReadable(ctx, this.getStore(ctx, "workspace")); + } catch { + return selected.filter((item) => !isWorkspaceItem(item)); + } + } + return selected; } } @@ -1807,44 +3458,6 @@ function sha256Hex(content: string): string { return createHash("sha256").update(content, "utf-8").digest("hex"); } -/** - * Refuse to COMMIT a mutation whose caller was torn down (r59/r61). Checked - * INSIDE the target mutation lock immediately before the first durable - * write; a mutation that already committed always journals (mutation → row - * → ack) so rollback lineage stays intact. Two teardown signals: - * - * - The caller's abort signal (r59): consolidation/refine passes receive no - * hard tool cancellation — an execution wedged in pre-commit I/O (e.g. a - * named pipe under a memory root) is detached by the caller's bounded - * drain, and once the I/O unblocks after workspace teardown it would - * still write durable memory AND append its refinement journal row into - * the deleted session directory, recreating it. - * - The durable removal tombstone (r61): with multiple backends over one - * Xum root, the remover cannot abort a dream/harvest run in ANOTHER - * process — that run's signal stays live after removal. The tombstone is - * published under the same memory target locks this check runs inside - * (see workspaceRemoval.ts), so a foreign backend's mutation observes - * removal here at commit time and refuses instead of recreating the - * deleted session directory via its write or journal append. - */ -async function assertMutationCommittable( - rootDir: string, - ctx: MemoryScopeContext, - signal: AbortSignal | undefined, - virtualPath: string -): Promise { - if (signal?.aborted === true) { - throw new MemoryCommandError( - `Mutation of ${virtualPath} was cancelled before commit (caller torn down)` - ); - } - if (ctx.workspaceId !== "" && (await isWorkspaceRemovalTombstoned(rootDir, ctx.workspaceId))) { - throw new MemoryCommandError( - `Workspace ${ctx.workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` - ); - } -} - /** * Deterministic, lenient hash of a physical subtree for delete-target change * detection (r55/r58, see MemoryService.fingerprintMutationTarget). Sorted walk; diff --git a/src/node/services/memoryWorkspaceOwner.test.ts b/src/node/services/memoryWorkspaceOwner.test.ts new file mode 100644 index 00000000000..af26d03554d --- /dev/null +++ b/src/node/services/memoryWorkspaceOwner.test.ts @@ -0,0 +1,102 @@ +import { describe, expect, it } from "bun:test"; +import type { Config } from "@/node/config"; +import { + pinDescendantWorkspaceMemoryOwners, + resolveWorkspaceMemoryOwnerId, +} from "./memoryWorkspaceOwner"; + +type ProjectsConfig = ReturnType; + +function topology( + workspaces: Array<{ id: string; parentWorkspaceId?: string; memoryOwnerWorkspaceId?: string }> +): ProjectsConfig { + return { + projects: new Map([ + ["/tmp/project", { workspaces: workspaces.map((ws) => ({ path: `/tmp/${ws.id}`, ...ws })) }], + ]), + } as unknown as ProjectsConfig; +} + +describe("pinDescendantWorkspaceMemoryOwners", () => { + it("pins each surviving child to the owner it resolves to now", () => { + const cfg = topology([ + { id: "ws-owner" }, + { id: "ws-other" }, + { id: "ws-mid", parentWorkspaceId: "ws-owner" }, + // No pin: the walk through ws-mid reaches ws-owner. + { id: "ws-plain", parentWorkspaceId: "ws-mid" }, + // Stale pin (its owner is gone): the resolver walks past it today, but + // once ws-mid is removed that walk would dangle — replaced. + { id: "ws-stale", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-gone" }, + // Pin to another live notebook while the parent is still registered: + // a state this code never writes (pins are recorded as an ancestor is + // removed). The live chain wins — the child has been using ws-owner's + // notebook — and the removal re-pins it to that (r84), rather than + // letting corrupt raw config redirect it across task trees. + { id: "ws-pinned", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-other" }, + // Not a child of the removed node: untouched. + { id: "ws-sibling", parentWorkspaceId: "ws-owner" }, + ]); + const before = Object.fromEntries( + ["ws-plain", "ws-stale", "ws-pinned"].map((id) => [ + id, + resolveWorkspaceMemoryOwnerId(cfg, id), + ]) + ); + expect(before).toEqual({ + "ws-plain": "ws-owner", + "ws-stale": "ws-owner", + "ws-pinned": "ws-owner", + }); + + const pinned = pinDescendantWorkspaceMemoryOwners(cfg, "ws-mid"); + expect(Object.fromEntries(pinned)).toEqual(before); + const entries = [...cfg.projects.values()][0].workspaces; + const pinOf = (id: string) => entries.find((ws) => ws.id === id)!.memoryOwnerWorkspaceId; + expect(pinOf("ws-plain")).toBe("ws-owner"); + expect(pinOf("ws-stale")).toBe("ws-owner"); + expect(pinOf("ws-pinned")).toBe("ws-owner"); + expect(pinOf("ws-sibling")).toBeUndefined(); + + // With ws-mid gone, every pinned child still resolves as before. + const after = topology( + entries + .filter((ws) => ws.id !== "ws-mid") + .map((ws) => ({ + id: ws.id!, + ...(ws.parentWorkspaceId === undefined + ? {} + : { parentWorkspaceId: ws.parentWorkspaceId }), + ...(ws.memoryOwnerWorkspaceId === undefined + ? {} + : { memoryOwnerWorkspaceId: ws.memoryOwnerWorkspaceId }), + })) + ); + for (const [id, owner] of Object.entries(before)) { + expect(resolveWorkspaceMemoryOwnerId(after, id)).toBe(owner); + } + }); + + it("honors a pin only once the recorded parent is gone", () => { + const live = topology([ + { id: "ws-owner" }, + { id: "ws-other" }, + { id: "ws-child", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-other" }, + { id: "ws-grand", parentWorkspaceId: "ws-child" }, + ]); + // Parent registered: the chain decides, for the child and everything below it. + expect(resolveWorkspaceMemoryOwnerId(live, "ws-child")).toBe("ws-owner"); + expect(resolveWorkspaceMemoryOwnerId(live, "ws-grand")).toBe("ws-owner"); + // Parent gone: the (live) pin decides; a pin whose owner is gone too + // leaves the child on its own store. + const dangling = topology([ + { id: "ws-other" }, + { id: "ws-child", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-other" }, + { id: "ws-grand", parentWorkspaceId: "ws-child" }, + { id: "ws-orphan", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-gone" }, + ]); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-child")).toBe("ws-other"); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-grand")).toBe("ws-other"); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-orphan")).toBe("ws-orphan"); + }); +}); diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts new file mode 100644 index 00000000000..5e23f9289d4 --- /dev/null +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -0,0 +1,187 @@ +import * as path from "node:path"; +import assert from "@/common/utils/assert"; +import type { Config, Workspace as WorkspaceConfigEntry } from "@/node/config"; +import { log } from "@/node/services/log"; + +type ProjectsConfig = ReturnType; + +/** + * The workspace whose /memory backs `/memories/workspace` for + * `workspaceId`: the root of its parentWorkspaceId chain. Sub-agents (and + * nested sub-agents) thereby share ONE notebook with the workspace that + * spawned the task tree, while their transcripts/session artifacts stay + * separate. Full `kind: "workspace"` tasks and forks carry no + * parentWorkspaceId and own their notes. + * + * Unknown IDs, dangling parents, cycles, and depth overflow resolve to the ID + * itself so a misconfigured tree degrades to per-workspace behavior instead + * of failing every memory command. Callers that need "is this a shared + * child?" must compare the result to the input rather than test + * parentWorkspaceId, so those fallbacks keep their private store usable. + * + * Pure over one config snapshot (indexed once per snapshot, see + * workspaceMemoryOwnerResolver); MemoryService memoizes it, removal and the + * rollback tooling call it directly. + */ +export function resolveWorkspaceMemoryOwnerId(cfg: ProjectsConfig, workspaceId: string): string { + return workspaceMemoryOwnerResolver(cfg)(workspaceId); +} + +/** + * Per-snapshot resolvers keyed by the config object: bulk passes (memo + * revalidation on every local config edit, launch sweep, change diffing) + * resolve many workspaces against one snapshot, and a linear + * findWorkspaceEntry per chain hop would make them O(n²) on the main process. + * The index is built once per snapshot and the snapshot is never mutated + * after load, so a WeakMap keyed by identity is safe. + */ +const resolversBySnapshot = new WeakMap string>(); + +export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: string) => string { + const cached = resolversBySnapshot.get(cfg); + if (cached !== undefined) return cached; + const byId = new Map(); + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.id !== undefined) byId.set(workspace.id, workspace); + } + } + const resolver = (workspaceId: string): string => { + assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); + let current = workspaceId; + const visited = new Set(); + for (let depth = 0; depth < 32; depth++) { + if (visited.has(current)) { + log.warn("[memory] parentWorkspaceId cycle; using acting workspace as memory owner", { + workspaceId, + }); + return workspaceId; + } + visited.add(current); + const entry = byId.get(current); + if (entry === undefined) { + // Only the chain root may be unknown without invalidating the walk: an + // unregistered starting workspace simply resolves to itself. + if (current !== workspaceId) { + log.warn("[memory] parentWorkspaceId points at an unknown workspace", { + workspaceId, + parentWorkspaceId: current, + }); + } + return workspaceId; + } + // A pinned owner is recorded when an intermediate ancestor is removed + // (pinDescendantWorkspaceMemoryOwners), so it only speaks for a chain + // that DANGLES: while the recorded parent is still registered the walk + // follows it, and a pin that disagrees with a live parent (raw config, + // never produced by this code) heals on the next removal instead of + // redirecting the child into an unrelated tree's notebook. With the + // parent gone, a live pin decides; a pin whose owner is gone too leaves + // the child on its own store. + const parentWorkspaceId = entry.parentWorkspaceId; + const parentLive = + parentWorkspaceId !== undefined && parentWorkspaceId !== "" && byId.has(parentWorkspaceId); + if (!parentLive) { + const pinned = entry.memoryOwnerWorkspaceId; + if (pinned !== undefined && pinned !== "" && byId.has(pinned)) return pinned; + } + if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; + current = parentWorkspaceId; + } + log.warn("[memory] parentWorkspaceId chain too deep; using acting workspace as memory owner", { + workspaceId, + }); + return workspaceId; + }; + resolversBySnapshot.set(cfg, resolver); + return resolver; +} + +/** + * Removal of `removedWorkspaceId`: pin each surviving direct child to the + * owner it resolves to NOW, so the notebook it uses stays the same once the + * chain through the removed node dangles. The pin is whatever the walk + * resolves to while the node is still registered — an existing pin is + * overwritten by it (a live parent takes precedence over a pin in the + * resolver, so that IS the notebook the child has been using), and a stale + * one (its owner gone) is replaced likewise. Mutates the entries in place; + * returns the pins written, for the caller's verified read-back. + */ +export function pinDescendantWorkspaceMemoryOwners( + cfg: ProjectsConfig, + removedWorkspaceId: string +): Map { + assert(removedWorkspaceId.length > 0, "pinDescendantWorkspaceMemoryOwners requires an id"); + const resolve = workspaceMemoryOwnerResolver(cfg); + const pinned = new Map(); + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.parentWorkspaceId !== removedWorkspaceId || workspace.id === undefined) { + continue; + } + // Resolved before this loop mutates anything: every child's chain runs + // through the removed node, never through a sibling being pinned. + const owner = resolve(workspace.id); + workspace.memoryOwnerWorkspaceId = owner; + pinned.set(workspace.id, owner); + } + } + return pinned; +} + +/** + * Session dirs of the OTHER registered members of `workspaceId`'s task tree — + * every workspace resolving to the same owner (the owner itself, siblings, + * descendants). They journal their own mutations of the shared + * `/memories/workspace` store, so a rollback in one member must consult all + * of them for later conflicting rows (refinementRollback.ts). Empty for a + * workspace that owns its store alone. + */ +export function sharedWorkspaceMemoryPeerSessionDirs( + cfg: ProjectsConfig, + sessionsDir: string, + workspaceId: string +): string[] { + assert(sessionsDir.length > 0, "sharedWorkspaceMemoryPeerSessionDirs requires sessionsDir"); + const resolve = workspaceMemoryOwnerResolver(cfg); + const owner = resolve(workspaceId); + const peers: string[] = []; + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.id === undefined || workspace.id === workspaceId) continue; + if (resolve(workspace.id) === owner) peers.push(path.join(sessionsDir, workspace.id)); + } + } + return peers; +} + +/** Owner root and live peers of a workspace sharing its notebook (rollback). */ +export interface SharedWorkspaceMemoryTopology { + /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ + ownerSessionDir: string | undefined; + /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ + peerSessionDirs: string[]; +} + +/** + * The rollback topology from ONE config snapshot that must prove itself: + * `loadExistingConfigOrThrow` throws when config.json is unreadable OR + * absent (mid-rewrite), so callers refuse instead of degrading to the + * fresh-install view in which the workspace owns its notebook — that "self" + * fallback would omit the owner root (a pre-sharing row's inverse then lands + * on the hidden legacy notebook instead of the owner's adopted copy) and the + * peer list (conflicting sibling rows go unseen). Peers are re-resolved per + * check by the engine (a member registered while waiting for the store lock + * must count), so callers hand it a callback that calls this again. + */ +export function resolveSharedWorkspaceMemoryTopology( + config: Pick, + workspaceId: string +): SharedWorkspaceMemoryTopology { + const cfg = config.loadExistingConfigOrThrow(); + const ownerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + return { + ownerSessionDir: ownerId === workspaceId ? undefined : path.join(config.sessionsDir, ownerId), + peerSessionDirs: sharedWorkspaceMemoryPeerSessionDirs(cfg, config.sessionsDir, workspaceId), + }; +} diff --git a/src/node/services/refinement/refinementJournal.test.ts b/src/node/services/refinement/refinementJournal.test.ts index 76145747e59..72949c2c1a5 100644 --- a/src/node/services/refinement/refinementJournal.test.ts +++ b/src/node/services/refinement/refinementJournal.test.ts @@ -60,6 +60,53 @@ describe("reclaimExcessRefinementInverseBlobs", () => { deleteSpy.mockRestore(); }); + test("the recovery sweep ranks unordered migrated rows behind the owner's dated rows", async () => { + using tmp = new DisposableTempDir("refinement-journal-test"); + const journal = new DurableEventJournal(tmp.path); + // Owner rows carry the shared store clock as sourceTs; the migrated row + // (a removed child's pre-sharing history, retargeted, private clock + // dropped) is appended LAST — its envelope `ts` is the newest of all. + const appendRow = async ( + content: string, + extra: { sourceTs?: number; migratedFrom?: string; orderUnknown?: true } + ): Promise => + await journal.withBlobLock(async () => { + const { ref } = await journal.blobs.put(content); + await journal.append({ + workspaceId: "ws-owner", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/n.md" }, + inverse: { op: "restore-files", files: [{ path: "/m/n.md", blobRef: ref }] }, + evidence: { workspaceId: "ws-owner", toolName: "test" }, + ...extra, + }, + }); + return ref; + }); + const ownerOld = await appendRow("owner-old", { sourceTs: 10 }); + const ownerNew = await appendRow("owner-new", { sourceTs: 20 }); + const migrated = await appendRow("child-legacy", { + migratedFrom: "ws-child:row-1", + orderUnknown: true, + }); + // Each payload charged at 0.4x quota: only two survive the sweep. + const size = spyOn(journal.blobs, "size").mockResolvedValue( + Math.ceil(REFINEMENT_INVERSE_BLOB_QUOTA_BYTES * 0.4) + ); + try { + await reclaimExcessRefinementInverseBlobs(journal, [], { resweep: true }); + } finally { + size.mockRestore(); + } + // The owner's genuinely recent payloads stay; the unordered migrated + // history is evicted first despite its newer append time. + expect(await journal.blobs.has(ownerOld)).toBe(true); + expect(await journal.blobs.has(ownerNew)).toBe(true); + expect(await journal.blobs.has(migrated)).toBe(false); + }); + test("a payload hash shared with another event kind survives eviction", async () => { using tmp = new DisposableTempDir("refinement-journal-test"); const journal = new DurableEventJournal(tmp.path); diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 522bbff3102..b9a9de49fb2 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -18,6 +18,7 @@ import { createHash } from "node:crypto"; import assert from "@/common/utils/assert"; +import { isValidSourceClock } from "@/common/types/durableEvent"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES, REFINEMENT_INVERSE_QUOTA_MIN_CHARGE_BYTES, @@ -26,6 +27,7 @@ import { type RefinementEvidence, type RefinementInverse, type RefinementPostState, + type RollbackRefinementAction, type SkillRefinementAction, } from "@/common/types/refinement"; import type { BlobStore } from "@/node/utils/journal/blobStore"; @@ -48,12 +50,28 @@ export interface RefinementFileCapture { content: string; } +/** + * A payload reference carried over as-is instead of content: shared-memory row + * migration uses it for a row whose inverse blob was already reclaimed in the + * source journal. The copied row keeps its paths and order for conflict + * detection (an audit record), and the rollback engine refuses it exactly as + * it refuses any evicted payload — the ref resolves to nothing. + */ +export interface RefinementFileReference { + path: string; + blobRef: string; +} + /** Inverse draft with captured contents inline; blob offload happens at append. */ export type RefinementInverseDraft = | { op: "delete-files"; paths: string[] } // deletePaths (r67): mixed force-apply pre-state — restore `files` AND // delete the paths the forced rollback created (see RefinementInverseSchema). - | { op: "restore-files"; files: RefinementFileCapture[]; deletePaths?: string[] } + | { + op: "restore-files"; + files: Array; + deletePaths?: string[]; + } | { op: "rename"; from: string; to: string }; export interface RefinementEmitArgs { @@ -61,7 +79,7 @@ export interface RefinementEmitArgs { sessionDir: string; workspaceId: string; kind: "memory" | "skill"; - action: MemoryRefinementAction | SkillRefinementAction; + action: MemoryRefinementAction | SkillRefinementAction | RollbackRefinementAction; inverse: RefinementInverseDraft; evidence: { toolName: string; toolCallId?: string; actor?: string }; /** @@ -70,6 +88,35 @@ export interface RefinementEmitArgs { * `postState` so rollback can detect out-of-band edits content-exactly. */ postFiles?: RefinementFileCapture[]; + /** + * Already-hashed post state, for re-appending an existing row whose + * contents are no longer available (shared-memory row migration). Ignored + * when `postFiles` is given. + */ + postState?: RefinementPostState; + /** Source identity of a row copied from a removed sub-agent's journal (see durableEvent.ts). */ + migratedFrom?: string; + /** + * For a migrated ROLLBACK row: the owner-journal id of the row it rolled + * back (the copy of its source target), so the lineage stays intact on the + * owner side. Only shared-memory row migration sets this; the rollback + * engine appends its own rows directly. + */ + rollbackOf?: string; + /** + * Cross-journal order key (see durableEvent.ts): the shared store's clock + * for a workspace-scope mutation (workspaceMemoryRevision.ts), or the + * source row's value/`ts` for a migrated row. + */ + sourceTs?: number; + /** See the durable event schema: the store clock write failed for this row. */ + orderUnknown?: true; + /** + * Original journal position of a migrated row (see the durable event + * schema); only shared-memory row migration sets these, together. + */ + originJournal?: string; + originSeq?: number; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -115,6 +162,9 @@ export async function resolveRefinementInverse( const publishedBlobs: BlobQuotaEntry[] = []; const files = await Promise.all( draft.files.map(async (file) => { + // A bare reference (payload already gone at the source) is neither + // stored nor quota-charged: there is nothing to retain or reclaim. + if (!("content" in file)) return { path: file.path, blobRef: file.blobRef }; const { ref, size } = await blobs.put(file.content); publishedBlobs.push({ ref, size: inverseQuotaCharge(size) }); return { path: file.path, blobRef: ref }; @@ -165,7 +215,17 @@ const reclamationStates = new WeakMap { await journal.withBlobLock(async () => { let state = reclamationStates.get(journal); @@ -179,13 +239,39 @@ export async function reclaimExcessRefinementInverseBlobs( // foreign CLI appended and must be re-derived from the journal. const epoch = journal.blobIndexEpoch; let entries: BlobQuotaEntry[]; - if (state.retainedInverseBlobs !== null && state.retainedEpoch === epoch) { + if ( + state.retainedInverseBlobs !== null && + state.retainedEpoch === epoch && + options?.resweep !== true + ) { entries = [...published, ...state.retainedInverseBlobs]; } else { - // Recovery sweep: walk refinement rows newest-first and re-derive the - // retained set. Rows never recorded payload sizes, so stat the blobs; - // a missing blob was already evicted (or never landed) — skip it. - const events = await journal.read(); + // Recovery sweep: walk refinement rows newest-first — by SOURCE time + // (`data.sourceTs ?? ts`, append sequence as tie-breaker), so migrated + // rows sit at their real chronological position — and re-derive the + // retained set. A migrated row WITHOUT a source time (pre-sharing or + // private-clock history; the migration omits the incomparable value and + // marks it orderUnknown) has only its append-time `ts`, which would rank + // that old history as the newest and let it evict the owner's genuinely + // recent payloads: rank it behind every dated row instead (evicted + // first). Rows never recorded payload sizes, so stat the blobs; a + // missing blob was already evicted (or never landed) — skip it. + const retentionTs = (event: { + ts: number; + data: { sourceTs?: number; migratedFrom?: string }; + }): number => + isValidSourceClock(event.data.sourceTs) + ? event.data.sourceTs + : event.data.migratedFrom !== undefined + ? Number.NEGATIVE_INFINITY + : event.ts; + const events = (await journal.read()) + .filter((event) => event.kind === "refinement") + .sort((left, right) => { + const leftTs = retentionTs(left); + const rightTs = retentionTs(right); + return leftTs !== rightTs ? (leftTs < rightTs ? -1 : 1) : left.seq - right.seq; + }); entries = []; for (let i = events.length - 1; i >= 0; i--) { const event = events[i]; @@ -228,55 +314,7 @@ export async function reclaimExcessRefinementInverseBlobs( */ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { try { - assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); - assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); - const journal = sharedDurableEventJournal(args.sessionDir); - // Inverse blob puts and the append referencing them run under the journal - // blob lock: a concurrent reclamation pass must never observe the - // put→append window (see DurableEventJournal.withBlobLock). - let publishedBlobs: BlobQuotaEntry[] = []; - await journal.withBlobLock(async () => { - const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); - const inverse = resolved.inverse; - publishedBlobs = resolved.publishedBlobs; - // Optional fields are spread conditionally: an explicit `undefined` value - // would fail the JsonValue schema validation on append and drop the row. - const evidence: RefinementEvidence = { - workspaceId: args.workspaceId, - toolName: args.evidence.toolName, - ...(args.evidence.toolCallId !== undefined ? { toolCallId: args.evidence.toolCallId } : {}), - ...(args.evidence.actor !== undefined ? { actor: args.evidence.actor } : {}), - }; - const postState: RefinementPostState | undefined = - args.postFiles !== undefined - ? { - files: args.postFiles.map((file) => ({ - path: file.path, - sha256: sha256Hex(file.content), - })), - } - : undefined; - await journal.append({ - workspaceId: args.workspaceId, - kind: "refinement", - data: { - kind: args.kind, - action: args.action, - inverse, - evidence, - ...(postState !== undefined ? { postState } : {}), - ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), - }, - }); - }); - // Bound retained inverse payloads per session AFTER releasing the publish - // lock (reclaim takes it itself; the mutex is non-reentrant). Best-effort: - // failure must never fail the mutation this row describes. - try { - await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs); - } catch (error) { - log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); - } + await appendRefinementEventOrThrow(args); } catch (error) { log.debug("[refinement] failed to journal refinement event; continuing", { kind: args.kind, @@ -286,6 +324,101 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { + assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); + const journal = sharedDurableEventJournal(args.sessionDir); + // Inverse blob puts and the append referencing them run under the journal + // blob lock: a concurrent reclamation pass must never observe the + // put→append window (see DurableEventJournal.withBlobLock). + const { publishedBlobs } = await journal.withBlobLock(() => + appendRefinementEventUnderBlobLock(journal, args) + ); + // Live rows may carry a store-clock `sourceTs` too (MemoryService); only + // MIGRATED rows are appended out of order and need the source-order resweep. + await reclaimRefinementInverseBlobsBestEffort(journal, publishedBlobs, { + resweep: args.migratedFrom !== undefined, + }); +} + +/** + * The locked leg of appendRefinementEventOrThrow, for callers that batch + * several appends (plus their own journal-state checks) under ONE + * `journal.withBlobLock` section — shared-memory row migration dedups + * against the owner journal and copies every row inside the same hold, so + * neither a concurrent duplicate migration nor a reclamation pass can + * interleave. The caller MUST hold `journal`'s blob lock (asserted) and MUST + * hand the returned payload entries to + * reclaimRefinementInverseBlobsBestEffort after releasing it. + */ +export async function appendRefinementEventUnderBlobLock( + journal: DurableEventJournal, + args: RefinementEmitArgs +): Promise<{ rowId: string; publishedBlobs: BlobQuotaEntry[] }> { + assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); + await journal.assertBlobLockOwned(); + const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); + const inverse = resolved.inverse; + // Optional fields are spread conditionally: an explicit `undefined` value + // would fail the JsonValue schema validation on append and drop the row. + const evidence: RefinementEvidence = { + workspaceId: args.workspaceId, + toolName: args.evidence.toolName, + ...(args.evidence.toolCallId !== undefined ? { toolCallId: args.evidence.toolCallId } : {}), + ...(args.evidence.actor !== undefined ? { actor: args.evidence.actor } : {}), + }; + const postState: RefinementPostState | undefined = + args.postFiles !== undefined + ? { + files: args.postFiles.map((file) => ({ + path: file.path, + sha256: sha256Hex(file.content), + })), + } + : args.postState; + const row = await journal.append({ + workspaceId: args.workspaceId, + kind: "refinement", + data: { + kind: args.kind, + action: args.action, + inverse, + evidence, + ...(postState !== undefined ? { postState } : {}), + ...(args.migratedFrom !== undefined ? { migratedFrom: args.migratedFrom } : {}), + ...(args.rollbackOf !== undefined ? { rollbackOf: args.rollbackOf } : {}), + ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), + ...(args.orderUnknown === true ? { orderUnknown: true } : {}), + ...(args.originJournal !== undefined && args.originSeq !== undefined + ? { originJournal: args.originJournal, originSeq: args.originSeq } + : {}), + ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), + }, + }); + return { rowId: row.id, publishedBlobs: resolved.publishedBlobs }; +} + +/** + * Bound retained inverse payloads per session AFTER the publish lock is + * released (reclaim takes it itself; the mutex is non-reentrant). Best-effort: + * failure must never fail the mutation the row(s) describe. + */ +export async function reclaimRefinementInverseBlobsBestEffort( + journal: DurableEventJournal, + publishedBlobs: BlobQuotaEntry[], + options: { resweep: boolean } +): Promise { + try { + await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs, options); + } catch (error) { + log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); + } +} + /** * Tool-side convenience wrapper: resolves the session journal from the tool * configuration. Skips (log-only) when the tool runs without a workspace diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 2c0cfa06d35..b10fe891150 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -6,6 +6,12 @@ import * as path from "node:path"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; import { Config } from "@/node/config"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, + type LegacyAdoptionRecord, +} from "@/node/services/memoryLegacyAdoption"; import { MemoryMetaService } from "@/node/services/memoryMeta"; import { MemoryService, type MemoryScopeContext } from "@/node/services/memoryService"; import { TestTempDir } from "@/node/services/tools/testHelpers"; @@ -873,6 +879,459 @@ describe("refinementRollback", () => { expect(await pathExists(path.join(fixture.muxHome, "memory", "global", "new.md"))).toBe(false); }); + it("retargets pre-sharing workspace rows to the adopted owner copy, refusing unadopted ones", async () => { + using fixture = await createFixture(); + // Rows journaled while the workspace owned its store address + // /memory (the legacy private notebook after an upgrade). + await fixture.service.create(fixture.ctx, "/memories/workspace/note.md", "v1\n", "agent"); + const createRow = await lastRow(fixture.sessionDir); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/note.md", + "v1", + "v2", + "agent" + ); + const editRow = await lastRow(fixture.sessionDir); + await fixture.service.create(fixture.ctx, "/memories/workspace/orphan.md", "o1\n", "agent"); + const orphanRow = await lastRow(fixture.sessionDir); + // The upgrade folded note.md into the task-tree owner's store (adoption + // manifest beside the legacy files); orphan.md could not be placed. + const ownerSessionDir = path.join(path.dirname(fixture.sessionDir), "ws-owner"); + const ownerRoot = path.join(ownerSessionDir, "memory"); + // Created records carry the generation of the copy the adoption wrote + // (LegacyAdoptionRecord.targetStamp); stamped from the file as it is now. + const writeManifest = async (records: Record) => { + for (const record of Object.values(records)) { + if ( + record.created === true && + record.deleted !== true && + record.targetStamp === undefined + ) { + record.targetStamp = + (await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/")))) ?? + undefined; + } + } + await fsPromises.writeFile( + legacyAdoptionManifestPath(fixture.sessionDir), + JSON.stringify(records) + ); + }; + await fsPromises.mkdir(path.join(ownerRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v2\n"); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + }); + // The retargeted row was journaled against this session's private clock: + // against an overlapping row of the OWNER's journal its order is unknown, + // so the rollback is refused unless forced. + await sharedDurableEventJournal(ownerSessionDir).append({ + workspaceId: "ws-owner", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/sub/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "sub", "note.md"), text: "v2\n" }], + }, + sourceTs: 1, + }, + }); + const unordered = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + }); + expect(unordered.success).toBe(false); + expect(unordered.success ? "" : unordered.error).toContain( + "order relative to this row is unknown" + ); + const result = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(result.success).toBe(true); + // The note the shared notebook serves is reverted; the hidden legacy file + // is left alone (it must keep matching the manifest, or the next adoption + // pass would re-import it as a conflicting duplicate). + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "sub", "note.md"), "utf-8") + ).toBe("v1\n"); + expect( + await fsPromises.readFile(path.join(fixture.sessionDir, "memory", "note.md"), "utf-8") + ).toBe("v2\n"); + // The rewrite is this lineage's own: the record is re-stamped to the new + // generation (r74), so the child's create row still maps onto the copy... + const restamped = ( + await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir)) + ).get("note.md")!; + expect(restamped.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "sub", "note.md"))) ?? undefined + ); + // ...but not after the owner replaced the copy outside the child's rows + // (a Memory-tab save is unjournaled and may keep the very same bytes): + // the file is the owner's now, and the create's delete-files is refused. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(ownerRoot, "sub", "note.md")); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v1\n"); + const replaced = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: createRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(replaced.success).toBe(false); + expect(replaced.success ? "" : replaced.error).toContain("since replaced"); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v1\n"); + const refused = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: orphanRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "not folded into the shared workspace store" + ); + expect(await pathExists(path.join(fixture.sessionDir, "memory", "orphan.md"))).toBe(true); + // A note the owner already had (adoption created nothing, `created` + // unset): the child's create row must not delete the owner's own file. + await fixture.service.create(fixture.ctx, "/memories/workspace/same.md", "same\n", "agent"); + const sameRow = await lastRow(fixture.sessionDir); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "same.md"), "same\n"); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + }); + const ownerOwned = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: sameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(ownerOwned.success).toBe(false); + expect(ownerOwned.success ? "" : ownerOwned.error).toContain("owner's own note"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "same.md"))).toBe(true); + // A note deleted on the downgraded build and reconciled out of the shared + // store keeps a tombstoned record: the child's pre-sharing delete row + // still maps, and rolling it back restores the note in the shared store. + await fixture.service.create(fixture.ctx, "/memories/workspace/gone.md", "g1\n", "agent"); + const goneCreateRow = await lastRow(fixture.sessionDir); + await fixture.service.deletePath(fixture.ctx, "/memories/workspace/gone.md", "agent"); + const deleteRow = await lastRow(fixture.sessionDir); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + }); + const restoredDelete = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: deleteRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(restoredDelete.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "gone.md"), "utf-8") + ).toBe("g1\n"); + // The recreated copy is stamped onto the tombstoned record, so the child's + // history over that note keeps unwinding: the create row now maps too. + expect( + (await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir))).get( + "gone.md" + )!.targetStamp + ).toBe((await adoptionTargetStamp(path.join(ownerRoot, "gone.md"))) ?? undefined); + const undoneGoneCreate = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: goneCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneGoneCreate.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "gone.md"))).toBe(false); + // A pre-sharing DIRECTORY rename: the manifest records files only, so the + // directory endpoints map through their adopted descendants (all landed + // at their own relPath as this adoption's copies). + await fixture.service.create(fixture.ctx, "/memories/workspace/olddir/a.md", "a\n", "agent"); + const olddirCreateRow = await lastRow(fixture.sessionDir); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/olddir", + "/memories/workspace/newdir", + "agent" + ); + const dirRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "newdir"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "a.md"), "a\n"); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + "newdir/a.md": { content: "x", sidecar: "", target: "newdir/a.md", created: true }, + "olddir/a.md": { + content: "x", + sidecar: "", + target: "olddir/a.md", + created: true, + deleted: true, + }, + }); + // An owner note added beside the adopted copies (no refinement row of + // its own) would travel with a structural rename: refused until the + // subtree is exactly the adopted descendants again. + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "owner.md"), "o\n"); + const extraFile = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(extraFile.success).toBe(false); + expect(extraFile.success ? "" : extraFile.error).toContain( + "not folded into the shared workspace store" + ); + await fsPromises.rm(path.join(ownerSessionDir, "memory", "newdir", "owner.md")); + // The same note landing while the rollback waits for the target lock: + // the plan-time directory proof is re-derived under the lock. + const lateExtraFile = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + testOnlyBeforeTargetLock: async () => { + await fsPromises.writeFile( + path.join(ownerSessionDir, "memory", "newdir", "late.md"), + "l\n" + ); + }, + }); + expect(lateExtraFile.success).toBe(false); + expect(lateExtraFile.success ? "" : lateExtraFile.error).toContain( + "not folded into the shared workspace store" + ); + expect(await pathExists(path.join(ownerSessionDir, "memory", "newdir", "late.md"))).toBe(true); + expect(await pathExists(path.join(ownerSessionDir, "memory", "olddir"))).toBe(false); + await fsPromises.rm(path.join(ownerSessionDir, "memory", "newdir", "late.md")); + const undoneRename = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneRename.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "olddir", "a.md"), "utf-8") + ).toBe("a\n"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "newdir"))).toBe(false); + // The generation moved with the file (r75): the vacated side's record + // loses its stamp, the tombstoned source record takes the moved file's — + // so the child's older rows at the restored name still map. + const afterRename = await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(fixture.sessionDir) + ); + expect(afterRename.get("newdir/a.md")!.targetStamp).toBeUndefined(); + expect(afterRename.get("olddir/a.md")!.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "olddir", "a.md"))) ?? undefined + ); + const undoneOlddirCreate = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: olddirCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneOlddirCreate.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "olddir", "a.md"))).toBe(false); + // A rename made BEFORE the first upgrade: adoption recorded only the + // post-rename names, so the inverse's destination (the vacated name) has + // no record — it still maps beside the adopted copies (r75). + await fixture.service.create(fixture.ctx, "/memories/workspace/dir2/b.md", "b\n", "agent"); + const dir2CreateRow = await lastRow(fixture.sessionDir); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/dir2", + "/memories/workspace/dir3", + "agent" + ); + const firstUpgradeRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerRoot, "dir3"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "dir3", "b.md"), "b\n"); + await writeManifest({ + "dir3/b.md": { content: "x", sidecar: "", target: "dir3/b.md", created: true }, + }); + const undoneFirstUpgradeRename = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: firstUpgradeRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneFirstUpgradeRename.success).toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "dir2", "b.md"), "utf-8")).toBe("b\n"); + expect(await pathExists(path.join(ownerRoot, "dir3"))).toBe(false); + // The restored side had no record: the moved copy gets a tombstoned one + // (r76) carrying its generation, so the child's older rows there map. + const manifest = () => + readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir)); + expect((await manifest()).get("dir3/b.md")!.targetStamp).toBeUndefined(); + const dir2Stamp = + (await adoptionTargetStamp(path.join(ownerRoot, "dir2", "b.md"))) ?? undefined; + expect((await manifest()).get("dir2/b.md")).toEqual({ + content: "x", + sidecar: "", + target: "dir2/b.md", + created: true, + deleted: true, + targetStamp: dir2Stamp, + }); + // Re-applying the rename through its rollback row (owner paths, no + // retargeting) moves the generation back; undoing that again restores it. + const renameRollbackRow = await lastRow(fixture.sessionDir); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: renameRollbackRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect((await manifest()).get("dir3/b.md")!.targetStamp).toBe(dir2Stamp); + expect((await manifest()).get("dir2/b.md")!.targetStamp).toBeUndefined(); + const reapplyRow = await lastRow(fixture.sessionDir); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: reapplyRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect((await manifest()).get("dir2/b.md")!.targetStamp).toBe(dir2Stamp); + const undoneDir2Create = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dir2CreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneDir2Create.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "dir2", "b.md"))).toBe(false); + // Same for a lone file whose adopted copy is a conflict import under + // imported// (the owner already had different content at the + // post-rename name): the moved copy gets its tombstoned record too (r77). + await fixture.service.create(fixture.ctx, "/memories/workspace/c.md", "c\n", "agent"); + const cCreateRow = await lastRow(fixture.sessionDir); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/c.md", + "/memories/workspace/d.md", + "agent" + ); + const fileRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerRoot, "imported", "child"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "imported", "child", "d.md"), "c\n"); + await fsPromises.writeFile(path.join(ownerRoot, "d.md"), "owner's own d\n"); + await writeManifest({ + "d.md": { content: "x", sidecar: "", target: "imported/child/d.md", created: true }, + }); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: fileRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "c.md"), "utf-8")).toBe("c\n"); + expect(await pathExists(path.join(ownerRoot, "imported", "child", "d.md"))).toBe(false); + expect(await fsPromises.readFile(path.join(ownerRoot, "d.md"), "utf-8")).toBe( + "owner's own d\n" + ); + expect((await manifest()).get("c.md")).toMatchObject({ + target: "c.md", + created: true, + deleted: true, + targetStamp: (await adoptionTargetStamp(path.join(ownerRoot, "c.md"))) ?? undefined, + }); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: cCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect(await pathExists(path.join(ownerRoot, "c.md"))).toBe(false); + }); + + it("keeps a retargeted row order-unknown against a peer row whose workspaceId is corrupted to its own", async () => { + using fixture = await createFixture(); + await fixture.service.create(fixture.ctx, "/memories/workspace/note.md", "v1\n", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/note.md", + "v1", + "v2", + "agent" + ); + const editRow = await lastRow(fixture.sessionDir); + const ownerSessionDir = path.join(path.dirname(fixture.sessionDir), "ws-owner"); + const ownerRoot = path.join(ownerSessionDir, "memory"); + await fsPromises.mkdir(path.join(ownerRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v2\n"); + const record: LegacyAdoptionRecord = { + content: "x", + sidecar: "", + target: "sub/note.md", + created: true, + targetStamp: (await adoptionTargetStamp(path.join(ownerRoot, "sub", "note.md"))) ?? undefined, + }; + await fsPromises.writeFile( + legacyAdoptionManifestPath(fixture.sessionDir), + JSON.stringify({ "note.md": record }) + ); + // The owner's row persisted with THIS workspace's id (corruption): it is + // still another journal's row, with the shared store's clock — a + // comparison of that clock against the retargeted row's private one + // would order the peer edit "earlier" and let the rollback overwrite it. + await sharedDurableEventJournal(ownerSessionDir).append({ + workspaceId: path.basename(fixture.sessionDir), + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/sub/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "sub", "note.md"), text: "v2\n" }], + }, + sourceTs: 1, + }, + }); + const unordered = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + }); + expect(unordered.success).toBe(false); + expect(unordered.success ? "" : unordered.error).toContain( + "order relative to this row is unknown" + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v2\n"); + }); + it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { using fixture = await createFixture(); const virtualPath = "/memories/global/order.md"; diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 1841c2eda19..76a07f80c57 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -29,7 +29,11 @@ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; -import type { DurableEvent } from "@/common/types/durableEvent"; +import { + isValidSourceClock, + refinementRowOrigin, + type DurableEvent, +} from "@/common/types/durableEvent"; import { MemoryRefinementActionSchema, RefinementInverseSchema, @@ -53,22 +57,79 @@ import { type RefinementInverseDraft, } from "./refinementJournal"; import { withTargetMutationLocks } from "./targetMutationLocks"; +import { advanceWorkspaceMemoryRevision } from "./workspaceMemoryRevision"; +import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, + refreshLegacyAdoptionTargetStamps, +} from "@/node/services/memoryLegacyAdoption"; export type RefinementEvent = Extract; +/** + * A rollback row the engine (and shared-memory row migration) may trust for + * lineage: `rollbackOf` set AND a parseable rollback action AND a parseable + * inverse (r78). Persisted rows are raw JSON, so a row can name a target while + * its payload is corrupt; counting such a row as "target rolled back" would + * hide a mutation that is still live on disk (its rollback row cannot itself + * be rolled back or copied), so it is treated like no rollback at all — + * conflict detection then sees the original as live (fail closed). + */ +export function isUsableRollbackRow(row: RefinementEvent): boolean { + if (row.data.rollbackOf === undefined) return false; + const action = RollbackRefinementActionSchema.safeParse(row.data.action); + // The two lineage fields must agree (r79): a row whose action names one + // target while `rollbackOf` names another is corrupt, and trusting either + // side would hide a live mutation behind the other. + return ( + action.success && + action.data.of === row.data.rollbackOf && + RefinementInverseSchema.safeParse(row.data.inverse).success + ); +} + /** All refinement rows in the session journal (byId-deduped, seq order). */ export async function listRefinements(sessionDir: string): Promise { assert(sessionDir.length > 0, "listRefinements requires a session dir"); const events = await sharedDurableEventJournal(sessionDir).read(); - return events.filter((event): event is RefinementEvent => event.kind === "refinement"); + const rows = events.filter((event): event is RefinementEvent => event.kind === "refinement"); + const journalWorkspaceId = path.basename(path.resolve(sessionDir)); + for (const row of rows) journalOfRow.set(row, journalWorkspaceId); + return rows; } +/** + * The journal (session workspace id) each row object was read from, recorded + * by listRefinements: origin comparisons (sameOriginOrder) must not trust a + * row's persisted `workspaceId` for that (r79). Rows obtained any other way + * have no known journal and compare as order-unknown. + */ +const journalOfRow = new WeakMap(); + export interface RollbackRefinementOptions { sessionDir: string; /** Envelope `id` of the refinement row to roll back. */ id: string; /** Apply despite detected divergence. Confinement is NEVER overridable. */ force?: boolean; + /** + * Session dir of the task-tree owner whose /memory backs this + * session's `/memories/workspace` (sub-agents only; omit when the session + * owns its store). Admits that one extra memory root for confinement. + */ + sharedWorkspaceMemorySessionDir?: string; + /** + * Session dirs of the OTHER live task-tree members sharing this session's + * `/memories/workspace` store (owner, siblings, descendants), resolved at + * rollback time. Each member journals only its own mutations of the shared + * store, so their rows are merged into divergence detection: a child's + * later edit under a path the owner renamed must surface as a conflict + * when the owner rolls the rename back. Omit when the store is private. + * MUST throw when membership cannot be established (unreadable config); + * the rollback is then refused instead of assuming an empty tree. + */ + listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; /** Attribution for the emitted rollback row. */ evidence: { toolName: string; toolCallId?: string; actor?: string }; /** Caller-supplied justification, recorded in the rollback row's action. */ @@ -244,7 +305,8 @@ function inferMemoryLayout(sessionDir: string): { muxRoot: string; sessionsDir: function resolveConfinementRoot( sessionDir: string, kind: "memory" | "skill", - filePath: string + filePath: string, + sharedWorkspaceMemorySessionDir?: string ): string { if (!path.isAbsolute(filePath)) { throw new RollbackError(`Refusing rollback: inverse path is not absolute: '${filePath}'`); @@ -307,22 +369,54 @@ function resolveConfinementRoot( } // /memory/ (workspace scope). Constrained to exactly // THIS session's memory subdir so a corrupted inverse can never touch other - // workspaces' memory or session artifacts (chat.jsonl, journals). - const workspaceMemoryRoot = path.join(path.resolve(sessionDir), "memory"); - const relToWorkspaceMemory = path.relative(workspaceMemoryRoot, resolved); - if (!relToWorkspaceMemory.startsWith("..") && !path.isAbsolute(relToWorkspaceMemory)) { - if (relToWorkspaceMemory.length > 0) { - return workspaceMemoryRoot; - } - throw new RollbackError( - `Refusing rollback: path targets a memory scope root, not a file inside it: '${filePath}'` + // workspaces' memory or session artifacts (chat.jsonl, journals). The one + // sanctioned second root is the task-tree owner's memory subdir, supplied + // by the CALLER (never read from the row): a sub-agent's workspace-scope + // writes physically land there (MemoryService.resolveWorkspaceMemoryOwnerId) + // while the row stays in the sub-agent's own journal. + const workspaceMemoryRoots = [path.join(path.resolve(sessionDir), "memory")]; + if (sharedWorkspaceMemorySessionDir !== undefined) { + const sharedSessionDir = path.resolve(sharedWorkspaceMemorySessionDir); + assert( + path.dirname(sharedSessionDir) === layout.sessionsDir, + "sharedWorkspaceMemorySessionDir must be a sibling session dir" ); + workspaceMemoryRoots.push(path.join(sharedSessionDir, "memory")); + } + for (const workspaceMemoryRoot of workspaceMemoryRoots) { + const relToWorkspaceMemory = path.relative(workspaceMemoryRoot, resolved); + if (!relToWorkspaceMemory.startsWith("..") && !path.isAbsolute(relToWorkspaceMemory)) { + if (relToWorkspaceMemory.length > 0) { + return workspaceMemoryRoot; + } + throw new RollbackError( + `Refusing rollback: path targets a memory scope root, not a file inside it: '${filePath}'` + ); + } } throw new RollbackError( `Refusing rollback: path is outside every memory scope root: '${filePath}'` ); } +/** + * Whether `root` (a confinement root from resolveConfinementRoot) is a + * workspace memory root — this session's own or the sanctioned owner's. Its + * parent is then the session dir that carries the store's clock + * (workspaceMemoryRevision.ts). + */ +function isWorkspaceMemoryRoot( + sessionDir: string, + root: string, + sharedWorkspaceMemorySessionDir: string | undefined +): boolean { + const candidates = [path.join(path.resolve(sessionDir), "memory")]; + if (sharedWorkspaceMemorySessionDir !== undefined) { + candidates.push(path.join(path.resolve(sharedWorkspaceMemorySessionDir), "memory")); + } + return candidates.includes(path.resolve(root)); +} + /** * The components of a confinement root that repo (or harness-writable) * content controls and could substitute with a symlink: `.mux`/`.agents` and @@ -472,11 +566,231 @@ interface InverseContentReader { * Collect divergence complaints for rolling back `target` given the current * filesystem + journal state. Empty array = safe to apply. */ +/** + * Two rows first appended to the same journal (a copy at its source position, + * see refinementRowOrigin): that store's mutation lock serialized clock and + * append, so the origin sequence is their mutation order whatever their clock + * values — pre-sharing history and clock-failed rows included. A copy without + * a carried origin has none (r73). Equal positions under distinct ids can + * only be corruption — no order evidence either. + */ +function sameOriginOrder( + row: RefinementEvent, + other: RefinementEvent +): { rowAfter: boolean } | null { + const rowOrigin = refinementRowOrigin(row, journalOfRow.get(row)); + const otherOrigin = refinementRowOrigin(other, journalOfRow.get(other)); + if (rowOrigin === null || otherOrigin === null || rowOrigin.journal !== otherOrigin.journal) { + return null; + } + if (rowOrigin.seq === otherOrigin.seq) return null; + return { rowAfter: rowOrigin.seq > otherOrigin.seq }; +} + +/** + * Journal order for conflict detection. Rows copied from a removed sub-agent's + * journal (sharedMemoryRowMigration.ts) were appended later than they + * happened; their `sourceTs` restores the mutation's real position relative + * to the owner's own rows. Same-instant ties fall back to append sequence. + * Rows of one origin order by that origin's sequence (sameOriginOrder). + */ +function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { + const sameOrigin = sameOriginOrder(row, other); + if (sameOrigin !== null) return sameOrigin.rowAfter; + const rowTs = isValidSourceClock(row.data.sourceTs) ? row.data.sourceTs : row.ts; + const otherTs = isValidSourceClock(other.data.sourceTs) ? other.data.sourceTs : other.ts; + return rowTs > otherTs || (rowTs === otherTs && row.seq > other.seq); +} + +/** + * A persisted `sourceTs` outside the clock's domain (zero, negative, a + * fraction, an unsafe integer): corruption, never order evidence — trusted, + * a later mutation corrupted to `-1` would sort before an older rename target + * and its file would move silently. + */ +function hasMalformedSourceClock(row: RefinementEvent): boolean { + return row.data.sourceTs !== undefined && !isValidSourceClock(row.data.sourceTs); +} + +/** + * Whether the order of two rows cannot be established: a row journaled while + * the shared store's clock write failed (`orderUnknown`) has only + * journal-local `ts`/`seq`, incomparable with other journals' rows — as does + * a row whose clock value is malformed. Callers fail closed — such a pair + * conflicts in either direction (force overrides). Rows of one origin are + * always ordered (sameOriginOrder), whatever their clock values. + */ +function orderUnknown( + row: RefinementEvent, + target: RefinementEvent, + targetRetargeted: boolean +): boolean { + if (sameOriginOrder(row, target) !== null) return false; + if (row.data.orderUnknown === true || target.data.orderUnknown === true) return true; + if (hasMalformedSourceClock(row) || hasMalformedSourceClock(target)) return true; + // A retargeted target (see wasRetargeted) vs. any row not proven to share + // its origin: the target's clock is a private store's, so no other clock + // orders it. Not decided by persisted `workspaceId`s (r79): a peer row's + // corrupted to the target's would otherwise be ordered by clock and a + // later peer mutation overwritten by the rollback. + return targetRetargeted; +} + +/** + * Memory rows from the other task-tree members' journals that touched the + * shared workspace store (owner's /memory). Read without their + * session locks: journals are append-only and self-healing on read, and a + * row landing after this read is caught by the fs-level checks like any + * other concurrent writer. + */ +async function readSharedMemoryPeerRows( + opts: RollbackRefinementOptions +): Promise { + // Membership must be established, not guessed: a resolver that cannot read + // the topology (config.json missing/malformed) throws, and the rollback is + // refused rather than proceeding with no peer journals — an owner would + // otherwise move a child's later edit without warning. + let peerDirs: string[]; + try { + peerDirs = opts.listSharedWorkspaceMemoryPeerSessionDirs?.() ?? []; + } catch (error) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the task tree sharing this workspace's memory store could not be resolved (${getErrorMessage(error)})` + ); + } + if (peerDirs.length === 0) return []; + const sharedRoot = path.join( + path.resolve(opts.sharedWorkspaceMemorySessionDir ?? opts.sessionDir), + "memory" + ); + const ownerSessionDir = path.dirname(sharedRoot); + const actingWorkspaceId = path.basename(path.resolve(opts.sessionDir)); + // The same duplicate seen from the other side: a removal that aborted after + // its pre-teardown pass leaves THIS (owner) journal holding copies of a + // still-registered child's rows. Read as that child's peer rows, the + // originals would count as separate later mutations of the very notes the + // copies describe — and a retargeted original is `orderUnknown`, so the + // copy could never be rolled back until the removal finally succeeds. + // Only a copy that still REPRESENTS its source stands in for it: a copy + // whose inverse is unparseable contributes nothing to divergence + // (collectDivergence skips it), so suppressing its intact original would + // let this rollback overwrite the peer mutation that original records. + const migratedHere = new Set(); + for (const row of await listRefinements(opts.sessionDir)) { + if ( + row.data.migratedFrom !== undefined && + row.data.kind === "memory" && + RefinementInverseSchema.safeParse(row.data.inverse).success + ) { + migratedHere.add(row.data.migratedFrom); + } + } + const peerRows: RefinementEvent[] = []; + for (const peerDir of peerDirs) { + assert( + path.resolve(peerDir) !== path.resolve(opts.sessionDir), + "peer session dirs exclude the acting session" + ); + // A peer sub-agent's pre-sharing rows address ITS legacy private + // notebook; the notes live in the shared store now (adoption manifest + // in the peer's session dir). Retarget them through that peer's manifest + // before the overlap test — the peer's later edit of an adopted note must + // surface against the owner's rollback like any shared-store row. The + // remapped inverse replaces the recorded one on the returned row, so the + // acting session's later checks (its own remapper is a no-op for owner + // paths) compare the paths the note actually lives at. Legacy paths the + // shared store never took address invisible files and are dropped. + // Strict: an unreadable peer manifest must refuse the rollback, not read + // as "nothing adopted" and silently drop that peer's later mutations. + let remap: RecordedPathRemapper; + if (path.resolve(peerDir) === path.resolve(ownerSessionDir)) { + remap = identityRemapper; + } else { + try { + remap = await createLegacyPathRemapper({ + childSessionDir: peerDir, + ownerSessionDir, + strict: true, + }); + } catch (error) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': a tree member's adoption manifest could not be read (${getErrorMessage(error)})` + ); + } + } + const peerWorkspaceId = path.basename(path.resolve(peerDir)); + for (const row of await listRefinements(peerDir)) { + if (row.data.kind !== "memory") continue; + // A removal that aborted after its pre-teardown pass leaves the owner + // journal holding COPIES of this session's rows (migratedFrom = + // ":") while this session lives on. They are + // this journal's rows seen twice, not later peer edits. + if (row.data.migratedFrom?.startsWith(`${actingWorkspaceId}:`) === true) continue; + // ...and the originals of copies this journal already holds (above). + if (migratedHere.has(`${peerWorkspaceId}:${row.id}`)) continue; + const original = RefinementInverseSchema.safeParse(row.data.inverse); + const parsed = parseRemappedInverse(row, remap); + if (parsed === null || !original.success) continue; + if (!inversePaths(parsed).some((p) => pathsOverlap(p, sharedRoot))) continue; + // A retargeted peer row carries its private clock: order unknown. + peerRows.push({ + ...row, + data: { + ...row.data, + inverse: parsed, + ...(wasRetargeted(original.data, parsed) ? { orderUnknown: true as const } : {}), + }, + }); + } + } + return peerRows; +} + +/** + * Path retargeting applied to every recorded path of the acting session's + * journal before it is compared or applied (see createLegacyPathRemapper): + * identity for a session that owns its store. Rows whose paths cannot be + * mapped (a legacy note the shared store never took) contribute nothing to + * divergence — they address an invisible file the target cannot overlap. + */ +interface RecordedPathRemapper { + path(filePath: string): string; + inverse(inverse: RefinementInverse): RefinementInverse; +} +const identityRemapper: RecordedPathRemapper = { + path: (filePath) => filePath, + inverse: (inverse) => inverse, +}; +function parseRemappedInverse( + row: RefinementEvent, + remap: RecordedPathRemapper +): RefinementInverse | null { + const parsed = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsed.success) return null; + try { + return remap.inverse(parsed.data); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } +} + +/** + * Whether retargeting changed the inverse's paths: such a row was journaled + * against a PRIVATE store (pre-sharing), so its `ts`/`sourceTs` belong to + * that store's clock and are incomparable with the shared store's rows. + */ +function wasRetargeted(original: RefinementInverse, remapped: RefinementInverse): boolean { + return JSON.stringify(inversePaths(original)) !== JSON.stringify(inversePaths(remapped)); +} + async function collectDivergence( rows: RefinementEvent[], target: RefinementEvent, inverse: RefinementInverse, - readContent: InverseContentReader + readContent: InverseContentReader, + remap: RecordedPathRemapper, + targetRetargeted: boolean ): Promise { const complaints: string[] = []; const targetPaths = inversePaths(inverse); @@ -488,19 +802,22 @@ async function collectDivergence( // live rollback chain only conflicts when its net effect differs from the // state the target left behind (see liveRowConflictsWithTarget). const rolledBackIds = new Set( - rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) + rows.filter(isUsableRollbackRow).map((row) => row.data.rollbackOf!) ); for (const row of rows) { - if (row.seq <= target.seq) continue; + if (row.id === target.id) continue; + if (!isAfter(row, target) && !orderUnknown(row, target, targetRetargeted)) continue; if (rolledBackIds.has(row.id)) continue; // Effect undone by a later rollback row. - if (!liveRowConflictsWithTarget(rows, row, target)) continue; - const parsed = RefinementInverseSchema.safeParse(row.data.inverse); - if (!parsed.success) continue; - const overlap = inversePaths(parsed.data).some((p) => - targetPaths.some((t) => pathsOverlap(p, t)) - ); + if (!liveRowConflictsWithTarget(rows, row, target, targetRetargeted)) continue; + const parsed = parseRemappedInverse(row, remap); + if (parsed === null) continue; + const overlap = inversePaths(parsed).some((p) => targetPaths.some((t) => pathsOverlap(p, t))); if (overlap) { - complaints.push(`later refinement row ${row.id} (seq ${row.seq}) touched the same paths`); + complaints.push( + orderUnknown(row, target, targetRetargeted) + ? `refinement row ${row.id} (seq ${row.seq}) touched the same paths and its order relative to this row is unknown (its store clock write failed)` + : `later refinement row ${row.id} (seq ${row.seq}) touched the same paths` + ); } } @@ -530,7 +847,7 @@ async function collectDivergence( // so the current state must still match that applied inverse — // content-exact where the original restored files. complaints.push( - ...(await collectRollbackTargetDivergence(rows, rollbackAction.data, readContent)) + ...(await collectRollbackTargetDivergence(rows, rollbackAction.data, readContent, remap)) ); break; } @@ -559,7 +876,7 @@ async function collectDivergence( // Content-exact check via the row's recorded post-action hashes: a manual // or cross-workspace edit after the target row never appears in this // session's journal, so the seq-based scan above cannot see it. - complaints.push(...(await collectPostStateDivergence(target))); + complaints.push(...(await collectPostStateDivergence(target, remap))); return complaints; } @@ -571,22 +888,27 @@ async function collectDivergence( * complaints — their expected post-edit contents cannot be reconstructed from * the journal, so the presence-only checks above are the best we can do. */ -async function collectPostStateDivergence(target: RefinementEvent): Promise { +async function collectPostStateDivergence( + target: RefinementEvent, + remap: RecordedPathRemapper +): Promise { const postState = RefinementPostStateSchema.safeParse(target.data.postState); if (!postState.success) { return []; } const complaints: string[] = []; for (const file of postState.data.files) { + let filePath: string; let current: string; try { - current = await fsPromises.readFile(file.path, "utf-8"); + filePath = remap.path(file.path); + current = await fsPromises.readFile(filePath, "utf-8"); } catch { - continue; // Missing files are already reported by the presence checks. + continue; // Missing (or unmappable) files are already reported by the presence checks. } if (sha256Hex(current) !== file.sha256) { complaints.push( - `'${file.path}' was modified after the target refinement (current content no longer matches the state it left behind)` + `'${filePath}' was modified after the target refinement (current content no longer matches the state it left behind)` ); } } @@ -607,7 +929,8 @@ async function collectPostStateDivergence(target: RefinementEvent): Promise([row.id]); while (current.data.rollbackOf !== undefined) { + // A corrupt rollback row anywhere in the chain (isUsableRollbackRow): the + // chain's net effect cannot be established — assume conflict. + if (!isUsableRollbackRow(current)) return true; const original = rows.find((r) => r.id === current.data.rollbackOf); if (original === undefined || seen.has(original.id)) { return true; // Corrupt chain (missing root or cycle): assume conflict. @@ -627,7 +953,9 @@ function liveRowConflictsWithTarget( if (rollbackCount % 2 === 0) { return true; // Even chain: the root row's edit was re-applied. } - return current.seq <= target.seq; // Odd chain: rewound to just before root. + // Odd chain: rewound to just before root — a conflict unless the root is + // provably after the target. + return !isAfter(current, target) || orderUnknown(current, target, targetRetargeted); } async function dirExists(target: string): Promise { @@ -648,20 +976,21 @@ async function dirExists(target: string): Promise { async function collectRollbackTargetDivergence( rows: RefinementEvent[], action: RollbackRefinementAction, - readContent: InverseContentReader + readContent: InverseContentReader, + remap: RecordedPathRemapper ): Promise { const original = rows.find((row) => row.id === action.of); if (original === undefined) { return [`the original row '${action.of}' this rollback applied is missing from the journal`]; } - const applied = RefinementInverseSchema.safeParse(original.data.inverse); - if (!applied.success) { - return [`the original row '${action.of}' has an unparseable inverse`]; + const applied = parseRemappedInverse(original, remap); + if (applied === null) { + return [`the original row '${action.of}' has an unparseable or unmappable inverse`]; } const complaints: string[] = []; - switch (applied.data.op) { + switch (applied.op) { case "delete-files": - for (const p of applied.data.paths) { + for (const p of applied.paths) { if (await fileExists(p)) { complaints.push( `expected '${p}' to be absent (the rollback deleted it), but it was recreated since` @@ -670,7 +999,7 @@ async function collectRollbackTargetDivergence( } break; case "restore-files": - for (const file of applied.data.files) { + for (const file of applied.files) { if (!(await fileExists(file.path))) { complaints.push( `expected '${file.path}' to exist (the rollback restored it), but it was deleted since` @@ -685,7 +1014,7 @@ async function collectRollbackTargetDivergence( } // Mixed force-apply inverse (r67): the rollback also deleted these // paths, so their recreation since is divergence too. - for (const p of applied.data.deletePaths ?? []) { + for (const p of applied.deletePaths ?? []) { if (await fileExists(p)) { complaints.push( `expected '${p}' to be absent (the rollback deleted it), but it was recreated since` @@ -748,7 +1077,13 @@ export async function rollbackRefinement( `Row '${opts.id}' was produced by a remote (SSH/Docker) workspace runtime; its paths are not addressable on this host. Remote skill rollbacks are not supported.` ); } - const existingRollback = rows.find((row) => row.data.rollbackOf === opts.id); + // Same predicate as liveness (isUsableRollbackRow): a corrupt rollback + // row cannot be rolled back "instead", and lineage treats its target as + // still live — so the target itself stays rollbackable (the divergence + // checks below decide whether the tree still matches its inverse). + const existingRollback = rows.find( + (row) => row.data.rollbackOf === opts.id && isUsableRollbackRow(row) + ); if (existingRollback !== undefined) { throw new RollbackError( `Row '${opts.id}' was already rolled back by row '${existingRollback.id}'. Roll back that row instead to re-apply.` @@ -761,7 +1096,35 @@ export async function rollbackRefinement( `Row '${opts.id}' has an unparseable inverse payload: ${parsedInverse.error.message}` ); } - const inverse = parsedInverse.data; + // Sub-agent whose `/memories/workspace` is the owner's store: rows + // journaled before sharing address its own /memory, whose + // notes were since folded into the owner's store (memoryLegacyAdoption). + // Every recorded path of this journal — the target's inverse, later rows + // for overlap, post-state hashes, a rollback chain's root — is retargeted + // to the adopted copy, so the rollback reverts the note the shared + // notebook serves. Mapped BEFORE confinement: the manifest's targets are + // checked like any other recorded path. + const remap: RecordedPathRemapper = + kind === "memory" && + opts.sharedWorkspaceMemorySessionDir !== undefined && + path.resolve(opts.sharedWorkspaceMemorySessionDir) !== path.resolve(opts.sessionDir) + ? await createLegacyPathRemapper({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + }) + : identityRemapper; + let inverse: RefinementInverse; + try { + inverse = remap.inverse(parsedInverse.data); + } catch (error) { + if (!(error instanceof LegacyPathNotAdoptedError)) throw error; + throw new RollbackError(`Refusing rollback of '${opts.id}': ${error.message}`); + } + // A retargeted target was journaled against this session's private + // clock: its order relative to OTHER journals' rows is unknown, so those + // are compared as order-unknown (conflict unless forced). Same-journal + // rows still order by their shared sequence. + const targetRetargeted = wasRetargeted(parsedInverse.data, inverse); // Confinement first — never overridable. A corrupted inverse must never // write outside the memory/skill roots (repo AGENTS.md, built-in skills, @@ -770,7 +1133,10 @@ export async function rollbackRefinement( // repo revision to swap a root for a symlink in the meantime. const roots = new Map(); for (const p of inversePaths(inverse)) { - roots.set(p, resolveConfinementRoot(opts.sessionDir, kind, p)); + roots.set( + p, + resolveConfinementRoot(opts.sessionDir, kind, p, opts.sharedWorkspaceMemorySessionDir) + ); } const assertConfinement = async (): Promise => { for (const [p, root] of roots) { @@ -800,7 +1166,19 @@ export async function rollbackRefinement( }, }; - const divergence = await collectDivergence(rows, target, inverse, readContent); + // Conflict detection sees this journal plus every live tree member's + // shared-store rows (see listSharedWorkspaceMemoryPeerSessionDirs); the + // store clock (`sourceTs`) orders rows across journals. Target lookup, + // rollbackOf checks and the appended row stay on this session's journal. + // Re-read under the target locks below before the apply. + const divergence = await collectDivergence( + kind === "memory" ? [...rows, ...(await readSharedMemoryPeerRows(opts))] : rows, + target, + inverse, + readContent, + remap, + targetRetargeted + ); if (divergence.length > 0 && opts.force !== true) { throw new RollbackError( `Refusing rollback of '${opts.id}': current state diverges from what the inverse expects:\n` + @@ -827,16 +1205,82 @@ export async function rollbackRefinement( // targetMutationLocks.ts. const targetLockRoot = inferMemoryLayout(opts.sessionDir)?.muxRoot ?? null; const applied = await withTargetMutationLocks(targetLockRoot, lockKeys, async () => { + // Teardown gates (r61), same tombstone MemoryService checks pre-commit, + // same lock. Acting workspace: removal's fail-closed orphan path leaves + // the journal on disk but the workspace tombstoned, and a rollback from + // it must not mutate anything nor append into the retained session. + // Shared-store owner: a delete inverse expects its target absent, so + // divergence alone would let a rollback that was waiting on this lock + // recreate the removed owner's /memory. + if (targetLockRoot !== null) { + const actingSessionDir = path.resolve(opts.sessionDir); + if (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(actingSessionDir))) { + throw new RollbackError(`Refusing rollback of '${opts.id}': this workspace was removed`); + } + if (opts.sharedWorkspaceMemorySessionDir !== undefined) { + const ownerSessionDir = path.resolve(opts.sharedWorkspaceMemorySessionDir); + const ownerMemoryRoot = path.join(ownerSessionDir, "memory"); + if ( + lockKeys.includes(ownerMemoryRoot) && + (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(ownerSessionDir))) + ) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the workspace owning the shared memory store was removed` + ); + } + } + } + // The remapper's directory proof (an adopted directory maps only while + // the owner's subtree is EXACTLY the adopted descendants) was computed + // before this lock: an owner note added beside the copies while the + // rollback waited would travel along with a legacy directory rename. + // Re-derive the mapping under the lock and require it to be identical; + // a mapping that no longer holds refuses like at plan time. + if (remap !== identityRemapper) { + assert( + opts.sharedWorkspaceMemorySessionDir !== undefined, + "a legacy remapper exists only for a sub-agent sharing its notebook" + ); + let relocked: RefinementInverse; + try { + relocked = ( + await createLegacyPathRemapper({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + }) + ).inverse(parsedInverse.data); + } catch (error) { + if (!(error instanceof LegacyPathNotAdoptedError)) throw error; + throw new RollbackError(`Refusing rollback of '${opts.id}': ${error.message}`); + } + if (JSON.stringify(inversePaths(relocked)) !== JSON.stringify(inversePaths(inverse))) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the shared-store mapping of its recorded paths changed while waiting for the target lock` + ); + } + } // Re-verify INSIDE the lock, immediately before mutating: a writer that // won the lock first has already landed, and its change must surface as - // divergence rather than be overwritten. `rows` is intentionally the - // pre-lock read — the fs-level checks (postState hashes, presence) are - // what detect concurrent mutations; force skips this exactly like the - // plan-time check. Cross-process residual: a writer in ANOTHER process - // (live app vs. debug CLI) does not contend on this in-process lock, so - // this re-verify narrows but cannot fully close that window. + // divergence rather than be overwritten. The journals are re-read here + // too: a rename row carries no post-state hash, so a tree member's edit + // beneath the renamed destination that journaled between the plan-time + // scan and this lock is visible only as its (now committed) row. Force + // skips this exactly like the plan-time check. Cross-process residual: + // a writer in ANOTHER process (live app vs. debug CLI) does not contend + // on this in-process lock, so this re-verify narrows but cannot fully + // close that window. if (opts.force !== true) { - const raced = await collectDivergence(rows, target, inverse, readContent); + const lockedRows = await listRefinements(opts.sessionDir); + const raced = await collectDivergence( + kind === "memory" + ? [...lockedRows, ...(await readSharedMemoryPeerRows(opts))] + : lockedRows, + target, + inverse, + readContent, + remap, + targetRetargeted + ); if (raced.length > 0) { throw new RollbackError( `Refusing rollback of '${opts.id}': a concurrent mutation landed before the apply:\n` + @@ -866,6 +1310,38 @@ export async function rollbackRefinement( // partial rollback behind (no rollbackOf row, and a retry refuses on the // resulting divergence). const applied: RollbackApplied = { rollbackRowId: null, restored: [], deleted: [] }; + // A sub-agent's apply on the shared store (r74): the adopted copies + // rewritten, removed or — after a failed apply — compensated are new + // generations of the files; re-stamp them so the child's remaining rows + // over the same notes still map (createLegacyPathRemapper refuses a copy + // that is no longer the recorded generation). A rename keeps inode and + // mtime but moves the generation to the other endpoint's records (r75). + // Not only retargeted inverses: the child's own rollback rows carry + // owner paths already, and re-applying one is the same lineage acting. + // Still under the owner-store lock. + const restampAdoptedCopies = async (): Promise => { + if ( + kind !== "memory" || + opts.sharedWorkspaceMemorySessionDir === undefined || + path.resolve(opts.sharedWorkspaceMemorySessionDir) === path.resolve(opts.sessionDir) + ) { + return; + } + try { + await refreshLegacyAdoptionTargetStamps({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + paths: [...applied.restored, ...applied.deleted], + ...(applied.renamed === undefined ? {} : { renamed: applied.renamed }), + }); + } catch (error) { + // Stale stamps only refuse later rollbacks (force overrides). + log.warn("[refinement] failed to re-stamp adopted legacy copies after a rollback", { + id: opts.id, + error, + }); + } + }; switch (inverse.op) { case "delete-files": try { @@ -875,6 +1351,7 @@ export async function rollbackRefinement( } } catch (error) { await compensatePartialApply(applied.deleted, newInverse); + await restampAdoptedCopies(); throw error; } break; @@ -908,6 +1385,7 @@ export async function rollbackRefinement( } } catch (error) { await compensatePartialApply([...applied.restored, ...applied.deleted], newInverse); + await restampAdoptedCopies(); throw error; } break; @@ -919,6 +1397,7 @@ export async function rollbackRefinement( applied.renamed = { from: inverse.from, to: inverse.to }; break; } + await restampAdoptedCopies(); // Commit point: even if two processes double-entered the critical section // (theoretically possible — plain POSIX files cannot make the guard's @@ -932,6 +1411,10 @@ export async function rollbackRefinement( await fileLock.assertStillOwned(); } catch (error) { await compensateApplied(applied, newInverse); + // The stamps above described the applied state; the compensated + // files are new generations again (r77). A reversed rename re-stamps + // both endpoints (nothing is synthesized: the restored side is empty). + await restampAdoptedCopies(); throw error; } if (opts.testOnlyBeforeRollbackJournal !== undefined) { @@ -958,6 +1441,27 @@ export async function rollbackRefinement( // Inverse blob puts + the append referencing them run under the // journal blob lock: a concurrent reclamation pass must never // observe the put→append window (see withBlobLock). + // A workspace-memory rollback is a store mutation like any other: + // advance the owner store's clock (under the target lock held here) + // for this row's cross-journal order AND as the cross-process change + // signal — the debug CLI reaches this engine without MemoryService, + // so nothing else would tell other backends' caches and Memory tabs. + const workspaceMemoryRoot = [...new Set(roots.values())].find((root) => + isWorkspaceMemoryRoot(opts.sessionDir, root, opts.sharedWorkspaceMemorySessionDir) + ); + let sourceTs: number | undefined; + let orderUnknownRow = false; + if (kind === "memory" && workspaceMemoryRoot !== undefined) { + try { + sourceTs = await advanceWorkspaceMemoryRevision(path.dirname(workspaceMemoryRoot)); + } catch (error) { + // The inverse is already applied, so the row must still be + // journaled — as order-unknown (see MemoryService.journalRefinement): + // its journal-local `ts` is incomparable with other journals' rows. + log.debug("[refinement] failed to advance workspace memory revision", { error }); + orderUnknownRow = true; + } + } let publishedBlobs: BlobQuotaEntry[] = []; const row = await journal.withBlobLock(async () => { const resolved = await resolveRefinementInverse(journal.blobs, newInverse); @@ -978,6 +1482,8 @@ export async function rollbackRefinement( ...(opts.evidence.actor !== undefined ? { actor: opts.evidence.actor } : {}), }, rollbackOf: opts.id, + ...(sourceTs !== undefined ? { sourceTs } : {}), + ...(orderUnknownRow ? { orderUnknown: true as const } : {}), }, }); }); @@ -1060,6 +1566,8 @@ async function compensatePartialApply( ? preState.files.find((file) => file.path === p) : undefined; if (prior !== undefined) { + // Captured from disk moments ago; only migrated rows carry bare references. + assert("content" in prior, "pre-rollback capture carries file contents"); await fsPromises.mkdir(path.dirname(p), { recursive: true }); await writeFileAtomic(p, prior.content, { encoding: "utf-8" }); } else { diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts new file mode 100644 index 00000000000..733c3814b54 --- /dev/null +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -0,0 +1,355 @@ +import * as path from "node:path"; +import assert from "@/common/utils/assert"; +import { isValidSourceClock, refinementRowOrigin } from "@/common/types/durableEvent"; +import { + MemoryRefinementActionSchema, + RefinementEvidenceSchema, + RefinementInverseSchema, + RefinementPostStateSchema, + RollbackRefinementActionSchema, + type MemoryRefinementAction, + type RefinementInverse, + type RollbackRefinementAction, +} from "@/common/types/refinement"; +import { log } from "@/node/services/log"; +import type { BlobQuotaEntry } from "@/node/utils/journal/blobReclamation"; +import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; +import { + appendRefinementEventUnderBlobLock, + reclaimRefinementInverseBlobsBestEffort, + type RefinementFileCapture, + type RefinementFileReference, + type RefinementInverseDraft, +} from "./refinementJournal"; +import { isUsableRollbackRow, listRefinements, type RefinementEvent } from "./refinementRollback"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, +} from "@/node/services/memoryLegacyAdoption"; + +function inversePaths(inverse: RefinementInverse): string[] { + switch (inverse.op) { + case "delete-files": + return inverse.paths; + case "restore-files": + return [...inverse.files.map((file) => file.path), ...(inverse.deletePaths ?? [])]; + case "rename": + return [inverse.from, inverse.to]; + } +} + +function isInside(root: string, filePath: string): boolean { + const rel = path.relative(root, path.resolve(filePath)); + return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel); +} + +/** + * Before a sub-agent's session directory is deleted, re-append its LIVE + * memory refinement rows that target the owner's shared `/memories/workspace` + * store into the OWNER's journal, copying the inverse blob payloads. The + * edits themselves already live in the owner's store; without this their + * audit trail and rollback IDs would vanish with the child's journal. Rows + * already rolled back and rows targeting other roots (global/project) are + * left alone — they die with the child as before. + * + * Rollback rows travel too, but only when their target has an owner copy, + * with `rollbackOf` remapped to that copy: removal runs this in two passes + * (pre-teardown, then a delta pass under the removal locks), and another + * backend can roll a row back in between. Its copy is already in the owner + * journal by then; without the rollback row following it, the owner journal + * would claim an edit whose inverse was already applied is still live and + * rollbackable. A row rolled back before its FIRST copy is simply dead and + * stays behind with its whole lineage. + * + * A row whose inverse payload was reclaimed is copied as an audit-only record + * (RefinementFileReference) so conflict detection keeps seeing the edit; a + * row that cannot be parsed at all is skipped with a log line — nothing + * durable exists to preserve. + * A row that CAN be reconstructed but cannot be persisted in the owner's + * journal throws: the caller must not delete the source journal, or the + * only inverse and rollback ID would be lost. Returns the number migrated. + */ +export async function migrateSharedMemoryRefinementRows(args: { + childSessionDir: string; + childWorkspaceId: string; + ownerSessionDir: string; + ownerWorkspaceId: string; +}): Promise { + assert( + args.childSessionDir.length > 0, + "migrateSharedMemoryRefinementRows requires childSessionDir" + ); + assert( + args.ownerSessionDir.length > 0, + "migrateSharedMemoryRefinementRows requires ownerSessionDir" + ); + assert( + args.ownerWorkspaceId.length > 0, + "migrateSharedMemoryRefinementRows requires ownerWorkspaceId" + ); + assert( + args.childWorkspaceId.length > 0, + "migrateSharedMemoryRefinementRows requires childWorkspaceId" + ); + const ownerMemoryRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const rows = await listRefinements(args.childSessionDir); + // Pre-sharing rows address the child's legacy private notebook; their notes + // live in the owner store now (adoption manifest, read while the child + // session still exists). Retargeted like the rollback engine does, so the + // adopted copy stays rollbackable once the child journal is gone; legacy + // paths the shared store never took are skipped below like other roots. + // Strict: an unreadable manifest must abort the removal (throws), not read + // as "nothing adopted" and let the child journal be deleted with the only + // rollback IDs and inverse payloads of adopted notes. + const remap = await createLegacyPathRemapper({ + childSessionDir: args.childSessionDir, + ownerSessionDir: args.ownerSessionDir, + strict: true, + }); + const remapInverse = (inverse: RefinementInverse): RefinementInverse | null => { + try { + return remap.inverse(inverse); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } + }; + // Liveness follows the whole rollback chain (rollback → rollback of the + // rollback re-applies): an original row is live when it has been rolled + // back an even number of times. Only a rollback row this migration could + // itself carry (parseable rollback action AND inverse) counts (r77): a + // corrupted one is skipped below, and letting it also kill its intact + // target would delete the child journal with neither inverse preserved. + // The copied target is then live on the owner side; its divergence checks + // refuse a re-apply that no longer matches the tree (force overrides). + // Only rollback rows this migration can carry count: a rollback row whose + // `kind` is not "memory" (corrupted, or another taxonomy's) is skipped by + // the copy loop below, so counting it here would mark its memory target + // dead and let removal delete the journal without copying either. + const isMigratableRollbackRow = (row: RefinementEvent): boolean => + row.data.kind === "memory" && isUsableRollbackRow(row); + const rollbackByTarget = new Map( + rows.filter(isMigratableRollbackRow).map((row) => [row.data.rollbackOf!, row] as const) + ); + // Returns null on a corrupted (cyclic / absurdly long) lineage: such a row + // is treated as non-migratable instead of hanging removal. + const isLive = (rowId: string): boolean | null => { + const visited = new Set([rowId]); + let depth = 0; + for ( + let next = rollbackByTarget.get(rowId); + next !== undefined; + next = rollbackByTarget.get(next.id) + ) { + if (visited.has(next.id) || depth >= 1024) { + log.warn("[refinement] corrupted rollback lineage; skipping row migration", { rowId }); + return null; + } + visited.add(next.id); + depth++; + } + return depth % 2 === 0; + }; + const childJournal = sharedDurableEventJournal(args.childSessionDir); + const ownerJournal = sharedDurableEventJournal(args.ownerSessionDir); + let migrated = 0; + const publishedBlobs: BlobQuotaEntry[] = []; + // One hold of the owner journal's publish lock for the whole batch: the + // dedup read and every append happen inside it, so a second backend + // removing the same child concurrently (or a retried removal — the child + // journal survives a retryable failure or a crash before deletion) sees the + // copied rows before it decides, and the owner journal is read once rather + // than once per row. Rows already copied are identified by their source + // identity on the owner side. + await ownerJournal.withBlobLock(async () => { + const ownerRows = await listRefinements(args.ownerSessionDir); + // Source identity → owner-journal id of its copy (earlier passes and this + // one). Only a copy that is still a USABLE memory row counts — parseable + // action (memory or rollback) and inverse: a copy whose persisted state + // is corrupted would otherwise make a retried removal skip its intact + // source, delete the child session, and leave the owner with nothing + // but an unusable rollback record. Such a source is copied again (the + // corrupted row stays behind as an audit record). A rollback copy is + // judged by the engine's own predicate (isUsableRollbackRow): its action + // must also name the `rollbackOf` target, or the engine rejects the copy + // while this pass would have counted it — and a retried removal would + // then skip the intact child source and delete its journal. + const ownerIdBySource = new Map(); + for (const ownerRow of ownerRows) { + if (ownerRow.data.migratedFrom === undefined || ownerRow.data.kind !== "memory") continue; + const usable = + ownerRow.data.rollbackOf === undefined + ? RefinementInverseSchema.safeParse(ownerRow.data.inverse).success && + MemoryRefinementActionSchema.safeParse(ownerRow.data.action).success + : isUsableRollbackRow(ownerRow); + if (usable) ownerIdBySource.set(ownerRow.data.migratedFrom, ownerRow.id); + } + // Owner rows already rolled back (by anyone): a second rollback row for + // the same target would corrupt the lineage the rollback engine walks. + // Only usable rollback rows count (r78): a corrupted rollback copy is + // excluded from ownerIdBySource above, so its intact source is copied + // again — and must not be blocked here by the bare `rollbackOf` of that + // very corruption, or the retry would leave the owner with an unusable + // rollback record over a target the engine then reads as live. + const ownerRollbackTargets = new Set( + ownerRows.filter(isMigratableRollbackRow).map((ownerRow) => ownerRow.data.rollbackOf!) + ); + for (const row of rows) { + if (row.data.kind !== "memory") continue; + const migratedFrom = `${args.childWorkspaceId}:${row.id}`; + if (ownerIdBySource.has(migratedFrom)) continue; + let action: MemoryRefinementAction | RollbackRefinementAction; + let rollbackOf: string | undefined; + if (row.data.rollbackOf === undefined) { + if (isLive(row.id) !== true) continue; + const parsed = MemoryRefinementActionSchema.safeParse(row.data.action); + if (!parsed.success) { + // A live edit whose action is corrupt but whose inverse still + // parses is evidence conflict detection needs (its inverse paths + // mark the child's later mutation over the same files); the owner + // journal cannot carry it without an action, so removal must not + // delete the only copy — throw, like a row that cannot be + // persisted. A forced removal accepts the loss explicitly. + if (RefinementInverseSchema.safeParse(row.data.inverse).success) { + throw new Error( + `refinement row ${row.id} of ${args.childWorkspaceId} is live but its action is malformed; it cannot be handed over to the owner journal` + ); + } + continue; + } + action = parsed.data; + } else { + // Child journal order puts a rollback row after its target, so the + // target's copy (from an earlier pass or this loop) is known here. + // The engine's own usability rule applies to the SOURCE row too: a + // row whose parseable action names another target than `rollbackOf` + // is corrupt and must not be turned into a usable owner rollback by + // rewriting `of` (that would suppress a possibly live mutation). + if (!isMigratableRollbackRow(row)) continue; + rollbackOf = ownerIdBySource.get(`${args.childWorkspaceId}:${row.data.rollbackOf}`); + if (rollbackOf === undefined || ownerRollbackTargets.has(rollbackOf)) continue; + const parsed = RollbackRefinementActionSchema.safeParse(row.data.action); + if (!parsed.success) continue; + action = { ...parsed.data, of: rollbackOf }; + } + const parsedInverse = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsedInverse.success) continue; + const remapped = remapInverse(parsedInverse.data); + if (remapped === null) continue; + // A row whose paths were retargeted was journaled against the child's + // PRIVATE store (pre-sharing, or a self-fallback while config.json was + // unreadable): any `sourceTs` it carries is that private clock's, not + // the owner's, so its order among the owner's rows is unknown. + const retargeted = + JSON.stringify(inversePaths(parsedInverse.data)) !== JSON.stringify(inversePaths(remapped)); + const inverse = { success: true as const, data: remapped }; + if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; + + let draft: RefinementInverseDraft; + if (inverse.data.op === "restore-files") { + const files: Array = []; + for (const file of inverse.data.files) { + // Contents are blob-offloaded at append (resolveRefinementInverse); older + // rows may carry them inline. + const content = + file.text ?? + (file.blobRef === undefined ? null : await childJournal.blobs.getText(file.blobRef)); + if (content !== null) { + files.push({ path: file.path, content }); + } else if (file.blobRef !== undefined) { + // Payload reclaimed under the child's inverse-blob quota. The row + // still travels as an audit record — its paths and source order + // are what conflict detection needs when the owner later rolls + // back an older edit over the same files (a directory rename has + // no post-state hash to notice the child's newer content by) — + // but it can no longer be rolled back: the reference resolves to + // nothing in the owner journal, which the rollback engine + // refuses exactly like an evicted payload of its own. + files.push({ path: file.path, blobRef: file.blobRef }); + } else { + break; // neither text nor blobRef: nothing durable to preserve + } + } + if (files.length !== inverse.data.files.length) { + log.debug("[refinement] skipping shared-memory row migration: inverse payload missing", { + rowId: row.id, + }); + continue; + } + draft = { + op: "restore-files", + files, + ...(inverse.data.deletePaths !== undefined + ? { deletePaths: inverse.data.deletePaths } + : {}), + }; + } else { + draft = inverse.data; + } + const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); + const postState = RefinementPostStateSchema.safeParse(row.data.postState); + const origin = refinementRowOrigin(row, args.childWorkspaceId); + // Throws: this is the only durable copy once the child's journal goes. + const appended = await appendRefinementEventUnderBlobLock(ownerJournal, { + sessionDir: args.ownerSessionDir, + workspaceId: args.ownerWorkspaceId, + kind: "memory", + action, + inverse: draft, + evidence: { + toolName: evidence.success ? evidence.data.toolName : "memory", + ...(evidence.success && evidence.data.toolCallId !== undefined + ? { toolCallId: evidence.data.toolCallId } + : {}), + ...(evidence.success && evidence.data.actor !== undefined + ? { actor: evidence.data.actor } + : {}), + }, + ...(postState.success + ? { + postState: { + files: postState.data.files.flatMap((file) => { + try { + return [{ ...file, path: remap.path(file.path) }]; + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return []; + throw error; + } + }), + }, + } + : {}), + migratedFrom, + ...(rollbackOf !== undefined ? { rollbackOf } : {}), + // A row without a store-clock value (pre-sharing, or its clock write + // failed) has only a journal-local `ts`, incomparable with the + // owner's clock-stamped rows: carried as order-unknown rather than + // dressed up as a clock value. + ...(isValidSourceClock(row.data.sourceTs) && !retargeted + ? { sourceTs: row.data.sourceTs } + : {}), + // A malformed clock value is copied as no clock at all (order unknown). + ...(row.data.orderUnknown === true || !isValidSourceClock(row.data.sourceTs) || retargeted + ? { orderUnknown: true as const } + : {}), + // Order-unknown against the owner's rows, but not against each other: + // the source position (this child's journal, or the first origin of a + // copy-of-copy) lets the owner unwind the child's own overlapping + // history LIFO — the copies' owner-side `seq` is migration order, + // which a retried pass can permute (a re-copied row lands last). + ...(origin !== null ? { originJournal: origin.journal, originSeq: origin.seq } : {}), + ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), + }); + publishedBlobs.push(...appended.publishedBlobs); + ownerIdBySource.set(migratedFrom, appended.rowId); + if (rollbackOf !== undefined) ownerRollbackTargets.add(rollbackOf); + migrated++; + } + }); + // Migrated rows carry sourceTs (appended out of chronological order), so + // the quota pass re-derives retention in source order. + if (publishedBlobs.length > 0) { + await reclaimRefinementInverseBlobsBestEffort(ownerJournal, publishedBlobs, { resweep: true }); + } + return migrated; +} diff --git a/src/node/services/refinement/workspaceMemoryRevision.test.ts b/src/node/services/refinement/workspaceMemoryRevision.test.ts new file mode 100644 index 00000000000..02374ade83c --- /dev/null +++ b/src/node/services/refinement/workspaceMemoryRevision.test.ts @@ -0,0 +1,41 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as fsPromises from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { + advanceWorkspaceMemoryRevision, + readWorkspaceMemoryRevision, + workspaceMemoryRevisionPath, +} from "./workspaceMemoryRevision"; + +describe("advanceWorkspaceMemoryRevision", () => { + let sessionDir: string; + beforeEach(async () => { + sessionDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "xum-memory-revision-")); + }); + afterEach(async () => { + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + }); + + it("advances monotonically past a clock that runs ahead of wall time", async () => { + const ahead = Date.now() + 60_000; + await fsPromises.writeFile(workspaceMemoryRevisionPath(sessionDir), String(ahead)); + expect(await advanceWorkspaceMemoryRevision(sessionDir)).toBe(ahead + 1); + expect(await readWorkspaceMemoryRevision(sessionDir)).toBe(ahead + 1); + }); + + it("refuses to advance an exhausted clock and leaves the file as it is", async () => { + // 2^53 - 1 reads as valid, but `+ 1` is no longer a safe integer: writing + // it would persist a value the strict reader rejects forever. + const revisionPath = workspaceMemoryRevisionPath(sessionDir); + await fsPromises.writeFile(revisionPath, String(Number.MAX_SAFE_INTEGER)); + const attempt = advanceWorkspaceMemoryRevision(sessionDir); + expect( + await attempt.then( + () => null, + (error: unknown) => String(error) + ) + ).toContain("exhausted"); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe(String(Number.MAX_SAFE_INTEGER)); + }); +}); diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts new file mode 100644 index 00000000000..acbb41f4810 --- /dev/null +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -0,0 +1,92 @@ +import * as fsPromises from "node:fs/promises"; +import writeFileAtomic from "write-file-atomic"; +import * as path from "node:path"; +import assert from "@/common/utils/assert"; +import { WORKSPACE_MEMORY_REVISION_FILE_NAME } from "@/common/constants/memory"; + +/** + * Per-owner clock for a shared `/memories/workspace` store, persisted at + * `/memory.revision`. Two jobs, one file: + * + * - Cross-journal order. Owner and sub-agent rows describing the same store + * live in DIFFERENT session journals, whose `seq`/`ts` are not comparable + * (a migrated child row gets a fresh owner sequence; same-millisecond + * edits tie on `ts`). Every store mutation — memory command or rollback — + * advances this clock while holding the store's target mutation lock and + * stamps the value as the row's `sourceTs`, so rollback's conflict + * detection sees one total order across the whole task tree. + * - Cross-process change signal. The value only ever grows, so consumers in + * other backends (AgentSession's cached memory context, the Memory tab) + * compare it before reusing a cached view (MemoryService.workspaceMemoryRevision). + * + * Millisecond domain: `max(now, previous + 1)` keeps it a real timestamp for + * ordinary spacing while guaranteeing strict monotonicity under the lock. + */ +export function workspaceMemoryRevisionPath(ownerSessionDir: string): string { + assert(ownerSessionDir.length > 0, "workspaceMemoryRevisionPath requires an owner session dir"); + return path.join(ownerSessionDir, WORKSPACE_MEMORY_REVISION_FILE_NAME); +} + +/** Current clock value, or null when the store was never written (or the file is unreadable). */ +export async function readWorkspaceMemoryRevision(ownerSessionDir: string): Promise { + try { + return await readWorkspaceMemoryRevisionStrict(ownerSessionDir); + } catch { + return null; + } +} + +/** + * `readWorkspaceMemoryRevision` that distinguishes a never-written clock + * (null: proven ENOENT) from one that exists but cannot be trusted — an + * unreadable file, or content that is not a positive safe integer — which + * throws. + */ +async function readWorkspaceMemoryRevisionStrict(ownerSessionDir: string): Promise { + const revisionPath = workspaceMemoryRevisionPath(ownerSessionDir); + let raw: string; + try { + raw = await fsPromises.readFile(revisionPath, "utf-8"); + } catch (error) { + if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return null; + throw error; + } + // The persisted format is exactly the decimal digits of a positive safe + // integer (parseInt would accept a numeric prefix of anything). + const trimmed = raw.trim(); + const value = /^[1-9][0-9]{0,15}$/.test(trimmed) ? Number(trimmed) : Number.NaN; + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error( + `workspace memory revision at ${revisionPath} is malformed: ${raw.slice(0, 32)}` + ); + } + return value; +} + +/** + * Advance and persist the clock; returns the new value. Callers MUST hold the + * store's target mutation lock (cross-process) — the read→write here is what + * that lock makes atomic. Throws when the owner session dir is missing: the + * file is never allowed to recreate a removed owner's directory. Throws too + * when an EXISTING clock cannot be read or parsed: advancing from 0 instead + * could persist a value below the prior counter (which may run ahead of wall + * time) and hand callers a `sourceTs` that orders the mutation before rows + * it followed — callers treat the throw as "order unknown". + */ +export async function advanceWorkspaceMemoryRevision(ownerSessionDir: string): Promise { + const previous = (await readWorkspaceMemoryRevisionStrict(ownerSessionDir)) ?? 0; + const next = Math.max(Date.now(), previous + 1); + // A persisted clock at 2^53 - 1 reads as valid but cannot advance: writing + // `previous + 1` would persist a value the strict reader rejects forever + // and hand out a malformed `sourceTs`. Refuse instead (callers treat the + // throw as "order unknown"), leaving the file as it is. + if (!Number.isSafeInteger(next)) { + throw new Error( + `workspace memory revision at ${workspaceMemoryRevisionPath(ownerSessionDir)} is exhausted: ${previous}` + ); + } + // Atomic: a crash mid-write must not leave a truncated value the strict + // reader would reject forever (every later row order-unknown). + await writeFileAtomic(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); + return next; +} diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4044f662853..f75e9af3a87 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -9337,6 +9337,10 @@ describe("TaskService", () => { ); slashCard.metadata = { ...slashCard.metadata, + // Like WorkspaceService's slash-command append, drop the builder's + // placeholder sequence: HistoryService assigns the real one (and refuses + // a counter it could not advance past). + historySequence: undefined, muxMetadata: { type: WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, runId: workflowRunId }, }; const appendCard = await historyService.appendToHistory(rootWorkspaceId, slashCard); diff --git a/src/node/services/toolAssembly.test.ts b/src/node/services/toolAssembly.test.ts index 5629b449a25..ad9a507be1e 100644 --- a/src/node/services/toolAssembly.test.ts +++ b/src/node/services/toolAssembly.test.ts @@ -1,5 +1,9 @@ import { resolveToolPolicyForAgent } from "./agentDefinitions/resolveToolPolicy"; -import { isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; +import { + isMemoryToolDisabled, + isSessionHistoryDisabled, + type ToolPolicy, +} from "@/common/utils/tools/toolPolicy"; import { resolveAgentFrontmatter } from "./agentDefinitions/agentDefinitionsService"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { ToolBridge } from "./ptc/toolBridge"; @@ -15,6 +19,7 @@ import { sandboxHostService } from "@/node/services/sandbox/sandboxHostService"; import { DisposableTempDir } from "@/node/services/tempDir"; import { appendRefinementEvent } from "@/node/services/refinement/refinementJournal"; import { listRefinements } from "@/node/services/refinement/refinementRollback"; +import type { MemoryService } from "@/node/services/memoryService"; function executableTool(description: string): Tool { return { @@ -318,6 +323,75 @@ describe("persistent kernel graduation (RLM mode)", () => { } }); + test("refinement_rollback refuses memory rows when policy denies the memory tool", async () => { + using tmp = new DisposableTempDir("tool-assembly-rlm-rollback-memory-policy"); + const scopeKey = "ws-tool-assembly-rlm-rollback-memory-policy"; + const sessionDir = path.join(tmp.path, "sessions", scopeKey); + const noteFile = path.join(sessionDir, "memory", "note.md"); + const notified: string[][] = []; + // Stand-in MemoryService: classifies every path as the workspace scope and + // records rollback announcements; the row's inverse deletes a note. + const memoryService = { + scopeOfPhysicalPath: () => "workspace" as const, + notifyExternalMutation: (_ctx: unknown, paths: string[]) => { + notified.push(paths); + return Promise.resolve(); + }, + } as unknown as MemoryService; + const memory = { + service: memoryService, + ctx: { runtime: null, checkoutCwd: "", workspaceId: scopeKey, projectPath: "" }, + // Exec-like class: read-write everywhere — the class alone must not decide. + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" } as const, + }; + const assemble = (policy: ToolPolicy | undefined): Promise> => + applyToolPolicyAndExperiments({ + allTools: { memory: executableTool("Memory"), file_read: executableTool("Read a file") }, + effectiveToolPolicy: policy, + experiments: { programmaticToolCalling: true, rlm: true }, + emitNestedToolEvent: () => undefined, + sandbox: { workspaceId: scopeKey, sessionDir, memory }, + }); + const seedRow = async () => { + await fsPromises.mkdir(path.dirname(noteFile), { recursive: true }); + await fsPromises.writeFile(noteFile, "note", "utf-8"); + await appendRefinementEvent({ + sessionDir, + workspaceId: scopeKey, + kind: "memory", + action: { op: "create", path: "/memories/workspace/note.md" }, + inverse: { op: "delete-files", paths: [noteFile] }, + evidence: { toolName: "memory" }, + }); + return (await listRefinements(sessionDir)).at(-1)!.id; + }; + const rollback = async (tools: Record, id: string) => + (await tools.refinement_rollback.execute!( + { id, reason: "test" }, + { toolCallId: "test-call-id", messages: [], context: undefined } + )) as { success: boolean; error?: string }; + try { + // Policy strips `memory` but leaves the rollback surface: a memory-row + // rollback is a memory write, so it is refused like a read-only scope. + const denied = await assemble([{ regex_match: "memory", action: "disable" }]); + expect(denied.memory).toBeUndefined(); + expect(denied.refinement_rollback).toBeDefined(); + const refused = await rollback(denied, await seedRow()); + expect(refused.success).toBe(false); + expect(refused.error).toContain("read-only"); + expect(await fsPromises.readFile(noteFile, "utf-8")).toBe("note"); + expect(notified).toEqual([]); + + // With the memory tool allowed, the same rollback proceeds and announces. + const allowed = await assemble(undefined); + const rolledBack = await rollback(allowed, await seedRow()); + expect(rolledBack.success).toBe(true); + expect(notified).toEqual([[noteFile]]); + } finally { + await sandboxHostService.disposeScope(scopeKey); + } + }); + test("MUX_SANDBOX_PERSISTENT_MOUNTS=1 still opts in without the rlm experiment", async () => { using tmp = new DisposableTempDir("tool-assembly-env-mounts"); const scopeKey = "ws-tool-assembly-env-mounts"; @@ -549,6 +623,30 @@ describe("resolveBackendGatedPtcExperiments", () => { }); describe("token budget history policy", () => { + test.each([ + { add: [], allowed: false }, + { add: ["file_read"], allowed: false }, + { add: ["memory"], allowed: true }, + { add: ["mem.*"], allowed: true }, + { add: [".*"], allowed: true }, + ])("harvest permission mirrors the assembled memory tool: $add", async ({ add, allowed }) => { + // The persisted workspaceMemoryWritable bit is derived from the policy + // before tool assembly; it must agree with whether `memory` survives it. + const policy = resolveToolPolicyForAgent({ + agents: [{ tools: { add } }], + isSubagent: true, + disableTaskToolsForDepth: false, + }); + const memory = executableTool("Memory"); + const result = await applyToolPolicyAndExperiments({ + allTools: { memory, file_read: executableTool("Read") }, + effectiveToolPolicy: policy, + emitNestedToolEvent: () => undefined, + }); + expect(isMemoryToolDisabled(policy)).toBe(!allowed); + expect(result.memory === undefined).toBe(!allowed); + }); + test.each([ { add: [], allowed: false }, { add: ["file_read"], allowed: false }, diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 30129df994e..41ccb0dd6da 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -35,7 +35,13 @@ import type { QuickJSRuntimeFactory } from "@/node/services/ptc/quickjsRuntime"; import type { ToolBridge } from "@/node/services/ptc/toolBridge"; import type { PTCExecutionResult } from "@/node/services/ptc/types"; import { sandboxHostService, type SandboxMount } from "@/node/services/sandbox/sandboxHostService"; -import { createRefinementRollbackTool } from "@/node/services/tools/refinement_rollback"; +import { + createRefinementRollbackTool, + type SharedWorkspaceMemoryTopologyResolver, +} from "@/node/services/tools/refinement_rollback"; +import { READ_ONLY_ACCESS } from "@/node/services/tools/memory"; +import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; import type { KernelFileLoader } from "@/node/services/tools/kernelFileLoad"; import { log } from "./log"; import type { MCPWorkspaceStats } from "@/node/services/mcpServerManager"; @@ -115,7 +121,15 @@ export interface ApplyToolPolicyAndExperimentsOptions { * caller from the workspace cwd/runtime pair the file tools use; only * honored in kernel mode with file_read bridged. */ - sandbox?: { workspaceId: string; sessionDir: string; kernelFileLoader?: KernelFileLoader }; + sandbox?: { + workspaceId: string; + sessionDir: string; + /** Shared-notebook topology resolver for refinement_rollback (see its ctx). */ + sharedWorkspaceMemory?: SharedWorkspaceMemoryTopologyResolver; + /** Lets refinement_rollback announce its direct-to-disk memory writes. */ + memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; + kernelFileLoader?: KernelFileLoader; + }; /** * Capability grants for this assembly (registry-with-filters posture). * Omitted = session-scope full grants (identical to pre-grants behavior). @@ -331,8 +345,19 @@ export async function applyToolPolicyAndExperiments( // disables the tool, e.g. a broad regex disable rule) must not gain a // harness-rollback surface. Unlike code_execution in exclusive mode, // rollback is never mandatory, so policy may freely remove it. + // A memory-row rollback is a memory WRITE. When policy or grants deny + // the memory tool itself (checked on the grant-and-policy-filtered base + // set: in PTC mode the tool may be bridged rather than model-visible), + // rollback must not become a side door into the (shared) notebook — + // hand it a view-only policy so every memory row is refused, exactly + // as an explore-like agent's would be. + const memoryToolAvailable = policyFilteredTools.memory !== undefined; let rollback: Record = { - refinement_rollback: createRefinementRollbackTool(sandbox), + refinement_rollback: createRefinementRollbackTool( + sandbox.memory === undefined || memoryToolAvailable + ? sandbox + : { ...sandbox, memory: { ...sandbox.memory, access: READ_ONLY_ACCESS } } + ), }; rollback = applyToolPolicy(rollback, effectiveToolPolicy); if (opts.capabilityGrants) { diff --git a/src/node/services/tools/memory.ts b/src/node/services/tools/memory.ts index 7fd482d26c3..9d9d48138ae 100644 --- a/src/node/services/tools/memory.ts +++ b/src/node/services/tools/memory.ts @@ -15,7 +15,7 @@ import { } from "@/node/services/memoryService"; /** Safe default: without an explicit policy, every scope is read-only. */ -const READ_ONLY_ACCESS: MemoryScopeAccess = { +export const READ_ONLY_ACCESS: MemoryScopeAccess = { global: "read", project: "read", workspace: "read", @@ -257,6 +257,13 @@ export async function executeMemoryCommand( * I/O unblocks. Ignored by reads. */ abortSignal?: AbortSignal; + /** + * Consolidation's pin protection (pinned files are editable but never + * deleted/renamed), enforced by MemoryService INSIDE its target mutation + * lock against the owner the command's store is bound to — see + * MemoryService.assertNotPinnedForRemoval. Ignored by other commands. + */ + rejectPinned?: boolean; } ): Promise { try { @@ -336,7 +343,8 @@ export async function executeMemoryCommand( "agent", toolCallId, options?.expectedTargetFingerprint, - options?.abortSignal + options?.abortSignal, + { rejectPinned: options?.rejectPinned } )) ); } @@ -355,7 +363,8 @@ export async function executeMemoryCommand( input.new_path, "agent", toolCallId, - options?.abortSignal + options?.abortSignal, + { rejectPinned: options?.rejectPinned } )) ); } diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 547af455fcf..be4169eed4a 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -1,8 +1,18 @@ +import * as path from "node:path"; import { tool, type Tool } from "ai"; import type { RefinementRollbackToolResult } from "@/common/types/tools"; import { TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; -import { rollbackRefinement } from "@/node/services/refinement/refinementRollback"; +import { listRefinements, rollbackRefinement } from "@/node/services/refinement/refinementRollback"; +import { RefinementInverseSchema, type RefinementInverse } from "@/common/types/refinement"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; +import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, +} from "@/node/services/memoryLegacyAdoption"; +import { getErrorMessage } from "@/common/utils/errors"; +import type { SharedWorkspaceMemoryTopology } from "@/node/services/memoryWorkspaceOwner"; interface RefinementRollbackToolArgs { id: string; @@ -17,9 +27,89 @@ interface RefinementRollbackToolArgs { * No force parameter on purpose: divergence overrides are a human decision * (debug CLI --force). The model gets the refusal text and can report it. */ +/** + * Policy gate for memory rows: every path the target row's inverse would touch + * must lie in a scope this agent may write. Unknown rows/inverses fall through + * (null) so the engine produces its canonical refusal. + */ +async function refuseReadOnlyMemoryRollback( + sessionDir: string, + sharedWorkspaceMemorySessionDir: string | undefined, + id: string, + memory: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess } +): Promise { + const row = (await listRefinements(sessionDir)).find((candidate) => candidate.id === id); + if (row?.data.kind !== "memory") return null; + const parsed = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsed.success) return null; + // The paths the engine will actually touch: a sub-agent's pre-sharing rows + // address its legacy private notebook, which the engine retargets to the + // adopted owner copy (refinementRollback.ts) — classify THOSE paths, or the + // legacy ones would read as unclassifiable and refuse every such row here. + // A legacy path the engine cannot map falls through to its refusal. + let inverse: RefinementInverse = parsed.data; + if ( + sharedWorkspaceMemorySessionDir !== undefined && + path.resolve(sharedWorkspaceMemorySessionDir) !== path.resolve(sessionDir) + ) { + const remap = await createLegacyPathRemapper({ + childSessionDir: sessionDir, + ownerSessionDir: sharedWorkspaceMemorySessionDir, + }); + try { + inverse = remap.inverse(parsed.data); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } + } + const paths = + inverse.op === "delete-files" + ? inverse.paths + : inverse.op === "rename" + ? [inverse.from, inverse.to] + : [...inverse.files.map((file) => file.path), ...(inverse.deletePaths ?? [])]; + for (const physicalPath of paths) { + const scope = memory.service.scopeOfPhysicalPath(memory.ctx, physicalPath); + // Fail closed: a memory row's paths always lie in some scope root, so + // "unclassifiable" means this context's roots no longer match the row + // (e.g. the owner root admitted at preparation time while the per-context + // resolver now falls back to self because config.json is unreadable) — + // the policy cannot be evaluated, so the write must not proceed. + if (scope === null) { + return `Cannot classify '${physicalPath}' against this agent's memory scopes; refusing to roll back '${id}'.`; + } + if (memory.access[scope] !== "readwrite") { + return `The ${scope} memory scope is read-only for this agent; rolling back '${id}' would write into it.`; + } + } + return null; +} + +export type SharedWorkspaceMemoryTopologyResolver = () => SharedWorkspaceMemoryTopology; + export function createRefinementRollbackTool(ctx: { workspaceId: string; sessionDir: string; + /** + * Task-tree topology of a workspace sharing its notebook, resolved PER + * EXECUTION (membership changes while the tool instance lives) from a + * config snapshot that must prove itself: a throw refuses the rollback. No + * fallback view is acceptable here — ownership read from a missing + * config.json resolves to "self", which would omit the owner root (a + * pre-sharing row's inverse then lands on the hidden legacy notebook + * instead of the owner's adopted copy) and the peer list (conflicting + * sibling rows go unseen). Omitted = the workspace owns its notebook. + */ + sharedWorkspaceMemory?: SharedWorkspaceMemoryTopologyResolver; + /** + * Memory integration: announces rolled-back memory files so shared-store + * readers refresh, and applies the agent's per-scope write policy — a + * rollback is a write into the scope, so a read-only scope (e.g. the shared + * workspace notebook for an explore-like sub-agent) refuses it, exactly as + * the memory tool would. + */ + memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; }): Tool { return tool({ description: TOOL_DEFINITIONS.refinement_rollback.description, @@ -28,8 +118,35 @@ export function createRefinementRollbackTool(ctx: { { id, reason }: RefinementRollbackToolArgs, { toolCallId } ): Promise => { + // Resolved once here for the owner root (stable for the session's + // lifetime) and the policy gate; peers are re-resolved by the engine per + // check through the same resolver (a throw there refuses too). + let topology: SharedWorkspaceMemoryTopology; + try { + topology = ctx.sharedWorkspaceMemory?.() ?? { + ownerSessionDir: undefined, + peerSessionDirs: [], + }; + } catch (error) { + return { + success: false, + error: `Cannot resolve this workspace's shared-memory ownership right now (${getErrorMessage(error)}); refusing to roll back '${id}'.`, + }; + } + if (ctx.memory !== undefined) { + const refusal = await refuseReadOnlyMemoryRollback( + ctx.sessionDir, + topology.ownerSessionDir, + id, + ctx.memory + ); + if (refusal !== null) return { success: false, error: refusal }; + } const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, + sharedWorkspaceMemorySessionDir: topology.ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => + ctx.sharedWorkspaceMemory?.().peerSessionDirs ?? [], id, reason, evidence: { toolName: "refinement_rollback", toolCallId, actor: "agent" }, @@ -37,6 +154,16 @@ export function createRefinementRollbackTool(ctx: { if (!result.success) { return { success: false, error: result.error }; } + // Rollback writes inverses straight to disk, bypassing MemoryService's + // change events; announce them so the (possibly shared) store's other + // readers — owner, siblings, open Memory tabs — do not keep stale context. + ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ + ...result.data.restored, + ...result.data.deleted, + ...(result.data.renamed === undefined + ? [] + : [result.data.renamed.from, result.data.renamed.to]), + ]); return { success: true, rollbackOf: id, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index be3ff56dcbc..4cca2a7e293 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -38,6 +38,11 @@ import type { SendMessageError } from "@/common/types/errors"; import type { GoalRecordV1 } from "@/common/types/goal"; import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/types/message"; import { createMuxMessage } from "@/common/types/message"; +import { + epochHasPriorTurnRows, + workspaceMemoryPolicyEpochOf, +} from "@/common/utils/messages/compactionBoundary"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; import { secretsToRecord } from "@/common/types/secrets"; import type { XumToolScope } from "@/common/types/toolScope"; @@ -124,8 +129,10 @@ import { type MCPWorkspaceStats, } from "@/node/services/mcpServerManager"; import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; +import { resolveSharedWorkspaceMemoryTopology } from "@/node/services/memoryWorkspaceOwner"; +import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; -import { resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; +import { READ_ONLY_ACCESS, resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; import { isWorkspaceTrustedForSharedExecution } from "@/node/services/utils/workspaceTrust"; import { MCP_OVERRIDES_READ_TIMEOUT_MS, @@ -259,7 +266,11 @@ export function resolveXumToolScope( import type { PostCompactionAttachment } from "@/common/types/attachment"; import type { ErrorEvent } from "@/common/types/stream"; -import { withContextBudgetFlushToolPolicy, type ToolPolicy } from "@/common/utils/tools/toolPolicy"; +import { + isMemoryToolDisabled, + withContextBudgetFlushToolPolicy, + type ToolPolicy, +} from "@/common/utils/tools/toolPolicy"; import type { FileState } from "@/node/services/agentSession"; import type { ActiveTurnThinkingOverride } from "@/node/services/thinkingOverride"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -541,8 +552,21 @@ type TurnRequestBuildOutcome = type: "ready"; turnExecutionOptions: TurnExecutionOptions; assistantMessageId: string; - deleteAbortedPlaceholder: (messageId: string) => Promise; + /** + * Remove the never-run turn's stamped placeholder, or deny its epoch + * when that fails; false when neither could be made durable — the + * caller must report WORKSPACE_MEMORY_POLICY_PERSIST_ERROR instead of + * its own outcome. + */ + deleteAbortedPlaceholder: (messageId: string) => Promise; logStartOutcome: (outcome: "started" | "stream_start_failed", errorType?: string) => void; + /** + * Durable side effects that must only land once the stream has actually + * started (a granted memory harvest permission): every earlier exit — + * append failure, late abort, thinking rebuild, stream start failure — + * then leaves nothing behind. Awaited by the caller after startStream. + */ + onStreamStarted?: () => Promise; }; export interface PreparedStreamMessage extends AsyncDisposable { @@ -553,6 +577,10 @@ export interface PreparedTurnRequest extends AsyncDisposable { start(thinkingOverride?: ActiveTurnThinkingOverride): Promise; } +/** Turn refused because a memory-policy DENY could not be made durable (see persistWorkspaceMemoryWritable). */ +export const WORKSPACE_MEMORY_POLICY_PERSIST_ERROR = + "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; + type PreparedTurnRequestOutcome = | Extract | { type: "prepared"; request: PreparedTurnRequest }; @@ -567,6 +595,23 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { onWorkflowRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; workflowResultContinuationSender?: WorkflowResultContinuationSender; workspaceHeartbeatService?: ToolConfiguration["workspaceHeartbeatService"]; + /** + * Receives each normal turn's workspace-memory write policy and persists it + * (see WorkspaceService.recordWorkspaceMemoryWritable); resolves false when + * the value could not be confirmed durable. + */ + workspaceMemoryPolicySink?: { + recordWorkspaceMemoryWritable( + workspaceId: string, + writable: boolean, + options: { + epochHasPriorTurns: boolean; + policyEpoch: number; + carriedPolicyEpochs?: number[]; + carriedPolicyUnknown?: boolean; + } + ): Promise; + }; analyticsService?: { executeRawQuery(sql: string): Promise }; desktopSessionManager?: DesktopSessionManager; } @@ -896,7 +941,7 @@ export class TurnRequestBuilder { const recordStartupPhaseTiming = context.recordStartupPhaseTiming; let pendingRunMetadataId: string | null = context.startupState.pendingRunMetadataId; - const deleteAbortedPlaceholder = async (messageId: string): Promise => { + const deleteAbortedPlaceholder = async (messageId: string): Promise => { const deleteResult = await this.dependencies.historyService.deleteMessage( workspaceId, messageId @@ -909,6 +954,7 @@ export class TurnRequestBuilder { deleteResult.error ); } + return deleteResult.success; }; // Mode (plan|exec|compact) is derived from the selected agent definition. const effectiveMuxProviderOptions: MuxProviderOptions = muxProviderOptions ?? {}; @@ -1474,6 +1520,125 @@ export class TurnRequestBuilder { const memoryAccess: MemoryScopeAccess = contextBudgetFlushTurn ? { global: "read", project: "read", workspace: agentMemoryAccess.workspace } : agentMemoryAccess; + // Post-compaction harvest writes to /memories/workspace on the agent's + // behalf; it must honor exactly what the memory tool enforces: the scope + // access AND the tool's presence in the final toolset. The tool exists + // only with the experiment + service (tools.ts) and survives only if the + // effective policy keeps it (applied later in tool assembly, mirrored + // here); request.assemble middleware can still strip it, so the FINAL + // check happens against the toolset of the request actually started (see + // persistWorkspaceMemoryWritable). The compaction turn itself runs the + // "compact" agent, so only normal turns record their policy (the session + // attaches it to the compaction completion). + const workspaceMemoryWritable = + memoryAccess.workspace === "readwrite" && + memoryExperimentEnabled && + this.dependencies.bindings.memoryService !== undefined && + !isMemoryToolDisabled(effectiveToolPolicy); + // Persisting the harvest permission is asymmetric because the two values + // fail differently when the turn then never streams: a stale DENY only + // fails closed (the preceding transcript is not harvested), a stale GRANT + // would let the preceding read-only transcript harvest into the (shared) + // workspace notebook after a restart. So a deny is persisted inside + // `start` before anything else (and refuses the turn when it cannot be + // made durable), while a grant is persisted only once the stream has + // actually started (onStreamStarted) — never at preparation time, where + // an admission-only candidate (prepareStreamMessage) may be rejected or + // disposed without running. Awaited (a config write happens only when + // the value changes). Returns false when a deny could not be persisted. + // Whether the active context already holds turns whose policy this + // process never recorded (see the unknown-history rule in + // WorkspaceService.recordWorkspaceMemoryWritable). The turn being started + // is its last user row plus that row's prelude snapshots; compaction + // request rows open an epoch rather than belong to one. + const epochHasPriorTurns = epochHasPriorTurnRows(activeContextMessages, { + userMessageId: latestUserMessage?.id, + preludeMessageIds: new Set( + getRequestPreludeMessageIds(latestUserMessage?.metadata?.requestPreludeMessageIds) + ), + }); + // The compaction epoch this turn's policy accumulates over: the latest + // durable boundary's history sequence (any kind), else the history + // segment's boundary-less identity — never reused across full clears + // (see workspaceMemoryPolicyEpochOf) — the same identity compaction + // completion reports as closingPolicyEpoch, so the completion-side + // observation and every backend's turn records agree on which epoch a + // value belongs to. + const policyEpoch = workspaceMemoryPolicyEpochOf(messages); + // A preserved-tail boundary (RLM keep-recent copies follow it) re-appends + // rows produced under EARLIER epochs' policies: those accumulators are + // part of this epoch. The compacting session re-binds the closing one to + // this epoch durably (AgentSession.carryWorkspaceMemoryWritable), but + // asynchronously — another backend's first turn here can precede that + // carry. Naming the carried epochs lets the sink AND their values + // directly (under whichever key each currently sits), so no window exists + // in which a read-only tail reads as writable. Each copy records the + // epoch it was originally produced under (compactionHandler stamps it; a + // copy of a copy keeps the first), so a chain of tail compactions names + // every epoch involved — `messages` here holds only the active epoch, so + // nothing about earlier boundaries can be derived from it. + const tailCopies = activeContextMessages.filter( + (message) => message.metadata?.rlmPreservedTailCopy === true + ); + // A usable source epoch is an integer below policyEpoch — a segment's + // boundary-less identity (negative), else a boundary's history sequence, + // both earlier than this epoch's (persisted history is unvalidated). A + // copy without one — persisted by a build + // before the field, a re-copy of such a copy, or a malformed value — + // carries a policy nobody can look up: it is excluded from the prior-turn + // check like every copy, so without this the epoch would grant on the + // strength of the turns it can see. Unknown fails closed — the epoch is + // denied until a no-tail boundary (or the tail turning over) leaves no + // such copy in the active context. + const usableSourceEpoch = (message: MuxMessage): number | undefined => { + const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; + return typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch < policyEpoch + ? epoch + : undefined; + }; + const carriedPolicyEpochs = [ + ...new Set( + tailCopies.flatMap((message) => { + const epoch = usableSourceEpoch(message); + return epoch === undefined ? [] : [epoch]; + }) + ), + ].sort((a, b) => a - b); + const carriedPolicyUnknown = tailCopies.some( + (message) => usableSourceEpoch(message) === undefined + ); + const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { + const sink = this.dependencies.bindings.workspaceMemoryPolicySink; + if (isCompactionRequest || !sink) return true; + if ( + await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { + epochHasPriorTurns, + policyEpoch, + ...(carriedPolicyEpochs.length === 0 ? {} : { carriedPolicyEpochs }), + ...(carriedPolicyUnknown ? { carriedPolicyUnknown } : {}), + }) + ) { + return true; + } + if (!writable) return false; + log.warn("Workspace memory write policy could not be persisted; harvests will fail closed", { + workspaceId, + }); + return true; + }; + // Placeholder of a turn that never ran (startup aborted or failed): it + // carries the request bound and the policy stamp the finalized row needs, + // so left behind it would present the user batch to the harvest gate as + // covered by a turn. When its removal fails (I/O, a concurrent rewrite) + // the epoch is denied instead — fail closed rather than trust a row + // nobody can confirm is gone. + // Returns false only when neither the deletion nor the deny could be made + // durable: the caller must then surface that instead of its own outcome, + // since a stamped placeholder nobody could remove or deny stays behind. + const discardPlaceholder = async (messageId: string): Promise => { + if (await deleteAbortedPlaceholder(messageId)) return true; + return persistWorkspaceMemoryWritable(false); + }; const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic // repo hook execution (tool_env/tool_pre/tool_post) while keeping @@ -2471,6 +2636,32 @@ export class TurnRequestBuilder { recordStartupPhaseTiming("getToolsForModelMs", getToolsStartedAt); } + // A sub-agent's workspace-scope memory rows point into its task-tree + // owner's session dir; rollback must admit that root (and only that), + // and announce its direct-to-disk writes through MemoryService so the + // shared store's readers refresh. Resolved per rollback (not per + // turn): tree membership changes as sub-agents are spawned and removed + // while the tool instance lives; a topology that cannot be proven + // refuses the rollback (see resolveSharedWorkspaceMemoryTopology). + const memoryService = this.dependencies.bindings.memoryService; + const sessionsDir = this.dependencies.config.sessionsDir; + const sharedWorkspaceMemory = + memoryService === undefined + ? undefined + : () => resolveSharedWorkspaceMemoryTopology(this.dependencies.config, workspaceId); + // Built anew for EVERY attempt (prepareModelRequest runs per primary / + // fallback request) from the never-mutated policy: refinement_rollback + // reads it by reference, and the request.assemble demotion below only + // touches this attempt's object, so a fallback whose middleware keeps + // `memory` starts writable again. + const attemptSandboxMemory = + memoryService === undefined + ? undefined + : { + service: memoryService, + ctx: memoryScopeContextFromToolConfig(toolsForModelConfig), + access: memoryAccess, + }; const applyPolicyStartedAt = Date.now(); let attemptTools = await applyToolPolicyAndExperiments({ allTools: this.dependencies.wrapToolsForDelegation( @@ -2484,7 +2675,9 @@ export class TurnRequestBuilder { emitNestedToolEvent: emitNestedPtcToolEvent, sandbox: { workspaceId, - sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), + sessionDir: path.join(sessionsDir, workspaceId), + sharedWorkspaceMemory, + memory: attemptSandboxMemory, kernelFileLoader, }, }); @@ -2583,6 +2776,13 @@ export class TurnRequestBuilder { if (!intuitionToolAvailable || attemptTools.memory === undefined) { delete attemptTools.intuition; } + // Nor may it leave refinement_rollback (RLM) as a side door into the + // notebook once `memory` is gone: the tool reads its memory policy + // from this per-attempt object by reference, so demote it to + // view-only the same way tool assembly does for policy-denied memory. + if (attemptTools.memory === undefined && attemptSandboxMemory !== undefined) { + attemptSandboxMemory.access = READ_ONLY_ACCESS; + } if (attemptTools.intuition === undefined) { assembleCtx.systemMessage = removeIntuitionGuidance( assembleCtx.systemMessage, @@ -2831,8 +3031,40 @@ export class TurnRequestBuilder { this.dependencies.createAbortedTurnHandle(assistantMessageId, combinedAbortSignal) ), }; + // Final toolset of the request being started: request.assemble + // middleware ran inside prepareModelRequest, and `memory` is a built-in + // that tool search never defers and PTC never bridges (ptcExcluded in + // toolDefinitions: its top-level presence keys the memory index / hot + // set), so its absence means denied. The deny lands now; the grant + // waits for onStreamStarted (see persistWorkspaceMemoryWritable). + const finalWorkspaceMemoryWritable = workspaceMemoryWritable && tools.memory !== undefined; + if (!finalWorkspaceMemoryWritable && !(await persistWorkspaceMemoryWritable(false))) { + const errorEvent = createErrorEvent(workspaceId, { + messageId: createAssistantMessageId(), + error: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR, + errorType: "unknown", + acpPromptId, + }); + if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); + onPreStartError?.(errorEvent); + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } + // Proof for the harvest gate that this turn's policy was recorded + // (above, or the grant at onStreamStarted) and for which epoch: a build + // without the sink — or an older build after a downgrade — leaves it + // unset and its turns' user rows stay unaccounted for. Carried by the + // placeholder AND the stream's initialMetadata below: StreamManager + // builds the final assistant row from the latter, not the placeholder. + const workspaceMemoryPolicyStamp = + !isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink + ? { workspaceMemoryPolicyEpoch: policyEpoch } + : {}; const assistantMessage = createMuxMessage(assistantMessageId, "assistant", "", { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), + ...workspaceMemoryPolicyStamp, timestamp: Date.now(), model: canonicalModelString, routedThroughGateway, @@ -2954,7 +3186,12 @@ export class TurnRequestBuilder { } if (combinedAbortSignal.aborted) { - await deleteAbortedPlaceholder(assistantMessageId); + if (!(await discardPlaceholder(assistantMessageId))) { + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } return { type: "finished", result: Ok( @@ -3082,6 +3319,19 @@ export class TurnRequestBuilder { if (error instanceof ContextBudgetExceededError) return Err(error.details); throw error; } + // The fallback runs its own request.assemble pass, which may + // strip or keep `memory` independently of the primary; the + // persisted harvest permission must follow the request + // actually streamed, in both directions. The stream is + // already running here, so a grant may land immediately. + if ( + !(await persistWorkspaceMemoryWritable( + workspaceMemoryWritable && nextRequest.tools.memory !== undefined + )) + ) { + runLanguageModelCleanup(nextRequest.model); + return Err(WORKSPACE_MEMORY_POLICY_PERSIST_ERROR); + } let nextHeaders = nextRequest.headers; if (pendingRunMetadataId != null) { nextHeaders = { @@ -3163,7 +3413,12 @@ export class TurnRequestBuilder { } catch (error) { if (error instanceof ContextBudgetExceededError) { runLanguageModelCleanup(modelResult.data.model); - await deleteAbortedPlaceholder(assistantMessageId); + if (!(await discardPlaceholder(assistantMessageId))) { + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } return { type: "finished", result: Err(error.details) }; } throw error; @@ -3196,6 +3451,7 @@ export class TurnRequestBuilder { contextBudgetLimit: primaryRequest.contextBudgetLimit, initialMetadata: { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), + ...workspaceMemoryPolicyStamp, systemMessageTokens, timestamp: Date.now(), agentId: effectiveAgentId, @@ -3261,8 +3517,18 @@ export class TurnRequestBuilder { type: "ready", turnExecutionOptions, assistantMessageId, - deleteAbortedPlaceholder, + deleteAbortedPlaceholder: discardPlaceholder, logStartOutcome, + ...(finalWorkspaceMemoryWritable + ? { + // Best-effort once streaming: the turn really runs with a + // writable memory tool, so an unpersisted grant only fails the + // harvest closed (persistWorkspaceMemoryWritable warns). + onStreamStarted: async () => { + await persistWorkspaceMemoryWritable(true); + }, + } + : {}), }; }; retained = true; diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts new file mode 100644 index 00000000000..1fa6b4b19e4 --- /dev/null +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -0,0 +1,226 @@ +/** + * Durable fallback for the workspace-memory write-policy DENY. + * + * The epoch accumulator normally lives on the workspace's config.json entry + * (`workspaceMemoryWritable`, see WorkspaceService.recordWorkspaceMemoryWritable). + * By the time a turn learns its final tool set has no `memory` tool, its user + * row is already durable in chat.jsonl — so if the config write fails, refusing + * the turn is not enough: after a restart the accumulator would be absent, a + * later writable turn would publish `true`, and the compaction harvest (which + * reads EVERY message of the epoch) would carry the refused turn's rows into + * the shared notebook. This marker lives in the session dir — the same + * durability domain as chat.jsonl — and is ANDed into the accumulator wherever + * it is consulted; it is cleared only at the destructive boundary that clears + * the config records. + * + * Like the config records, the marker is bound to its compaction epoch (the + * opening boundary's history sequence, -1 before any boundary) and holds one + * entry per epoch: a reader consulting it for another epoch ignores it, so a + * backend starting the new epoch cannot inherit a stale deny — and a deny that + * backend records for the new epoch cannot displace the closing epoch's deny + * before every compacting backend observes it (see + * workspaceMemoryPolicyEpochs.ts). + * + * Fail-closed by construction: a missing marker (or one without an entry for + * this epoch) is "no deny"; a present entry for this epoch, a wildcard entry + * (inherited from a malformed marker some later writer replaced), or a + * malformed/unreadable marker, is a deny. + */ +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; +import writeFileAtomic from "write-file-atomic"; +import assert from "@/common/utils/assert"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; + +export const WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME = "memory-policy-deny.json"; + +export function workspaceMemoryDenyMarkerPath(sessionDir: string): string { + assert(sessionDir.length > 0, "workspaceMemoryDenyMarkerPath requires a session dir"); + return path.join(sessionDir, WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME); +} + +/** Durable-or-throw: verified by reading the marker back. Adds `epoch` to the recorded set. */ +export async function writeWorkspaceMemoryDenyMarker( + sessionDir: string, + epoch: number +): Promise { + assert(Number.isInteger(epoch), "workspace memory deny marker epoch must be an integer"); + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + await fsPromises.mkdir(sessionDir, { recursive: true }); + const record = await readMarkerRecord(markerPath); + if (record === "unreadable") { + throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); + } + // A malformed marker was a deny for EVERY epoch's reader while it existed, + // possibly the only evidence of some other backend's read-only turn in + // the closing epoch. This writer, recording a deny for its own epoch, + // cannot know which epoch that was, so the well-formed file it leaves + // behind carries the deny forward as a wildcard until a destructive + // boundary clears it (clearWorkspaceMemoryDenyMarker). + const epochs = record === "absent" || record === null ? [] : record.epochs; + const wildcard = record === null || (record !== "absent" && record.wildcard); + await writeMarkerRecord(markerPath, [...epochs.filter((e) => e !== epoch), epoch], wildcard); + if (!(await readWorkspaceMemoryDenyMarker(sessionDir, epoch))) { + throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); + } +} + +/** `wildcard`: a deny of unknown epoch (inherited from a malformed marker), denying every reader. */ +async function writeMarkerRecord( + markerPath: string, + epochs: readonly number[], + wildcard: boolean +): Promise { + // Entries are removed only by a destructive boundary, never by count or by + // the observation of a compaction boundary — see workspaceMemoryPolicyEpochs.ts. + const retained = [...new Set(epochs)].sort((a, b) => b - a); + await writeFileAtomic( + markerPath, + JSON.stringify({ deniedAt: Date.now(), epochs: retained, wildcard }), + { encoding: "utf-8" } + ); +} + +/** + * Parsed marker; "absent" on a proven ENOENT; null when the content is + * malformed (readers deny; a boundary clear heals it); "unreadable" when the + * file could not be read at all (EACCES, I/O) — readers deny, and mutators + * throw rather than replace or delete epoch state they could not see. + */ +async function readMarkerRecord( + markerPath: string +): Promise<{ epochs: number[]; wildcard: boolean } | "absent" | "unreadable" | null> { + let raw: string; + try { + raw = await fsPromises.readFile(markerPath, "utf-8"); + } catch (error) { + return hasErrorCode(error, "ENOENT") ? "absent" : "unreadable"; + } + try { + const parsed: unknown = JSON.parse(raw); + if (typeof parsed !== "object" || parsed === null) return null; + const { epochs, wildcard } = parsed as { epochs?: unknown; wildcard?: unknown }; + // Epochs are a segment's boundary-less identity (negative) or a + // boundary's history sequence (workspaceMemoryPolicyEpochOf): anything + // outside the integer domain is corruption, read as malformed (deny). + if ( + !Array.isArray(epochs) || + !epochs.every( + (epoch): epoch is number => typeof epoch === "number" && Number.isSafeInteger(epoch) + ) + ) { + return null; + } + // `wildcard` may be omitted (false) but, when present, must be a boolean: + // a corrupted `"true"` read as false would drop the only surviving deny + // of an epoch the `epochs` list does not name. + if (wildcard !== undefined && typeof wildcard !== "boolean") return null; + return { epochs, wildcard: wildcard === true }; + } catch { + return null; + } +} + +/** + * True when a deny is recorded for `epoch` (or the marker cannot be trusted: + * unreadable or malformed); false when absent or recorded only for other + * epochs. Without `epoch`, any present marker is a deny (epoch-agnostic + * existence check). + */ +export async function readWorkspaceMemoryDenyMarker( + sessionDir: string, + epoch?: number +): Promise { + return readWorkspaceMemoryDenyMarkerForEpochs( + sessionDir, + epoch === undefined ? undefined : [epoch] + ); +} + +/** + * `readWorkspaceMemoryDenyMarker` for several epochs from ONE read of the + * marker: a deny recorded for any of them. A preserved-tail turn consults its + * own epoch and the epochs its tail copies were produced under; the carry + * (carryWorkspaceMemoryDenyMarker) re-stamps an entry from the closing epoch + * to the new one atomically, so two separate reads could each miss it (old + * gone, new not yet seen) while a single snapshot always holds it under one + * key or the other. Without `epochs`, any present marker is a deny. + */ +export async function readWorkspaceMemoryDenyMarkerForEpochs( + sessionDir: string, + epochs?: readonly number[] +): Promise { + const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); + if (record === "absent") return false; + if (record === null || record === "unreadable" || epochs === undefined) return true; + return record.wildcard || epochs.some((epoch) => record.epochs.includes(epoch)); +} + +/** + * Destructive boundary (/clear, context reset, history replace): every + * epoch's deny goes, durable-or-throw (verified absent). Compaction boundaries + * clear nothing: the closing epoch's entry may still be observed by another + * backend's boundary closing the same epoch (r77; see + * AgentSession.resetWorkspaceMemoryWritable), and a malformed marker stays a + * deny for every epoch until the transcript it may describe is discarded. + * Unreadable is not malformed: a marker that cannot be read is refused rather + * than deleted (the caller's reset fails and is retried). + */ +export async function clearWorkspaceMemoryDenyMarker( + rootDir: string, + sessionDir: string +): Promise { + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + // Under the session-dir target lock the writer holds + // (WorkspaceService.recordWorkspaceMemoryWritable's fallback). + await withTargetMutationLock(rootDir, sessionDir, async () => { + if ((await readMarkerRecord(markerPath)) === "unreadable") { + throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); + } + await fsPromises.rm(markerPath, { force: true }); + if (await readWorkspaceMemoryDenyMarker(sessionDir)) { + throw new Error(`Workspace memory deny marker could not be removed at ${markerPath}`); + } + }); +} + +/** + * A preserved-tail compaction carries the closing epoch's policy into the new + * one (the tail copies were produced under it): a deny recorded for + * `closingEpoch` is re-stamped as `nextEpoch` so readers of the new epoch + * keep seeing it. Entries of other epochs and absent markers are left alone; + * a malformed one stays a deny for every reader regardless. Same lock as the + * clear, for the same read→write reason. + */ +export async function carryWorkspaceMemoryDenyMarker( + rootDir: string, + sessionDir: string, + closingEpoch: number, + nextEpoch: number +): Promise { + await withTargetMutationLock(rootDir, sessionDir, async () => { + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + const record = await readMarkerRecord(markerPath); + if (record === "unreadable") { + throw new Error( + `Workspace memory deny marker is unreadable at ${workspaceMemoryDenyMarkerPath(sessionDir)}` + ); + } + if (record === "absent" || record === null) return; + if (!record.wildcard && !record.epochs.includes(closingEpoch)) return; + // The closing entry (or the wildcard, which denied the closing epoch too) + // is copied, not moved: another backend's boundary closing the same + // epoch must still find it (r77). The wildcard stays a deny of unknown + // epoch — narrowing it to this boundary's epochs would drop the deny + // for whichever epoch it actually recorded. + await writeMarkerRecord( + markerPath, + [...record.epochs.filter((epoch) => epoch !== nextEpoch), nextEpoch], + record.wildcard + ); + if (!(await readWorkspaceMemoryDenyMarker(sessionDir, nextEpoch))) { + throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); + } + }); +} diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts new file mode 100644 index 00000000000..fe7c8519407 --- /dev/null +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -0,0 +1,90 @@ +/** + * Per-epoch storage of the workspace-memory write-policy accumulator on a + * workspace's config.json entry (`workspaceMemoryWritableByEpoch`, see + * WorkspaceService.recordWorkspaceMemoryWritable). + * + * One record per compaction epoch rather than a single slot: with several + * backends over one chat.jsonl (XUM_ALLOW_MULTIPLE_INSTANCES), a backend can + * start the NEW epoch's first turn between another backend's compaction + * boundary and that backend's completion-side read of the CLOSING epoch's + * value. A single slot would be overwritten by the new epoch's grant, dropping + * a deny recorded for the closing epoch — and the compacting backend's own + * mirror (writable) would then grant the harvest of a read-only turn. A + * record is removed only by a destructive boundary (AgentSession. + * resetWorkspaceMemoryWritable), never by a compaction boundary's observation + * and never by count: two backends can each commit a boundary closing the + * same epoch, and a backend suspended between persisting its boundary and + * observing the closing policy must still find the record however many + * epochs other backends opened meanwhile. Readers key strictly by epoch and + * epoch keys (boundary history sequences; -1 only before the first boundary + * of a segment) never recur before the destructive boundary that drops every + * record, so retained records are inert — one boolean per compaction epoch + * in config.json until then. + */ +import type { Workspace as WorkspaceConfigEntry } from "@/node/config"; +import assert from "@/common/utils/assert"; + +function epochKey(epoch: number): string { + assert(Number.isInteger(epoch), "workspace memory policy epoch must be an integer"); + return String(epoch); +} + +/** + * The recorded policy for `epoch`: `undefined` when none was recorded. Config + * entries are loaded from raw JSON without schema validation, so anything + * present that is not an actual boolean (a corrupted `"false"` or `null`) + * reads as a DENY — a truthy string or a null-coalesced default would + * otherwise turn corrupted deny state into a grant. + */ +export function workspaceMemoryWritableForEpoch( + entry: WorkspaceConfigEntry, + epoch: number +): boolean | undefined { + const records = policyRecords(entry); + if (records === undefined) return undefined; + // A container of the wrong shape (null, array, string) is corruption too: + // fail closed for every epoch rather than throw out of every turn start + // (Object.hasOwn(null) would). The next write or forget heals it. + if (records === null) return false; + const key = epochKey(epoch); + if (!Object.hasOwn(records, key)) return undefined; + const value = records[key]; + return typeof value === "boolean" ? value : false; +} + +/** + * Whether the persisted container is corrupt (present but not a plain + * object). Every epoch then reads as a deny; the next write replaces the + * container (setWorkspaceMemoryWritableForEpoch), so writers must not take + * a no-write fast path on the strength of that deny or the corruption — and + * the blanket deny — would outlive every epoch until a destructive clear. + */ +export function hasMalformedWorkspaceMemoryPolicyRecords(entry: WorkspaceConfigEntry): boolean { + return policyRecords(entry) === null; +} + +/** + * The persisted container: `undefined` when absent, `null` when present but + * not a plain object (raw JSON, no schema validation upstream). + */ +function policyRecords(entry: WorkspaceConfigEntry): Record | null | undefined { + const records: unknown = entry.workspaceMemoryWritableByEpoch; + if (records === undefined) return undefined; + return typeof records === "object" && records !== null && !Array.isArray(records) + ? (records as Record) + : null; +} + +/** Record `writable` for `epoch`. */ +export function setWorkspaceMemoryWritableForEpoch( + entry: WorkspaceConfigEntry, + epoch: number, + writable: boolean +): void { + // A malformed container is replaced, not spread (spreading a string would + // persist its characters as epoch keys). + entry.workspaceMemoryWritableByEpoch = { + ...(policyRecords(entry) === null ? {} : entry.workspaceMemoryWritableByEpoch), + [epochKey(epoch)]: writable, + }; +} diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 970ccea6d11..f08fa07f997 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -11,10 +11,12 @@ import { import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, + historyWriteLockPath, isWorkspaceRemovalTombstoned, refineApplyLockPath, REMOVAL_TOMBSTONE_HEAL_MIN_AGE_MS, removeSessionDirUnderMemoryLocks, + SharedMemoryRemovalAbortedError, rollbackRemovalTombstoneIfOwned, TombstoneNotDurableError, workspaceRemovalTombstonePath, @@ -80,6 +82,197 @@ describe("workspaceRemoval", () => { expect((JSON.parse(raw) as { workspaceId: string }).workspaceId).toBe(workspaceId); }); + test("sub-agent removal also waits for a writer holding the OWNER's memory store lock", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // A sub-agent's admitted workspace-memory write holds the OWNER store's + // lock (that is where the shared notebook lives), not the child's. + let releaseWriter!: () => void; + const writerGate = new Promise((resolve) => (releaseWriter = resolve)); + let writerEntered!: () => void; + const entered = new Promise((resolve) => (writerEntered = resolve)); + const writer = withTargetMutationLock( + rootDir, + path.join(ownerSessionDir, "memory"), + async () => { + writerEntered(); + await writerGate; + } + ); + await entered; + + const removal = removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + // Without the owner lock the child tombstone would already be published + // here, between the writer's commit check and its durable write. + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + + releaseWriter(); + await writer; + await removal; + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(true); + // Only the child's session dir is deleted; the owner keeps its notebook. + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(false); + expect( + await fsPromises.access(path.join(ownerSessionDir, "memory")).then( + () => true, + () => false + ) + ).toBe(true); + }); + + test("runs beforeTombstone under the locks and aborts a shared-store removal when it throws", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-hook"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // The hook observes the locked section: the owner store lock is held, so a + // concurrent writer cannot enter while it runs. + let writerRanDuringHook = false; + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + beforeTombstone: async () => { + const writer = withTargetMutationLock( + rootDir, + path.join(ownerSessionDir, "memory"), + () => { + writerRanDuringHook = true; + return Promise.resolve(); + } + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(writerRanDuringHook).toBe(false); + // Let the writer settle later (after the locks release) and abort. + void writer; + throw new Error("migration failed"); + }, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); + expect(String(thrown)).toContain("migration failed"); + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(true); + // The queued writer runs once removal released the locks. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(writerRanDuringHook).toBe(true); + }); + + test("sub-agent removal aborts (no tombstone) when the owner store lock cannot be acquired", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-locked"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // A foreign process holds the OWNER store's cross-process lock (as in the + // r62 test): acquisition times out instead of reclaiming. + const lockPath = targetMutationLockFilePath(rootDir, path.join(ownerSessionDir, "memory")); + await fsPromises.mkdir(path.dirname(lockPath), { recursive: true }); + const birth = getProcessBirth(process.pid); + const token = + birth === null + ? `${process.pid}:feed` + : `${process.pid}:feed:${Buffer.from(birth).toString("hex")}`; + await fsPromises.writeFile(lockPath, token, { flag: "wx" }); + + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); + // Unlike the own-store orphan path, NO tombstone is published: the + // wedged holder targets the owner's live notebook, so the child must + // stay registered and removal be retried. + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(true); + }, 20_000); + + test("sub-agent removal aborts when a failure lands after the target locks but before the tombstone", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-history-locked"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // Target locks succeed; the history write lock (taken INSIDE them) is + // held by a foreign process and times out. The old orphan path would now + // publish the tombstone with the owner-store lock already released. + const historyLock = await acquireProcessFileLock({ + lockPath: historyWriteLockPath(rootDir, childId), + timeoutMs: 1_000, + label: "history write lock (test holder)", + }); + try { + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + } finally { + await historyLock[Symbol.asyncDispose](); + } + }, 30_000); + test("waits on the refine lock BEFORE taking the teardown target locks (r67)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); @@ -259,6 +452,45 @@ describe("workspaceRemoval", () => { expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); }); + test("a sealed removal republishes its own tombstone over a foreign attempt's marker", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const sessionDir = path.join(tmp.path, "sessions", "ws-sealed"); + const workspaceId = "ws-sealed"; + await fsPromises.mkdir(sessionDir, { recursive: true }); + const tombstonePath = workspaceRemovalTombstonePath(rootDir, workspaceId); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + // Two backends sealed the same sub-agent; B's marker overwrote A's. + await fsPromises.writeFile( + tombstonePath, + JSON.stringify({ workspaceId, removedAt: Date.now(), attemptId: "attempt-B" }) + ); + // A's deletion must not rely on B's marker (B's rollback may delete it): + // it republishes its own before deleting the session. + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir, + workspaceId, + attemptId: "attempt-A", + tombstoneSealed: true, + }); + expect( + (JSON.parse(await fsPromises.readFile(tombstonePath, "utf-8")) as { attemptId: string }) + .attemptId + ).toBe("attempt-A"); + // B's compensating rollback now leaves A's marker alone. + expect( + await rollbackRemovalTombstoneIfOwned({ + rootDir, + sessionDir, + workspaceId, + attemptId: "attempt-B", + workspaceStillRegistered: () => true, + }) + ).toBe(false); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(true); + }); + test("startup heal reclaims old tombstones only for still-registered workspaces (r63)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 154c7311626..6a9495d960b 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -49,6 +49,26 @@ import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; * ENOSPC) clears — while the workspace no longer exists anywhere else. * Keeping the workspace registered keeps removal retryable instead. */ +/** + * A sub-agent's removal failed before publishing its tombstone under its + * memory owner's store lock (lock unavailable, history lock timeout, or the + * in-lock shared-row migration failing). The caller must ABORT the removal — + * no tombstone, no deregistration — because the orphan fallback would leave + * an admitted child write free to land in the owner's live notebook after + * removal, or delete the only copy of a shared-memory refinement row. + */ +export class SharedMemoryRemovalAbortedError extends Error { + constructor(workspaceId: string, options?: ErrorOptions) { + super( + `Removing ${workspaceId} failed before its tombstone could be published under the shared workspace-memory store lock (${ + options?.cause instanceof Error ? options.cause.message : String(options?.cause) + }); removal aborted and can be retried`, + options + ); + this.name = "SharedMemoryRemovalAbortedError"; + } +} + export class TombstoneNotDurableError extends Error { constructor(workspaceId: string, options?: ErrorOptions) { super( @@ -133,6 +153,14 @@ export async function removeSessionDirUnderMemoryLocks(args: { rootDir: string; sessionDir: string; workspaceId: string; + /** + * The tombstone was already published under these same locks by + * sealSubAgentForRemovalUnderMemoryLocks (sub-agents: before the checkout + * was deleted). Nothing fallible remains before the rm, and a lock failure + * here takes the orphan path instead of aborting a removal whose checkout + * is already gone. + */ + tombstoneSealed?: boolean; /** * Unique ID of THIS removal attempt, stamped into the tombstone (r66). * The caller's compensating rollback deletes the marker only while it @@ -142,6 +170,21 @@ export async function removeSessionDirUnderMemoryLocks(args: { * succeeding or still-active) attempt relies on. */ attemptId: string; + /** + * Session dir of the task-tree owner whose /memory backs this + * workspace's `/memories/workspace` (sub-agents only). Its store lock is + * held too, so a sub-agent's admitted write into the shared notebook + * either commits before the tombstone or re-checks and refuses. + */ + sharedWorkspaceMemorySessionDir?: string; + /** + * Runs INSIDE the target locks, immediately before the tombstone is + * published: shared-memory refinement rows are copied to the owner here, + * so a child write that landed after any earlier scan (another backend) is + * still captured — the locks guarantee no further write can slip in. A + * throw aborts the removal (see SharedMemoryRemovalAbortedError). + */ + beforeTombstone?: () => Promise; }): Promise { assert(args.sessionDir.length > 0, "removeSessionDirUnderMemoryLocks requires a session dir"); // Crash clearly on a malformed config (test stubs, future refactors): an @@ -151,6 +194,139 @@ export async function removeSessionDirUnderMemoryLocks(args: { typeof args.rootDir === "string" && args.rootDir.length > 0, "removeSessionDirUnderMemoryLocks requires a rootDir" ); + assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); + let tombstonePublishedUnderLocks = args.tombstoneSealed === true; + // A sealed tombstone is not rewritten while it is still THIS attempt's: the + // redundant write could fail (storage read-only/full after the checkout + // deletion) and would then abort a removal whose durable marker is already + // in place. It IS republished when another attempt's marker sits there + // (two backends sealing the same sub-agent): relying on a foreign marker + // would let that attempt's compensating rollback delete the only tombstone + // while this one proceeds to delete the session and deregister. + const publish = async (): Promise => { + if ( + args.tombstoneSealed === true && + (await readRemovalTombstoneAttemptId(args.rootDir, args.workspaceId)) === args.attemptId + ) { + return; + } + await publishRemovalTombstone(args); + }; + try { + await withRemovalLocks(args, async () => { + await args.beforeTombstone?.(); + // Tombstone BEFORE rm: once the locks release, any waiting writer + // re-checks it pre-commit (inside its own lock) and refuses, so the + // deleted directory cannot be recreated by a late mutation or + // journal append. + await publish(); + tombstonePublishedUnderLocks = true; + await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); + }); + } catch (error) { + // The orphan path below assumes a wedged writer's target is THIS + // workspace's retained session dir. A sub-agent's admitted memory write + // targets its OWNER's live notebook instead, so unless the tombstone was + // already published UNDER the owner-store lock, publishing it outside + // (after the lock was released, or never taken) would let a holder that + // passed its commit check — or a new writer — finish after removal. + // Abort instead: the workspace stays registered and removal is retried. + if (args.sharedWorkspaceMemorySessionDir !== undefined && !tombstonePublishedUnderLocks) { + throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); + } + // Fail-closed orphan path (r62): a wedged writer blocks the deletion, + // but the caller proceeds to deregister the workspace regardless — so + // the terminal marker must still become durable or a foreign backend + // would keep mutating memory and journaling into the retained orphan + // forever. Publishing outside the locks is safe on THIS path precisely + // because the directory is not deleted: a writer mid-commit lands in + // the orphan, and every later mutation observes the tombstone. + try { + await publish(); + } catch (publishError) { + // No durable marker could be written at all (r63, e.g. ENOSPC): + // deregistering now would leave the orphan writable again the moment + // the transient failure clears. Signal the caller to ABORT the + // removal so the workspace stays registered and retryable. + throw new TombstoneNotDurableError(args.workspaceId, { cause: publishError }); + } + throw error; + } +} + +/** + * Sub-agent removal, BEFORE the checkout is deleted: run the final, fallible + * shared-memory handover (legacy-notebook adoption + refinement-row delta) + * under the full removal lock set and publish the removal tombstone in the + * same critical section. From here on no backend — including a downgraded + * one, which honors the same locks and tombstone at its memory commit + * points — can add to the child's notebooks, so the later session-dir + * deletion has nothing fallible left in front of it. Runs before runtime + * deletion so a handover failure aborts with the checkout intact (the + * caller rolls the tombstone back if the checkout deletion is then refused). + * Any failure aborts the removal (SharedMemoryRemovalAbortedError). + */ +export async function sealSubAgentForRemovalUnderMemoryLocks(args: { + rootDir: string; + sessionDir: string; + workspaceId: string; + attemptId: string; + sharedWorkspaceMemorySessionDir: string; + beforeTombstone: () => Promise; +}): Promise { + try { + await withRemovalLocks(args, async () => { + await args.beforeTombstone(); + await publishRemovalTombstone(args); + }); + } catch (error) { + throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); + } +} + +/** The attempt ID stamped in the tombstone, or null when missing, unreadable or malformed. */ +async function readRemovalTombstoneAttemptId( + rootDir: string, + workspaceId: string +): Promise { + try { + const parsed = JSON.parse( + await fsPromises.readFile(workspaceRemovalTombstonePath(rootDir, workspaceId), "utf-8") + ) as { attemptId?: unknown }; + return typeof parsed.attemptId === "string" ? parsed.attemptId : null; + } catch { + return null; + } +} + +async function publishRemovalTombstone(args: { + rootDir: string; + workspaceId: string; + attemptId: string; +}): Promise { + assert(args.attemptId.length > 0, "removal tombstone requires an attemptId"); + const tombstonePath = workspaceRemovalTombstonePath(args.rootDir, args.workspaceId); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await writeFileAtomic( + tombstonePath, + JSON.stringify({ + workspaceId: args.workspaceId, + removedAt: Date.now(), + attemptId: args.attemptId, + }) + ); +} + +/** The removal critical section's lock set (see removeSessionDirUnderMemoryLocks). */ +async function withRemovalLocks( + args: { + rootDir: string; + sessionDir: string; + workspaceId: string; + sharedWorkspaceMemorySessionDir?: string; + }, + body: () => Promise +): Promise { // Same key derivations as MemoryService.storeLockKey: the workspace store // root lives inside the session directory; global/project mutations hold // the coarse `/memory` key while journaling into this session dir. @@ -159,24 +335,20 @@ export async function removeSessionDirUnderMemoryLocks(args: { path.join(args.sessionDir, "memory") ); const sharedMemoryKey = memoryMutationLockKey(args.rootDir, path.join(args.rootDir, "memory")); + const ownerMemoryKeys = + args.sharedWorkspaceMemorySessionDir === undefined + ? [] + : [ + memoryMutationLockKey( + args.rootDir, + path.join(args.sharedWorkspaceMemorySessionDir, "memory") + ), + ]; // The session dir itself is a third target key (r63): session-scoped // sidecar writers (headless usage) serialize their tombstone check + // commit against this same key, closing their check→write window. const sessionDirKey = path.resolve(args.sessionDir); - assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); - const publishTombstone = async (): Promise => { - const tombstonePath = workspaceRemovalTombstonePath(args.rootDir, args.workspaceId); - await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); - await writeFileAtomic( - tombstonePath, - JSON.stringify({ - workspaceId: args.workspaceId, - removedAt: Date.now(), - attemptId: args.attemptId, - }) - ); - }; - try { + { // Refine serialization (r66) — acquired FIRST (r67): a /refine apply in // ANOTHER backend is untouched by the remover's process-local // cancellation and holds this same (session-dir-external) lock across @@ -200,7 +372,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { }); await withTargetMutationLocks( args.rootDir, - [sessionDirKey, workspaceMemoryKey, sharedMemoryKey], + [sessionDirKey, workspaceMemoryKey, sharedMemoryKey, ...ownerMemoryKeys], async () => { // History append serialization (r63): a foreign backend's in-flight // stream can be mid-append under the history write lock; acquiring @@ -212,32 +384,9 @@ export async function removeSessionDirUnderMemoryLocks(args: { timeoutMs: 10_000, label: "history write lock (removal)", }); - // Tombstone BEFORE rm: once the locks release, any waiting writer - // re-checks it pre-commit (inside its own lock) and refuses, so the - // deleted directory cannot be recreated by a late mutation or - // journal append. - await publishTombstone(); - await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); + await body(); } ); - } catch (error) { - // Fail-closed orphan path (r62): a wedged writer blocks the deletion, - // but the caller proceeds to deregister the workspace regardless — so - // the terminal marker must still become durable or a foreign backend - // would keep mutating memory and journaling into the retained orphan - // forever. Publishing outside the locks is safe on THIS path precisely - // because the directory is not deleted: a writer mid-commit lands in - // the orphan, and every later mutation observes the tombstone. - try { - await publishTombstone(); - } catch (publishError) { - // No durable marker could be written at all (r63, e.g. ENOSPC): - // deregistering now would leave the orphan writable again the moment - // the transient failure clears. Signal the caller to ABORT the - // removal so the workspace stays registered and retryable. - throw new TombstoneNotDurableError(args.workspaceId, { cause: publishError }); - } - throw error; } } diff --git a/src/node/services/workspaceService.multiProject.test.ts b/src/node/services/workspaceService.multiProject.test.ts index 59b956df223..aca12866496 100644 --- a/src/node/services/workspaceService.multiProject.test.ts +++ b/src/node/services/workspaceService.multiProject.test.ts @@ -82,9 +82,25 @@ function createMockAIService(metadata?: WorkspaceMetadata): AIService { off: mock(() => undefined), } as unknown as AIService; } +/** + * Mock configs model no file on disk. Removal's shared-memory handover + * requires an EXISTING config.json (Config.loadExistingConfigOrThrow); for a + * mock that only provides loadConfigOrDefault, treat that snapshot as the + * existing file (same shim as workspaceService.test.ts). + */ +function withExistingConfigLoader(config: Partial): Partial { + if ( + typeof config.loadExistingConfigOrThrow !== "function" && + typeof config.loadConfigOrDefault === "function" + ) { + const load = config.loadConfigOrDefault.bind(config); + return { ...config, loadExistingConfigOrThrow: () => load({ throwOnError: true }) }; + } + return config; +} function createWorkspaceServiceForTest(options: WorkspaceServiceTestOptions): WorkspaceService { return new WorkspaceService( - options.config as Config, + withExistingConfigLoader(options.config) as Config, options.historyService, options.aiService ?? createMockAIService(), options.initStateManager ?? createMockInitStateManager(), diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 03e18ec9e7a..82e7fb65e36 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,3 +1,20 @@ +import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + clearWorkspaceMemoryDenyMarker, + readWorkspaceMemoryDenyMarker, + readWorkspaceMemoryDenyMarkerForEpochs, + workspaceMemoryDenyMarkerPath, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; +import { + workspaceRemovalTombstonePath, + isWorkspaceRemovalTombstoned, +} from "@/node/services/workspaceRemoval"; +import { + setWorkspaceMemoryWritableForEpoch, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -206,6 +223,23 @@ function createMockAIService(overrides: Partial = {}): AIService { } as unknown as AIService; } +/** + * Mock configs model no file on disk. Production requires an EXISTING + * config.json for removal's shared-memory handover and the memory policy + * accumulator (Config.loadExistingConfigOrThrow); for a mock that only + * provides loadConfigOrDefault, treat that snapshot as the existing file. + */ +function withExistingConfigLoader>(config: T): T { + if ( + typeof config.loadExistingConfigOrThrow !== "function" && + typeof config.loadConfigOrDefault === "function" + ) { + const load = config.loadConfigOrDefault.bind(config); + return { ...config, loadExistingConfigOrThrow: () => load({ throwOnError: true }) }; + } + return config; +} + function createWorkspaceServiceForTest(options: { config: | (Partial & { getEffectiveSecrets?: SecretsStore["getEffectiveSecrets"] }) @@ -227,7 +261,7 @@ function createWorkspaceServiceForTest(options: { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const defaultHistoryService: HistoryService = {} as HistoryService; return new WorkspaceService( - options.config as Config, + withExistingConfigLoader(options.config as Partial) as Config, options.historyService ?? defaultHistoryService, options.aiService ?? createMockAIService(), options.initStateManager ?? (mockInitStateManager as InitStateManager), @@ -7369,7 +7403,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { getInitState: mock(() => null), } as unknown as InitStateManager; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, aiService, initStateManager, @@ -7530,8 +7564,11 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }); test("destructive clear waits for startup monitor recovery discovery", async () => { - const { historyService, workspaceService, cleanup } = await createServices(); + const { config, historyService, workspaceService, cleanup } = await createServices(); const workspaceId = "clear-waits-for-monitor-recovery"; + // A destructive clear proves its policy reset against an EXISTING + // config.json (an absent one is a transient state, not the empty default). + await config.editConfig((cfg) => cfg); const recovery = createDeferred(); const internal = workspaceService as unknown as { bashMonitorRecoveryPromise: Promise; @@ -9294,6 +9331,568 @@ describe("WorkspaceService initialize", () => { } }); + test("accumulates the workspace memory write policy fail-closed in config across an epoch", async () => { + const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); + const scratchDir = path.join(realConfig.rootDir, "scratch", "policy-scratch"); + await fsPromises.mkdir(scratchDir, { recursive: true }); + await realConfig.editConfig((cfg) => { + cfg.projects.set(SCRATCH_PROJECT_CONFIG_KEY, { + workspaces: [ + { + kind: "scratch", + path: scratchDir, + id: "policy-scratch", + name: "scratch-policy-scratch", + runtimeConfig: { type: "local" }, + }, + ], + projectKind: "system", + trusted: true, + }); + return cfg; + }); + const aiService = { + ...createStreamLifecycleMocks(), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + const service = createWorkspaceServiceForTest({ + config: realConfig, + historyService, + aiService, + initStateManager: mockInitStateManager as InitStateManager, + }); + const persistedFor = (epoch: number) => + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace + .workspaceMemoryWritableByEpoch?.[String(epoch)]; + const persisted = () => persistedFor(-1); + const EPOCH0 = { epochHasPriorTurns: false, policyEpoch: -1 }; + try { + // The durable bit is the epoch accumulator: the harvest reads every + // message of the epoch, so a read-only turn denies the epoch even when + // a writable turn follows — across restarts and backends, since the + // conjunction lives in config.json rather than in one process. + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + + // New epoch (field cleared at the boundary), then ANOTHER backend records + // a read-only turn straight into config: this backend's next grant must + // not publish false→true. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(true); + await realConfig.editConfig((cfg) => { + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch = { + "-1": false, + }; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + + // New epoch again, and now config.json cannot take the deny: the turn's + // user row is already durable, so the deny falls back to the session + // dir (same durability domain) and the record still succeeds. A later + // writable turn — even from a fresh process with no mirror — stays + // denied by that marker, and it is ANDed into the compaction + // observation as well. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + const sessionDir = path.join(realConfig.sessionsDir, "policy-scratch"); + spyOn(realConfig, "editConfig").mockImplementationOnce(() => + Promise.reject(new Error("disk full")) + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + // Malformed marker still denies; only a DESTRUCTIVE boundary heals it + // (a compaction boundary clears nothing: another backend's boundary + // closing the same epoch must still find every entry). + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + // Another backend's new-epoch deny write must not heal the malformed + // marker away (it may be the only evidence of a read-only turn in the + // closing epoch): the deny is carried as a wildcard, denying every + // epoch, until the destructive boundary. + await writeWorkspaceMemoryDenyMarker(sessionDir, 7); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 3)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + // A present but non-boolean wildcard is corruption, not "false": read as + // malformed (deny for every epoch) rather than dropping the only + // surviving deny of an epoch the list does not name. + await fsPromises.writeFile( + workspaceMemoryDenyMarkerPath(sessionDir), + JSON.stringify({ epochs: [3], wildcard: "true" }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); + await fsPromises.writeFile( + workspaceMemoryDenyMarkerPath(sessionDir), + JSON.stringify({ epochs: [3] }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + // Unreadable is not malformed: a marker that cannot be read may hold + // denies, so the clear refuses instead of deleting it. + await writeWorkspaceMemoryDenyMarker(sessionDir, 9); + const unreadableMarker = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + const refusedClear = await clearWorkspaceMemoryDenyMarker( + realConfig.rootDir, + sessionDir + ).then( + () => null, + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ); + expect(refusedClear).toContain("unreadable"); + unreadableMarker.mockRestore(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(true); + // Entries of several epochs coexist; readers of any other epoch ignore + // each, and all of them go together at the destructive boundary. + await writeWorkspaceMemoryDenyMarker(sessionDir, -1); + await writeWorkspaceMemoryDenyMarker(sessionDir, 7); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + + // The durable bit is bound to its epoch too: the closing epoch's + // `false` is invisible to the first turn of the next epoch on ANOTHER + // backend (which cannot await this backend's boundary reset), so it + // grants under its own epoch's record — WITHOUT displacing the closing + // epoch's deny, which the compacting backend's completion observation + // may not have read yet (a single slot would let its writable mirror + // grant the harvest of the read-only turn). + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": false }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 12, + }) + ).toBe(true); + expect(persistedFor(12)).toBe(true); + expect(persisted()).toBe(false); + // A PRESERVED-TAIL epoch names the epoch its tail was copied out of: + // that epoch's deny is ANDed in directly, under whichever key it sits — + // the closing key (compacting backend's carry not landed yet), the new + // key (carry landed), or the session-dir marker — so no window exists + // in which the read-only tail reads as writable to another backend. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": false }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 14, + carriedPolicyEpochs: [-1], + }) + ).toBe(true); + expect(persistedFor(14)).toBe(false); + expect(persisted()).toBe(false); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "16": true }; + return cfg; + }); + await writeWorkspaceMemoryDenyMarker(sessionDir, 12); + // One snapshot answers for every epoch consulted (a carry re-stamping + // 12 → 16 between two separate reads could hide the entry from both). + expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1, 12])).toBe(true); + expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1])).toBe(false); + // Epochs outside the integer domain (a segment's negative boundary-less + // identity or a history sequence) are corruption: the marker reads as + // malformed, a deny for every epoch. + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + const savedMarker = await fsPromises.readFile(markerPath, "utf-8"); + await fsPromises.writeFile( + markerPath, + JSON.stringify({ deniedAt: Date.now(), epochs: [2.5], wildcard: false }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 16)).toBe(true); + await fsPromises.writeFile(markerPath, savedMarker); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 16, + carriedPolicyEpochs: [-1, 12], + }) + ).toBe(true); + expect(persistedFor(16)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + // A carried grant (or no carried record at all) changes nothing. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": true }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 18, + carriedPolicyEpochs: [-1], + }) + ).toBe(true); + expect(persistedFor(18)).toBe(true); + // A carried epoch with no record anywhere (no carried key, nothing under + // this epoch, no marker — e.g. the first tail compaction after an + // upgrade) is unknown history: denied. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + delete entry.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 24, + carriedPolicyEpochs: [-1], + }) + ).toBe(true); + expect(persistedFor(24)).toBe(false); + // A chain of tail compactions names several epochs: one recorded grant + // does not vouch for a sibling epoch with no record anywhere. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": true }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 26, + carriedPolicyEpochs: [-1, 5], + }) + ).toBe(true); + expect(persistedFor(26)).toBe(false); + // A tail copy whose source epoch is unknown (persisted before the field + // existed) carries a policy nobody can look up: denied, like unknown + // history, even for an otherwise writable first turn. + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 22, + carriedPolicyUnknown: true, + }) + ).toBe(true); + expect(persistedFor(22)).toBe(false); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": false, "12": true }; + return cfg; + }); + // ...while a turn of the closing epoch itself still sees its deny. + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + expect(persistedFor(12)).toBe(true); + // Records are never pruned by count: a suspended compacting backend + // must still find its closing epoch's record however many epochs + // others opened meanwhile. + for (const policyEpoch of [20, 30, 40]) { + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch, + }) + ).toBe(true); + } + expect( + Object.keys( + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")!.workspace + .workspaceMemoryWritableByEpoch! + ).sort() + ).toEqual(["-1", "12", "20", "30", "40"]); + // Raw config is not schema-validated: a corrupted non-boolean record + // reads as a deny, never as a grant. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + (entry.workspaceMemoryWritableByEpoch as Record)["50"] = "false"; + (entry.workspaceMemoryWritableByEpoch as Record)["51"] = null; + return cfg; + }); + for (const policyEpoch of [50, 51]) { + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch, + }) + ).toBe(true); + // The deny stands (no grant written over the corrupted value)... + expect(persistedFor(policyEpoch)).not.toBe(true); + // ...and every reader sees it as false. + expect( + workspaceMemoryWritableForEpoch( + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")!.workspace, + policyEpoch + ) + ).toBe(false); + } + // A corrupted CONTAINER (null, array, string) must not throw out of + // every turn start; it reads as a deny for every epoch and is healed + // (replaced, not spread) by the next write. + for (const container of [null, "false", ["-1"]]) { + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace as Record< + string, + unknown + >; + entry.workspaceMemoryWritableByEpoch = container; + return cfg; + }); + const corrupted = findWorkspaceEntry( + realConfig.loadConfigOrDefault(), + "policy-scratch" + )!.workspace; + expect(workspaceMemoryWritableForEpoch(corrupted, 60)).toBe(false); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 60, + }) + ).toBe(true); + expect(persistedFor(60)).not.toBe(true); + // Healed on that write (no fast path on a corrupt container's blanket + // deny, r83): the container is a plain object again holding this + // epoch's deny, and the next epoch's first turn grants normally. + const healed = findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")! + .workspace.workspaceMemoryWritableByEpoch; + expect(healed).toEqual({ "60": false }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 62, + }) + ).toBe(true); + expect(persistedFor(62)).toBe(true); + setWorkspaceMemoryWritableForEpoch(corrupted, 61, true); + expect(corrupted.workspaceMemoryWritableByEpoch).toEqual({ "61": true }); + } + + // Unknown history fails closed: no accumulator, no marker, no mirror + // (this service never recorded this epoch), yet the epoch already holds + // turns — the record was lost (a deny that could not be made durable + // anywhere before a restart, an upgrade mid-epoch). A writable turn + // must not grant the whole epoch; the first turn of a fresh epoch does. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: true, + policyEpoch: -1, + }) + ).toBe(true); + expect(persisted()).toBe(false); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: -1, + }) + ).toBe(true); + expect(persisted()).toBe(true); + + // A record must wait for the session's in-flight epoch reset (a no-tail + // compaction clearing the closing epoch's accumulator/marker), or the + // first turn of the new epoch would AND itself with the stale deny. + let releaseReset!: () => void; + const resetInFlight = new Promise((resolve) => (releaseReset = resolve)); + const fakeSession = { + settleWorkspaceMemoryPolicyEpoch: () => resetInFlight, + workspaceMemoryWritableMirror: () => undefined, + recordWorkspaceMemoryWritable: () => undefined, + }; + (service as unknown as { sessions: Map }).sessions.set( + "policy-scratch", + fakeSession + ); + await realConfig.editConfig((cfg) => { + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch = { + "-1": false, + }; + return cfg; + }); + let settled = false; + const pendingRecord = service + .recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0) + .then((ok) => { + settled = true; + return ok; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); + // The reset finishes (field cleared) and only then does the record read. + await realConfig.editConfig((cfg) => { + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + releaseReset(); + expect(await pendingRecord).toBe(true); + expect(persisted()).toBe(true); + (service as unknown as { sessions: Map }).sessions.delete("policy-scratch"); + + // An unreadable config.json (missing/malformed while the turn starts) + // must not make the still-registered workspace look unregistered: the + // deny takes the session-dir fallback instead of being reported durable + // without a record anywhere, and a grant is reported unpersisted (the + // harvest stays closed) rather than "done". + await realConfig.editConfig((cfg) => { + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + const unreadable = () => { + throw new Error("config.json: unexpected token"); + }; + spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + false + ); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + // A MISSING config.json (mid-rewrite by another backend) is not a fresh + // install either: strict mode alone would read it as one and take the + // "unregistered" shortcut. Same fallback as unreadable. + const configFile = path.join(realConfig.rootDir, "config.json"); + await fsPromises.rename(configFile, `${configFile}.parked`); + try { + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + false + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + } finally { + await fsPromises.rename(`${configFile}.parked`, configFile); + } + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(persisted()).toBeUndefined(); + + // A late deny reaching the marker fallback after the workspace was + // removed (tombstoned, session dir deleted) must not recreate the + // session dir as an orphan; nothing is left to harvest, so it is done. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritableByEpoch; + return cfg; + }); + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + const tombstonePath = workspaceRemovalTombstonePath(realConfig.rootDir, "policy-scratch"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "policy-scratch" })); + spyOn(realConfig, "editConfig").mockImplementationOnce(() => + Promise.reject(new Error("disk full")) + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect( + await fsPromises.stat(sessionDir).then( + () => true, + () => false + ) + ).toBe(false); + } finally { + await cleanup(); + } + }); + + test("the fenced deny-marker clear cannot delete a new-epoch deny written under the lock", async () => { + const { config: realConfig, cleanup } = await createTestHistoryService(); + try { + const sessionDir = path.join(realConfig.sessionsDir, "policy-lock"); + await writeWorkspaceMemoryDenyMarker(sessionDir, -1); + // Another backend's deny writer holds the session-dir lock while the + // destructive boundary reset starts: the reset must queue behind it + // (a write landing between its rm and its verification would read as + // a failed removal) and then discard that deny with the rest of the + // discarded transcript's entries. + let cleared = false; + let clear: Promise | undefined; + await withTargetMutationLock(realConfig.rootDir, sessionDir, async () => { + clear = clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir).then(() => { + cleared = true; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(cleared).toBe(false); + await writeWorkspaceMemoryDenyMarker(sessionDir, 5); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); + }); + await clear; + expect(cleared).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + } finally { + await cleanup(); + } + }); + test("removes stale orphaned scratch workdirs but keeps referenced and recent ones", async () => { const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); const scratchDirFor = (id: string) => path.join(realConfig.rootDir, "scratch", id); @@ -11348,7 +11947,7 @@ describe("WorkspaceService pending auto-title", () => { }; workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, aiService, mockInitStateManager as InitStateManager, @@ -14295,7 +14894,7 @@ describe("WorkspaceService remove timing rollup", () => { }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -14323,6 +14922,169 @@ describe("WorkspaceService remove timing rollup", () => { }); }); +describe("WorkspaceService remove sub-agent handover ordering", () => { + // A sub-agent's final shared-memory handover + removal tombstone are sealed + // under the removal locks BEFORE the checkout is deleted: a handover the + // owner store cannot take aborts with the checkout intact, and a refused + // checkout deletion rolls the tombstone back. + const projectPath = "/tmp/proj-handover"; + const workspaceId = "child-handover"; + const ownerId = "owner-handover"; + const workspacePath = path.join(projectPath, "child-ws"); + const runtimeConfig = { type: "worktree" as const, srcBaseDir: "/tmp/src" }; + let rootDir: string; + + beforeEach(async () => { + rootDir = path.join(tmpdir(), "mux-handover-order", `root-${crypto.randomUUID()}`); + await fsPromises.mkdir(path.join(rootDir, "sessions", workspaceId), { recursive: true }); + }); + afterEach(async () => { + await fsPromises.rm(rootDir, { recursive: true, force: true }); + }); + + function buildConfig(): Partial { + const topology = { + projects: new Map([ + [ + projectPath, + { + trusted: true, + workspaces: [ + { + id: ownerId, + name: "owner", + path: path.join(projectPath, "owner-ws"), + runtimeConfig, + }, + { + id: workspaceId, + name: "child", + path: workspacePath, + runtimeConfig, + parentWorkspaceId: ownerId, + }, + ], + }, + ], + ]), + }; + return { + rootDir, + srcDir: "/tmp/src", + sessionsDir: path.join(rootDir, "sessions"), + removeWorkspace: mock(() => Promise.resolve()), + findWorkspace: mock(() => ({ workspacePath, projectPath })), + loadConfigOrDefault: mock(() => topology), + editConfig: mock((edit: (cfg: typeof topology) => typeof topology) => + Promise.resolve(edit(topology)) + ), + } as unknown as Partial; + } + + function buildAiService(): AIService { + return { + ...createStreamLifecycleMocks(), + isStreaming: mock(() => false), + stopStream: mock(() => Promise.resolve(Ok(undefined))), + getWorkspaceMetadata: mock(() => + Promise.resolve( + Ok({ + id: workspaceId, + name: "child", + projectPath, + projectName: "proj", + runtimeConfig, + parentWorkspaceId: ownerId, + }) + ) + ), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + } + + test("a handover the owner cannot take aborts before the checkout is deleted", async () => { + const deleteWorkspace = mock(() => + Promise.resolve({ success: true as const, deletedPath: workspacePath }) + ); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockReturnValue({ + deleteWorkspace, + } as unknown as ReturnType); + try { + const workspaceService = createWorkspaceServiceForTest({ + config: buildConfig(), + aiService: buildAiService(), + }); + let adoptions = 0; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: (_child, _owner, options) => { + adoptions++; + // The unlocked pre-pass succeeds; the late note appears for the + // locked pass, which cannot place it. + return options?.locksHeld + ? Promise.reject(new Error("1 legacy note could not be folded")) + : Promise.resolve(); + }, + }); + const result = await workspaceService.remove(workspaceId); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("could not be folded"); + expect(adoptions).toBe(2); + expect(deleteWorkspace).not.toHaveBeenCalled(); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); + // force accepts the loss and completes the removal. + expect((await workspaceService.remove(workspaceId, true)).success).toBe(true); + expect(deleteWorkspace).toHaveBeenCalledTimes(1); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(false); + } finally { + createRuntimeSpy.mockRestore(); + } + }); + + test("a refused checkout deletion rolls the sealed tombstone back", async () => { + let refuse = true; + const deleteWorkspace = mock(() => + Promise.resolve( + refuse + ? { success: false as const, error: "Workspace has uncommitted changes" } + : { success: true as const, deletedPath: workspacePath } + ) + ); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockReturnValue({ + deleteWorkspace, + } as unknown as ReturnType); + try { + const workspaceService = createWorkspaceServiceForTest({ + config: buildConfig(), + aiService: buildAiService(), + }); + let sealedTombstone = false; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => Promise.resolve(), + }); + deleteWorkspace.mockImplementation(async () => { + // Runtime deletion runs with the tombstone already sealed. + sealedTombstone = await isWorkspaceRemovalTombstoned(rootDir, workspaceId); + return refuse + ? { success: false as const, error: "Workspace has uncommitted changes" } + : { success: true as const, deletedPath: workspacePath }; + }); + const refused = await workspaceService.remove(workspaceId); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("uncommitted changes"); + expect(sealedTombstone).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(true); + refuse = false; + expect((await workspaceService.remove(workspaceId)).success).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(true); + } finally { + createRuntimeSpy.mockRestore(); + } + }); +}); + describe("WorkspaceService remove shared-workspace guard", () => { const projectPath = "/tmp/proj-shared"; const workspaceId = "child-shared"; @@ -14639,6 +15401,82 @@ describe("WorkspaceService remove desktop session cleanup", () => { expect(reopened).toEqual([workspaceId]); }); + test("remove() lifts the consolidation teardown gate only when it aborts before committing", async () => { + const calls: string[] = []; + workspaceService.setMemoryConsolidationService({ + triggerInBackground: () => undefined, + triggerHarvestThenSweepInBackground: () => undefined, + cancelInFlightConsolidation: () => { + calls.push("cancel"); + return Promise.resolve(); + }, + releaseRemovalCancellation: () => { + calls.push("release"); + }, + finalizeHarvestsForRemoval: () => Promise.resolve(), + }); + // Aborted before the point of no return (live descendant tasks): the + // workspace stays intact, so any teardown gate is lifted again. + let descendants = true; + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ hasDescendantAgentTasks: () => descendants }) + ); + const aborted = await workspaceService.remove(workspaceId); + expect(aborted.success).toBe(false); + expect(calls).toEqual(["release"]); + // Aborted inside the locked handover (a late legacy note the owner store + // cannot take), i.e. after the drain but BEFORE the tombstone: the session + // directory survives, so the gate is lifted too. + descendants = false; + calls.length = 0; + const sessionDir = path.join(tempRoot, "sessions", workspaceId); + await fsPromises.mkdir(sessionDir, { recursive: true }); + const topology = { + projects: new Map([ + [ + "/tmp/src/project", + { + workspaces: [ + { path: "/tmp/src/project/owner", id: "ws-owner" }, + { path: "/tmp/src/project/child", id: workspaceId, parentWorkspaceId: "ws-owner" }, + ], + }, + ], + ]), + }; + // The service holds its own copy of the mock config (createWorkspaceServiceForTest). + const config = (workspaceService as unknown as { config: MockWorkspaceConfig }).config; + const previousLoad = config.loadConfigOrDefault; + const previousLoadExisting = config.loadExistingConfigOrThrow; + config.loadConfigOrDefault = (() => topology) as MockWorkspaceConfig["loadConfigOrDefault"]; + config.loadExistingConfigOrThrow = (() => + topology) as MockWorkspaceConfig["loadExistingConfigOrThrow"]; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => + Promise.reject(new Error("1 legacy note could not be folded into the shared notebook")), + }); + try { + const lockedAbort = await workspaceService.remove(workspaceId); + expect(lockedAbort.success).toBe(false); + if (!lockedAbort.success) expect(lockedAbort.error).toContain("tombstone could be published"); + expect(existsSync(sessionDir)).toBe(true); + expect(calls).toContain("cancel"); + expect(calls).toContain("release"); + } finally { + config.loadConfigOrDefault = previousLoad; + config.loadExistingConfigOrThrow = previousLoadExisting; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => Promise.resolve(), + }); + } + // Committed removal: cancelled (drained) and never released. + calls.length = 0; + const removed = await workspaceService.remove(workspaceId); + expect(removed.success).toBe(true); + expect(calls.filter((call) => call === "cancel").length).toBeGreaterThan(0); + expect(calls).not.toContain("release"); + }); + test("remove() flushes the timeline before deleting the session directory", async () => { const sessionDir = path.join(tempRoot, "sessions", workspaceId); await fsPromises.mkdir(sessionDir, { recursive: true }); @@ -14722,7 +15560,7 @@ describe("WorkspaceService metadata listeners", () => { }; new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -14785,7 +15623,7 @@ describe("WorkspaceService metadata listeners", () => { }; new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -16638,7 +17476,7 @@ describe("WorkspaceService archive init cancellation", () => { } as unknown as AIService; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -18812,7 +19650,7 @@ describe("WorkspaceService init cancellation", () => { try { const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -18898,9 +19736,12 @@ describe("WorkspaceService init cancellation", () => { sessionsDir: tempRoot, removeWorkspace: mock(() => Promise.resolve()), findWorkspace: mock(() => null), + // The metadata-less removal path resolves the shared-memory owner + // strictly from config; an unreadable config aborts the removal. + loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager, @@ -18976,7 +19817,7 @@ describe("WorkspaceService init cancellation", () => { findWorkspace: mock(() => null), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager, @@ -19048,7 +19889,7 @@ describe("WorkspaceService init cancellation", () => { loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19128,7 +19969,7 @@ describe("WorkspaceService init cancellation", () => { loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19495,7 +20336,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19625,7 +20466,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19744,7 +20585,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19857,7 +20698,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19968,7 +20809,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20078,7 +20919,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20424,7 +21265,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { const mockBackgroundProcessManager = {}; const { historyService } = await createTestHistoryService(); return new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20557,7 +21398,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { }; const { historyService } = await createTestHistoryService(); const service = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20791,7 +21632,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { const mockExtensionMetadataService = {}; const mockBackgroundProcessManager = {}; return new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -21867,12 +22708,57 @@ describe("WorkspaceService.fork branch-summary rollback ordering", () => { }); }); +describe("WorkspaceService phantom removal probes", () => { + test("skips teardown only on PROVEN absence; an unreadable probe aborts the removal", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const service = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + getWorkspaceMetadata: mock(() => Promise.resolve(Err("not found"))), + }), + }); + const workspaceId = "phantom-probe"; + const sessionDir = path.join(config.sessionsDir, workspaceId); + try { + // No config.json and no session dir: nothing to tear down (idempotent). + expect((await service.remove(workspaceId, true)).success).toBe(true); + // (Deregistration wrote config.json; take it away again so the probe + // pair is exercised.) The session dir probe now fails for a reason + // other than absence: the removal must not deregister on a guess — + // abort, retryable. + await fsPromises.rm(path.join(config.rootDir, "config.json"), { force: true }); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "stat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === sessionDir + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realStat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + const aborted = await service.remove(workspaceId, true); + expect(aborted.success).toBe(false); + expect(aborted.success ? "" : aborted.error).toContain("removal aborted"); + } finally { + unreadable.mockRestore(); + } + } finally { + await cleanup(); + } + }); +}); + describe("WorkspaceService disposal ownership", () => { test.each([false, true])( "leased cleanup removes real session files without a task-tree self-join (external=%s)", async (externalRemoval) => { const h = await createAgentSessionHarness({ workspaceId: "leased-removal" }); const workspaceId = "leased-removal"; + // Removal requires an EXISTING config.json (an absent one reads as + // mid-rewrite, not as a fresh install); this workspace is simply not + // registered in it, exercising the phantom (metadata-less) path. + await h.config.editConfig((cfg) => cfg); const service = createWorkspaceServiceForTest({ config: h.config, historyService: h.historyService, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 205e7a2126e..a67a65a8b92 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -33,7 +33,13 @@ import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import { STOP_UNRECORDED_MESSAGE } from "@/common/constants/workspace"; import { MULTI_PROJECT_CONFIG_KEY } from "@/common/constants/multiProject"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; -import { ProvidersConfigStore, SecretsStore, type Config } from "@/node/config"; +import { + ProvidersConfigStore, + SecretsStore, + configFilePath, + type Config, + type Workspace as WorkspaceConfigEntry, +} from "@/node/config"; import type { ProjectsConfig, Workspace } from "@/common/types/project"; import type { Result } from "@/common/types/result"; import { Ok, Err } from "@/common/types/result"; @@ -114,6 +120,7 @@ import { extractEditedFilePaths } from "@/common/utils/messages/extractEditedFil import { buildCompactionMessageText } from "@/common/utils/compaction/compactionPrompt"; import { CONTEXT_BOUNDARY_KINDS, + compactionClosingPolicyEpoch, hasProviderEligibleMessages, isDurableCompactedMarker, sliceMessagesForProviderFromLatestContextBoundary, @@ -130,12 +137,33 @@ import { } from "@/node/services/branchSummary"; import { healRemovalTombstonesForRegisteredWorkspaces, + isWorkspaceRemovalTombstoned, removeSessionDirUnderMemoryLocks, + sealSubAgentForRemovalUnderMemoryLocks, + SharedMemoryRemovalAbortedError, refineApplyLockPath, rollbackRemovalTombstoneIfOwned, startRemovalTombstoneLease, TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; +import { + pinDescendantWorkspaceMemoryOwners, + resolveWorkspaceMemoryOwnerId, +} from "@/node/services/memoryWorkspaceOwner"; +import { + readWorkspaceMemoryDenyMarkerForEpochs, + readWorkspaceMemoryDenyMarker, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; +import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; +import type { MemoryService } from "@/node/services/memoryService"; + +/** MemoryService methods removal needs (see MemoryService.adoptLegacyPrivateStoreForRemoval). */ +type SharedWorkspaceMemoryStoreForRemoval = Pick< + MemoryService, + "adoptLegacyPrivateStoreForRemoval" +>; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -337,6 +365,11 @@ import { type WorkspaceLiveActivity, } from "@/node/services/taskWorkspaceSeam"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + setWorkspaceMemoryWritableForEpoch, + hasMalformedWorkspaceMemoryPolicyRecords, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; import type { WorktreeArchiveSnapshotService } from "@/node/services/worktreeArchiveSnapshotService"; import type { DevToolsService } from "@/node/services/devToolsService"; import type { WorkspaceMcpOverridesService } from "@/node/services/workspaceMcpOverridesService"; @@ -376,6 +409,7 @@ import { upsertSubagentTranscriptArtifactIndexEntry, } from "@/node/services/subagentTranscriptArtifacts"; import { getErrorMessage } from "@/common/utils/errors"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; /** Maximum number of retry attempts when workspace name collides */ const MAX_WORKSPACE_NAME_COLLISION_RETRIES = 3; @@ -2740,7 +2774,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + releaseRemovalCancellation(workspaceId: string): void; + finalizeHarvestsForRemoval(workspaceId: string): Promise; }; + /** Narrow MemoryService surface for removal's shared-memory handover; wired by coreServices. */ + private sharedWorkspaceMemoryStore?: SharedWorkspaceMemoryStoreForRemoval; private worktreeArchiveSnapshotService?: WorktreeArchiveSnapshotLifecycleService; private agentTaskIntegration?: AgentTaskIntegration; private workspaceGoalService?: WorkspaceGoalService; @@ -3083,10 +3121,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + releaseRemovalCancellation(workspaceId: string): void; + finalizeHarvestsForRemoval(workspaceId: string): Promise; }): void { this.memoryConsolidationService = service; } + setSharedWorkspaceMemoryStore(store: SharedWorkspaceMemoryStoreForRemoval): void { + this.sharedWorkspaceMemoryStore = store; + } + setWorkspaceLifecycleHooks(hooks: WorkspaceLifecycleHooks): void { this.workspaceLifecycleHooks = hooks; } @@ -4198,6 +4242,308 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { for (const session of this.sessions.values()) session.beginShutdown(); } + /** + * Sub-agents share their task-tree owner's /memories/workspace store, so a + * workspace-scope write by one tree member stales the cached memory context + * of every live session in that tree, not just the acting one (which already + * clears its own cache on tool-call-end). `isAffected` decides membership. + */ + invalidateMemoryContextWhere(isAffected: (workspaceId: string) => boolean): void { + // Startup-recovery sessions are live too and may be promoted with their cache. + for (const registry of [this.sessions, this.transientStartupRecoverySessions]) { + for (const [workspaceId, session] of registry) { + if (isAffected(workspaceId)) session.invalidateMemoryContext(); + } + } + } + + /** + * Config snapshot for resolving a removal's shared-memory owner: strict + * load, so a transiently unreadable config.json aborts the removal (the + * workspace stays registered and retryable) rather than yielding the + * fresh-install default whose empty topology would resolve every sub-agent + * to itself. + */ + private loadConfigForRemovalOrAbort(workspaceId: string): ProjectsConfig { + try { + // Existing file required: strict mode alone reads ENOENT as a fresh + // install, which would verify this child as its own owner mid-rewrite. + return this.config.loadExistingConfigOrThrow(); + } catch (error: unknown) { + throw new SharedMemoryRemovalAbortedError(workspaceId, { cause: error }); + } + } + + /** + * TurnRequestBuilder → session: the agent's workspace-memory write policy + * for this turn. Also persisted on the workspace config entry (only when it + * changes) so a harvest completing in a fresh session after a restart can + * still be gated; see onCompactionComplete. Awaited by the builder and + * VERIFIED by reading the config back: Config swallows write failures, and + * a stale persisted `true` would let a now read-only agent's transcript + * harvest into the shared notebook after a restart. A deny that config.json + * cannot hold is recorded in the session dir instead + * (workspaceMemoryDenyMarker.ts); resolves false only when the new value + * could not be confirmed durable anywhere. + */ + async recordWorkspaceMemoryWritable( + workspaceId: string, + writable: boolean, + options: { + epochHasPriorTurns: boolean; + policyEpoch: number; + carriedPolicyEpochs?: number[]; + /** The epoch holds tail copies whose source epoch is unknown: denied (see TurnRequestBuilder). */ + carriedPolicyUnknown?: boolean; + } + ): Promise { + // The accumulator (config bit and deny marker alike) is bound to the + // compaction epoch it accumulates over — the opening boundary's history + // sequence, -1 before any boundary — so a value recorded under another + // epoch reads as absent here. Without this, a backend starting the FIRST + // turn of a new epoch could read the closing epoch's `false` before the + // compacting backend's durable reset landed (the reset is awaited only by + // that backend's own session) and carry it, via its mirror, through an + // otherwise all-writable epoch. + const { policyEpoch } = options; + const carriedPolicyEpochs = options.carriedPolicyEpochs ?? []; + assert(Number.isInteger(policyEpoch), "policyEpoch must be an integer"); + assert( + carriedPolicyEpochs.every((epoch) => Number.isInteger(epoch) && epoch < policyEpoch), + "carriedPolicyEpochs must be earlier epochs" + ); + const session = + this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); + // A no-tail compaction's durable epoch reset may still be in flight: read + // nothing of the closing epoch (accumulator, marker) before it settled. + await session?.settleWorkspaceMemoryPolicyEpoch(); + const mirror = session?.workspaceMemoryWritableMirror(policyEpoch); + const sessionDir = path.join(this.config.sessionsDir, workspaceId); + // A deny that config.json cannot hold falls back to the session dir: the + // turn's user row is already durable in chat.jsonl there, so the deny + // must become durable in the same place or the epoch could later be + // harvested as writable after a restart (the mirror is process-local). + // `effective` is what the caller computed for the epoch so far; a grant + // never takes the fallback (harvest stays closed on the unpersisted bit). + let effective = false; + const denyDurableFallback = async (cause: string): Promise => { + if (effective) return false; + try { + // Late session-dir writer (like headless usage): with several + // backends, this turn may reach the fallback after a remover has + // tombstoned, deleted and deregistered the workspace — the marker's + // mkdir would recreate the session dir as an orphan. Gate + write run + // inside the session-dir target lock that removal's tombstone+delete + // critical section also holds, so the check cannot go stale. A + // removed workspace has nothing left to harvest: recorded as done. + const written = await withTargetMutationLock(this.config.rootDir, sessionDir, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return false; + await writeWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); + return true; + }); + if (!written) { + log.debug("Skipping workspace memory deny marker for removed workspace", { workspaceId }); + session?.recordWorkspaceMemoryWritable(false, policyEpoch); + return true; + } + } catch (markerError: unknown) { + log.error("Workspace memory deny could not be made durable anywhere", { + workspaceId, + cause, + error: getErrorMessage(markerError), + }); + // Process-local floor: this session at least keeps refusing until + // the next successful persist writes the false it now mirrors. + session?.recordWorkspaceMemoryWritable(false, policyEpoch); + return false; + } + session?.recordWorkspaceMemoryWritable(false, policyEpoch); + return true; + }; + // Strict load of an EXISTING file: a config.json that is missing (strict + // mode alone reads ENOENT as a fresh install) or malformed right now + // reads as the fresh-install default, in which this still-registered + // workspace is absent — the "unregistered" shortcut below would then + // report a deny as durable without persisting anything, while another + // backend keeps a prior durable grant. Treat an unreadable config like a + // failed config write: the deny takes the session-dir fallback + // (tombstone-gated, so a genuinely deregistered workspace still skips), a + // grant leaves the harvest closed. + let before: ReturnType; + try { + before = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), workspaceId); + } catch (error: unknown) { + log.error("Workspace memory write policy: config unreadable", { + workspaceId, + writable, + error: getErrorMessage(error), + }); + effective = writable; + return denyDurableFallback(`config unreadable: ${getErrorMessage(error)}`); + } + // Unregistered workspace: nothing durable to update and no stale + // permission to invalidate (harvests fail closed on the missing value). + if (before === null) { + session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable, policyEpoch); + return true; + } + // The persisted bit is the epoch accumulator, fail-closed: the harvest + // reads every message of the compaction epoch, so one read-only turn + // denies the whole epoch even if writable turns follow. Durable so it + // survives a restart mid-epoch AND so backends sharing one chat.jsonl + // (multi-instance) contribute to the same conjunction; it restarts at + // context boundaries (AgentSession.resetWorkspaceMemoryWritable). The + // conjunction is computed INSIDE the config transaction (registration + // lock, cross-process) from the value current at write time — two + // backends reading an absent bit concurrently could otherwise publish + // false→true. The session additionally contributes its own mirror: a + // deny it observed survives even if another backend's boundary reset + // removed the durable field underneath it. + // A fourth input: the session-dir deny marker, the durable fallback taken + // when config.json could not record a deny (denyDurableFallback above). + // Preserved-tail epoch: the accumulators of the epochs its tail copies + // were produced under are part of this one. The compacting session's + // carry moves a record/marker from the carried key to this one + // asynchronously; reading EVERY key (records inside the transaction + // below, marker entries from ONE snapshot here) makes the conjunction + // independent of that carry's timing — a deny is visible under one key + // or another at every instant, never under none. One read for all the + // epochs: separate reads could straddle the carry's atomic re-stamp and + // each miss the entry. + const denyMarker = await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [ + policyEpoch, + ...carriedPolicyEpochs, + ]); + // undefined: no carried epoch recorded anything; false: some carried deny. + const carriedFor = (entry: WorkspaceConfigEntry): boolean | undefined => { + let carried: boolean | undefined; + for (const epoch of carriedPolicyEpochs) { + const value = workspaceMemoryWritableForEpoch(entry, epoch); + if (value === undefined) continue; + carried = (carried ?? true) && value; + } + return carried; + }; + // Unknown history fails closed, like the harvest's own unknown → closed + // rule: with no durable accumulator, no marker and no mirror, an epoch + // that already holds turns has a policy nobody recorded — the record was + // lost (a deny that could not be made durable anywhere before this + // process died, an upgrade mid-epoch) — so this epoch's harvest is denied + // until the next boundary rather than granted by whichever turn comes + // first. The first turn of a fresh epoch has no prior turns and grants + // normally. + const storedFor = (entry: WorkspaceConfigEntry): boolean | undefined => + workspaceMemoryWritableForEpoch(entry, policyEpoch); + const stored = storedFor(before.workspace); + // A carried epoch whose policy was never recorded anywhere — no record + // under ITS key, none under this epoch's (where a completed carry would + // have moved it), no marker — is unknown history too: the tail copies ARE + // turns of that epoch (excluded from epochHasPriorTurns by design), e.g. + // the first tail compaction after upgrading a chat. EVERY carried epoch + // must be represented: with a chain like [-1, 5], a recorded -1 says + // nothing about 5, and one recorded grant must not mask the epoch whose + // record is missing. (The first turn recording a deny for this reason + // persists it under this epoch's key, so a later turn that finds a + // record here inherits the verdict rather than re-deriving it.) + const carriedUnrecorded = + stored === undefined && + !denyMarker && + carriedPolicyEpochs.some( + (epoch) => workspaceMemoryWritableForEpoch(before.workspace, epoch) === undefined + ); + const unknownHistory = + (stored === undefined && mirror === undefined && options.epochHasPriorTurns) || + options.carriedPolicyUnknown === true || + carriedUnrecorded; + const conjunction = (durable: boolean | undefined, carried: boolean | undefined): boolean => + !denyMarker && + !unknownHistory && + (durable ?? true) && + (carried ?? true) && + (mirror ?? true) && + writable; + // Fast path (no write): the outcome cannot differ from the stored value — + // it is already false, or already true and this turn grants. Not when + // the stored "false" is a corrupt container's blanket deny: the write + // below replaces the container with a healed record (this epoch stays + // denied; later epochs read their own records again). + if ( + !hasMalformedWorkspaceMemoryPolicyRecords(before.workspace) && + (stored === false || (stored === true && conjunction(stored, carriedFor(before.workspace)))) + ) { + session?.recordWorkspaceMemoryWritable(stored, policyEpoch); + return true; + } + effective = conjunction(stored, carriedFor(before.workspace)); + try { + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, workspaceId); + if (current !== null) { + effective = conjunction(storedFor(current.workspace), carriedFor(current.workspace)); + // Per-epoch record: never overwrites the closing epoch's value, + // which the compacting backend may not have observed yet + // (workspaceMemoryPolicyEpochs.ts). + setWorkspaceMemoryWritableForEpoch(current.workspace, policyEpoch, effective); + } + return cfg; + }); + } catch (error: unknown) { + log.error("Failed to persist workspace memory write policy", { + workspaceId, + writable, + error: getErrorMessage(error), + }); + return denyDurableFallback(getErrorMessage(error)); + } + session?.recordWorkspaceMemoryWritable(effective, policyEpoch); + const persistedEntry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + const persisted = persistedEntry === null ? undefined : storedFor(persistedEntry.workspace); + if (persisted !== effective) { + log.error("Workspace memory write policy did not persist (config write swallowed?)", { + workspaceId, + writable, + persisted, + }); + return denyDurableFallback("config write swallowed"); + } + return true; + } + + /** + * Removal's in-lock shared-memory handover for a sub-agent (owner store's + * lock AND the child's own held by the caller): legacy-notebook adoption — + * a self-fallback backend's late note into /memory either landed + * before this pass or is refused — then the refinement-row delta. Throws + * so removal aborts with the session intact; `force` logs and proceeds + * instead, accepting the loss (the user asked for the deletion regardless). + */ + private async lockedSharedMemoryHandover( + workspaceId: string, + ownerWorkspaceId: string, + force: boolean + ): Promise { + try { + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + ownerWorkspaceId, + { locksHeld: true } + ); + await migrateSharedMemoryRefinementRows({ + childSessionDir: path.join(this.config.sessionsDir, workspaceId), + childWorkspaceId: workspaceId, + ownerSessionDir: path.join(this.config.sessionsDir, ownerWorkspaceId), + ownerWorkspaceId, + }); + } catch (error) { + if (!force) throw error; + log.warn("Forced removal: locked shared-memory handover to the owner failed", { + workspaceId, + ownerWorkspaceId, + error: getErrorMessage(error), + }); + } + } + /** Transfer destructive cleanup out of a callback that still owns a session lease. */ deferWorkspaceCleanup(run: () => Promise): void { this.trackWorkspaceCleanup(run).catch((error: unknown) => @@ -4298,7 +4644,59 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.schedulePostCompactionMetadataRefresh(workspaceId); // Compaction marks a long session with accumulated learnings: harvest // the compacted epoch first, then let Dream sweep/merge the candidates. - this.memoryConsolidationService?.triggerHarvestThenSweepInBackground(metadata); + // Two observations of the epoch policy — the session's mirror (attached + // to the completion) and the durable accumulator (which other backends + // sharing this chat.jsonl also write) — and either deny is + // authoritative; both unknown (fresh recovery session, field absent) → + // the harvest fails closed. The durable value counts only under the + // CLOSING epoch's key (see recordWorkspaceMemoryWritable). Strict + // load: an unreadable config.json would read as the empty default and + // silently drop another backend's persisted deny, letting this + // session's own writable mirror grant the harvest — skip it instead + // (fail closed; nothing is recorded, the epoch is simply not harvested). + const closingEpoch = compactionClosingPolicyEpoch(metadata); + let persistedWritable: boolean | undefined; + try { + const entry = findWorkspaceEntry( + this.config.loadExistingConfigOrThrow(), + workspaceId + )?.workspace; + persistedWritable = + entry === undefined ? undefined : workspaceMemoryWritableForEpoch(entry, closingEpoch); + } catch (error: unknown) { + log.warn("Skipping post-compaction memory harvest: config.json unreadable", { + workspaceId, + error: getErrorMessage(error), + }); + return; + } + // Third observation: the session-dir deny marker (fallback taken when + // config.json could not record a deny; see recordWorkspaceMemoryWritable). + // Returned so the session orders its epoch reset (which clears the + // marker) after this observation; the harvest runs in the background. + return readWorkspaceMemoryDenyMarker( + path.join(this.config.sessionsDir, workspaceId), + closingEpoch + ) + .then((denyMarker) => { + const observed = [ + metadata.workspaceMemoryWritable, + persistedWritable, + ...(denyMarker ? [false] : []), + ].filter((value): value is boolean => value !== undefined); + this.memoryConsolidationService?.triggerHarvestThenSweepInBackground({ + ...metadata, + ...(observed.length > 0 + ? { workspaceMemoryWritable: observed.every((value) => value) } + : {}), + }); + }) + .catch((error: unknown) => { + log.warn("Skipping post-compaction memory harvest: deny marker unreadable", { + workspaceId, + error: getErrorMessage(error), + }); + }); }, onIdleCompactionOutcome: (success) => { // Reports the *persisted* idle-compaction outcome (success only after the summary @@ -5822,6 +6220,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { .filter((session): session is AgentSession => session != null) .map((session) => session.holdTurnAdmission()); + // Set once removal passes its point of no return (session teardown and + // tombstone follow unconditionally); an abort before that leaves the + // workspace registered and intact, so the finally lifts the consolidation + // teardown gate the drains below installed. + let removalCommitted = false; + // r66: identifies THIS removal attempt in the durable tombstone so the + // compensating rollback below cannot delete a concurrent backend + // attempt's marker. + const removalAttemptId = crypto.randomUUID(); + // Sub-agents: the tombstone was published (with the final shared-memory + // handover) BEFORE the checkout deletion; an abort between the two rolls + // it back so the intact workspace stays usable. + let sealedForRemoval = false; // Removal deletes the checkout: hold THIS workspace's MCP-overrides lock // like rename does (see rename), so a settings save that verified the // checkout's existence cannot have its `mkdir -p` recreate the deleted @@ -5834,6 +6245,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { if (this.agentTaskIntegration?.hasDescendantAgentTasks(workspaceId) === true) { return Err(DESCENDANT_WORKSPACE_REMOVE_ERROR); } + const sessionDir = path.join(this.config.sessionsDir, workspaceId); + // r65: keep renewing the removal tombstone's mtime until this removal + // settles so a foreign backend's startup self-heal cannot mistake a + // merely SLOW removal (a hung runtime deletion or MCP server close) for + // crash residue and delete the marker while removal is live — a healed + // marker would readmit child writes after the final shared-memory + // handover (sealSubAgentForRemovalUnderMemoryLocks), which the later + // republish (tombstoneSealed) does not migrate. Held from before the + // earliest publish point: ticks against a not-yet-published marker are + // swallowed ENOENTs, as are ticks after a rollback deleted it, and + // disposal at scope exit (after deregistration or its rollback) is safe + // since a late renewal of a retained terminal marker is meaningless. + using _tombstoneLease = startRemovalTombstoneLease(this.config.rootDir, workspaceId); // Forced removals too (routine task cleanup uses force): proceeding // while a stalled writer still owns the lock would let it resume after // the deletion and recreate the removed path. The acquisition is @@ -5875,6 +6299,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let parentWorkspaceId: string | null = null; let childTaskModelString: string | undefined; let childTaskThinkingLevel: ThinkingLevel | undefined; + // Shared-memory owner as verified by the pre-teardown handover below; + // the destructive step reuses it rather than re-resolving (see there). + let verifiedSharedMemoryOwnerId: string | null = null; const metadataResult = await this.aiService.getWorkspaceMetadata(workspaceId); if (metadataResult.success) { @@ -5929,9 +6356,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // with no second rollup). // Trade-off: a force=false deletion failure below keeps the // workspace but its producers were already drained. That loss is - // recoverable (rerun /refine, refork); a checkout write racing - // deletion is not. Both calls are idempotent; they run again later - // for the phantom-metadata path. + // recoverable (rerun /refine, refork; the consolidation teardown gate + // is lifted again in the finally); a checkout write racing deletion + // is not. Both calls are idempotent; they run again later for the + // phantom-metadata path. // Dream/harvest consolidation is a third producer (r60): its runs // ride only a hard timeout, so removal must abort them explicitly or // a detached run could mutate memory and journal into the deleted @@ -5942,6 +6370,90 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await clearPendingBranchSummary(workspaceId); await this.refinePassCanceller?.cancelInFlightRefinePass(workspaceId); + // Shared workspace memory (sub-agents write into their task-tree + // owner's store). BEFORE any destructive step — so a failure leaves a + // fully intact, retryable workspace: + // - pin the owner on surviving descendants (their parent chain is + // about to lose this node), verified by reading the config back + // because Config swallows write failures; + // - hand this workspace's live shared-memory refinement rows over to + // the owner's journal (idempotent via `migratedFrom`). A second, + // delta pass runs under the removal locks below so a write that + // lands in between is captured too; that late pass only has the + // few rows appended since this one, keeping the fallible work at + // the point of no return minimal. + // Strict load: a config.json that is missing or malformed right now + // would read as the fresh-install default and resolve this child to + // ITSELF — dropping the owner-store lock and the row handover at the + // destructive step below, which reuses this value. Nothing has been + // torn down yet, so aborting here leaves the workspace intact. + const sharedMemoryOwnerId = resolveWorkspaceMemoryOwnerId( + this.loadConfigForRemovalOrAbort(workspaceId), + workspaceId + ); + verifiedSharedMemoryOwnerId = sharedMemoryOwnerId; + if (sharedMemoryOwnerId !== workspaceId) { + try { + let pinnedOwners = new Map(); + await this.config.editConfig((cfg) => { + pinnedOwners = pinDescendantWorkspaceMemoryOwners(cfg, workspaceId); + return cfg; + }); + const persisted = this.config.loadConfigOrDefault(); + for (const [id, owner] of pinnedOwners) { + const entry = findWorkspaceEntry(persisted, id); + if (entry?.workspace.memoryOwnerWorkspaceId !== owner) { + throw new Error(`memory owner pin for descendant ${id} did not persist`); + } + } + // A pre-sharing build kept this child's notebook in its OWN + // session dir (//memory); access-time adoption + // may never have run for a child removed right after the upgrade, + // and the deletion below would take those notes with it. + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + sharedMemoryOwnerId + ); + await migrateSharedMemoryRefinementRows({ + childSessionDir: path.join(this.config.sessionsDir, workspaceId), + childWorkspaceId: workspaceId, + ownerSessionDir: path.join(this.config.sessionsDir, sharedMemoryOwnerId), + ownerWorkspaceId: sharedMemoryOwnerId, + }); + } catch (error) { + if (!force) { + return Err( + `Failed to hand this sub-agent's shared workspace memory over to its owner (${getErrorMessage(error)}); the workspace was left intact — retry the removal` + ); + } + log.warn("Forced removal: shared-memory handover to the owner failed", { + workspaceId, + sharedMemoryOwnerId, + error: getErrorMessage(error), + }); + } + // Final handover + tombstone under the removal locks, BEFORE the + // checkout is deleted (sealSubAgentForRemovalUnderMemoryLocks): a + // late legacy note the owner store cannot take must abort while the + // checkout still exists, and once sealed no backend can add + // another (they honor the tombstone at their commit points), so the + // session-dir deletion after runtime deletion has nothing fallible + // left. `force` accepts the loss of notes the handover cannot place. + await sealSubAgentForRemovalUnderMemoryLocks({ + rootDir: this.config.rootDir, + sessionDir, + workspaceId, + attemptId: removalAttemptId, + sharedWorkspaceMemorySessionDir: path.join( + this.config.sessionsDir, + sharedMemoryOwnerId + ), + beforeTombstone: () => + this.lockedSharedMemoryHandover(workspaceId, sharedMemoryOwnerId, force), + }); + sealedForRemoval = true; + } + if (isMultiProject(metadata)) { const projects = getProjects(metadata); const deleteErrors: string[] = []; @@ -6237,6 +6749,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // // Intentionally deferred until we're committed to removal: if runtime deletion fails with // force=false we return early and keep init state intact so init-end can refresh metadata. + removalCommitted = true; this.initStateManager.clearInMemoryState(workspaceId); // Dispose the session before deleting its directory: disposal aborts the active stream, and @@ -6246,7 +6759,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // Same for in-flight dream/harvest consolidation (r60): abort + drain // before the session directory disappears (idempotent; normally - // already cancelled before the usage rollup above). + // already cancelled before the usage rollup above). Retryable harvest + // records are finalized too: their transcript goes with the session. await this.memoryConsolidationService?.cancelInFlightConsolidation(workspaceId); // Cancel and drain any background branch-summary writer BEFORE deleting @@ -6287,11 +6801,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); // Remove session data - const sessionDir = path.join(this.config.sessionsDir, workspaceId); - // r66: identifies THIS removal attempt in the durable tombstone so the - // compensating rollback below cannot delete a concurrent backend - // attempt's marker. - const removalAttemptId = crypto.randomUUID(); try { if (parentWorkspaceId) { try { @@ -6321,32 +6830,93 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // directory. Fail-closed on a wedged writer: the catch below keeps // the directory as a recoverable orphan instead of deleting it out // from under a live commit. - await removeSessionDirUnderMemoryLocks({ - rootDir: this.config.rootDir, - sessionDir, - workspaceId, - attemptId: removalAttemptId, - }); + // A sub-agent's workspace memory lives in its task-tree owner's + // session dir; hold that store's lock as well so an admitted child + // write cannot slip between this tombstone and its commit check, and + // run the delta pass of the refinement-row handover inside those + // locks (rows appended since the pre-teardown pass; the phantom, + // metadata-less path gets its full pass here). The owner verified by + // that earlier pass is retained: a config.json that turns missing or + // malformed in between would otherwise resolve the child to ITSELF + // here and drop the owner-store lock, letting a foreign child write + // admitted under the real owner lock recreate the deleted session dir. + // Without an earlier pass, resolve strictly — an unreadable config + // aborts the removal (retryable) instead of guessing the topology. + // Idempotent no-op short of that: a root whose config.json was never + // written (fresh install, nothing registered) and no session dir + // means there is no topology to resolve and nothing a tombstone + // could protect — removing an unknown id must still succeed. + // Proven absence only: a probe that fails for any other reason + // (EACCES, EIO) says nothing, and declaring "nothing to tear down" + // on it would deregister the workspace without deleting its session + // or publishing a tombstone — an orphan foreign writers keep mutating. + const provenAbsent = (target: string): Promise => + fsPromises.stat(target).then( + () => false, + (error: unknown) => { + if (hasErrorCode(error, "ENOENT") || hasErrorCode(error, "ENOTDIR")) return true; + throw new SharedMemoryRemovalAbortedError(workspaceId, { cause: error }); + } + ); + const nothingToTearDown = + verifiedSharedMemoryOwnerId === null && + (await provenAbsent(configFilePath(this.config.rootDir))) && + (await provenAbsent(sessionDir)); + if (nothingToTearDown) { + log.debug("Skipping session teardown: no config.json and no session dir", { + workspaceId, + }); + } else { + const memoryOwnerId = + verifiedSharedMemoryOwnerId ?? + resolveWorkspaceMemoryOwnerId( + this.loadConfigForRemovalOrAbort(workspaceId), + workspaceId + ); + const ownerSessionDir = + memoryOwnerId === workspaceId + ? undefined + : path.join(this.config.sessionsDir, memoryOwnerId); + await removeSessionDirUnderMemoryLocks({ + rootDir: this.config.rootDir, + sessionDir, + workspaceId, + attemptId: removalAttemptId, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + tombstoneSealed: sealedForRemoval, + // Sealed above (metadata path): handover done and tombstone + // published under these locks already. Otherwise (phantom, + // metadata-less path) the handover runs here, inside the locks + // and right before the tombstone. Throws → removal aborts, + // session intact. + beforeTombstone: + ownerSessionDir === undefined || sealedForRemoval + ? undefined + : () => this.lockedSharedMemoryHandover(workspaceId, memoryOwnerId, force), + }); + // Only once the session (and with it the transcript) is gone are the + // retryable harvest records truly unrecoverable; an aborted removal + // above must leave them retryable. + await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); + } } catch (error) { // r63: without a durable tombstone the retained orphan stays // writable by foreign backends forever — abort the removal (the // workspace stays registered and retryable) instead of proceeding // to deregistration below. - if (error instanceof TombstoneNotDurableError) { + if ( + error instanceof TombstoneNotDurableError || + error instanceof SharedMemoryRemovalAbortedError + ) { + // No durable tombstone was published (the locked handover or the + // tombstone write itself failed): the workspace stays registered + // with its session directory intact, so the consolidation teardown + // gate is lifted again in the finally like any pre-commit abort. + removalCommitted = false; throw error; } log.error(`Failed to remove session directory for ${workspaceId}:`, error); } - // r65: the tombstone is durable here (both the locked path and the - // orphan fallback published it). Keep renewing its mtime until this - // removal settles so a foreign backend's startup self-heal cannot - // mistake a merely SLOW removal (e.g. a hung MCP server close below) - // for crash residue and delete the marker while removal is live. - // Disposal at scope exit (after deregistration or its rollback) is - // safe: a late renewal of a retained terminal marker is meaningless, - // and utimes on a rolled-back (deleted) marker is a swallowed ENOENT. - using _tombstoneLease = startRemovalTombstoneLease(this.config.rootDir, workspaceId); - // The on-disk devtools.jsonl died with the session directory above; also drop any // in-memory DevTools state so stale runs cannot outlive the workspace. try { @@ -6458,6 +7028,29 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const message = getErrorMessage(error); return Err(`Failed to remove workspace: ${message}`); } finally { + if (!removalCommitted) { + // A sealed sub-agent whose checkout deletion was then refused + // (force=false) keeps its session dir and config entry: lift the + // tombstone again (ownership-checked, r66) so it stays usable. + if (sealedForRemoval) { + try { + await rollbackRemovalTombstoneIfOwned({ + rootDir: this.config.rootDir, + sessionDir: path.join(this.config.sessionsDir, workspaceId), + workspaceId, + attemptId: removalAttemptId, + workspaceStillRegistered: () => this.config.findWorkspace(workspaceId) != null, + }); + } catch (rollbackError) { + log.error( + "Failed to roll back the removal tombstone after an aborted removal; " + + "the startup self-heal will reclaim it", + { workspaceId, rollbackError } + ); + } + } + this.memoryConsolidationService?.releaseRemovalCancellation(workspaceId); + } if (releaseOverridesLock !== undefined) { try { await releaseOverridesLock();