From 4049e6db209ca8f4b865b9567c2c3ef6f4903c5d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 11:04:25 +0000 Subject: [PATCH 01/98] =?UTF-8?q?=F0=9F=A4=96=20feat:=20share=20workspace?= =?UTF-8?q?=20memory=20notes=20across=20a=20sub-agent=20task=20tree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sub-agent child workspaces (parentWorkspaceId set) now resolve /memories/workspace/... to the task-tree root's /memory, so an owner and all of its (nested) sub-agents read and write one notebook while their transcripts and session artifacts stay separate. - MemoryService.resolveWorkspaceMemoryOwnerId walks parentWorkspaceId to the root (cycle/depth guarded, unknown ids resolve to themselves) and is used for the workspace store root, sidecar pin/usage keys, change events, the workspace-scope refinement journal (owner session, where rollback confinement holds), and the removal tombstone check (acting + owner). - Memory tab subscriptions/list/pin resolve the owner too. - Dream consolidation refuses sub-agent children and the launch sweep skips them; children still harvest into the shared inbox, the owner sweeps it. - A workspace-scope write invalidates the cached memory context of every live session in the same tree. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$1.38`_ --- src/common/constants/memory.ts | 3 +- src/common/utils/tools/toolDefinitions.ts | 2 +- src/node/orpc/routerSubscriptions.test.ts | 52 ++++- src/node/orpc/routerSubscriptions.ts | 7 +- .../agentSession.memoryContext.test.ts | 6 + src/node/services/agentSession.ts | 11 + src/node/services/di/layers/core.ts | 11 +- src/node/services/memoryConsolidation.ts | 6 +- .../memoryConsolidationService.test.ts | 40 +++- .../services/memoryConsolidationService.ts | 15 ++ src/node/services/memoryOperations.ts | 8 +- src/node/services/memoryService.test.ts | 134 +++++++++++- src/node/services/memoryService.ts | 204 +++++++++++++----- src/node/services/workspaceService.ts | 12 ++ 14 files changed, 447 insertions(+), 64 deletions(-) diff --git a/src/common/constants/memory.ts b/src/common/constants/memory.ts index bf650683d65..35b1ad3f9f8 100644 --- a/src/common/constants/memory.ts +++ b/src/common/constants/memory.ts @@ -9,7 +9,8 @@ * per-project notes; never committed to the repo, survives * workspaces; carried by the settings backup only when the * user opts into the project bundle) - * - workspace -> /memory/ (host-local, deleted with the workspace) + * - workspace -> /memory/ of the task-tree OWNER (host-local, deleted + * with that workspace; sub-agents share their parent's store) */ /** Virtual root prefix all memory paths are expressed under. */ diff --git a/src/common/utils/tools/toolDefinitions.ts b/src/common/utils/tools/toolDefinitions.ts index 974a79cede0..f196ba55205 100644 --- a/src/common/utils/tools/toolDefinitions.ts +++ b/src/common/utils/tools/toolDefinitions.ts @@ -2485,7 +2485,7 @@ export const TOOL_DEFINITIONS = { "Scopes (all paths are virtual):\n" + "- /memories/global/... — personal, permanent, shared across all projects\n" + "- /memories/project/... — private notes about this project; host-local, never committed to the repo (included in the settings backup only when the user opts in), survives workspaces\n" + - "- /memories/workspace/... — scratch state for this workspace; deleted with the workspace\n" + + "- /memories/workspace/... — scratch state for this workspace, shared with its sub-agents (a sub-agent reads and writes its parent's workspace notes); deleted with the owning workspace\n" + "Commands:\n" + "- view: list a directory (up to 2 levels, dotfiles excluded) or show a file with line numbers (offset/limit supported)\n" + "- create: create a new file; ERRORS if the file already exists (to overwrite: delete first, then create)\n" + diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 2e454232851..9353991d62f 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -13,7 +13,11 @@ import { TestClock } from "effect/testing"; import { SUBSCRIPTION_HEARTBEAT_INTERVAL_MS } from "@/common/utils/withQueueHeartbeat"; import { disposeAppRuntime, makeAppRuntime } from "@/node/services/di/appRuntime"; import type { ORPCContext } from "./context"; -import { subscribeWorkspaceActivity, subscribeDesignExperiment } from "./routerSubscriptions"; +import { + subscribeWorkspaceActivity, + subscribeDesignExperiment, + subscribeMemoryChanges, +} from "./routerSubscriptions"; test("subscription handlers forward the oRPC runtime Clock", async () => { const app = makeAppRuntime(TestClock.layer()); @@ -37,6 +41,52 @@ test("subscription handlers forward the oRPC runtime Clock", async () => { expect(workspaceService.listenerCount("activity")).toBe(0); }); +test("memory subscriptions match workspace-scope events on the shared memory owner", async () => { + // Workspace-scope change events carry the memory OWNER (task-tree root); + // a sub-agent's subscription must see edits to the notebook it shares. + const app = makeAppRuntime(TestClock.layer()); + const memoryService = new EventEmitter(); + const memoryConsolidationService = new EventEmitter(); + const controller = new AbortController(); + const context = { + "effect/context": app.context, + workspaceService: { getInfo: () => Promise.resolve(null) }, + memoryService: Object.assign(memoryService, { + resolveWorkspaceMemoryOwnerId: (workspaceId: string) => + workspaceId === "ws-child" ? "ws-owner" : workspaceId, + }), + memoryConsolidationService, + } as unknown as ORPCContext; + const stream = subscribeMemoryChanges(context, "ws-child", controller.signal); + try { + const first = stream.next(); + // The listener attaches once the generator has started running. + while (memoryService.listenerCount("change") === 0) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + const base = { + scope: "workspace", + path: "/memories/workspace/n.md", + actor: "agent", + projectPath: "", + }; + memoryService.emit("change", { ...base, workspaceId: "ws-other" }); + memoryService.emit("change", { ...base, workspaceId: "ws-child" }); + memoryService.emit("change", { ...base, workspaceId: "ws-owner" }); + // Global scope is never filtered: it marks the end of the batch, so + // receiving it second proves the other/child events were dropped. + const marker = { ...base, scope: "global", path: "/memories/global/g.md", workspaceId: "" }; + memoryService.emit("change", marker); + expect((await first).value).toEqual({ ...base, workspaceId: "ws-owner" }); + expect((await stream.next()).value).toEqual(marker); + } finally { + controller.abort(); + await stream.return(undefined); + await disposeAppRuntime(app.managed); + } + expect(memoryService.listenerCount("change")).toBe(0); +}); + test("Design subscriptions publish sibling changes only after client shutdown", async () => { using temp = new DisposableTempDir("design-subscription"); const flags = path.join(temp.path, EXPERIMENT_OVERRIDES_FILE_NAME); diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 8c0a374f894..83a3108c10a 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -207,11 +207,16 @@ export function subscribeMemoryChanges( validate?.(); const metadata = workspaceId ? await context.workspaceService.getInfo(workspaceId) : null; const projectPath = metadata ? resolveMemoryProjectIdentity(metadata) : null; + // Workspace-scope events carry the memory OWNER (task-tree root), which is + // also the store this subscriber's workspace displays. + const ownerWorkspaceId = workspaceId + ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) + : null; yield* runtimeSubscription(context, { signal, subscribe: (emit) => { const onChange = (event: MemoryChangeEvent) => { - if (event.scope === "workspace" && event.workspaceId !== workspaceId) return; + if (event.scope === "workspace" && event.workspaceId !== ownerWorkspaceId) return; if (event.scope === "project" && event.projectPath !== projectPath) return; emit.push(event); }; diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 0e9d2077634..7a6ef196c02 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -127,6 +127,12 @@ describe("AgentSession memory context", () => { expect(await priv.resolveMemoryContext("test-model")).toEqual(context); expect(await priv.resolveMemoryContext("test-model")).toEqual(context); expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + + // A write by another task-tree member to the shared workspace notebook + // invalidates from outside; the next resolve rebuilds from disk. + session.invalidateMemoryContext(); + expect(await priv.resolveMemoryContext("test-model")).toEqual(context); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); } finally { await session.dispose(); } diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 2866c5d4b03..06b7d5347fd 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1034,6 +1034,17 @@ export class AgentSession { * prompt-cache-stable bytes without preserving stale files forever. */ private memoryContextByModelString = new Map(); + + /** + * Drop the cached memory context so the next stream rebuilds the index and + * hot set from disk. Own memory tool calls clear it on tool-call-end; this + * entry point is for writes by OTHER sessions to a store this session also + * reads (a sub-agent editing the task tree's shared workspace notes). + */ + invalidateMemoryContext(): void { + this.memoryContextByModelString.clear(); + } + /** * Cache the last-known experiment state so we don't spam metadata refresh * when post-compaction context is disabled. diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index b651cdbe9c5..30d10efafbe 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -57,7 +57,7 @@ import { MCPConfigService } from "@/node/services/mcpConfigService"; import { MCPServerManager } from "@/node/services/mcpServerManager"; import { MemoryConsolidationService } from "@/node/services/memoryConsolidationService"; import { MemoryMetaService } from "@/node/services/memoryMeta"; -import { MemoryService } from "@/node/services/memoryService"; +import { MemoryService, type MemoryChangeEvent } from "@/node/services/memoryService"; import { ProviderService } from "@/node/services/providerService"; import { SessionUsageService } from "@/node/services/sessionUsageService"; import { StreamManager } from "@/node/services/streamManager"; @@ -543,6 +543,15 @@ export const CoreWiringLive: Layer.Layer< workspaceService.emitWorkflowRunActivity(event); turnRequestBuilderBindings.workflowResultContinuationSender = workspaceService; workspaceService.setMemoryConsolidationService(memoryConsolidationService); + // Workspace-scope change events carry the memory OWNER (task-tree root); + // every live session resolving to that owner reads the same notebook. + memoryService.on("change", (event: MemoryChangeEvent) => { + if (event.scope !== "workspace" || event.workspaceId === "") return; + workspaceService.invalidateMemoryContextWhere( + (workspaceId) => + memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) === event.workspaceId + ); + }); if (opts.devToolsService) { // DevTools debug-log cleanup when workspaces are archived/removed. workspaceService.setDevToolsService(opts.devToolsService); diff --git a/src/node/services/memoryConsolidation.ts b/src/node/services/memoryConsolidation.ts index 509da74270c..cf86bd37f12 100644 --- a/src/node/services/memoryConsolidation.ts +++ b/src/node/services/memoryConsolidation.ts @@ -253,7 +253,11 @@ export function createConsolidationMemoryTool(args: { const entries = await metaService.getEntries(); const key = memoryLogicalKey(scope, relPath, { projectPath: ctx.projectPath, - workspaceId: ctx.workspaceId, + // Sidecar keys follow the shared store's owner (MemoryService.logicalKeyFor). + workspaceId: + ctx.workspaceId === "" + ? "" + : memoryService.resolveWorkspaceMemoryOwnerId(ctx.workspaceId), }); const subtreePrefix = `${key}/`; for (const [entryKey, entry] of entries) { diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 4f9fdaca800..725716b485d 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -204,7 +204,10 @@ interface Fixture extends Disposable { setEnabled: (enabled: boolean) => void; /** When true, scripted runs emit a fatal stream error instead of finishing. */ setStreamFailing: (failing: boolean) => void; - addWorkspace: (id: string, opts?: { archivedAt?: string }) => Promise; + addWorkspace: ( + id: string, + opts?: { archivedAt?: string; parentWorkspaceId?: string } + ) => Promise; addMultiProjectWorkspace: (id: string, opts?: { bucket?: string }) => Promise; } @@ -283,6 +286,7 @@ async function createFixture(options?: { name: id, path: `/projects/demo/${id}`, archivedAt: opts?.archivedAt, + parentWorkspaceId: opts?.parentWorkspaceId, }); return cfg; }); @@ -1280,6 +1284,40 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(1); }); + it("refuses Dream runs for sub-agent children and lets the owner sweep their shared writes", async () => { + using fixture = await createFixture(); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + + // Every trigger, including an explicit /dream, is refused on the child: + // its /memories/workspace IS the owner's store. + for (const trigger of ["manual", "compaction", "archive"] as const) { + const result = await fixture.service.maybeRun("ws-sub", trigger); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("owner"); + } + expect(fixture.modelCalls).toHaveLength(0); + + // A child's workspace write is keyed under the owner (MemoryService + // resolution), so an idle owner qualifies for the launch sweep while the + // idle child is skipped even though it is listed. + await fixture.memoryService.create( + { runtime: null, checkoutCwd: "", workspaceId: "ws-sub", projectPath: "" }, + "/memories/workspace/from-child.md", + "shared lesson", + "agent" + ); + const dayAgo = Date.now() - 25 * 60 * 60 * 1000; + await fixture.service.runLaunchSweep( + new Map([ + ["ws-sub", dayAgo], + ["ws-dream", dayAgo], + ]) + ); + expect(fixture.modelCalls).toHaveLength(1); + expect(await fixture.service.getRecord("ws-sub")).toBeNull(); + expect(await fixture.service.getRecord("ws-dream")).not.toBeNull(); + }); + it("launch sweep skips archived workspaces and caps runs per launch", async () => { using fixture = await createFixture(); const dayAgo = Date.now() - 25 * 60 * 60 * 1000; diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 9e830b3ccbf..81bc6f91e8a 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -733,6 +733,16 @@ export class MemoryConsolidationService extends EventEmitter { const workspace = self.config.findWorkspace(workspaceId); if (!workspace) return Err(`workspace not found: ${workspaceId}`); + // Sub-agents share their task-tree owner's /memories/workspace store + // (MemoryService.resolveWorkspaceMemoryOwnerId), so a Dream run from a + // child would consolidate the owner's notebook concurrently with the + // owner's own runs. Children still harvest into the shared inbox; only + // the owner sweeps it. + if (workspace.parentWorkspaceId) { + return Err( + "sub-agent workspaces share their owner's workspace memory; the owner consolidates it" + ); + } const agentBody = yield* Effect.promise(() => resolveDreamAgentBody(self.config.rootDir)); if (agentBody === null) return Err("dream agent definition is missing"); @@ -1192,10 +1202,12 @@ export class MemoryConsolidationService extends EventEmitter { // covering pass, not one per idle workspace in the same sweep. let globalLastRunAt = findNewestWorkspaceRecord(sidecar.workspaces)?.lastRunAt ?? 0; const archivedById = new Map(); + const subAgentIds = new Set(); const projectPathByWorkspace = new Map(); for (const [configProjectPath, project] of self.config.loadConfigOrDefault().projects) { for (const workspace of project.workspaces) { if (workspace.id === undefined) continue; + if (workspace.parentWorkspaceId) subAgentIds.add(workspace.id); archivedById.set( workspace.id, isWorkspaceArchived(workspace.archivedAt, workspace.unarchivedAt) @@ -1221,6 +1233,9 @@ export class MemoryConsolidationService extends EventEmitter { if (started >= MEMORY_CONSOLIDATION_LAUNCH_SWEEP_CAP) break; if (now - recency < MEMORY_CONSOLIDATION_IDLE_MS) continue; if (archivedById.get(workspaceId) === true) continue; + // Shared store: the owner's own sweep covers a child's writes (they + // are keyed under the owner), and runLockedEffect refuses children. + if (subAgentIds.has(workspaceId)) continue; const lastRunAt = sidecar.workspaces[workspaceId]?.lastRunAt ?? 0; const projectPath = projectPathByWorkspace.get(workspaceId) ?? ""; const projectRunAt = projectPath === "" ? 0 : (projectLastRunAt.get(projectPath) ?? 0); diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index 0837e5250ba..e487ff8ab89 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -57,6 +57,8 @@ function workspaceNotFound(workspaceId: string | null | undefined): string { interface ResolvedMemoryScope { projectPath: string; + /** Task-tree root whose store backs workspace scope ("" without a workspace); keys sidecar pins/stats. */ + ownerWorkspaceId: string; scopeCtx: MemoryScopeContext; } @@ -73,6 +75,7 @@ function resolveMemoryScope( if (workspaceId == null) return { projectPath: "", + ownerWorkspaceId: "", scopeCtx: { runtime: null, checkoutCwd: "", workspaceId: "", projectPath: "" }, }; const metadata = yield* Effect.promise(() => context.workspaceService.getInfo(workspaceId)); @@ -80,6 +83,7 @@ function resolveMemoryScope( const projectPath = resolveMemoryProjectIdentity(metadata); return { projectPath, + ownerWorkspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), scopeCtx: { runtime: createRuntimeForWorkspace(metadata), checkoutCwd: "", @@ -102,7 +106,7 @@ export function listMemoryEffect(context: MemoryContext, input: Input { }); }); + describe("sub-agent workspace memory sharing", () => { + /** Register owner → child → grandchild so parentWorkspaceId chains resolve. */ + async function registerTaskTree(fixture: MemoryFixture): Promise { + await fixture.config.editConfig((cfg) => { + cfg.projects.set(FIXTURE_PROJECT_PATH, { + workspaces: [ + { id: "ws-owner", name: "owner", path: "/checkouts/owner" }, + { + id: "ws-child", + name: "child", + path: "/checkouts/child", + parentWorkspaceId: "ws-owner", + }, + { + id: "ws-grandchild", + name: "grandchild", + path: "/checkouts/grandchild", + parentWorkspaceId: "ws-child", + }, + { id: "ws-solo", name: "solo", path: "/checkouts/solo" }, + ], + }); + return cfg; + }); + } + + it("resolves the task-tree root as the owner; unknown and parentless ids resolve to themselves", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-owner")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-owner"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-solo")).toBe("ws-solo"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-unregistered")).toBe( + "ws-unregistered" + ); + }); + + it("stores a sub-agent's workspace notes in the owner's session dir, visible to the whole tree", async () => { + using fixture = await createFixture("ws-grandchild"); + await registerTaskTree(fixture); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/context-notes.md", + "found the bug in parser.ts", + "agent" + ); + expect(created.success).toBe(true); + + // Physically in the OWNER's session dir, not the grandchild's. + const ownerPhysical = path.join( + fixture.config.sessionsDir, + "ws-owner", + "memory", + "context-notes.md" + ); + expect(await fsPromises.readFile(ownerPhysical, "utf-8")).toBe("found the bug in parser.ts"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-grandchild", "memory")) + ).toBe(false); + + // Owner and sibling child read the same file through their own contexts. + for (const workspaceId of ["ws-owner", "ws-child"]) { + const viewed = await fixture.service.view( + { ...fixture.ctx, workspaceId }, + "/memories/workspace/context-notes.md" + ); + expect(viewed.success).toBe(true); + if (viewed.success) expect(viewed.output).toContain("found the bug in parser.ts"); + } + // An unrelated workspace does not see it. + const solo = await fixture.service.view( + { ...fixture.ctx, workspaceId: "ws-solo" }, + "/memories/workspace/context-notes.md" + ); + expect(solo.success).toBe(false); + + // Change events name the owner so the owner's Memory tab (and every + // tree member's) refreshes; sidecar stats are keyed by the owner too. + expect(events).toEqual([ + { + scope: "workspace", + path: "/memories/workspace/context-notes.md", + actor: "agent", + workspaceId: "ws-owner", + projectPath: FIXTURE_PROJECT_PATH, + }, + ]); + const meta = await fixture.metaService.getEntries(); + expect( + meta.get( + memoryLogicalKey("workspace", "context-notes.md", { + projectPath: "", + workspaceId: "ws-owner", + }) + )?.lastWriteAt + ).not.toBeNull(); + expect( + meta.has( + memoryLogicalKey("workspace", "context-notes.md", { + projectPath: "", + workspaceId: "ws-grandchild", + }) + ) + ).toBe(false); + }); + + it("journals a sub-agent's workspace-scope mutation into the owner's session (where rollback is confined)", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + // Global scope stays attributed to the acting child. + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "mine", "agent"); + + const ownerEvents = await readRefinementEvents( + path.join(fixture.config.sessionsDir, "ws-owner") + ); + expect(ownerEvents.map((event) => event.data.action)).toEqual([ + { op: "create", path: "/memories/workspace/n.md" }, + ]); + const childEvents = await readRefinementEvents( + path.join(fixture.config.sessionsDir, "ws-child") + ); + expect(childEvents.map((event) => event.data.action)).toEqual([ + { op: "create", path: "/memories/global/g.md" }, + ]); + }); + }); + describe("memory index entries", () => { it("lists files across scopes with sanitized frontmatter descriptions", async () => { using fixture = await createFixture(); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8ea24fa0bd4..e3d077e1a8f 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -46,6 +46,7 @@ import { withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; +import { findWorkspaceEntry } from "@/node/services/taskUtils"; import { REFINEMENT_CAPTURE_MAX_FILES, REFINEMENT_CAPTURE_MAX_TOTAL_BYTES, @@ -70,7 +71,12 @@ export interface MemoryScopeContext { runtime: Runtime | null; /** Workspace checkout cwd. Kept in the context shape for existing callers; storage ignores it. */ checkoutCwd: string; - /** Workspace ID; workspace scope root is /memory. */ + /** + * ACTING workspace ID. The workspace scope root is the memory OWNER's + * /memory, where the owner is the task-tree root: sub-agent + * children (parentWorkspaceId set) share their parent's workspace notes + * (see MemoryService.resolveWorkspaceMemoryOwnerId). + */ workspaceId: string; /** * Stable project identity from Xum config (the project root path, never the @@ -589,12 +595,79 @@ export class MemoryService extends EventEmitter { // Best-effort: stats failures must never break a memory command. // ------------------------------------------------------------------------- + /** + * Positive-only memo for resolveWorkspaceMemoryOwnerId: a workspace's + * parentWorkspaceId is fixed at creation and IDs are never reused, so a + * resolved chain stays valid for the process lifetime. Unknown IDs are not + * cached — the workspace may simply not be registered yet. + */ + private readonly workspaceMemoryOwnerById = new Map(); + + /** + * The workspace whose /memory backs `/memories/workspace` for + * `workspaceId`: the root of its parentWorkspaceId chain. Sub-agents (and + * nested sub-agents) thereby share ONE notebook with the workspace that + * spawned the task tree, while their transcripts/session artifacts stay + * separate. Full `kind: "workspace"` tasks and forks carry no + * parentWorkspaceId and own their notes. Unknown IDs, cycles, and depth + * overflow resolve to the ID itself so a misconfigured tree degrades to + * today's per-workspace behavior instead of failing every memory command. + */ + resolveWorkspaceMemoryOwnerId(workspaceId: string): string { + assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); + const cached = this.workspaceMemoryOwnerById.get(workspaceId); + if (cached !== undefined) return cached; + const cfg = this.config.loadConfigOrDefault(); + let current = workspaceId; + const visited = new Set(); + for (let depth = 0; depth < 32; depth++) { + if (visited.has(current)) { + log.warn( + "[MemoryService] parentWorkspaceId cycle; using acting workspace as memory owner", + { + workspaceId, + } + ); + return workspaceId; + } + visited.add(current); + const entry = findWorkspaceEntry(cfg, current); + if (entry === null) { + // Only the chain root may be unknown without invalidating the walk: + // an unregistered starting workspace resolves to itself (not cached). + if (current === workspaceId) return workspaceId; + log.warn("[MemoryService] parentWorkspaceId points at an unknown workspace", { + workspaceId, + parentWorkspaceId: current, + }); + return workspaceId; + } + const parentWorkspaceId = entry.workspace.parentWorkspaceId; + if (parentWorkspaceId === undefined || parentWorkspaceId === "") { + this.workspaceMemoryOwnerById.set(workspaceId, current); + return current; + } + current = parentWorkspaceId; + } + log.warn("[MemoryService] parentWorkspaceId chain too deep; using acting workspace", { + workspaceId, + }); + return workspaceId; + } + + /** Owner of the workspace scope for this context ("" when there is no workspace). */ + private ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { + return ctx.workspaceId === "" ? "" : this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + } + /** Logical sidecar key, or null when the scope has no stable identity. */ private logicalKeyFor(ctx: MemoryScopeContext, scope: MemoryScope, relPath: string) { if (scope === "project" && ctx.projectPath === "") return null; return memoryLogicalKey(scope, relPath, { projectPath: ctx.projectPath, - workspaceId: ctx.workspaceId, + // Pins/usage stats follow the physical file, so a shared notebook has + // one ranking regardless of which tree member touched it. + workspaceId: this.ownerWorkspaceIdFor(ctx), }); } @@ -697,7 +770,9 @@ export class MemoryService extends EventEmitter { "Workspace memory is unavailable: no workspace is associated with this session" ); } - return new LocalMemoryStore(path.join(this.config.sessionsDir, ctx.workspaceId, "memory")); + return new LocalMemoryStore( + path.join(this.config.sessionsDir, this.ownerWorkspaceIdFor(ctx), "memory") + ); } } } @@ -750,12 +825,17 @@ export class MemoryService extends EventEmitter { * Rows land in the ACTING workspace's session journal even though memory * files can be global/project-scoped: the journal is per-session, so * cross-workspace edits to a shared file are attributed to (and invertible - * from) whichever workspace made them — the intended v1 scope. When the - * context has no workspace, there is no session journal; skip (log-only). + * from) whichever workspace made them — the intended v1 scope. The one + * exception is workspace scope written by a sub-agent: the file lives in + * the OWNER's /memory and rollback confinement only admits a + * journal's own session memory root, so those rows go to the owner's + * journal (where they are actually invertible). When the context has no + * workspace, there is no session journal; skip (log-only). * Never throws: journaling failures must not fail the memory command. */ private async journalRefinement( ctx: MemoryScopeContext, + scope: MemoryScope, action: MemoryRefinementAction, inverse: RefinementInverseDraft, actor: MemoryActor, @@ -768,9 +848,11 @@ export class MemoryService extends EventEmitter { }); return; } + const journalWorkspaceId = + scope === "workspace" ? this.ownerWorkspaceIdFor(ctx) : ctx.workspaceId; await appendRefinementEvent({ - sessionDir: path.join(this.config.sessionsDir, ctx.workspaceId), - workspaceId: ctx.workspaceId, + sessionDir: path.join(this.config.sessionsDir, journalWorkspaceId), + workspaceId: journalWorkspaceId, kind: "memory", action, inverse, @@ -783,6 +865,51 @@ export class MemoryService extends EventEmitter { }); } + /** + * Refuse to COMMIT a mutation whose caller was torn down (r59/r61). Checked + * INSIDE the target mutation lock immediately before the first durable + * write; a mutation that already committed always journals (mutation → row + * → ack) so rollback lineage stays intact. Two teardown signals: + * + * - The caller's abort signal (r59): consolidation/refine passes receive no + * hard tool cancellation — an execution wedged in pre-commit I/O (e.g. a + * named pipe under a memory root) is detached by the caller's bounded + * drain, and once the I/O unblocks after workspace teardown it would + * still write durable memory AND append its refinement journal row into + * the deleted session directory, recreating it. + * - The durable removal tombstone (r61): with multiple backends over one + * Xum root, the remover cannot abort a dream/harvest run in ANOTHER + * process — that run's signal stays live after removal. The tombstone is + * published under the same memory target locks this check runs inside + * (see workspaceRemoval.ts), so a foreign backend's mutation observes + * removal here at commit time and refuses instead of recreating the + * deleted session directory via its write or journal append. + * + * Both the acting workspace and the workspace-memory owner are checked: a + * removed sub-agent must not keep writing into its parent's notebook, and + * a removed owner must not have its session directory recreated by a + * lingering child's write. + */ + private async assertMutationCommittable( + ctx: MemoryScopeContext, + signal: AbortSignal | undefined, + virtualPath: string + ): Promise { + if (signal?.aborted === true) { + throw new MemoryCommandError( + `Mutation of ${virtualPath} was cancelled before commit (caller torn down)` + ); + } + if (ctx.workspaceId === "") return; + for (const workspaceId of new Set([ctx.workspaceId, this.ownerWorkspaceIdFor(ctx)])) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + throw new MemoryCommandError( + `Workspace ${workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` + ); + } + } + } + /** * Capture the restore payload for a delete (file or recursive directory) * BEFORE it is removed. Returns null when capture fails or the subtree @@ -901,7 +1028,9 @@ export class MemoryService extends EventEmitter { scope, path: toVirtualPath(scope, relPath), actor, - workspaceId: ctx.workspaceId, + // Owner, not actor: subscribers filter workspace-scope events by the + // store they display, and every tree member displays the owner's. + workspaceId: this.ownerWorkspaceIdFor(ctx), projectPath: ctx.projectPath, }; this.emit("change", event); @@ -1004,7 +1133,7 @@ export class MemoryService extends EventEmitter { // only INSIDE the lock and after the removal check (r62), so the // mkdir serializes with removal's locked deletion and cannot // recreate a removed session directory. - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.ensureRoot(); const existing = await store.kind(parsed.relPath); if (existing !== null) { @@ -1018,11 +1147,12 @@ export class MemoryService extends EventEmitter { `The ${scope} memory scope is full (${MEMORY_MAX_FILES_PER_SCOPE} files); delete unused files first` ); } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.writeFile(parsed.relPath, fileText); // Row is written before the create is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + scope, { op: "create", path: toVirtualPath(scope, parsed.relPath) }, { op: "delete-files", paths: [store.physicalPath(parsed.relPath)] }, actor, @@ -1059,11 +1189,12 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const updated = computeStrReplaceUpdate(content, oldStr, newStr, virtualPath); assertWithinFileSizeCap(updated); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + scope, { op: "str_replace", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1112,11 +1243,12 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const { updated, insertedLineCount } = computeInsertUpdate(content, insertLine, insertText); assertWithinFileSizeCap(updated); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + scope, { op: "insert", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1295,11 +1427,12 @@ export class MemoryService extends EventEmitter { // Prior contents must be captured before removal; the row itself is // written after the mutation succeeds and before it is acknowledged. const inverse = await this.captureDeleteInverse(store, parsed.relPath, kind); - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.remove(parsed.relPath); if (inverse !== null) { await this.journalRefinement( ctx, + scope, { op: "delete", path: toVirtualPath(scope, parsed.relPath) }, inverse, actor, @@ -1357,11 +1490,12 @@ export class MemoryService extends EventEmitter { if (newKind !== null) { throw new MemoryCommandError(`Destination ${newVirtualPath} already exists`); } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, oldVirtualPath); + await this.assertMutationCommittable(ctx, abortSignal, oldVirtualPath); await store.rename(oldParsed.relPath, newParsed.relPath); // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + scope, { op: "rename", path: toVirtualPath(scope, oldParsed.relPath), @@ -1481,7 +1615,7 @@ export class MemoryService extends EventEmitter { async () => { // UI save can create new files: materialize the scope root on // first use — in-lock, after the removal check (r62; see create). - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.ensureRoot(); const kind = await store.kind(parsed.relPath); if (kind === "dir") { @@ -1508,7 +1642,7 @@ export class MemoryService extends EventEmitter { ); } } - await assertMutationCommittable(this.config.rootDir, ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); @@ -1682,44 +1816,6 @@ function sha256Hex(content: string): string { return createHash("sha256").update(content, "utf-8").digest("hex"); } -/** - * Refuse to COMMIT a mutation whose caller was torn down (r59/r61). Checked - * INSIDE the target mutation lock immediately before the first durable - * write; a mutation that already committed always journals (mutation → row - * → ack) so rollback lineage stays intact. Two teardown signals: - * - * - The caller's abort signal (r59): consolidation/refine passes receive no - * hard tool cancellation — an execution wedged in pre-commit I/O (e.g. a - * named pipe under a memory root) is detached by the caller's bounded - * drain, and once the I/O unblocks after workspace teardown it would - * still write durable memory AND append its refinement journal row into - * the deleted session directory, recreating it. - * - The durable removal tombstone (r61): with multiple backends over one - * Xum root, the remover cannot abort a dream/harvest run in ANOTHER - * process — that run's signal stays live after removal. The tombstone is - * published under the same memory target locks this check runs inside - * (see workspaceRemoval.ts), so a foreign backend's mutation observes - * removal here at commit time and refuses instead of recreating the - * deleted session directory via its write or journal append. - */ -async function assertMutationCommittable( - rootDir: string, - ctx: MemoryScopeContext, - signal: AbortSignal | undefined, - virtualPath: string -): Promise { - if (signal?.aborted === true) { - throw new MemoryCommandError( - `Mutation of ${virtualPath} was cancelled before commit (caller torn down)` - ); - } - if (ctx.workspaceId !== "" && (await isWorkspaceRemovalTombstoned(rootDir, ctx.workspaceId))) { - throw new MemoryCommandError( - `Workspace ${ctx.workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` - ); - } -} - /** * Deterministic, lenient hash of a physical subtree for delete-target change * detection (r55/r58, see MemoryService.fingerprintMutationTarget). Sorted walk; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2d0eceaddde..3d4c74f496e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4189,6 +4189,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { for (const session of this.sessions.values()) session.beginShutdown(); } + /** + * Sub-agents share their task-tree owner's /memories/workspace store, so a + * workspace-scope write by one tree member stales the cached memory context + * of every live session in that tree, not just the acting one (which already + * clears its own cache on tool-call-end). `isAffected` decides membership. + */ + invalidateMemoryContextWhere(isAffected: (workspaceId: string) => boolean): void { + for (const [workspaceId, session] of this.sessions) { + if (isAffected(workspaceId)) session.invalidateMemoryContext(); + } + } + /** Transfer destructive cleanup out of a callback that still owns a session lease. */ deferWorkspaceCleanup(run: () => Promise): void { this.trackWorkspaceCleanup(run).catch((error: unknown) => From 0145db85616845a563ca63e95f0b119cfad87193 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 12:41:44 +0000 Subject: [PATCH 02/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20Codex=20r?= =?UTF-8?q?eview=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removal of a sub-agent also holds the task-tree owner's memory store lock, closing the tombstone-vs-admitted-write window. - Workspace-scope rows stay in the acting session's journal (refine audit correlation); rollback admits the owner memory root via a caller-supplied sharedWorkspaceMemorySessionDir (tool ctx, debug CLI). - Memory-context invalidation bumps a generation so an in-flight build cannot repopulate the cache with a pre-write snapshot. - Cross-session invalidation resolves owners from one config snapshot. - Dream/sweep suppression compares the resolved owner, so dangling parent chains keep their private store consolidatable. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$2.30`_ --- src/cli/debug/refinements.ts | 11 +++ .../agentSession.memoryContext.test.ts | 15 ++++ src/node/services/agentSession.ts | 8 ++ src/node/services/di/layers/core.ts | 6 +- .../memoryConsolidationService.test.ts | 7 ++ .../services/memoryConsolidationService.ts | 13 +-- src/node/services/memoryService.test.ts | 54 ++++++++---- src/node/services/memoryService.ts | 85 +++++-------------- src/node/services/memoryWorkspaceOwner.ts | 57 +++++++++++++ .../services/refinement/refinementRollback.ts | 46 +++++++--- src/node/services/toolAssembly.ts | 8 +- .../services/tools/refinement_rollback.ts | 3 + src/node/services/turnRequestBuilder.ts | 10 +++ src/node/services/workspaceRemoval.test.ts | 56 ++++++++++++ src/node/services/workspaceRemoval.ts | 18 +++- src/node/services/workspaceService.ts | 12 +++ 16 files changed, 312 insertions(+), 97 deletions(-) create mode 100644 src/node/services/memoryWorkspaceOwner.ts diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index d90ca13f4f4..512504d9ec9 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -1,5 +1,6 @@ import * as path from "path"; import { defaultConfig } from "@/node/config"; +import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; import { MemoryRefinementActionSchema, RollbackRefinementActionSchema, @@ -50,8 +51,18 @@ export async function refinementsCommand( const sessionDir = opts.sessionDir ?? path.join(defaultConfig.sessionsDir, workspaceId); if (opts.rollback !== undefined) { + // Sub-agents journal workspace-scope rows that point into the owner's + // session dir; admit that root the same way the in-app tool does. + const memoryOwnerId = resolveWorkspaceMemoryOwnerId( + defaultConfig.loadConfigOrDefault(), + workspaceId + ); const result = await rollbackRefinement({ sessionDir, + sharedWorkspaceMemorySessionDir: + memoryOwnerId === workspaceId + ? undefined + : path.join(defaultConfig.sessionsDir, memoryOwnerId), id: opts.rollback, force: opts.force, evidence: { toolName: "debug-cli", actor: "user" }, diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 7a6ef196c02..10b536918bc 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -133,6 +133,21 @@ describe("AgentSession memory context", () => { session.invalidateMemoryContext(); expect(await priv.resolveMemoryContext("test-model")).toEqual(context); expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + + // Invalidation DURING a build: the pre-write snapshot is served once but + // must not be cached, so the following resolve rebuilds again. + let finishBuild!: () => void; + buildMemorySessionContext.mockImplementationOnce( + () => new Promise((resolve) => (finishBuild = () => resolve(context))) + ); + session.invalidateMemoryContext(); + const inFlight = priv.resolveMemoryContext("test-model"); + await Promise.resolve(); + session.invalidateMemoryContext(); + finishBuild(); + expect(await inFlight).toEqual(context); + expect(await priv.resolveMemoryContext("test-model")).toEqual(context); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(4); } finally { await session.dispose(); } diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 06b7d5347fd..adc5e2221ac 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1043,8 +1043,13 @@ export class AgentSession { */ invalidateMemoryContext(): void { this.memoryContextByModelString.clear(); + // A build already awaiting buildMemorySessionContext read the pre-write + // files; bumping the generation stops it from repopulating the cache. + this.memoryContextGeneration++; } + private memoryContextGeneration = 0; + /** * Cache the last-known experiment state so we don't spam metadata refresh * when post-compaction context is disabled. @@ -9829,6 +9834,7 @@ export class AgentSession { return cached.context ?? undefined; } + const generation = this.memoryContextGeneration; // Guard for test mocks that may not implement buildMemorySessionContext. const context = typeof this.aiService.buildMemorySessionContext === "function" @@ -9837,6 +9843,8 @@ export class AgentSession { tokenBudgetActive, }) : null; + // Invalidated mid-build: serve this snapshot once but do not cache it. + if (generation !== this.memoryContextGeneration) return context ?? undefined; cache.set(modelString, { context, includesHotMemories: includeHotMemories, diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index 30d10efafbe..6eddf23ebc6 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -547,9 +547,13 @@ export const CoreWiringLive: Layer.Layer< // every live session resolving to that owner reads the same notebook. memoryService.on("change", (event: MemoryChangeEvent) => { if (event.scope !== "workspace" || event.workspaceId === "") return; + // One config snapshot for the whole pass: cold owner lookups would + // otherwise parse the config once per live session, synchronously. + let cfg: ReturnType | undefined; + const loadConfig = () => (cfg ??= config.loadConfigOrDefault()); workspaceService.invalidateMemoryContextWhere( (workspaceId) => - memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) === event.workspaceId + memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, loadConfig) === event.workspaceId ); }); if (opts.devToolsService) { diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 725716b485d..4e4337dac9f 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1316,6 +1316,13 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(1); expect(await fixture.service.getRecord("ws-sub")).toBeNull(); expect(await fixture.service.getRecord("ws-dream")).not.toBeNull(); + + // A dangling parent chain resolves to a PRIVATE store (owner == self), so + // that workspace must remain consolidatable rather than orphaned forever. + await fixture.addWorkspace("ws-orphan", { parentWorkspaceId: "ws-gone" }); + const orphan = await fixture.service.maybeRun("ws-orphan", "manual"); + expect(orphan.success).toBe(true); + expect(fixture.modelCalls).toHaveLength(2); }); it("launch sweep skips archived workspaces and caps runs per launch", async () => { diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 81bc6f91e8a..70fb78a559b 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -738,7 +738,9 @@ export class MemoryConsolidationService extends EventEmitter { // child would consolidate the owner's notebook concurrently with the // owner's own runs. Children still harvest into the shared inbox; only // the owner sweeps it. - if (workspace.parentWorkspaceId) { + // Compared by resolved owner, not parentWorkspaceId: a dangling/cyclic + // chain falls back to a private store that must stay consolidatable. + if (self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) !== workspaceId) { return Err( "sub-agent workspaces share their owner's workspace memory; the owner consolidates it" ); @@ -1202,12 +1204,13 @@ export class MemoryConsolidationService extends EventEmitter { // covering pass, not one per idle workspace in the same sweep. let globalLastRunAt = findNewestWorkspaceRecord(sidecar.workspaces)?.lastRunAt ?? 0; const archivedById = new Map(); - const subAgentIds = new Set(); const projectPathByWorkspace = new Map(); - for (const [configProjectPath, project] of self.config.loadConfigOrDefault().projects) { + const cfg = self.config.loadConfigOrDefault(); + const sharesOwnerStore = (workspaceId: string) => + self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, () => cfg) !== workspaceId; + for (const [configProjectPath, project] of cfg.projects) { for (const workspace of project.workspaces) { if (workspace.id === undefined) continue; - if (workspace.parentWorkspaceId) subAgentIds.add(workspace.id); archivedById.set( workspace.id, isWorkspaceArchived(workspace.archivedAt, workspace.unarchivedAt) @@ -1235,7 +1238,7 @@ export class MemoryConsolidationService extends EventEmitter { if (archivedById.get(workspaceId) === true) continue; // Shared store: the owner's own sweep covers a child's writes (they // are keyed under the owner), and runLockedEffect refuses children. - if (subAgentIds.has(workspaceId)) continue; + if (sharesOwnerStore(workspaceId)) continue; const lastRunAt = sidecar.workspaces[workspaceId]?.lastRunAt ?? 0; const projectPath = projectPathByWorkspace.get(workspaceId) ?? ""; const projectRunAt = projectPath === "" ? 0 : (projectLastRunAt.get(projectPath) ?? 0); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index ceaf2677b37..1ef2d3adc10 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -23,6 +23,7 @@ import { RefinementInverseSchema, } from "@/common/types/refinement"; import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; +import { rollbackRefinement } from "./refinement/refinementRollback"; import { TestTempDir } from "./tools/testHelpers"; function pathExists(target: string): Promise { @@ -889,25 +890,46 @@ describe("MemoryService", () => { ).toBe(false); }); - it("journals a sub-agent's workspace-scope mutation into the owner's session (where rollback is confined)", async () => { + it("journals a sub-agent's workspace-scope mutation in its own session and rolls it back via the owner root", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); - await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); - // Global scope stays attributed to the acting child. - await fixture.service.create(fixture.ctx, "/memories/global/g.md", "mine", "agent"); - - const ownerEvents = await readRefinementEvents( - path.join(fixture.config.sessionsDir, "ws-owner") - ); - expect(ownerEvents.map((event) => event.data.action)).toEqual([ - { op: "create", path: "/memories/workspace/n.md" }, - ]); - const childEvents = await readRefinementEvents( - path.join(fixture.config.sessionsDir, "ws-child") + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/n.md", + "shared", + "agent" ); - expect(childEvents.map((event) => event.data.action)).toEqual([ - { op: "create", path: "/memories/global/g.md" }, - ]); + expect(created.success).toBe(true); + + // Attribution stays with the acting workspace: the row is in the + // child's journal but its inverse points into the owner's memory dir. + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const events = await readRefinementEvents(childSessionDir); + expect(events).toHaveLength(1); + expect(await readRefinementEvents(ownerSessionDir)).toHaveLength(0); + const physical = path.join(ownerSessionDir, "memory", "n.md"); + expect(events[0].data.inverse).toEqual({ op: "delete-files", paths: [physical] }); + + // Confinement: the child's own memory root does not admit the path... + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + id: events[0].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("outside every memory scope root"); + expect(await pathExists(physical)).toBe(true); + + // ...the caller-supplied owner session dir does. + const rolledBack = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: events[0].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await pathExists(physical)).toBe(false); }); }); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index e3d077e1a8f..0663431f975 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -47,6 +47,7 @@ import { } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; import { REFINEMENT_CAPTURE_MAX_FILES, REFINEMENT_CAPTURE_MAX_TOTAL_BYTES, @@ -604,55 +605,24 @@ export class MemoryService extends EventEmitter { private readonly workspaceMemoryOwnerById = new Map(); /** - * The workspace whose /memory backs `/memories/workspace` for - * `workspaceId`: the root of its parentWorkspaceId chain. Sub-agents (and - * nested sub-agents) thereby share ONE notebook with the workspace that - * spawned the task tree, while their transcripts/session artifacts stay - * separate. Full `kind: "workspace"` tasks and forks carry no - * parentWorkspaceId and own their notes. Unknown IDs, cycles, and depth - * overflow resolve to the ID itself so a misconfigured tree degrades to - * today's per-workspace behavior instead of failing every memory command. + * Memoized resolveWorkspaceMemoryOwnerId (see memoryWorkspaceOwner.ts). The + * config is only loaded on a memo miss; callers resolving many workspaces + * in one synchronous pass supply a shared `loadConfig` so a cold pass parses + * the config once instead of once per workspace. */ - resolveWorkspaceMemoryOwnerId(workspaceId: string): string { - assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); + resolveWorkspaceMemoryOwnerId( + workspaceId: string, + loadConfig: () => ReturnType = () => + this.config.loadConfigOrDefault() + ): string { const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; - const cfg = this.config.loadConfigOrDefault(); - let current = workspaceId; - const visited = new Set(); - for (let depth = 0; depth < 32; depth++) { - if (visited.has(current)) { - log.warn( - "[MemoryService] parentWorkspaceId cycle; using acting workspace as memory owner", - { - workspaceId, - } - ); - return workspaceId; - } - visited.add(current); - const entry = findWorkspaceEntry(cfg, current); - if (entry === null) { - // Only the chain root may be unknown without invalidating the walk: - // an unregistered starting workspace resolves to itself (not cached). - if (current === workspaceId) return workspaceId; - log.warn("[MemoryService] parentWorkspaceId points at an unknown workspace", { - workspaceId, - parentWorkspaceId: current, - }); - return workspaceId; - } - const parentWorkspaceId = entry.workspace.parentWorkspaceId; - if (parentWorkspaceId === undefined || parentWorkspaceId === "") { - this.workspaceMemoryOwnerById.set(workspaceId, current); - return current; - } - current = parentWorkspaceId; + const cfg = loadConfig(); + const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + if (findWorkspaceEntry(cfg, workspaceId) !== null) { + this.workspaceMemoryOwnerById.set(workspaceId, owner); } - log.warn("[MemoryService] parentWorkspaceId chain too deep; using acting workspace", { - workspaceId, - }); - return workspaceId; + return owner; } /** Owner of the workspace scope for this context ("" when there is no workspace). */ @@ -823,19 +793,17 @@ export class MemoryService extends EventEmitter { * Append the invertible `refinement` row for one memory mutation (RLM r2). * * Rows land in the ACTING workspace's session journal even though memory - * files can be global/project-scoped: the journal is per-session, so + * files can be global/project-scoped — or, for a sub-agent's workspace + * scope, live in the OWNER's session dir: the journal is per-session, so * cross-workspace edits to a shared file are attributed to (and invertible - * from) whichever workspace made them — the intended v1 scope. The one - * exception is workspace scope written by a sub-agent: the file lives in - * the OWNER's /memory and rollback confinement only admits a - * journal's own session memory root, so those rows go to the owner's - * journal (where they are actually invertible). When the context has no - * workspace, there is no session journal; skip (log-only). + * from) whichever workspace made them — the intended v1 scope. Rollback of + * a sub-agent's workspace-scope row admits the owner's memory root via + * RollbackRefinementOptions.sharedWorkspaceMemorySessionDir. When the + * context has no workspace, there is no session journal; skip (log-only). * Never throws: journaling failures must not fail the memory command. */ private async journalRefinement( ctx: MemoryScopeContext, - scope: MemoryScope, action: MemoryRefinementAction, inverse: RefinementInverseDraft, actor: MemoryActor, @@ -848,11 +816,9 @@ export class MemoryService extends EventEmitter { }); return; } - const journalWorkspaceId = - scope === "workspace" ? this.ownerWorkspaceIdFor(ctx) : ctx.workspaceId; await appendRefinementEvent({ - sessionDir: path.join(this.config.sessionsDir, journalWorkspaceId), - workspaceId: journalWorkspaceId, + sessionDir: path.join(this.config.sessionsDir, ctx.workspaceId), + workspaceId: ctx.workspaceId, kind: "memory", action, inverse, @@ -1152,7 +1118,6 @@ export class MemoryService extends EventEmitter { // Row is written before the create is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, - scope, { op: "create", path: toVirtualPath(scope, parsed.relPath) }, { op: "delete-files", paths: [store.physicalPath(parsed.relPath)] }, actor, @@ -1194,7 +1159,6 @@ export class MemoryService extends EventEmitter { // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, - scope, { op: "str_replace", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1248,7 +1212,6 @@ export class MemoryService extends EventEmitter { // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, - scope, { op: "insert", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1432,7 +1395,6 @@ export class MemoryService extends EventEmitter { if (inverse !== null) { await this.journalRefinement( ctx, - scope, { op: "delete", path: toVirtualPath(scope, parsed.relPath) }, inverse, actor, @@ -1495,7 +1457,6 @@ export class MemoryService extends EventEmitter { // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, - scope, { op: "rename", path: toVirtualPath(scope, oldParsed.relPath), diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts new file mode 100644 index 00000000000..d83bc3f5759 --- /dev/null +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -0,0 +1,57 @@ +import assert from "@/common/utils/assert"; +import type { Config } from "@/node/config"; +import { log } from "@/node/services/log"; +import { findWorkspaceEntry } from "@/node/services/taskUtils"; + +type ProjectsConfig = ReturnType; + +/** + * The workspace whose /memory backs `/memories/workspace` for + * `workspaceId`: the root of its parentWorkspaceId chain. Sub-agents (and + * nested sub-agents) thereby share ONE notebook with the workspace that + * spawned the task tree, while their transcripts/session artifacts stay + * separate. Full `kind: "workspace"` tasks and forks carry no + * parentWorkspaceId and own their notes. + * + * Unknown IDs, dangling parents, cycles, and depth overflow resolve to the ID + * itself so a misconfigured tree degrades to per-workspace behavior instead + * of failing every memory command. Callers that need "is this a shared + * child?" must compare the result to the input rather than test + * parentWorkspaceId, so those fallbacks keep their private store usable. + * + * Pure over one config snapshot; MemoryService memoizes it, removal and the + * rollback tooling call it directly. + */ +export function resolveWorkspaceMemoryOwnerId(cfg: ProjectsConfig, workspaceId: string): string { + assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); + let current = workspaceId; + const visited = new Set(); + for (let depth = 0; depth < 32; depth++) { + if (visited.has(current)) { + log.warn("[memory] parentWorkspaceId cycle; using acting workspace as memory owner", { + workspaceId, + }); + return workspaceId; + } + visited.add(current); + const entry = findWorkspaceEntry(cfg, current); + if (entry === null) { + // Only the chain root may be unknown without invalidating the walk: an + // unregistered starting workspace simply resolves to itself. + if (current !== workspaceId) { + log.warn("[memory] parentWorkspaceId points at an unknown workspace", { + workspaceId, + parentWorkspaceId: current, + }); + } + return workspaceId; + } + const parentWorkspaceId = entry.workspace.parentWorkspaceId; + if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; + current = parentWorkspaceId; + } + log.warn("[memory] parentWorkspaceId chain too deep; using acting workspace as memory owner", { + workspaceId, + }); + return workspaceId; +} diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 1841c2eda19..6afec72fc62 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -69,6 +69,12 @@ export interface RollbackRefinementOptions { id: string; /** Apply despite detected divergence. Confinement is NEVER overridable. */ force?: boolean; + /** + * Session dir of the task-tree owner whose /memory backs this + * session's `/memories/workspace` (sub-agents only; omit when the session + * owns its store). Admits that one extra memory root for confinement. + */ + sharedWorkspaceMemorySessionDir?: string; /** Attribution for the emitted rollback row. */ evidence: { toolName: string; toolCallId?: string; actor?: string }; /** Caller-supplied justification, recorded in the rollback row's action. */ @@ -244,7 +250,8 @@ function inferMemoryLayout(sessionDir: string): { muxRoot: string; sessionsDir: function resolveConfinementRoot( sessionDir: string, kind: "memory" | "skill", - filePath: string + filePath: string, + sharedWorkspaceMemorySessionDir?: string ): string { if (!path.isAbsolute(filePath)) { throw new RollbackError(`Refusing rollback: inverse path is not absolute: '${filePath}'`); @@ -307,16 +314,30 @@ function resolveConfinementRoot( } // /memory/ (workspace scope). Constrained to exactly // THIS session's memory subdir so a corrupted inverse can never touch other - // workspaces' memory or session artifacts (chat.jsonl, journals). - const workspaceMemoryRoot = path.join(path.resolve(sessionDir), "memory"); - const relToWorkspaceMemory = path.relative(workspaceMemoryRoot, resolved); - if (!relToWorkspaceMemory.startsWith("..") && !path.isAbsolute(relToWorkspaceMemory)) { - if (relToWorkspaceMemory.length > 0) { - return workspaceMemoryRoot; - } - throw new RollbackError( - `Refusing rollback: path targets a memory scope root, not a file inside it: '${filePath}'` + // workspaces' memory or session artifacts (chat.jsonl, journals). The one + // sanctioned second root is the task-tree owner's memory subdir, supplied + // by the CALLER (never read from the row): a sub-agent's workspace-scope + // writes physically land there (MemoryService.resolveWorkspaceMemoryOwnerId) + // while the row stays in the sub-agent's own journal. + const workspaceMemoryRoots = [path.join(path.resolve(sessionDir), "memory")]; + if (sharedWorkspaceMemorySessionDir !== undefined) { + const sharedSessionDir = path.resolve(sharedWorkspaceMemorySessionDir); + assert( + path.dirname(sharedSessionDir) === layout.sessionsDir, + "sharedWorkspaceMemorySessionDir must be a sibling session dir" ); + workspaceMemoryRoots.push(path.join(sharedSessionDir, "memory")); + } + for (const workspaceMemoryRoot of workspaceMemoryRoots) { + const relToWorkspaceMemory = path.relative(workspaceMemoryRoot, resolved); + if (!relToWorkspaceMemory.startsWith("..") && !path.isAbsolute(relToWorkspaceMemory)) { + if (relToWorkspaceMemory.length > 0) { + return workspaceMemoryRoot; + } + throw new RollbackError( + `Refusing rollback: path targets a memory scope root, not a file inside it: '${filePath}'` + ); + } } throw new RollbackError( `Refusing rollback: path is outside every memory scope root: '${filePath}'` @@ -770,7 +791,10 @@ export async function rollbackRefinement( // repo revision to swap a root for a symlink in the meantime. const roots = new Map(); for (const p of inversePaths(inverse)) { - roots.set(p, resolveConfinementRoot(opts.sessionDir, kind, p)); + roots.set( + p, + resolveConfinementRoot(opts.sessionDir, kind, p, opts.sharedWorkspaceMemorySessionDir) + ); } const assertConfinement = async (): Promise => { for (const [p, root] of roots) { diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 6e7e58f35e5..e6ca8d6605b 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -115,7 +115,13 @@ export interface ApplyToolPolicyAndExperimentsOptions { * caller from the workspace cwd/runtime pair the file tools use; only * honored in kernel mode with file_read bridged. */ - sandbox?: { workspaceId: string; sessionDir: string; kernelFileLoader?: KernelFileLoader }; + sandbox?: { + workspaceId: string; + sessionDir: string; + /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ + sharedWorkspaceMemorySessionDir?: string; + kernelFileLoader?: KernelFileLoader; + }; /** * Capability grants for this assembly (registry-with-filters posture). * Omitted = session-scope full grants (identical to pre-grants behavior). diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 547af455fcf..db04cf8004d 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -20,6 +20,8 @@ interface RefinementRollbackToolArgs { export function createRefinementRollbackTool(ctx: { workspaceId: string; sessionDir: string; + /** Owner session dir when this workspace is a sub-agent sharing its notebook. */ + sharedWorkspaceMemorySessionDir?: string; }): Tool { return tool({ description: TOOL_DEFINITIONS.refinement_rollback.description, @@ -30,6 +32,7 @@ export function createRefinementRollbackTool(ctx: { ): Promise => { const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, + sharedWorkspaceMemorySessionDir: ctx.sharedWorkspaceMemorySessionDir, id, reason, evidence: { toolName: "refinement_rollback", toolCallId, actor: "agent" }, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index c6bdeee8271..a9791928496 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2324,6 +2324,15 @@ export class TurnRequestBuilder { recordStartupPhaseTiming("getToolsForModelMs", getToolsStartedAt); } + // A sub-agent's workspace-scope memory rows point into its task-tree + // owner's session dir; rollback must admit that root (and only that). + const memoryOwnerId = + this.dependencies.bindings.memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId) ?? + workspaceId; + const sharedWorkspaceMemorySessionDir = + memoryOwnerId === workspaceId + ? undefined + : path.join(this.dependencies.config.sessionsDir, memoryOwnerId); const applyPolicyStartedAt = Date.now(); let attemptTools = await applyToolPolicyAndExperiments({ allTools: this.dependencies.wrapToolsForDelegation( @@ -2338,6 +2347,7 @@ export class TurnRequestBuilder { sandbox: { workspaceId, sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), + sharedWorkspaceMemorySessionDir, kernelFileLoader, }, }); diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 970ccea6d11..75abc00ae01 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -80,6 +80,62 @@ describe("workspaceRemoval", () => { expect((JSON.parse(raw) as { workspaceId: string }).workspaceId).toBe(workspaceId); }); + test("sub-agent removal also waits for a writer holding the OWNER's memory store lock", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // A sub-agent's admitted workspace-memory write holds the OWNER store's + // lock (that is where the shared notebook lives), not the child's. + let releaseWriter!: () => void; + const writerGate = new Promise((resolve) => (releaseWriter = resolve)); + let writerEntered!: () => void; + const entered = new Promise((resolve) => (writerEntered = resolve)); + const writer = withTargetMutationLock( + rootDir, + path.join(ownerSessionDir, "memory"), + async () => { + writerEntered(); + await writerGate; + } + ); + await entered; + + const removal = removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + // Without the owner lock the child tombstone would already be published + // here, between the writer's commit check and its durable write. + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + + releaseWriter(); + await writer; + await removal; + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(true); + // Only the child's session dir is deleted; the owner keeps its notebook. + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(false); + expect( + await fsPromises.access(path.join(ownerSessionDir, "memory")).then( + () => true, + () => false + ) + ).toBe(true); + }); + test("waits on the refine lock BEFORE taking the teardown target locks (r67)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 154c7311626..fd05d5c29b6 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -142,6 +142,13 @@ export async function removeSessionDirUnderMemoryLocks(args: { * succeeding or still-active) attempt relies on. */ attemptId: string; + /** + * Session dir of the task-tree owner whose /memory backs this + * workspace's `/memories/workspace` (sub-agents only). Its store lock is + * held too, so a sub-agent's admitted write into the shared notebook + * either commits before the tombstone or re-checks and refuses. + */ + sharedWorkspaceMemorySessionDir?: string; }): Promise { assert(args.sessionDir.length > 0, "removeSessionDirUnderMemoryLocks requires a session dir"); // Crash clearly on a malformed config (test stubs, future refactors): an @@ -159,6 +166,15 @@ export async function removeSessionDirUnderMemoryLocks(args: { path.join(args.sessionDir, "memory") ); const sharedMemoryKey = memoryMutationLockKey(args.rootDir, path.join(args.rootDir, "memory")); + const ownerMemoryKeys = + args.sharedWorkspaceMemorySessionDir === undefined + ? [] + : [ + memoryMutationLockKey( + args.rootDir, + path.join(args.sharedWorkspaceMemorySessionDir, "memory") + ), + ]; // The session dir itself is a third target key (r63): session-scoped // sidecar writers (headless usage) serialize their tombstone check + // commit against this same key, closing their check→write window. @@ -200,7 +216,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { }); await withTargetMutationLocks( args.rootDir, - [sessionDirKey, workspaceMemoryKey, sharedMemoryKey], + [sessionDirKey, workspaceMemoryKey, sharedMemoryKey, ...ownerMemoryKeys], async () => { // History append serialization (r63): a foreign backend's in-flight // stream can be mid-append under the history write lock; acquiring diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3d4c74f496e..ecb8259435c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -136,6 +136,7 @@ import { startRemovalTombstoneLease, TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; +import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -6309,11 +6310,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // directory. Fail-closed on a wedged writer: the catch below keeps // the directory as a recoverable orphan instead of deleting it out // from under a live commit. + // A sub-agent's workspace memory lives in its task-tree owner's + // session dir; hold that store's lock as well so an admitted child + // write cannot slip between this tombstone and its commit check. + const memoryOwnerId = resolveWorkspaceMemoryOwnerId( + this.config.loadConfigOrDefault(), + workspaceId + ); await removeSessionDirUnderMemoryLocks({ rootDir: this.config.rootDir, sessionDir, workspaceId, attemptId: removalAttemptId, + sharedWorkspaceMemorySessionDir: + memoryOwnerId === workspaceId + ? undefined + : path.join(this.config.sessionsDir, memoryOwnerId), }); } catch (error) { // r63: without a durable tombstone the retained orphan stays From 71493b3936d9d688ca20aede1457b1a00d7bf289 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 13:09:08 +0000 Subject: [PATCH 03/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20second=20?= =?UTF-8?q?Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Owner memo is cleared on config changes so a child of a removed owner re-resolves to its own store instead of the tombstoned owner. - Rollback re-checks the shared owner's removal tombstone inside the target lock before applying an inverse into the owner's memory root. - refinement_rollback announces its direct-to-disk writes through MemoryService.notifyExternalMutation (subscriptions + tree-wide cache invalidation); startup-recovery sessions are invalidated too. - Sub-agent Dream runs (manual/compaction, incl. post-harvest) redirect to the owner under the owner's in-flight lock; archive is refused; the child's status view reports the owner's workspace record. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$3.10`_ --- .../memoryConsolidationService.test.ts | 40 +++++++---- .../services/memoryConsolidationService.ts | 40 ++++++++--- src/node/services/memoryService.test.ts | 68 +++++++++++++++++++ src/node/services/memoryService.ts | 31 +++++++++ .../services/refinement/refinementRollback.ts | 17 +++++ src/node/services/toolAssembly.ts | 3 + .../services/tools/refinement_rollback.ts | 13 ++++ src/node/services/turnRequestBuilder.ts | 17 ++++- src/node/services/workspaceService.ts | 7 +- 9 files changed, 206 insertions(+), 30 deletions(-) diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 4e4337dac9f..4d3e3b19c99 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1284,22 +1284,13 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(1); }); - it("refuses Dream runs for sub-agent children and lets the owner sweep their shared writes", async () => { + it("redirects a sub-agent's Dream runs and status to the memory owner", async () => { using fixture = await createFixture(); await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); - // Every trigger, including an explicit /dream, is refused on the child: - // its /memories/workspace IS the owner's store. - for (const trigger of ["manual", "compaction", "archive"] as const) { - const result = await fixture.service.maybeRun("ws-sub", trigger); - expect(result.success).toBe(false); - if (!result.success) expect(result.error).toContain("owner"); - } - expect(fixture.modelCalls).toHaveLength(0); - - // A child's workspace write is keyed under the owner (MemoryService - // resolution), so an idle owner qualifies for the launch sweep while the - // idle child is skipped even though it is listed. + // Launch sweep: a child's workspace write is keyed under the owner + // (MemoryService resolution), so the idle owner qualifies while the idle + // child is skipped even though it is listed. await fixture.memoryService.create( { runtime: null, checkoutCwd: "", workspaceId: "ws-sub", projectPath: "" }, "/memories/workspace/from-child.md", @@ -1317,12 +1308,31 @@ describe("MemoryConsolidationService", () => { expect(await fixture.service.getRecord("ws-sub")).toBeNull(); expect(await fixture.service.getRecord("ws-dream")).not.toBeNull(); + // Manual/compaction runs from the child consolidate the OWNER's store + // under the owner's lock; the record lands on the owner and the child's + // status view reports it (the tab shows the shared store). + const manual = await fixture.service.maybeRun("ws-sub", "manual"); + expect(manual.success).toBe(true); + expect(fixture.modelCalls).toHaveLength(2); + expect(await fixture.service.getRecord("ws-sub")).toBeNull(); + expect((await fixture.service.getStatus("ws-sub")).workspaceRecord).toEqual( + await fixture.service.getRecord("ws-dream") + ); + + // Archive is the owner's own one-shot promotion pass: a child archive is + // refused instead of running it. + const archive = await fixture.service.maybeRun("ws-sub", "archive"); + expect(archive.success).toBe(false); + if (!archive.success) expect(archive.error).toContain("owner"); + expect(fixture.modelCalls).toHaveLength(2); + // A dangling parent chain resolves to a PRIVATE store (owner == self), so - // that workspace must remain consolidatable rather than orphaned forever. + // that workspace consolidates itself rather than being orphaned forever. await fixture.addWorkspace("ws-orphan", { parentWorkspaceId: "ws-gone" }); const orphan = await fixture.service.maybeRun("ws-orphan", "manual"); expect(orphan.success).toBe(true); - expect(fixture.modelCalls).toHaveLength(2); + expect(await fixture.service.getRecord("ws-orphan")).not.toBeNull(); + expect(fixture.modelCalls).toHaveLength(3); }); it("launch sweep skips archived workspaces and caps runs per launch", async () => { diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 70fb78a559b..abb255b92c6 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -414,8 +414,12 @@ export class MemoryConsolidationService extends EventEmitter { const workspace = self.config.findWorkspace(workspaceId); const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); const globalRecord = findNewestWorkspaceRecord(file.workspaces); + // The workspace record describes the STORE the tab shows: for a + // sub-agent that is the owner's (runs are redirected there, see + // maybeRun); harvests stay per acting workspace. + const ownerWorkspaceId = self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); return { - workspaceRecord: file.workspaces[workspaceId] ?? null, + workspaceRecord: file.workspaces[ownerWorkspaceId] ?? null, projectRecord: projectPath === "" ? null : (file.projects[projectPath] ?? null), globalRecord, latestHarvestRecord: findNewestHarvestRecord(file.harvestsByWorkspace[workspaceId]), @@ -667,6 +671,23 @@ export class MemoryConsolidationService extends EventEmitter { options: MemoryConsolidationRunOptions = {} ): Promise> { if (!this.enabled()) return Err("memory-consolidation experiment is disabled"); + // Sub-agents share their task-tree owner's /memories/workspace store + // (MemoryService.resolveWorkspaceMemoryOwnerId): a child's manual or + // post-compaction run consolidates the OWNER's notebook under the owner's + // in-flight lock, so it never races the owner's own runs and the child's + // harvested candidates are actually swept. Archive is the owner's own + // one-shot promotion pass; a child archive must not trigger it. Compared + // by resolved owner, not parentWorkspaceId: a dangling/cyclic chain falls + // back to a private store that must stay consolidatable. + const ownerWorkspaceId = this.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); + if (ownerWorkspaceId !== workspaceId) { + if (trigger === "archive") { + return Err( + "sub-agent workspaces share their owner's workspace memory; the owner's archive pass promotes it" + ); + } + return this.maybeRun(ownerWorkspaceId, trigger, options); + } if (this.removalCancelled.has(workspaceId)) { return Err("workspace is being removed; consolidation refused"); } @@ -738,13 +759,6 @@ export class MemoryConsolidationService extends EventEmitter { // child would consolidate the owner's notebook concurrently with the // owner's own runs. Children still harvest into the shared inbox; only // the owner sweeps it. - // Compared by resolved owner, not parentWorkspaceId: a dangling/cyclic - // chain falls back to a private store that must stay consolidatable. - if (self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) !== workspaceId) { - return Err( - "sub-agent workspaces share their owner's workspace memory; the owner consolidates it" - ); - } const agentBody = yield* Effect.promise(() => resolveDreamAgentBody(self.config.rootDir)); if (agentBody === null) return Err("dream agent definition is missing"); @@ -969,7 +983,13 @@ export class MemoryConsolidationService extends EventEmitter { .pipe(Effect.catch(journalHarvestFailure), Effect.catchDefect(journalHarvestFailure)); } - return yield* Effect.promise(() => self.runCompactionSweepAfterHarvest(metadata.workspaceId)); + // A sub-agent's inbox lives in the owner's store: wait on and run the + // OWNER's consolidation (see maybeRun) so the harvest is actually swept. + return yield* Effect.promise(() => + self.runCompactionSweepAfterHarvest( + self.memoryService.resolveWorkspaceMemoryOwnerId(metadata.workspaceId) + ) + ); }); } @@ -1237,7 +1257,7 @@ export class MemoryConsolidationService extends EventEmitter { if (now - recency < MEMORY_CONSOLIDATION_IDLE_MS) continue; if (archivedById.get(workspaceId) === true) continue; // Shared store: the owner's own sweep covers a child's writes (they - // are keyed under the owner), and runLockedEffect refuses children. + // are keyed under the owner); running the child would just redirect. if (sharesOwnerStore(workspaceId)) continue; const lastRunAt = sidecar.workspaces[workspaceId]?.lastRunAt ?? 0; const projectPath = projectPathByWorkspace.get(workspaceId) ?? ""; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 1ef2d3adc10..3b72ca4a646 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -13,6 +13,7 @@ import { MemoryService, projectMemoryDirName, resolveMemoryProjectIdentity, + type MemoryChangeEvent, type MemoryScopeContext, } from "./memoryService"; import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; @@ -24,6 +25,7 @@ import { } from "@/common/types/refinement"; import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; +import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; import { TestTempDir } from "./tools/testHelpers"; function pathExists(target: string): Promise { @@ -931,6 +933,72 @@ describe("MemoryService", () => { expect(rolledBack.success).toBe(true); expect(await pathExists(physical)).toBe(false); }); + it("re-resolves the owner after config changes so a removed owner's child falls back to its own store", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + + // The owner is deregistered (removal with a live shared-checkout child); + // the dangling chain must not keep pointing at the tombstoned owner. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/after.md", + "own store now", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "after.md")) + ).toBe(true); + }); + + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const [row] = await readRefinementEvents(childSessionDir); + + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: row.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + }); + + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const events: MemoryChangeEvent[] = []; + fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); + const ownerMemory = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + fixture.service.notifyExternalMutation(fixture.ctx, [ + path.join(ownerMemory, "a.md"), + path.join(ownerMemory, "dir", "b.md"), + path.join(fixture.xumHome, "memory", "global", "g.md"), + path.join(fixture.xumHome, "elsewhere", "x.md"), + ownerMemory, // the root itself is not a file inside the scope + ]); + expect(events.map((event) => [event.scope, event.path, event.workspaceId])).toEqual([ + ["global", "/memories/global", "ws-owner"], + ["workspace", "/memories/workspace", "ws-owner"], + ]); + }); }); describe("memory index entries", () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 0663431f975..7e728b90177 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -587,6 +587,11 @@ export class MemoryService extends EventEmitter { private readonly metaService: MemoryMetaService ) { super(); + // Parent links are immutable, but an OWNER can be removed while a + // shared-checkout descendant keeps running; its deregistration lands as a + // config change, after which the child must re-resolve (and fall back to + // its own store) instead of writing into the tombstoned owner forever. + this.config.onConfigChanged(() => this.workspaceMemoryOwnerById.clear()); } // ------------------------------------------------------------------------- @@ -1010,6 +1015,32 @@ export class MemoryService extends EventEmitter { * scope root: subscribers refetch the whole scope per event, so per-file events for a * bulk restore would only multiply identical refreshes. */ + /** + * Announces memory files mutated outside this service by a refinement + * rollback (which applies inverses straight to disk). Physical paths are + * classified against this context's scope roots and one root-addressed + * event per touched scope is emitted, so Memory tabs refresh and — for the + * shared workspace store — every task-tree session drops its cached + * context (see the change listener wired in di/layers/core.ts). + */ + notifyExternalMutation(ctx: MemoryScopeContext, physicalPaths: readonly string[]): void { + const touched = new Set(); + for (const scope of MEMORY_SCOPES) { + let root: string; + try { + root = this.getStore(ctx, scope).physicalRoot; + } catch (error) { + if (error instanceof MemoryCommandError) continue; // scope unavailable in this context + throw error; + } + for (const physicalPath of physicalPaths) { + const rel = path.relative(root, path.resolve(physicalPath)); + if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) touched.add(scope); + } + } + for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); + } + notifyExternalProjectChange(projectPath: string): void { const event: MemoryChangeEvent = { scope: "project", diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 6afec72fc62..b8f6da5ded7 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -53,6 +53,7 @@ import { type RefinementInverseDraft, } from "./refinementJournal"; import { withTargetMutationLocks } from "./targetMutationLocks"; +import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; export type RefinementEvent = Extract; @@ -851,6 +852,22 @@ export async function rollbackRefinement( // targetMutationLocks.ts. const targetLockRoot = inferMemoryLayout(opts.sessionDir)?.muxRoot ?? null; const applied = await withTargetMutationLocks(targetLockRoot, lockKeys, async () => { + // Shared-store owner teardown gate: a delete inverse expects its target + // absent, so divergence alone would let a rollback that was waiting on + // this lock recreate the removed owner's /memory. Same + // tombstone MemoryService checks pre-commit (r61), same lock. + if (opts.sharedWorkspaceMemorySessionDir !== undefined && targetLockRoot !== null) { + const ownerSessionDir = path.resolve(opts.sharedWorkspaceMemorySessionDir); + const ownerMemoryRoot = path.join(ownerSessionDir, "memory"); + if ( + lockKeys.includes(ownerMemoryRoot) && + (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(ownerSessionDir))) + ) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the workspace owning the shared memory store was removed` + ); + } + } // Re-verify INSIDE the lock, immediately before mutating: a writer that // won the lock first has already landed, and its change must surface as // divergence rather than be overwritten. `rows` is intentionally the diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index e6ca8d6605b..58d229fb89a 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -36,6 +36,7 @@ import type { ToolBridge } from "@/node/services/ptc/toolBridge"; import type { PTCExecutionResult } from "@/node/services/ptc/types"; import { sandboxHostService, type SandboxMount } from "@/node/services/sandbox/sandboxHostService"; import { createRefinementRollbackTool } from "@/node/services/tools/refinement_rollback"; +import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; import type { KernelFileLoader } from "@/node/services/tools/kernelFileLoad"; import { log } from "./log"; import type { MCPWorkspaceStats } from "@/node/services/mcpServerManager"; @@ -120,6 +121,8 @@ export interface ApplyToolPolicyAndExperimentsOptions { sessionDir: string; /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; + /** Lets refinement_rollback announce its direct-to-disk memory writes. */ + memory?: { service: MemoryService; ctx: MemoryScopeContext }; kernelFileLoader?: KernelFileLoader; }; /** diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index db04cf8004d..ecafdb13f42 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -3,6 +3,7 @@ import { tool, type Tool } from "ai"; import type { RefinementRollbackToolResult } from "@/common/types/tools"; import { TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; import { rollbackRefinement } from "@/node/services/refinement/refinementRollback"; +import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; interface RefinementRollbackToolArgs { id: string; @@ -22,6 +23,8 @@ export function createRefinementRollbackTool(ctx: { sessionDir: string; /** Owner session dir when this workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; + /** Announces rolled-back memory files so shared-store readers refresh. */ + memory?: { service: MemoryService; ctx: MemoryScopeContext }; }): Tool { return tool({ description: TOOL_DEFINITIONS.refinement_rollback.description, @@ -40,6 +43,16 @@ export function createRefinementRollbackTool(ctx: { if (!result.success) { return { success: false, error: result.error }; } + // Rollback writes inverses straight to disk, bypassing MemoryService's + // change events; announce them so the (possibly shared) store's other + // readers — owner, siblings, open Memory tabs — do not keep stale context. + ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ + ...result.data.restored, + ...result.data.deleted, + ...(result.data.renamed === undefined + ? [] + : [result.data.renamed.from, result.data.renamed.to]), + ]); return { success: true, rollbackOf: id, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index a9791928496..b2c24f2b469 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -114,6 +114,7 @@ import { } from "@/common/utils/providers/customProviders"; import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpServerManager"; import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; +import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; import { resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; import { isWorkspaceTrustedForSharedExecution } from "@/node/services/utils/workspaceTrust"; @@ -2325,14 +2326,23 @@ export class TurnRequestBuilder { } // A sub-agent's workspace-scope memory rows point into its task-tree - // owner's session dir; rollback must admit that root (and only that). + // owner's session dir; rollback must admit that root (and only that), + // and announce its direct-to-disk writes through MemoryService so the + // shared store's readers refresh. + const memoryService = this.dependencies.bindings.memoryService; const memoryOwnerId = - this.dependencies.bindings.memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId) ?? - workspaceId; + memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId) ?? workspaceId; const sharedWorkspaceMemorySessionDir = memoryOwnerId === workspaceId ? undefined : path.join(this.dependencies.config.sessionsDir, memoryOwnerId); + const sandboxMemory = + memoryService === undefined + ? undefined + : { + service: memoryService, + ctx: memoryScopeContextFromToolConfig(toolsForModelConfig), + }; const applyPolicyStartedAt = Date.now(); let attemptTools = await applyToolPolicyAndExperiments({ allTools: this.dependencies.wrapToolsForDelegation( @@ -2348,6 +2358,7 @@ export class TurnRequestBuilder { workspaceId, sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), sharedWorkspaceMemorySessionDir, + memory: sandboxMemory, kernelFileLoader, }, }); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ecb8259435c..044d325c671 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4197,8 +4197,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * clears its own cache on tool-call-end). `isAffected` decides membership. */ invalidateMemoryContextWhere(isAffected: (workspaceId: string) => boolean): void { - for (const [workspaceId, session] of this.sessions) { - if (isAffected(workspaceId)) session.invalidateMemoryContext(); + // Startup-recovery sessions are live too and may be promoted with their cache. + for (const registry of [this.sessions, this.transientStartupRecoverySessions]) { + for (const [workspaceId, session] of registry) { + if (isAffected(workspaceId)) session.invalidateMemoryContext(); + } } } From 0c87b76305020ca23e5ab9a8569a1401cd0d9083 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 13:30:22 +0000 Subject: [PATCH 04/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20third=20C?= =?UTF-8?q?odex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Owner memo is validated against a durable config.json stamp (Config.configFileStamp) so another backend's owner removal is picked up, not only local editConfig notifications. - Memory change subscriptions resolve the subscriber's owner per event. - Pin toggles emit an owner-addressed change event so other tabs on the shared store refetch. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$3.70`_ --- src/node/config/index.ts | 15 +++++++++ src/node/orpc/routerSubscriptions.ts | 16 +++++++--- src/node/services/memoryOperations.test.ts | 27 ++++++++++++++++ src/node/services/memoryOperations.ts | 7 ++++- src/node/services/memoryService.test.ts | 26 ++++++++++++++++ src/node/services/memoryService.ts | 36 +++++++++++++++++----- 6 files changed, 113 insertions(+), 14 deletions(-) diff --git a/src/node/config/index.ts b/src/node/config/index.ts index 63a1f6424f2..c46a983b0e9 100644 --- a/src/node/config/index.ts +++ b/src/node/config/index.ts @@ -934,6 +934,21 @@ export class Config { readonly sessionsDir: string; readonly srcDir: string; private readonly configFile: string; + /** + * Cheap durable change signal for config.json: one stat, no parse. Any + * backend's rewrite changes size/mtime (atomic replace also changes the + * inode), so a memo built against a previous stamp knows to rebuild — + * unlike onConfigChanged, which only fires for THIS process's edits. A + * missing/unreadable file yields a distinct stamp. + */ + configFileStamp(): string { + try { + const st = fs.statSync(this.configFile, { bigint: true }); + return `${st.dev}:${st.ino}:${st.size}:${st.mtimeNs}`; + } catch { + return "missing"; + } + } private readonly providersConfigStore: ProvidersConfigStore; private readonly emitter = new EventEmitter(); /** diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 83a3108c10a..6f6ec4298b2 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -208,15 +208,21 @@ export function subscribeMemoryChanges( const metadata = workspaceId ? await context.workspaceService.getInfo(workspaceId) : null; const projectPath = metadata ? resolveMemoryProjectIdentity(metadata) : null; // Workspace-scope events carry the memory OWNER (task-tree root), which is - // also the store this subscriber's workspace displays. - const ownerWorkspaceId = workspaceId - ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) - : null; + // also the store this subscriber's workspace displays. Resolved per event + // (memoized, cheap): the owner can change while the tab stays open — a + // removed owner makes the child fall back to its own store. yield* runtimeSubscription(context, { signal, subscribe: (emit) => { const onChange = (event: MemoryChangeEvent) => { - if (event.scope === "workspace" && event.workspaceId !== ownerWorkspaceId) return; + if ( + event.scope === "workspace" && + event.workspaceId !== + (workspaceId + ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) + : null) + ) + return; if (event.scope === "project" && event.projectPath !== projectPath) return; emit.push(event); }; diff --git a/src/node/services/memoryOperations.test.ts b/src/node/services/memoryOperations.test.ts index deee04bd986..20ad73f028f 100644 --- a/src/node/services/memoryOperations.test.ts +++ b/src/node/services/memoryOperations.test.ts @@ -198,6 +198,33 @@ describe("memory operations", () => { expect(await memoryMetaService.getPinnedKeys()).toEqual(new Set(["global:prefs.md"])); }); + test("setPinned broadcasts a change event so other tabs on the same store refetch", async () => { + const context = createContext({ enabled: true }); + await saveMemory(context, { + workspaceId: "ws-mem", + path: "/memories/workspace/pinned.md", + content: "x", + expectedSha256: null, + }); + const events: MemoryChangeEvent[] = []; + memoryService.on("change", (event: MemoryChangeEvent) => events.push(event)); + const result = await setMemoryPinned(context, { + workspaceId: "ws-mem", + path: "/memories/workspace/pinned.md", + pinned: true, + }); + expect(result).toEqual({ success: true, data: undefined }); + expect(events).toEqual([ + { + scope: "workspace", + path: "/memories/workspace/pinned.md", + actor: "user", + workspaceId: "ws-mem", + projectPath, + }, + ]); + }); + test("list exposes usage stats; UI reads do not count as uses", async () => { const client = createClient({ enabled: true }); await client.memory.save({ diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index e487ff8ab89..fcbe43b1de0 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -240,7 +240,12 @@ export function setMemoryPinnedEffect( input.pinned ) .pipe( - Effect.map(() => ({ success: true as const, data: undefined })), + Effect.map(() => { + // Pins live in the sidecar, not the store, so nothing else emits a + // change: notify so the other tree members' tabs refetch too. + context.memoryService.notifyPinChange(resolved.scopeCtx, input.path); + return { success: true as const, data: undefined }; + }), // Sidecar write failures (disk full, permissions) arrive as the typed // MemoryMetaWriteError and map onto the legacy string error channel // instead of escaping as an untyped INTERNAL_SERVER_ERROR rejection. diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 3b72ca4a646..76a62a7f707 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -958,6 +958,32 @@ describe("MemoryService", () => { ).toBe(true); }); + it("re-resolves the owner after an EXTERNAL config rewrite (another backend removed it)", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + + // Rewrite config.json directly: no local onConfigChanged fires, only the + // file's stamp changes — as when a second backend deregisters the owner. + const configFile = path.join(fixture.xumHome, "config.json"); + // On disk, projects are [path, project] tuples. + const raw = JSON.parse(await fsPromises.readFile(configFile, "utf-8")) as { + projects: Array<[string, { workspaces: Array<{ id: string }> }]>; + }; + const project = raw.projects.find(([projectPath]) => projectPath === FIXTURE_PROJECT_PATH); + expect(project).toBeDefined(); + project![1].workspaces = project![1].workspaces.filter((ws) => ws.id !== "ws-owner"); + await fsPromises.writeFile(configFile, JSON.stringify(raw, null, 2)); + // Same-tick same-size rewrites can leave mtime unchanged; force a distinct stamp. + await fsPromises.utimes( + configFile, + new Date(Date.now() + 5_000), + new Date(Date.now() + 5_000) + ); + + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 7e728b90177..a1ccd75d907 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -591,6 +591,8 @@ export class MemoryService extends EventEmitter { // shared-checkout descendant keeps running; its deregistration lands as a // config change, after which the child must re-resolve (and fall back to // its own store) instead of writing into the tombstoned owner forever. + // Local edits notify here; edits by ANOTHER backend (multi-instance) are + // caught by the config-file stamp check in resolveWorkspaceMemoryOwnerId. this.config.onConfigChanged(() => this.workspaceMemoryOwnerById.clear()); } @@ -608,6 +610,8 @@ export class MemoryService extends EventEmitter { * cached — the workspace may simply not be registered yet. */ private readonly workspaceMemoryOwnerById = new Map(); + /** Config-file stamp (Config.configFileStamp) the memo was built against. */ + private workspaceMemoryOwnerConfigStamp: string | null = null; /** * Memoized resolveWorkspaceMemoryOwnerId (see memoryWorkspaceOwner.ts). The @@ -620,6 +624,11 @@ export class MemoryService extends EventEmitter { loadConfig: () => ReturnType = () => this.config.loadConfigOrDefault() ): string { + const stamp = this.config.configFileStamp(); + if (stamp !== this.workspaceMemoryOwnerConfigStamp) { + this.workspaceMemoryOwnerById.clear(); + this.workspaceMemoryOwnerConfigStamp = stamp; + } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; const cfg = loadConfig(); @@ -1007,14 +1016,6 @@ export class MemoryService extends EventEmitter { this.emit("change", event); } - /** - * Announces that a project's memory was mutated outside this service. The settings-backup - * restore writes memory files directly (under the shared memory mutation lock), and - * subscribers only refresh from disk on change events, so without this an open memory - * browser keeps showing pre-restore contents. One event per project, addressed to the - * scope root: subscribers refetch the whole scope per event, so per-file events for a - * bulk restore would only multiply identical refreshes. - */ /** * Announces memory files mutated outside this service by a refinement * rollback (which applies inverses straight to disk). Physical paths are @@ -1041,6 +1042,25 @@ export class MemoryService extends EventEmitter { for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); } + /** + * Announces a sidecar-only change (pin toggled from the Memory tab) so other + * subscribers of the same store — for the shared workspace notebook, every + * task-tree member's tab — refetch their listing. + */ + notifyPinChange(ctx: MemoryScopeContext, virtualPath: string): void { + const parsed = parseMemoryPath(virtualPath); + const scope = this.requireFilePath(parsed, virtualPath); + this.emitChange(ctx, scope, parsed.relPath, "user"); + } + + /** + * Announces that a project's memory was mutated outside this service. The settings-backup + * restore writes memory files directly (under the shared memory mutation lock), and + * subscribers only refresh from disk on change events, so without this an open memory + * browser keeps showing pre-restore contents. One event per project, addressed to the + * scope root: subscribers refetch the whole scope per event, so per-file events for a + * bulk restore would only multiply identical refreshes. + */ notifyExternalProjectChange(projectPath: string): void { const event: MemoryChangeEvent = { scope: "project", From b5370dedfa5963ef022d198a83cd361e1c633198 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 13:49:21 +0000 Subject: [PATCH 05/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fourth=20?= =?UTF-8?q?Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Owner memo invalidation emits `ownersInvalidated` for formerly-shared children; core.ts drops those live sessions' cached memory context. - Caller-supplied config snapshots are used for resolution but never populate the memo (a snapshot can predate the current file stamp). - Harvest recovery on an owner run retries every tree member's bucket, so a sub-agent's failed/stale harvest is retried through the redirected run. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$4.20`_ --- src/node/services/di/layers/core.ts | 10 ++++-- .../memoryConsolidationService.test.ts | 35 +++++++++++++++++++ .../services/memoryConsolidationService.ts | 18 ++++++++-- src/node/services/memoryService.test.ts | 4 +++ src/node/services/memoryService.ts | 35 ++++++++++++++----- 5 files changed, 89 insertions(+), 13 deletions(-) diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index 6eddf23ebc6..1f467047687 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -550,12 +550,18 @@ export const CoreWiringLive: Layer.Layer< // One config snapshot for the whole pass: cold owner lookups would // otherwise parse the config once per live session, synchronously. let cfg: ReturnType | undefined; - const loadConfig = () => (cfg ??= config.loadConfigOrDefault()); + const snapshot = () => (cfg ??= config.loadConfigOrDefault()); workspaceService.invalidateMemoryContextWhere( (workspaceId) => - memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, loadConfig) === event.workspaceId + memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, snapshot) === event.workspaceId ); }); + // Ownership itself changed (an owner was removed while its sub-agents + // live on): those sessions' cached contexts still describe the old store. + memoryService.on("ownersInvalidated", (workspaceIds: string[]) => { + const affected = new Set(workspaceIds); + workspaceService.invalidateMemoryContextWhere((workspaceId) => affected.has(workspaceId)); + }); if (opts.devToolsService) { // DevTools debug-log cleanup when workspaces are archived/removed. workspaceService.setDevToolsService(opts.devToolsService); diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 4d3e3b19c99..5c4ca7e0a00 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1083,6 +1083,41 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls).toHaveLength(3); }); + it("recovers a sub-agent's failed harvest through the owner's run", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + + // The child's manual run redirects to the owner; the owner's recovery + // must still retry the CHILD's bucket (the launch sweep never visits it). + expect((await fixture.service.maybeRun("ws-sub", "manual")).success).toBe(true); + const status = await fixture.service.getStatus("ws-sub"); + expect(status.latestHarvestRecord?.status).toBe("completed"); + expect(status.latestHarvestRecord?.attemptCount).toBe(2); + }); + it("normalizes stale max-attempt pending harvest records to failed", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); const metadata = await seedCompactionEpoch(fixture); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index abb255b92c6..9e7952f1a96 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -498,10 +498,21 @@ export class MemoryConsolidationService extends EventEmitter { const self = this; return Effect.gen(function* () { const sidecar = yield* self.loadEffect(); - const records = sidecar.harvestsByWorkspace[workspaceId]; - if (records === undefined) return; + // Harvest buckets are keyed by the ACTING workspace, but a sub-agent's + // runs redirect to its owner (see maybeRun) and the launch sweep skips + // children, so an owner run must retry every tree member's bucket or a + // child's failed/stale harvest would never be recovered. + const cfg = self.config.loadConfigOrDefault(); + const records = Object.entries(sidecar.harvestsByWorkspace) + .filter( + ([bucketId]) => + bucketId === workspaceId || + self.memoryService.resolveWorkspaceMemoryOwnerId(bucketId, () => cfg) === workspaceId + ) + .flatMap(([, bucket]) => Object.values(bucket)); + if (records.length === 0) return; - const retryable = Object.values(records) + const retryable = records .filter((record) => { if (record.completionMetadata === undefined) return false; if (record.attemptCount >= HARVEST_MAX_ATTEMPTS) return false; @@ -686,6 +697,7 @@ export class MemoryConsolidationService extends EventEmitter { "sub-agent workspaces share their owner's workspace memory; the owner's archive pass promotes it" ); } + // The owner run's recovery covers this child's harvest bucket too. return this.maybeRun(ownerWorkspaceId, trigger, options); } if (this.removalCancelled.has(workspaceId)) { diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 76a62a7f707..34f1f7bf0dd 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -937,6 +937,8 @@ describe("MemoryService", () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); // The owner is deregistered (removal with a live shared-checkout child); // the dangling chain must not keep pointing at the tombstoned owner. @@ -945,6 +947,8 @@ describe("MemoryService", () => { project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); return cfg; }); + // Live sessions of formerly-shared children are told to drop their cache. + expect(invalidated).toEqual([["ws-child"]]); expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); const created = await fixture.service.create( fixture.ctx, diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index a1ccd75d907..d193fa93640 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -593,7 +593,7 @@ export class MemoryService extends EventEmitter { // its own store) instead of writing into the tombstoned owner forever. // Local edits notify here; edits by ANOTHER backend (multi-instance) are // caught by the config-file stamp check in resolveWorkspaceMemoryOwnerId. - this.config.onConfigChanged(() => this.workspaceMemoryOwnerById.clear()); + this.config.onConfigChanged(() => this.invalidateWorkspaceMemoryOwnerMemo()); } // ------------------------------------------------------------------------- @@ -615,30 +615,49 @@ export class MemoryService extends EventEmitter { /** * Memoized resolveWorkspaceMemoryOwnerId (see memoryWorkspaceOwner.ts). The - * config is only loaded on a memo miss; callers resolving many workspaces - * in one synchronous pass supply a shared `loadConfig` so a cold pass parses - * the config once instead of once per workspace. + * config is only loaded on a memo miss. Callers resolving many workspaces + * in one synchronous pass may supply a shared `snapshot`; results derived + * from a caller snapshot are NOT memoized, because the snapshot can predate + * the current file stamp (another backend rewriting config.json mid-pass) + * and would otherwise be cached under the newer stamp. */ resolveWorkspaceMemoryOwnerId( workspaceId: string, - loadConfig: () => ReturnType = () => - this.config.loadConfigOrDefault() + snapshot?: () => ReturnType ): string { const stamp = this.config.configFileStamp(); if (stamp !== this.workspaceMemoryOwnerConfigStamp) { - this.workspaceMemoryOwnerById.clear(); this.workspaceMemoryOwnerConfigStamp = stamp; + this.invalidateWorkspaceMemoryOwnerMemo(); } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; - const cfg = loadConfig(); + if (snapshot !== undefined) return resolveWorkspaceMemoryOwnerId(snapshot(), workspaceId); + const cfg = this.config.loadConfigOrDefault(); const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + // Only positive (registered) results are memoized; the workspace may + // simply not be registered yet. if (findWorkspaceEntry(cfg, workspaceId) !== null) { this.workspaceMemoryOwnerById.set(workspaceId, owner); } return owner; } + /** + * Drop the owner memo and tell listeners which workspaces were sharing a + * store: their live sessions hold a memory context built from an owner + * that may just have been removed, so core.ts invalidates those caches + * (there is no memory-change event for a removal to ride on). Idempotent + * and cheap when nothing was shared. + */ + private invalidateWorkspaceMemoryOwnerMemo(): void { + const shared = [...this.workspaceMemoryOwnerById] + .filter(([workspaceId, owner]) => workspaceId !== owner) + .map(([workspaceId]) => workspaceId); + this.workspaceMemoryOwnerById.clear(); + if (shared.length > 0) this.emit("ownersInvalidated", shared); + } + /** Owner of the workspace scope for this context ("" when there is no workspace). */ private ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { return ctx.workspaceId === "" ? "" : this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); From 17af5145eaaab4b33ab7482a5a1e9c60ff2a999b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 14:05:42 +0000 Subject: [PATCH 06/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifth=20C?= =?UTF-8?q?odex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AgentSession probes memory ownership (AIService.probeMemoryOwnership → stamp-validated owner lookup) before consulting its cached context, so a removed owner rebuilds the CURRENT request's index. - Rollback also refuses when the acting workspace itself is tombstoned (orphaned journal after a fail-closed removal). --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$4.60`_ --- .../agentSession.memoryContext.test.ts | 39 +++++++++++++++++++ src/node/services/agentSession.ts | 7 ++++ src/node/services/aiService.ts | 12 ++++++ src/node/services/memoryService.test.ts | 26 +++++++++++++ .../services/refinement/refinementRollback.ts | 37 +++++++++++------- 5 files changed, 107 insertions(+), 14 deletions(-) diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 10b536918bc..7313b91d163 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -32,6 +32,7 @@ function createSession(args: { historyService: HistoryService; sessionDir: string; buildMemorySessionContext: AIService["buildMemorySessionContext"]; + probeMemoryOwnership?: AIService["probeMemoryOwnership"]; isExperimentEnabled?: AIService["isExperimentEnabled"]; }): AgentSession { const aiEmitter = new EventEmitter(); @@ -50,6 +51,7 @@ function createSession(args: { ), stopStream: mock(() => Promise.resolve({ success: true as const, data: undefined })), buildMemorySessionContext: args.buildMemorySessionContext, + probeMemoryOwnership: args.probeMemoryOwnership, isExperimentEnabled: args.isExperimentEnabled ?? (() => false), } as unknown as AIService; @@ -153,6 +155,43 @@ describe("AgentSession memory context", () => { } }); + test("probes memory ownership before consulting the cache so a removed owner rebuilds this request", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-probe"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + const context: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => Promise.resolve(context)); + // Simulates MemoryService's stamp check finding a removed owner: the + // synchronous ownersInvalidated → core.ts → invalidateMemoryContext chain. + let ownerRemoved = false; + const sessionRef: { current?: AgentSession } = {}; + const probeMemoryOwnership = mock(() => { + if (ownerRemoved) sessionRef.current?.invalidateMemoryContext(); + }); + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryOwnership, + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + sessionRef.current = session; + const priv = session as unknown as PrivateSessionAccess; + try { + await priv.resolveMemoryContext("test-model"); + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + expect(probeMemoryOwnership).toHaveBeenCalledTimes(2); + + ownerRemoved = true; + // The probe runs before the cache read, so THIS request rebuilds. + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + } finally { + await session.dispose(); + } + }); + test("upgrades an index-only memory context when hot memories are requested", async () => { using sessionDir = new DisposableTempDir("agent-session-memory-context-upgrade"); const { historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index adc5e2221ac..fab51654694 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -676,6 +676,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { replayStream?(workspaceId: string, options?: { afterTimestamp?: number }): Promise; getProvidersConfig(): ProvidersConfigMap | null; isExperimentEnabled(experimentId: ExperimentId): boolean; + probeMemoryOwnership?(workspaceId: string): void; buildMemorySessionContext?( workspaceId: string, modelString: string, @@ -9823,6 +9824,12 @@ export class AgentSession { this.aiService.isExperimentEnabled(id); const memoryEnabled = enabled(EXPERIMENT_IDS.MEMORY); const hotSetEnabled = enabled(EXPERIMENT_IDS.MEMORY_HOT_SET); + // Ownership probe first: a removed owner invalidates this cache + // synchronously (see AIService.probeMemoryOwnership), so the lookup below + // never serves an index built from a store this workspace no longer reads. + if (memoryEnabled && typeof this.aiService.probeMemoryOwnership === "function") { + this.aiService.probeMemoryOwnership(this.workspaceId); + } const cached = cache.get(modelString); // Policy changes must not retain a previously injected extra (including index-only lookups). if ( diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 03e55934c4d..9416f11890e 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -246,6 +246,18 @@ export class AIService extends EventEmitter { return this.experimentsService?.isExperimentEnabled(experimentId) === true; } + /** + * Re-check who owns this workspace's `/memories/workspace` store. Memoized + * and stamp-validated in MemoryService, so this is one stat per call; when + * another backend removed the owner since the last turn, the resulting + * `ownersInvalidated` event clears the affected sessions' cached context + * synchronously — AgentSession calls this BEFORE consulting its cache so the + * current request, not the next one, rebuilds from the right store. + */ + probeMemoryOwnership(workspaceId: string): void { + this.turnRequestBuilderBindings.memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId); + } + /** * Build the session-segment memory context: the index snapshot advertised * in the memory tool description, plus the hot-memories block (pinned + diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 34f1f7bf0dd..e625de8e523 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1011,6 +1011,32 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); }); + it("refuses a rollback from a tombstoned acting workspace (orphaned journal)", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const [row] = await readRefinementEvents(childSessionDir); + + // Removal took the orphan path: the child's journal stays on disk, but + // the child is tombstoned and must not mutate the owner's live notebook. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: row.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("this workspace was removed"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + expect(await readRefinementEvents(childSessionDir)).toHaveLength(1); + }); + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index b8f6da5ded7..ca75ecf183f 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -852,20 +852,29 @@ export async function rollbackRefinement( // targetMutationLocks.ts. const targetLockRoot = inferMemoryLayout(opts.sessionDir)?.muxRoot ?? null; const applied = await withTargetMutationLocks(targetLockRoot, lockKeys, async () => { - // Shared-store owner teardown gate: a delete inverse expects its target - // absent, so divergence alone would let a rollback that was waiting on - // this lock recreate the removed owner's /memory. Same - // tombstone MemoryService checks pre-commit (r61), same lock. - if (opts.sharedWorkspaceMemorySessionDir !== undefined && targetLockRoot !== null) { - const ownerSessionDir = path.resolve(opts.sharedWorkspaceMemorySessionDir); - const ownerMemoryRoot = path.join(ownerSessionDir, "memory"); - if ( - lockKeys.includes(ownerMemoryRoot) && - (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(ownerSessionDir))) - ) { - throw new RollbackError( - `Refusing rollback of '${opts.id}': the workspace owning the shared memory store was removed` - ); + // Teardown gates (r61), same tombstone MemoryService checks pre-commit, + // same lock. Acting workspace: removal's fail-closed orphan path leaves + // the journal on disk but the workspace tombstoned, and a rollback from + // it must not mutate anything nor append into the retained session. + // Shared-store owner: a delete inverse expects its target absent, so + // divergence alone would let a rollback that was waiting on this lock + // recreate the removed owner's /memory. + if (targetLockRoot !== null) { + const actingSessionDir = path.resolve(opts.sessionDir); + if (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(actingSessionDir))) { + throw new RollbackError(`Refusing rollback of '${opts.id}': this workspace was removed`); + } + if (opts.sharedWorkspaceMemorySessionDir !== undefined) { + const ownerSessionDir = path.resolve(opts.sharedWorkspaceMemorySessionDir); + const ownerMemoryRoot = path.join(ownerSessionDir, "memory"); + if ( + lockKeys.includes(ownerMemoryRoot) && + (await isWorkspaceRemovalTombstoned(targetLockRoot, path.basename(ownerSessionDir))) + ) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the workspace owning the shared memory store was removed` + ); + } } } // Re-verify INSIDE the lock, immediately before mutating: a writer that From bb4583837151cfdfc54f5b0dddca053f643dc479 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 14:24:24 +0000 Subject: [PATCH 07/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixth=20C?= =?UTF-8?q?odex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Launch sweep folds every tree member's recency into the owner's row, so an active sub-agent keeps the shared notebook from being swept as idle. - Memory subscriptions forward `ownersInvalidated` for the subscribed workspace as a root-addressed change plus a consolidation-status refresh. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$5.00`_ --- src/node/orpc/routerSubscriptions.test.ts | 21 ++++++++++++++++++- src/node/orpc/routerSubscriptions.ts | 18 ++++++++++++++++ .../memoryConsolidationService.test.ts | 9 ++++++++ .../services/memoryConsolidationService.ts | 17 +++++++++------ src/node/services/memoryService.ts | 2 +- 5 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 9353991d62f..8eb45a00f79 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -48,12 +48,13 @@ test("memory subscriptions match workspace-scope events on the shared memory own const memoryService = new EventEmitter(); const memoryConsolidationService = new EventEmitter(); const controller = new AbortController(); + const ownerOf = new Map([["ws-child", "ws-owner"]]); const context = { "effect/context": app.context, workspaceService: { getInfo: () => Promise.resolve(null) }, memoryService: Object.assign(memoryService, { resolveWorkspaceMemoryOwnerId: (workspaceId: string) => - workspaceId === "ws-child" ? "ws-owner" : workspaceId, + ownerOf.get(workspaceId) ?? workspaceId, }), memoryConsolidationService, } as unknown as ORPCContext; @@ -79,6 +80,24 @@ test("memory subscriptions match workspace-scope events on the shared memory own memoryService.emit("change", marker); expect((await first).value).toEqual({ ...base, workspaceId: "ws-owner" }); expect((await stream.next()).value).toEqual(marker); + + // Ownership change for THIS workspace (owner removed): synthesized + // root-addressed refresh + status refresh, now addressed to the new owner. + ownerOf.set("ws-child", "ws-child"); + memoryService.emit("ownersInvalidated", ["ws-unrelated"]); + memoryService.emit("ownersInvalidated", ["ws-child"]); + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "user", + workspaceId: "ws-child", + projectPath: "", + }); + expect((await stream.next()).value).toEqual({ + kind: "consolidation_status", + workspaceId: "ws-child", + projectPath: "", + }); } finally { controller.abort(); await stream.return(undefined); diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 6f6ec4298b2..537c7ae8311 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -24,6 +24,7 @@ import type { LogEntry } from "@/node/services/logBuffer"; import { subscribeLogFeed } from "@/node/services/logBuffer"; import { resolveMemoryProjectIdentity, + toVirtualPath, type MemoryChangeEvent, } from "@/node/services/memoryService"; @@ -228,10 +229,27 @@ export function subscribeMemoryChanges( }; const onStatusChange = (event: MemoryConsolidationStatusChangeEventPayload) => emit.push(event); + // Ownership changed for this workspace (its owner was removed; it + // now reads its own store). No mutation event accompanies a removal, + // so synthesize a root-addressed refresh for the file list and the + // consolidation status, both of which described the old store. + const onOwnersInvalidated = (workspaceIds: string[]) => { + if (!workspaceId || !workspaceIds.includes(workspaceId)) return; + emit.push({ + scope: "workspace", + path: toVirtualPath("workspace", ""), + actor: "user", + workspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), + projectPath: projectPath ?? "", + }); + emit.push({ kind: "consolidation_status", workspaceId, projectPath: projectPath ?? "" }); + }; context.memoryService.on("change", onChange); + context.memoryService.on("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.on("statusChange", onStatusChange); return () => { context.memoryService.off("change", onChange); + context.memoryService.off("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.off("statusChange", onStatusChange); }; }, diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 5c4ca7e0a00..967f8452418 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1333,6 +1333,15 @@ describe("MemoryConsolidationService", () => { "agent" ); const dayAgo = Date.now() - 25 * 60 * 60 * 1000; + // A recently-used child keeps the whole tree's notebook "not idle": the + // owner must not be swept on its own stale recency. + await fixture.service.runLaunchSweep( + new Map([ + ["ws-sub", Date.now()], + ["ws-dream", dayAgo], + ]) + ); + expect(fixture.modelCalls).toHaveLength(0); await fixture.service.runLaunchSweep( new Map([ ["ws-sub", dayAgo], diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 9e7952f1a96..12671eccd37 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -1238,8 +1238,6 @@ export class MemoryConsolidationService extends EventEmitter { const archivedById = new Map(); const projectPathByWorkspace = new Map(); const cfg = self.config.loadConfigOrDefault(); - const sharesOwnerStore = (workspaceId: string) => - self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, () => cfg) !== workspaceId; for (const [configProjectPath, project] of cfg.projects) { for (const workspace of project.workspaces) { if (workspace.id === undefined) continue; @@ -1263,14 +1261,21 @@ export class MemoryConsolidationService extends EventEmitter { ]) ); - let started = 0; + // Shared stores: a child's writes are keyed under its owner, so the + // owner's row absorbs every tree member's recency (a notebook is idle + // only when the WHOLE tree is) and child rows are then dropped — + // running a child would just redirect to the owner anyway. + const treeRecency = new Map(); for (const [workspaceId, recency] of recencyByWorkspace) { + const ownerId = self.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId, () => cfg); + treeRecency.set(ownerId, Math.max(treeRecency.get(ownerId) ?? 0, recency)); + } + + let started = 0; + for (const [workspaceId, recency] of treeRecency) { if (started >= MEMORY_CONSOLIDATION_LAUNCH_SWEEP_CAP) break; if (now - recency < MEMORY_CONSOLIDATION_IDLE_MS) continue; if (archivedById.get(workspaceId) === true) continue; - // Shared store: the owner's own sweep covers a child's writes (they - // are keyed under the owner); running the child would just redirect. - if (sharesOwnerStore(workspaceId)) continue; const lastRunAt = sidecar.workspaces[workspaceId]?.lastRunAt ?? 0; const projectPath = projectPathByWorkspace.get(workspaceId) ?? ""; const projectRunAt = projectPath === "" ? 0 : (projectLastRunAt.get(projectPath) ?? 0); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index d193fa93640..cd15e3da89e 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -332,7 +332,7 @@ export function projectMemoryDirName(projectPath: string): string { return `${base}-${hash}`; } -function toVirtualPath(scope: MemoryScope, relPath: string): string { +export function toVirtualPath(scope: MemoryScope, relPath: string): string { return relPath === "" ? `${MEMORY_VIRTUAL_ROOT}/${scope}` : `${MEMORY_VIRTUAL_ROOT}/${scope}/${relPath}`; From 6921a48b283fad4509ba2d1b50993fa12879d3bf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 14:43:34 +0000 Subject: [PATCH 08/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventh?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sub-agent removal aborts (SharedMemoryLockUnavailableError, no tombstone, stays registered) when the owner store lock cannot be acquired instead of taking the orphan path, which would let an admitted child write land in the owner's live notebook after removal. - A removal-cancelled owner run no longer starts tree-wide harvest recovery; cancelled buckets are skipped. - Open memory subscriptions probe ownership every 30s (one stat) so an external owner removal refreshes an idle tab. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$5.50`_ --- src/node/orpc/routerSubscriptions.ts | 14 ++++++ .../memoryConsolidationService.test.ts | 32 +++++++++++++ .../services/memoryConsolidationService.ts | 10 ++-- src/node/services/workspaceRemoval.test.ts | 46 +++++++++++++++++++ src/node/services/workspaceRemoval.ts | 27 +++++++++++ src/node/services/workspaceService.ts | 6 ++- 6 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 537c7ae8311..9dc55e2e7d6 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -198,6 +198,9 @@ export function subscribeLogs( }); } +/** How often an open memory subscription re-checks its workspace's memory owner. */ +const MEMORY_OWNERSHIP_PROBE_INTERVAL_MS = 30_000; + export function subscribeMemoryChanges( context: ORPCContext, workspaceId: string | null, @@ -244,10 +247,21 @@ export function subscribeMemoryChanges( }); emit.push({ kind: "consolidation_status", workspaceId, projectPath: projectPath ?? "" }); }; + // ownersInvalidated is emitted lazily, when something probes ownership. + // Another backend (multi-instance) removing the owner leaves an idle + // tab with nothing to trigger that probe, so probe here: one stat of + // config.json per interval (see Config.configFileStamp). + const ownershipProbe = workspaceId + ? setInterval( + () => context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), + MEMORY_OWNERSHIP_PROBE_INTERVAL_MS + ).unref() + : undefined; context.memoryService.on("change", onChange); context.memoryService.on("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.on("statusChange", onStatusChange); return () => { + if (ownershipProbe !== undefined) clearInterval(ownershipProbe); context.memoryService.off("change", onChange); context.memoryService.off("ownersInvalidated", onOwnersInvalidated); context.memoryConsolidationService.off("statusChange", onStatusChange); diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 967f8452418..a4820bb55c6 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -399,6 +399,30 @@ describe("MemoryConsolidationService", () => { }, }), }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const childMetadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [childMetadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: childMetadata.summaryMessageId, + compactionEpoch: childMetadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: childMetadata, + }, + }, + }, + }) + ); const run = fixture.service.maybeRun("ws-dream", "manual"); await started; await fixture.service.cancelInFlightConsolidation("ws-dream"); @@ -409,6 +433,14 @@ describe("MemoryConsolidationService", () => { if (!result.success) { expect(result.error).toContain("stream failed"); } + // The cancelled owner's continuation must not start recovery of a + // sub-agent's retryable harvest (work outside the drained registry). + const childRecords = ( + JSON.parse( + await fsPromises.readFile(path.join(fixture.xumHome, "memory-consolidation.json"), "utf-8") + ) as { harvestsByWorkspace: Record> } + ).harvestsByWorkspace["ws-sub"]; + expect(Object.values(childRecords).map((record) => record.attemptCount)).toEqual([1]); // Idempotent with nothing in flight (the phantom-metadata removal path). await fixture.service.cancelInFlightConsolidation("ws-dream"); }); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 12671eccd37..785a86ccfc7 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -506,8 +506,9 @@ export class MemoryConsolidationService extends EventEmitter { const records = Object.entries(sidecar.harvestsByWorkspace) .filter( ([bucketId]) => - bucketId === workspaceId || - self.memoryService.resolveWorkspaceMemoryOwnerId(bucketId, () => cfg) === workspaceId + !self.removalCancelled.has(bucketId) && + (bucketId === workspaceId || + self.memoryService.resolveWorkspaceMemoryOwnerId(bucketId, () => cfg) === workspaceId) ) .flatMap(([, bucket]) => Object.values(bucket)); if (records.length === 0) return; @@ -731,7 +732,10 @@ export class MemoryConsolidationService extends EventEmitter { this.inFlight.delete(workspaceId); removal.dispose(); } - if (options.skipHarvestRecovery !== true) { + // Removal cancelled this run: recovery would spawn child harvests outside + // the cancellation registry being drained (their controllers were never + // registered), mutating the shared inbox during destructive teardown. + if (options.skipHarvestRecovery !== true && !this.removalCancelled.has(workspaceId)) { await Effect.runPromise(this.recoverRetryableHarvestsEffect(workspaceId)); } return result; diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 75abc00ae01..1bb902f07b5 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -15,6 +15,7 @@ import { refineApplyLockPath, REMOVAL_TOMBSTONE_HEAL_MIN_AGE_MS, removeSessionDirUnderMemoryLocks, + SharedMemoryLockUnavailableError, rollbackRemovalTombstoneIfOwned, TombstoneNotDurableError, workspaceRemovalTombstonePath, @@ -136,6 +137,51 @@ describe("workspaceRemoval", () => { ).toBe(true); }); + test("sub-agent removal aborts (no tombstone) when the owner store lock cannot be acquired", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-locked"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // A foreign process holds the OWNER store's cross-process lock (as in the + // r62 test): acquisition times out instead of reclaiming. + const lockPath = targetMutationLockFilePath(rootDir, path.join(ownerSessionDir, "memory")); + await fsPromises.mkdir(path.dirname(lockPath), { recursive: true }); + const birth = getProcessBirth(process.pid); + const token = + birth === null + ? `${process.pid}:feed` + : `${process.pid}:feed:${Buffer.from(birth).toString("hex")}`; + await fsPromises.writeFile(lockPath, token, { flag: "wx" }); + + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryLockUnavailableError); + // Unlike the own-store orphan path, NO tombstone is published: the + // wedged holder targets the owner's live notebook, so the child must + // stay registered and removal be retried. + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(true); + }, 20_000); + test("waits on the refine lock BEFORE taking the teardown target locks (r67)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index fd05d5c29b6..1e605a370da 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -49,6 +49,22 @@ import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; * ENOSPC) clears — while the workspace no longer exists anywhere else. * Keeping the workspace registered keeps removal retryable instead. */ +/** + * A sub-agent's removal could not take its memory owner's store lock (r61 + * for shared stores). The caller must ABORT the removal — no tombstone, no + * deregistration — because the orphan fallback would leave an admitted child + * write free to land in the owner's live notebook after removal. + */ +export class SharedMemoryLockUnavailableError extends Error { + constructor(workspaceId: string, options?: ErrorOptions) { + super( + `Could not lock the shared workspace-memory store while removing ${workspaceId}; removal aborted`, + options + ); + this.name = "SharedMemoryLockUnavailableError"; + } +} + export class TombstoneNotDurableError extends Error { constructor(workspaceId: string, options?: ErrorOptions) { super( @@ -192,6 +208,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { }) ); }; + let targetLocksHeld = false; try { // Refine serialization (r66) — acquired FIRST (r67): a /refine apply in // ANOTHER backend is untouched by the remover's process-local @@ -218,6 +235,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { args.rootDir, [sessionDirKey, workspaceMemoryKey, sharedMemoryKey, ...ownerMemoryKeys], async () => { + targetLocksHeld = true; // History append serialization (r63): a foreign backend's in-flight // stream can be mid-append under the history write lock; acquiring // that same (session-dir-external) lock here means the append either @@ -237,6 +255,15 @@ export async function removeSessionDirUnderMemoryLocks(args: { } ); } catch (error) { + // The orphan path below assumes a wedged writer's target is THIS + // workspace's retained session dir. A sub-agent's admitted memory write + // targets its OWNER's live notebook instead, so if the owner-store lock + // could not be taken, publishing the tombstone outside it would let a + // holder that already passed its commit check finish after removal. + // Abort instead: the workspace stays registered and removal is retried. + if (ownerMemoryKeys.length > 0 && !targetLocksHeld) { + throw new SharedMemoryLockUnavailableError(args.workspaceId, { cause: error }); + } // Fail-closed orphan path (r62): a wedged writer blocks the deletion, // but the caller proceeds to deregister the workspace regardless — so // the terminal marker must still become durable or a foreign backend diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 044d325c671..d82d7bd47b9 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -131,6 +131,7 @@ import { import { healRemovalTombstonesForRegisteredWorkspaces, removeSessionDirUnderMemoryLocks, + SharedMemoryLockUnavailableError, refineApplyLockPath, rollbackRemovalTombstoneIfOwned, startRemovalTombstoneLease, @@ -6335,7 +6336,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // writable by foreign backends forever — abort the removal (the // workspace stays registered and retryable) instead of proceeding // to deregistration below. - if (error instanceof TombstoneNotDurableError) { + if ( + error instanceof TombstoneNotDurableError || + error instanceof SharedMemoryLockUnavailableError + ) { throw error; } log.error(`Failed to remove session directory for ${workspaceId}:`, error); From 340239df5458e20b79194a95f8b6e021b30d95e8 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 15:04:59 +0000 Subject: [PATCH 09/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighth=20?= =?UTF-8?q?Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sub-agent removal aborts unless the tombstone was published UNDER the owner-store lock (a post-lock failure such as a history-lock timeout no longer falls into the unlocked orphan path). - Before a sub-agent's session dir is deleted, its live memory refinement rows targeting the owner's shared store are re-appended (payload blobs copied, postState preserved) into the owner's journal, where the owner can inspect and roll them back. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$6.10`_ --- src/node/services/memoryService.test.ts | 62 +++++++++ .../services/refinement/refinementJournal.ts | 8 +- .../refinement/sharedMemoryRowMigration.ts | 128 ++++++++++++++++++ src/node/services/workspaceRemoval.test.ts | 38 ++++++ src/node/services/workspaceRemoval.ts | 13 +- src/node/services/workspaceService.ts | 18 +++ 6 files changed, 260 insertions(+), 7 deletions(-) create mode 100644 src/node/services/refinement/sharedMemoryRowMigration.ts diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index e625de8e523..e074699eb5d 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -25,6 +25,7 @@ import { } from "@/common/types/refinement"; import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; +import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; import { TestTempDir } from "./tools/testHelpers"; @@ -1037,6 +1038,67 @@ describe("MemoryService", () => { expect(await readRefinementEvents(childSessionDir)).toHaveLength(1); }); + it("migrates a removed sub-agent's live shared-memory rows into the owner's journal, rollbackable there", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // Shared-store edit (migrates), an edit already rolled back (skipped), + // and a global edit (not the owner's store: stays with the child). + await fixture.service.create(fixture.ctx, "/memories/workspace/keep.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/keep.md", + "v1", + "v2", + "agent" + ); + await fixture.service.create(fixture.ctx, "/memories/workspace/undone.md", "x", "agent"); + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); + const childRows = await readRefinementEvents(childSessionDir); + const undone = childRows.find( + (row) => (row.data.action as { path: string }).path === "/memories/workspace/undone.md" + )!; + const rolledBack = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: undone.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(2); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect( + ownerRows.map((row) => [ + (row.data.action as { op: string }).op, + (row.data.action as { path: string }).path, + (row.data.evidence as { workspaceId: string }).workspaceId, + ]) + ).toEqual([ + ["create", "/memories/workspace/keep.md", "ws-owner"], + ["str_replace", "/memories/workspace/keep.md", "ws-owner"], + ]); + + // The child is gone; the owner rolls the edit back from its own journal + // (payload blobs were copied, postState hashes preserved). + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const keep = path.join(ownerSessionDir, "memory", "keep.md"); + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRows[1].id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(true); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + }); + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 522bbff3102..93f3cc8c7c9 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -70,6 +70,12 @@ export interface RefinementEmitArgs { * `postState` so rollback can detect out-of-band edits content-exactly. */ postFiles?: RefinementFileCapture[]; + /** + * Already-hashed post state, for re-appending an existing row whose + * contents are no longer available (shared-memory row migration). Ignored + * when `postFiles` is given. + */ + postState?: RefinementPostState; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -255,7 +261,7 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise file.path), ...(inverse.deletePaths ?? [])]; + case "rename": + return [inverse.from, inverse.to]; + } +} + +function isInside(root: string, filePath: string): boolean { + const rel = path.relative(root, path.resolve(filePath)); + return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel); +} + +/** + * Before a sub-agent's session directory is deleted, re-append its LIVE + * memory refinement rows that target the owner's shared `/memories/workspace` + * store into the OWNER's journal, copying the inverse blob payloads. The + * edits themselves already live in the owner's store; without this their + * audit trail and rollback IDs would vanish with the child's journal. Rows + * already rolled back (or rollback rows themselves) and rows targeting other + * roots (global/project) are left alone — they die with the child as before. + * + * Best-effort per row: a row whose payload cannot be reconstructed (evicted + * blob, unparseable action) is skipped with a log line rather than failing + * the removal. Returns the number of rows migrated. + */ +export async function migrateSharedMemoryRefinementRows(args: { + childSessionDir: string; + ownerSessionDir: string; + ownerWorkspaceId: string; +}): Promise { + assert( + args.childSessionDir.length > 0, + "migrateSharedMemoryRefinementRows requires childSessionDir" + ); + assert( + args.ownerSessionDir.length > 0, + "migrateSharedMemoryRefinementRows requires ownerSessionDir" + ); + assert( + args.ownerWorkspaceId.length > 0, + "migrateSharedMemoryRefinementRows requires ownerWorkspaceId" + ); + const ownerMemoryRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const rows = await listRefinements(args.childSessionDir); + const rolledBack = new Set( + rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) + ); + const childJournal = sharedDurableEventJournal(args.childSessionDir); + let migrated = 0; + for (const row of rows) { + if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined || rolledBack.has(row.id)) { + continue; + } + const inverse = RefinementInverseSchema.safeParse(row.data.inverse); + const action = MemoryRefinementActionSchema.safeParse(row.data.action); + if (!inverse.success || !action.success) continue; + if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; + + let draft: RefinementInverseDraft; + if (inverse.data.op === "restore-files") { + const files: Array<{ path: string; content: string }> = []; + for (const file of inverse.data.files) { + // Contents are blob-offloaded at append (resolveRefinementInverse); older + // rows may carry them inline. + const content = + file.text ?? + (file.blobRef === undefined ? null : await childJournal.blobs.getText(file.blobRef)); + if (content === null) break; + files.push({ path: file.path, content }); + } + if (files.length !== inverse.data.files.length) { + log.debug("[refinement] skipping shared-memory row migration: inverse payload missing", { + rowId: row.id, + }); + continue; + } + draft = { + op: "restore-files", + files, + ...(inverse.data.deletePaths !== undefined + ? { deletePaths: inverse.data.deletePaths } + : {}), + }; + } else { + draft = inverse.data; + } + const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); + const postState = RefinementPostStateSchema.safeParse(row.data.postState); + await appendRefinementEvent({ + sessionDir: args.ownerSessionDir, + workspaceId: args.ownerWorkspaceId, + kind: "memory", + action: action.data, + inverse: draft, + evidence: { + toolName: evidence.success ? evidence.data.toolName : "memory", + ...(evidence.success && evidence.data.toolCallId !== undefined + ? { toolCallId: evidence.data.toolCallId } + : {}), + ...(evidence.success && evidence.data.actor !== undefined + ? { actor: evidence.data.actor } + : {}), + }, + ...(postState.success ? { postState: postState.data } : {}), + ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), + }); + migrated++; + } + return migrated; +} diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 1bb902f07b5..771b287269f 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -11,6 +11,7 @@ import { import { acquireProcessFileLock, getProcessBirth } from "@/node/utils/concurrency/fileLock"; import { healRemovalTombstonesForRegisteredWorkspaces, + historyWriteLockPath, isWorkspaceRemovalTombstoned, refineApplyLockPath, REMOVAL_TOMBSTONE_HEAL_MIN_AGE_MS, @@ -182,6 +183,43 @@ describe("workspaceRemoval", () => { ).toBe(true); }, 20_000); + test("sub-agent removal aborts when a failure lands after the target locks but before the tombstone", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-history-locked"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // Target locks succeed; the history write lock (taken INSIDE them) is + // held by a foreign process and times out. The old orphan path would now + // publish the tombstone with the owner-store lock already released. + const historyLock = await acquireProcessFileLock({ + lockPath: historyWriteLockPath(rootDir, childId), + timeoutMs: 1_000, + label: "history write lock (test holder)", + }); + try { + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryLockUnavailableError); + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + } finally { + await historyLock[Symbol.asyncDispose](); + } + }, 30_000); + test("waits on the refine lock BEFORE taking the teardown target locks (r67)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 1e605a370da..5cc4b2eded1 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -208,7 +208,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { }) ); }; - let targetLocksHeld = false; + let tombstonePublishedUnderLocks = false; try { // Refine serialization (r66) — acquired FIRST (r67): a /refine apply in // ANOTHER backend is untouched by the remover's process-local @@ -235,7 +235,6 @@ export async function removeSessionDirUnderMemoryLocks(args: { args.rootDir, [sessionDirKey, workspaceMemoryKey, sharedMemoryKey, ...ownerMemoryKeys], async () => { - targetLocksHeld = true; // History append serialization (r63): a foreign backend's in-flight // stream can be mid-append under the history write lock; acquiring // that same (session-dir-external) lock here means the append either @@ -251,17 +250,19 @@ export async function removeSessionDirUnderMemoryLocks(args: { // deleted directory cannot be recreated by a late mutation or // journal append. await publishTombstone(); + tombstonePublishedUnderLocks = true; await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); } ); } catch (error) { // The orphan path below assumes a wedged writer's target is THIS // workspace's retained session dir. A sub-agent's admitted memory write - // targets its OWNER's live notebook instead, so if the owner-store lock - // could not be taken, publishing the tombstone outside it would let a - // holder that already passed its commit check finish after removal. + // targets its OWNER's live notebook instead, so unless the tombstone was + // already published UNDER the owner-store lock, publishing it outside + // (after the lock was released, or never taken) would let a holder that + // passed its commit check — or a new writer — finish after removal. // Abort instead: the workspace stays registered and removal is retried. - if (ownerMemoryKeys.length > 0 && !targetLocksHeld) { + if (ownerMemoryKeys.length > 0 && !tombstonePublishedUnderLocks) { throw new SharedMemoryLockUnavailableError(args.workspaceId, { cause: error }); } // Fail-closed orphan path (r62): a wedged writer blocks the deletion, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d82d7bd47b9..7d64826da41 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -138,6 +138,7 @@ import { TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -6321,6 +6322,23 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.config.loadConfigOrDefault(), workspaceId ); + if (memoryOwnerId !== workspaceId) { + // The child's memory edits live on in the owner's store; keep their + // audit trail / rollback IDs there too before the journal is deleted. + try { + await migrateSharedMemoryRefinementRows({ + childSessionDir: sessionDir, + ownerSessionDir: path.join(this.config.sessionsDir, memoryOwnerId), + ownerWorkspaceId: memoryOwnerId, + }); + } catch (error) { + log.warn("Failed to migrate shared-memory refinement rows to the owner", { + workspaceId, + memoryOwnerId, + error: getErrorMessage(error), + }); + } + } await removeSessionDirUnderMemoryLocks({ rootDir: this.config.rootDir, sessionDir, From 08ef9f2b2ef1f9241492c1d114da71362fb7862a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 15:21:14 +0000 Subject: [PATCH 10/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20ninth=20C?= =?UTF-8?q?odex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The pre-commit tombstone check derives the owner from the store the command already bound to (not a re-resolution), so an ownership change between resolution and lock acquisition cannot recreate a removed owner. - Shared-memory row migration is idempotent (rows carry a `migratedFrom` source identity, retried removals skip them) and liveness follows the full rollback chain, so a rollback-of-rollback re-apply is preserved. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$6.60`_ --- src/common/types/durableEvent.ts | 6 +++ src/node/services/memoryService.test.ts | 37 ++++++++++++++++-- src/node/services/memoryService.ts | 36 +++++++++++------- .../services/refinement/refinementJournal.ts | 3 ++ .../refinement/sharedMemoryRowMigration.ts | 38 +++++++++++++++++-- src/node/services/workspaceService.ts | 1 + 6 files changed, 102 insertions(+), 19 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index fb75e794c35..704a8465861 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -87,6 +87,12 @@ export const RefinementDataSchema = z.object({ evidence: JsonValueSchema.optional(), /** Envelope `id` of the entry this one rolls back. */ rollbackOf: z.string().optional(), + /** + * Stable source identity (`:`) when this row was + * copied from a removed sub-agent's journal into its memory owner's + * (sharedMemoryRowMigration.ts); lets a retried migration skip it. + */ + migratedFrom: z.string().optional(), /** Expected post-action file hashes (RefinementPostStateSchema in refinement.ts). */ postState: JsonValueSchema.optional(), /** diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index e074699eb5d..5be01b6d86d 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1067,13 +1067,42 @@ describe("MemoryService", () => { }); expect(rolledBack.success).toBe(true); + // A rollback of the rollback re-applies "redone.md": it is live again. + await fixture.service.create(fixture.ctx, "/memories/workspace/redone.md", "r", "agent"); + const redone = (await readRefinementEvents(childSessionDir)).find( + (row) => + (row.data.action as { path?: string }).path === "/memories/workspace/redone.md" && + row.data.rollbackOf === undefined + )!; + const undoRedone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: redone.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undoRedone.success).toBe(true); + if (!undoRedone.success) return; expect( - await migrateSharedMemoryRefinementRows({ + ( + await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: undoRedone.data.rollbackRowId ?? "", + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + + const migrate = () => + migrateSharedMemoryRefinementRows({ childSessionDir, + childWorkspaceId: "ws-child", ownerSessionDir, ownerWorkspaceId: "ws-owner", - }) - ).toBe(2); + }); + expect(await migrate()).toBe(3); + // Idempotent: a retried removal migrates nothing twice. + expect(await migrate()).toBe(0); const ownerRows = await readRefinementEvents(ownerSessionDir); expect( ownerRows.map((row) => [ @@ -1084,7 +1113,9 @@ describe("MemoryService", () => { ).toEqual([ ["create", "/memories/workspace/keep.md", "ws-owner"], ["str_replace", "/memories/workspace/keep.md", "ws-owner"], + ["create", "/memories/workspace/redone.md", "ws-owner"], ]); + expect(ownerRows.every((row) => row.data.migratedFrom?.startsWith("ws-child:"))).toBe(true); // The child is gone; the owner rolls the edit back from its own journal // (payload blobs were copied, postState hashes preserved). diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index cd15e3da89e..42f2b999c95 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -884,13 +884,17 @@ export class MemoryService extends EventEmitter { * removal here at commit time and refuses instead of recreating the * deleted session directory via its write or journal append. * - * Both the acting workspace and the workspace-memory owner are checked: a - * removed sub-agent must not keep writing into its parent's notebook, and - * a removed owner must not have its session directory recreated by a - * lingering child's write. + * Both the acting workspace and the workspace that physically owns the + * RESOLVED store are checked: a removed sub-agent must not keep writing + * into its parent's notebook, and a removed owner must not have its session + * directory recreated by a lingering child's write. The owner is derived + * from the store the command already bound to — not re-resolved — so an + * ownership change between resolution and lock acquisition cannot make the + * check pass for the new owner while the write lands in the old one. */ private async assertMutationCommittable( ctx: MemoryScopeContext, + store: MemoryStore, signal: AbortSignal | undefined, virtualPath: string ): Promise { @@ -900,7 +904,13 @@ export class MemoryService extends EventEmitter { ); } if (ctx.workspaceId === "") return; - for (const workspaceId of new Set([ctx.workspaceId, this.ownerWorkspaceIdFor(ctx)])) { + const guarded = new Set([ctx.workspaceId]); + // //memory → owner; global/project roots live elsewhere. + const rel = path.relative(this.config.sessionsDir, store.physicalRoot); + if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) { + guarded.add(rel.split(path.sep)[0]); + } + for (const workspaceId of guarded) { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { throw new MemoryCommandError( `Workspace ${workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` @@ -1169,7 +1179,7 @@ export class MemoryService extends EventEmitter { // only INSIDE the lock and after the removal check (r62), so the // mkdir serializes with removal's locked deletion and cannot // recreate a removed session directory. - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.ensureRoot(); const existing = await store.kind(parsed.relPath); if (existing !== null) { @@ -1183,7 +1193,7 @@ export class MemoryService extends EventEmitter { `The ${scope} memory scope is full (${MEMORY_MAX_FILES_PER_SCOPE} files); delete unused files first` ); } - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, fileText); // Row is written before the create is acknowledged (mutation → row → ack). await this.journalRefinement( @@ -1224,7 +1234,7 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const updated = computeStrReplaceUpdate(content, oldStr, newStr, virtualPath); assertWithinFileSizeCap(updated); - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( @@ -1277,7 +1287,7 @@ export class MemoryService extends EventEmitter { const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); const { updated, insertedLineCount } = computeInsertUpdate(content, insertLine, insertText); assertWithinFileSizeCap(updated); - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( @@ -1460,7 +1470,7 @@ export class MemoryService extends EventEmitter { // Prior contents must be captured before removal; the row itself is // written after the mutation succeeds and before it is acknowledged. const inverse = await this.captureDeleteInverse(store, parsed.relPath, kind); - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.remove(parsed.relPath); if (inverse !== null) { await this.journalRefinement( @@ -1522,7 +1532,7 @@ export class MemoryService extends EventEmitter { if (newKind !== null) { throw new MemoryCommandError(`Destination ${newVirtualPath} already exists`); } - await this.assertMutationCommittable(ctx, abortSignal, oldVirtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, oldVirtualPath); await store.rename(oldParsed.relPath, newParsed.relPath); // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( @@ -1646,7 +1656,7 @@ export class MemoryService extends EventEmitter { async () => { // UI save can create new files: materialize the scope root on // first use — in-lock, after the removal check (r62; see create). - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.ensureRoot(); const kind = await store.kind(parsed.relPath); if (kind === "dir") { @@ -1673,7 +1683,7 @@ export class MemoryService extends EventEmitter { ); } } - await this.assertMutationCommittable(ctx, abortSignal, virtualPath); + await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 93f3cc8c7c9..19aa60eb6b3 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -76,6 +76,8 @@ export interface RefinementEmitArgs { * when `postFiles` is given. */ postState?: RefinementPostState; + /** Source identity of a row copied from a removed sub-agent's journal (see durableEvent.ts). */ + migratedFrom?: string; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -271,6 +273,7 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { @@ -58,17 +59,47 @@ export async function migrateSharedMemoryRefinementRows(args: { args.ownerWorkspaceId.length > 0, "migrateSharedMemoryRefinementRows requires ownerWorkspaceId" ); + assert( + args.childWorkspaceId.length > 0, + "migrateSharedMemoryRefinementRows requires childWorkspaceId" + ); const ownerMemoryRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); const rows = await listRefinements(args.childSessionDir); - const rolledBack = new Set( - rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) + // Liveness follows the whole rollback chain (rollback → rollback of the + // rollback re-applies): an original row is live when it has been rolled + // back an even number of times. Rollback rows themselves are never copied. + const rollbackByTarget = new Map( + rows + .filter((row) => row.data.rollbackOf !== undefined) + .map((row) => [row.data.rollbackOf as string, row] as const) + ); + const isLive = (rowId: string): boolean => { + let depth = 0; + for ( + let next = rollbackByTarget.get(rowId); + next !== undefined; + next = rollbackByTarget.get(next.id) + ) { + depth++; + } + return depth % 2 === 0; + }; + // Idempotent across retried removals (the child journal survives a + // retryable removal failure or a crash before deletion): rows already + // copied are identified by their source identity on the owner side. + const alreadyMigrated = new Set( + (await listRefinements(args.ownerSessionDir)) + .map((row) => row.data.migratedFrom) + .filter((id): id is string => id !== undefined) ); const childJournal = sharedDurableEventJournal(args.childSessionDir); let migrated = 0; for (const row of rows) { - if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined || rolledBack.has(row.id)) { + if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined || !isLive(row.id)) { continue; } + const migratedFrom = `${args.childWorkspaceId}:${row.id}`; + if (alreadyMigrated.has(migratedFrom)) continue; const inverse = RefinementInverseSchema.safeParse(row.data.inverse); const action = MemoryRefinementActionSchema.safeParse(row.data.action); if (!inverse.success || !action.success) continue; @@ -120,6 +151,7 @@ export async function migrateSharedMemoryRefinementRows(args: { : {}), }, ...(postState.success ? { postState: postState.data } : {}), + migratedFrom, ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); migrated++; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7d64826da41..e4898575efe 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6328,6 +6328,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { try { await migrateSharedMemoryRefinementRows({ childSessionDir: sessionDir, + childWorkspaceId: workspaceId, ownerSessionDir: path.join(this.config.sessionsDir, memoryOwnerId), ownerWorkspaceId: memoryOwnerId, }); From 31cca377cea2cb70f4add33fa1875386af105ac6 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 15:22:15 +0000 Subject: [PATCH 11/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20lint=20(non-null=20?= =?UTF-8?q?assertion=20in=20shared-memory=20row=20migration)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$6.70`_ --- src/node/services/refinement/sharedMemoryRowMigration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index f5d72064913..6be394ce0d7 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -71,7 +71,7 @@ export async function migrateSharedMemoryRefinementRows(args: { const rollbackByTarget = new Map( rows .filter((row) => row.data.rollbackOf !== undefined) - .map((row) => [row.data.rollbackOf as string, row] as const) + .map((row) => [row.data.rollbackOf!, row] as const) ); const isLive = (rowId: string): boolean => { let depth = 0; From 5e3f111bc675f639311a4033036f4cf5e04d2abe Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 15:42:41 +0000 Subject: [PATCH 12/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20tenth=20C?= =?UTF-8?q?odex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Owner resolution is cached per scope-context object and skipped for global/project keys, so a hot-set build stats config.json once, not once per candidate. - Migrated rows carry `sourceTs`; rollback conflict ordering uses the source timestamp so a migrated (older) child row does not read as a later conflicting mutation of the owner's edit. - Migration appends via a throwing variant; a failure aborts removal (SharedMemoryRowMigrationError) unless forced, so the only durable inverse is never deleted silently. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$7.20`_ --- src/common/types/durableEvent.ts | 2 + src/node/services/memoryService.test.ts | 50 +++++++++++++++++++ src/node/services/memoryService.ts | 21 ++++++-- .../services/refinement/refinementJournal.ts | 26 +++++++--- .../services/refinement/refinementRollback.ts | 16 +++++- .../refinement/sharedMemoryRowMigration.ts | 25 ++++++++-- src/node/services/workspaceService.ts | 17 +++++-- 7 files changed, 138 insertions(+), 19 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 704a8465861..01d23d9482e 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -93,6 +93,8 @@ export const RefinementDataSchema = z.object({ * (sharedMemoryRowMigration.ts); lets a retried migration skip it. */ migratedFrom: z.string().optional(), + /** Source row's `ts`, so rollback ordering keeps the mutation's real cross-session position. */ + sourceTs: z.number().optional(), /** Expected post-action file hashes (RefinementPostStateSchema in refinement.ts). */ postState: JsonValueSchema.optional(), /** diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 5be01b6d86d..4cf3204fa7c 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1130,6 +1130,56 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); }); + it("migrated rows keep their real order relative to the owner's own later edits", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Child edits first, owner edits the same file later, THEN the child is + // removed: the migrated (older) child row is appended after the owner's. + await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "c1", "agent"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/shared.md", + "c1", + "o2", + "agent" + ); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const ownerEdit = ownerRows.find((row) => row.data.migratedFrom === undefined)!; + const migrated = ownerRows.find((row) => row.data.migratedFrom !== undefined)!; + expect(migrated.seq).toBeGreaterThan(ownerEdit.seq); + + // LIFO unrolling works without force: the owner's edit is the newest + // mutation of the file, so it rolls back first... + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + const shared = path.join(ownerSessionDir, "memory", "shared.md"); + expect(await fsPromises.readFile(shared, "utf-8")).toBe("c1"); + // ...and then the migrated child create. + const second = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: migrated.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(second.success).toBe(true); + expect(await pathExists(shared)).toBe(false); + }); + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 42f2b999c95..424dbb2b30d 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -658,9 +658,23 @@ export class MemoryService extends EventEmitter { if (shared.length > 0) this.emit("ownersInvalidated", shared); } + /** + * Per-context owner cache: a context object is created per command / per + * index+hot-set build and reused for every entry within it, so the stamp + * stat behind resolveWorkspaceMemoryOwnerId runs once per operation instead + * of once per candidate file. Staleness is bounded to that one operation; + * writes are still gated by the store-bound tombstone check. + */ + private readonly ownerByContext = new WeakMap(); + /** Owner of the workspace scope for this context ("" when there is no workspace). */ private ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { - return ctx.workspaceId === "" ? "" : this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + if (ctx.workspaceId === "") return ""; + const cached = this.ownerByContext.get(ctx); + if (cached !== undefined) return cached; + const owner = this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + this.ownerByContext.set(ctx, owner); + return owner; } /** Logical sidecar key, or null when the scope has no stable identity. */ @@ -669,8 +683,9 @@ export class MemoryService extends EventEmitter { return memoryLogicalKey(scope, relPath, { projectPath: ctx.projectPath, // Pins/usage stats follow the physical file, so a shared notebook has - // one ranking regardless of which tree member touched it. - workspaceId: this.ownerWorkspaceIdFor(ctx), + // one ranking regardless of which tree member touched it. Only the + // workspace key embeds the id; skip the lookup for the other scopes. + workspaceId: scope === "workspace" ? this.ownerWorkspaceIdFor(ctx) : ctx.workspaceId, }); } diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 19aa60eb6b3..84fe46f028a 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -78,6 +78,8 @@ export interface RefinementEmitArgs { postState?: RefinementPostState; /** Source identity of a row copied from a removed sub-agent's journal (see durableEvent.ts). */ migratedFrom?: string; + /** Source row's `ts` for a migrated row (see durableEvent.ts). */ + sourceTs?: number; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -236,6 +238,23 @@ export async function reclaimExcessRefinementInverseBlobs( */ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { try { + await appendRefinementEventOrThrow(args); + } catch (error) { + log.debug("[refinement] failed to journal refinement event; continuing", { + kind: args.kind, + workspaceId: args.workspaceId, + error, + }); + } +} + +/** + * Same as appendRefinementEvent but propagates failures: for callers whose + * row is the ONLY durable copy (shared-memory row migration before the + * source journal is deleted) a swallowed failure would silently lose it. + */ +export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Promise { + { assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); const journal = sharedDurableEventJournal(args.sessionDir); @@ -274,6 +293,7 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise otherTs || (rowTs === otherTs && row.seq > other.seq); +} + async function collectDivergence( rows: RefinementEvent[], target: RefinementEvent, @@ -513,7 +525,7 @@ async function collectDivergence( rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) ); for (const row of rows) { - if (row.seq <= target.seq) continue; + if (!isAfter(row, target)) continue; if (rolledBackIds.has(row.id)) continue; // Effect undone by a later rollback row. if (!liveRowConflictsWithTarget(rows, row, target)) continue; const parsed = RefinementInverseSchema.safeParse(row.data.inverse); @@ -649,7 +661,7 @@ function liveRowConflictsWithTarget( if (rollbackCount % 2 === 0) { return true; // Even chain: the root row's edit was re-applied. } - return current.seq <= target.seq; // Odd chain: rewound to just before root. + return !isAfter(current, target); // Odd chain: rewound to just before root. } async function dirExists(target: string): Promise { diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 6be394ce0d7..f3695e47718 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -9,9 +9,20 @@ import { } from "@/common/types/refinement"; import { log } from "@/node/services/log"; import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; -import { appendRefinementEvent, type RefinementInverseDraft } from "./refinementJournal"; +import { appendRefinementEventOrThrow, type RefinementInverseDraft } from "./refinementJournal"; import { listRefinements } from "./refinementRollback"; +/** Removal must abort: a live shared-memory row could not be persisted in the owner's journal. */ +export class SharedMemoryRowMigrationError extends Error { + constructor(workspaceId: string, options?: ErrorOptions) { + super( + `Could not preserve ${workspaceId}'s shared-memory refinement rows in its owner's journal; removal aborted`, + options + ); + this.name = "SharedMemoryRowMigrationError"; + } +} + function inversePaths(inverse: RefinementInverse): string[] { switch (inverse.op) { case "delete-files": @@ -37,9 +48,11 @@ function isInside(root: string, filePath: string): boolean { * already rolled back (or rollback rows themselves) and rows targeting other * roots (global/project) are left alone — they die with the child as before. * - * Best-effort per row: a row whose payload cannot be reconstructed (evicted - * blob, unparseable action) is skipped with a log line rather than failing - * the removal. Returns the number of rows migrated. + * A row whose payload cannot be reconstructed (evicted blob, unparseable + * action) is skipped with a log line — nothing durable exists to preserve. + * A row that CAN be reconstructed but cannot be persisted in the owner's + * journal throws: the caller must not delete the source journal, or the + * only inverse and rollback ID would be lost. Returns the number migrated. */ export async function migrateSharedMemoryRefinementRows(args: { childSessionDir: string; @@ -135,7 +148,8 @@ export async function migrateSharedMemoryRefinementRows(args: { } const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); const postState = RefinementPostStateSchema.safeParse(row.data.postState); - await appendRefinementEvent({ + // Throws: this is the only durable copy once the child's journal goes. + await appendRefinementEventOrThrow({ sessionDir: args.ownerSessionDir, workspaceId: args.ownerWorkspaceId, kind: "memory", @@ -152,6 +166,7 @@ export async function migrateSharedMemoryRefinementRows(args: { }, ...(postState.success ? { postState: postState.data } : {}), migratedFrom, + sourceTs: row.data.sourceTs ?? row.ts, ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); migrated++; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e4898575efe..3e2f74ac8df 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -138,7 +138,10 @@ import { TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; -import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; +import { + migrateSharedMemoryRefinementRows, + SharedMemoryRowMigrationError, +} from "@/node/services/refinement/sharedMemoryRowMigration"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -6333,7 +6336,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ownerWorkspaceId: memoryOwnerId, }); } catch (error) { - log.warn("Failed to migrate shared-memory refinement rows to the owner", { + // The child's journal is the only copy of these rows: do not + // delete it. Abort (workspace stays registered, retryable) unless + // the caller forces removal, in which case the audit trail is + // knowingly given up. + if (!force) { + throw new SharedMemoryRowMigrationError(workspaceId, { cause: error }); + } + log.warn("Forced removal: shared-memory refinement rows could not be migrated", { workspaceId, memoryOwnerId, error: getErrorMessage(error), @@ -6357,7 +6367,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // to deregistration below. if ( error instanceof TombstoneNotDurableError || - error instanceof SharedMemoryLockUnavailableError + error instanceof SharedMemoryLockUnavailableError || + error instanceof SharedMemoryRowMigrationError ) { throw error; } From 9f36514aa46f75fd2b21196af3b619eede87d9b1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 16:18:16 +0000 Subject: [PATCH 13/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eleventh?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removing an intermediate sub-agent pins `memoryOwnerWorkspaceId` on its surviving children so nested descendants stay on the root store; the resolver honors the pin while that owner is registered. - Migration lineage walk is cycle/depth guarded. - A removed workspace's retryable harvest records are finalized (its transcript goes with the session), so no unrecoverable bucket lingers. - memoryOperations binds sidecar keys, store, and notify to one per-context owner resolution. - Security: TurnRequestBuilder records each normal turn's workspace-memory write policy on the session; compaction completions carry `workspaceMemoryWritable`, and harvest/sweep is refused when it is false, so a read-only (explore-like) sub-agent cannot write the owner's notebook. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$8.00`_ --- src/common/orpc/schemas/memory.ts | 1 + src/common/schemas/project.ts | 4 ++ src/common/types/compaction.ts | 8 +++ src/node/services/agentSession.ts | 19 +++++- src/node/services/di/layers/core.ts | 1 + .../memoryConsolidationService.test.ts | 59 +++++++++++++++++++ .../services/memoryConsolidationService.ts | 42 ++++++++++++- src/node/services/memoryOperations.ts | 17 +++--- src/node/services/memoryService.test.ts | 35 +++++++++++ src/node/services/memoryService.ts | 8 ++- src/node/services/memoryWorkspaceOwner.ts | 7 +++ .../refinement/sharedMemoryRowMigration.ts | 15 +++-- src/node/services/turnRequestBuilder.ts | 14 +++++ src/node/services/workspaceService.ts | 28 ++++++++- 14 files changed, 242 insertions(+), 16 deletions(-) diff --git a/src/common/orpc/schemas/memory.ts b/src/common/orpc/schemas/memory.ts index 0bea6f09b40..d7310437d50 100644 --- a/src/common/orpc/schemas/memory.ts +++ b/src/common/orpc/schemas/memory.ts @@ -96,6 +96,7 @@ export type MemoryConsolidationRecordPayload = z.infer 0 ? metadata.summaryMessageId : null ); - onCompactionComplete?.(metadata); + onCompactionComplete?.({ + ...metadata, + ...(this.workspaceMemoryWritable !== undefined + ? { workspaceMemoryWritable: this.workspaceMemoryWritable } + : {}), + }); }, onIdleCompactionOutcome, }); diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index 1f467047687..ab1f4e70d25 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -537,6 +537,7 @@ export const CoreWiringLive: Layer.Layer< }); turnRequestBuilderBindings.workspaceHeartbeatService = workspaceService; + turnRequestBuilderBindings.workspaceMemoryPolicySink = workspaceService; // Tool-started workflows share the same sidebar activity cache as ORPC-started workflows, // so terminal updates must prune active run counts regardless of launch path. turnRequestBuilderBindings.onWorkflowRunStatusChanged = (event) => diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index a4820bb55c6..37dc14fb3d8 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -17,6 +17,7 @@ import { import { Ok } from "@/common/types/result"; import { Config } from "@/node/config"; import { + HARVEST_MAX_ATTEMPTS, MemoryConsolidationService, resolveDreamAgentBody, resolveDreamModelString, @@ -1150,6 +1151,64 @@ describe("MemoryConsolidationService", () => { expect(status.latestHarvestRecord?.attemptCount).toBe(2); }); + it("refuses to harvest (and sweep) for an agent whose workspace memory is read-only", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + const metadata = await seedCompactionEpoch(fixture); + const refused = await fixture.service.maybeHarvestThenSweep({ + ...metadata, + workspaceMemoryWritable: false, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("read-only"); + expect(fixture.modelCalls).toHaveLength(0); + expect((await fixture.service.getStatus("ws-dream")).latestHarvestRecord).toBeNull(); + + // Explicitly writable (and legacy records without the flag) harvest as before. + const allowed = await fixture.service.maybeHarvestThenSweep({ + ...metadata, + workspaceMemoryWritable: true, + }); + expect(allowed.success).toBe(true); + expect(fixture.modelCalls.length).toBeGreaterThan(0); + }); + + it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + await fsPromises.writeFile( + path.join(fixture.xumHome, "memory-consolidation.json"), + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + const record = (await fixture.service.getStatus("ws-sub")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + // The owner's run no longer sees a retryable child bucket. + expect((await fixture.service.maybeRun("ws-dream", "manual")).success).toBe(true); + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe( + HARVEST_MAX_ATTEMPTS + ); + }); + it("normalizes stale max-attempt pending harvest records to failed", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); const metadata = await seedCompactionEpoch(fixture); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 785a86ccfc7..5c4478959f8 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -285,7 +285,7 @@ function pruneHarvestRecords(records: Record): void } } -const HARVEST_MAX_ATTEMPTS = 3; +export const HARVEST_MAX_ATTEMPTS = 3; export class MemoryConsolidationService extends EventEmitter { private readonly sidecarPath: string; @@ -598,6 +598,40 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.runPromise(this.cancelInFlightConsolidationEffect(workspaceId)); } + /** + * Removal teardown for harvest state: the workspace's transcript is about + * to be deleted, so its failed/stale-pending harvest records can never be + * retried (recovery needs the compaction epoch's messages) — and once the + * config entry is gone they could not even be associated with the memory + * owner. Mark them terminal now so nothing lingers as "retryable". + */ + async finalizeHarvestsForRemoval(workspaceId: string): Promise { + const sidecar = await this.load(); + const records = sidecar.harvestsByWorkspace[workspaceId]; + if (records === undefined) return; + const workspace = this.config.findWorkspace(workspaceId); + const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); + for (const [boundaryKey, record] of Object.entries(records)) { + if (record.status === "completed") continue; + if (record.status === "failed" && record.attemptCount >= HARVEST_MAX_ATTEMPTS) continue; + await Effect.runPromise( + this.saveHarvestRecordEffect( + workspaceId, + boundaryKey, + { + ...record, + status: "failed", + completedAt: record.completedAt ?? Date.now(), + attemptCount: HARVEST_MAX_ATTEMPTS, + error: + "workspace removed before the harvest could be retried; transcript no longer available", + }, + projectPath + ) + ); + } + } + /** * Teardown pipeline: uninterruptible end-to-end so the r61 mark, the abort * loop, and the residual-run handoff can never be separated, with the @@ -879,6 +913,12 @@ export class MemoryConsolidationService extends EventEmitter { if (this.removalCancelled.has(metadata.workspaceId)) { return Err("workspace is being removed; harvest refused"); } + // The harvest writes /memories/workspace on the agent's behalf — for a + // sub-agent, into the OWNER's shared notebook — and then sweeps it. A + // read-only (explore-like) agent's transcript must not reach either. + if (metadata.workspaceMemoryWritable === false) { + return Err("workspace memory is read-only for this agent; harvest and sweep refused"); + } const boundaryRunKey = `${metadata.workspaceId}:${metadata.summaryMessageId}`; const active = this.harvestInFlight.get(boundaryRunKey); diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index fcbe43b1de0..e892ca299f7 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -81,15 +81,18 @@ function resolveMemoryScope( const metadata = yield* Effect.promise(() => context.workspaceService.getInfo(workspaceId)); if (!metadata) return yield* Effect.fail(new MemoryWorkspaceNotFoundError({ workspaceId })); const projectPath = resolveMemoryProjectIdentity(metadata); + const scopeCtx: MemoryScopeContext = { + runtime: createRuntimeForWorkspace(metadata), + checkoutCwd: "", + workspaceId, + projectPath, + }; return { projectPath, - ownerWorkspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), - scopeCtx: { - runtime: createRuntimeForWorkspace(metadata), - checkoutCwd: "", - workspaceId, - projectPath, - }, + // Same per-context resolution the store/notify paths use, so sidecar keys + // and the physical store never disagree about the owner within a request. + ownerWorkspaceId: context.memoryService.ownerWorkspaceIdFor(scopeCtx), + scopeCtx, }; }); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 4cf3204fa7c..e112aa98c08 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -821,6 +821,41 @@ describe("MemoryService", () => { ); }); + it("keeps a grandchild on the root store via the pinned owner after its parent is removed", async () => { + using fixture = await createFixture("ws-grandchild"); + await registerTaskTree(fixture); + // Removal of the intermediate "ws-child" pins memoryOwnerWorkspaceId on + // its children before deregistering it (WorkspaceService.remove). + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + for (const ws of project.workspaces) { + if (ws.parentWorkspaceId === "ws-child") ws.memoryOwnerWorkspaceId = "ws-owner"; + } + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-child"); + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-owner"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/still-shared.md", + "root store", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "still-shared.md") + ) + ).toBe(true); + // A pinned owner that is itself gone falls back to self. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild")).toBe("ws-grandchild"); + }); + it("stores a sub-agent's workspace notes in the owner's session dir, visible to the whole tree", async () => { using fixture = await createFixture("ws-grandchild"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 424dbb2b30d..74417f02b2a 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -667,8 +667,12 @@ export class MemoryService extends EventEmitter { */ private readonly ownerByContext = new WeakMap(); - /** Owner of the workspace scope for this context ("" when there is no workspace). */ - private ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { + /** + * Owner of the workspace scope for this context ("" when there is no + * workspace). Public so callers that key sidecar metadata for the same + * context (memoryOperations) bind to the exact owner the store resolved to. + */ + ownerWorkspaceIdFor(ctx: MemoryScopeContext): string { if (ctx.workspaceId === "") return ""; const cached = this.ownerByContext.get(ctx); if (cached !== undefined) return cached; diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index d83bc3f5759..e1a1952d83c 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -46,6 +46,13 @@ export function resolveWorkspaceMemoryOwnerId(cfg: ProjectsConfig, workspaceId: } return workspaceId; } + // A pinned owner (recorded when an intermediate ancestor was removed) + // short-circuits the walk; if that owner is itself gone, fall through to + // the parent chain, which then dangles and resolves to self. + const pinned = entry.workspace.memoryOwnerWorkspaceId; + if (pinned !== undefined && pinned !== "" && findWorkspaceEntry(cfg, pinned) !== null) { + return pinned; + } const parentWorkspaceId = entry.workspace.parentWorkspaceId; if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; current = parentWorkspaceId; diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index f3695e47718..ec4cd7dd991 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -86,13 +86,21 @@ export async function migrateSharedMemoryRefinementRows(args: { .filter((row) => row.data.rollbackOf !== undefined) .map((row) => [row.data.rollbackOf!, row] as const) ); - const isLive = (rowId: string): boolean => { + // Returns null on a corrupted (cyclic / absurdly long) lineage: such a row + // is treated as non-migratable instead of hanging removal. + const isLive = (rowId: string): boolean | null => { + const visited = new Set([rowId]); let depth = 0; for ( let next = rollbackByTarget.get(rowId); next !== undefined; next = rollbackByTarget.get(next.id) ) { + if (visited.has(next.id) || depth >= 1024) { + log.warn("[refinement] corrupted rollback lineage; skipping row migration", { rowId }); + return null; + } + visited.add(next.id); depth++; } return depth % 2 === 0; @@ -108,9 +116,8 @@ export async function migrateSharedMemoryRefinementRows(args: { const childJournal = sharedDurableEventJournal(args.childSessionDir); let migrated = 0; for (const row of rows) { - if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined || !isLive(row.id)) { - continue; - } + if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined) continue; + if (isLive(row.id) !== true) continue; const migratedFrom = `${args.childWorkspaceId}:${row.id}`; if (alreadyMigrated.has(migratedFrom)) continue; const inverse = RefinementInverseSchema.safeParse(row.data.inverse); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index b2c24f2b469..5b597b375f9 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -536,6 +536,10 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { onWorkflowRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; workflowResultContinuationSender?: WorkflowResultContinuationSender; workspaceHeartbeatService?: ToolConfiguration["workspaceHeartbeatService"]; + /** Receives each normal turn's workspace-memory write policy (see recordWorkspaceMemoryWritable). */ + workspaceMemoryPolicySink?: { + recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): void; + }; analyticsService?: { executeRawQuery(sql: string): Promise }; desktopSessionManager?: DesktopSessionManager; } @@ -1416,6 +1420,16 @@ export class TurnRequestBuilder { planLike: agentIsPlanLike, editingCapable: isExecLikeEditingCapableInResolvedChain(agentInheritanceChain), }); + // Post-compaction harvest writes to /memories/workspace on the agent's + // behalf; it must honor the same policy the memory tool enforces. The + // compaction turn itself runs the "compact" agent, so record only normal + // turns' policy (the session attaches it to the compaction completion). + if (!isCompactionRequest) { + this.dependencies.bindings.workspaceMemoryPolicySink?.recordWorkspaceMemoryWritable( + workspaceId, + memoryAccess.workspace === "readwrite" + ); + } const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic // repo hook execution (tool_env/tool_pre/tool_post) while keeping diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 3e2f74ac8df..aedc7163da7 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2737,6 +2737,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + finalizeHarvestsForRemoval(workspaceId: string): Promise; }; private worktreeArchiveSnapshotService?: WorktreeArchiveSnapshotLifecycleService; private agentTaskIntegration?: AgentTaskIntegration; @@ -3080,6 +3081,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + finalizeHarvestsForRemoval(workspaceId: string): Promise; }): void { this.memoryConsolidationService = service; } @@ -4210,6 +4212,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } + /** TurnRequestBuilder → session: the agent's workspace-memory write policy for this turn. */ + recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): void { + ( + this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) + )?.recordWorkspaceMemoryWritable(writable); + } + /** Transfer destructive cleanup out of a callback that still owns a session lease. */ deferWorkspaceCleanup(run: () => Promise): void { this.trackWorkspaceCleanup(run).catch((error: unknown) => @@ -6243,8 +6252,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // Same for in-flight dream/harvest consolidation (r60): abort + drain // before the session directory disappears (idempotent; normally - // already cancelled before the usage rollup above). + // already cancelled before the usage rollup above). Retryable harvest + // records are finalized too: their transcript goes with the session. await this.memoryConsolidationService?.cancelInFlightConsolidation(workspaceId); + await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); // Cancel and drain any background branch-summary writer BEFORE deleting // the session directory: a mid-flight append could otherwise recreate @@ -6326,6 +6337,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceId ); if (memoryOwnerId !== workspaceId) { + // Descendants that outlive this intermediate node would otherwise + // lose their path to the root: pin the owner on them first. + await this.config.editConfig((cfg) => { + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if ( + workspace.parentWorkspaceId === workspaceId && + !workspace.memoryOwnerWorkspaceId + ) { + workspace.memoryOwnerWorkspaceId = memoryOwnerId; + } + } + } + return cfg; + }); // The child's memory edits live on in the owner's store; keep their // audit trail / rollback IDs there too before the journal is deleted. try { From 16682a2287f339be72ac8484a056274403d0e5b1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 16:49:12 +0000 Subject: [PATCH 14/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twelfth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removal: owner pinning + refinement-row migration now run BEFORE runtime deletion and abort cleanly (Err, workspace intact) on failure unless forced; retryable harvests are finalized only after the locked session deletion commits. - Security: the refinement_rollback tool applies the agent's per-scope memory write policy (MemoryService.scopeOfPhysicalPath) and refuses memory inverses into a read-only scope. - Security: the workspace-memory write policy is persisted on the workspace config entry (when it changes) and used as the fallback when a compaction completes in a fresh session; harvest fails closed when still unknown and records the refusal as a terminal harvest record. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$8.90`_ --- src/common/schemas/project.ts | 4 + .../memoryConsolidationService.test.ts | 29 +++- .../services/memoryConsolidationService.ts | 46 +++++- src/node/services/memoryService.test.ts | 44 +++++- src/node/services/memoryService.ts | 22 ++- .../refinement/sharedMemoryRowMigration.ts | 11 -- src/node/services/toolAssembly.ts | 3 +- .../services/tools/refinement_rollback.ts | 47 +++++- src/node/services/turnRequestBuilder.ts | 1 + src/node/services/workspaceService.ts | 134 +++++++++++------- 10 files changed, 261 insertions(+), 80 deletions(-) diff --git a/src/common/schemas/project.ts b/src/common/schemas/project.ts index 9d575be479b..0e40e3c2d1c 100644 --- a/src/common/schemas/project.ts +++ b/src/common/schemas/project.ts @@ -105,6 +105,10 @@ export const WorkspaceConfigSchema = z.object({ description: "If set, this workspace is a child workspace spawned from the parent workspaceId (enables nesting in UI and backend orchestration).", }), + workspaceMemoryWritable: z.boolean().optional().meta({ + description: + "Whether this workspace's agent may write /memories/workspace, as resolved on its last normal turn. Persisted so a post-compaction memory harvest that resumes in a fresh session (restart, recovery) still knows the policy; harvest fails closed when unknown.", + }), memoryOwnerWorkspaceId: z.string().optional().meta({ description: "Memory owner pinned when an intermediate ancestor was removed while this descendant stayed alive: the parentWorkspaceId chain no longer reaches the task-tree root, so this keeps /memories/workspace bound to the root's store (memoryWorkspaceOwner.ts). Set only by workspace removal.", diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 37dc14fb3d8..f72a352efb3 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -343,6 +343,9 @@ async function seedCompactionEpoch( expect(typeof summaryHistorySequence).toBe("number"); return { workspaceId, + // A normal editing-capable agent; the read-only / unknown gates are + // exercised explicitly where they matter. + workspaceMemoryWritable: true, summaryMessageId: "summary-1", summaryHistorySequence: summaryHistorySequence ?? -1, compactionEpoch: 1, @@ -1156,20 +1159,34 @@ describe("MemoryConsolidationService", () => { const metadata = await seedCompactionEpoch(fixture); const refused = await fixture.service.maybeHarvestThenSweep({ ...metadata, + summaryMessageId: "summary-readonly", workspaceMemoryWritable: false, }); expect(refused.success).toBe(false); if (!refused.success) expect(refused.error).toContain("read-only"); expect(fixture.modelCalls).toHaveLength(0); - expect((await fixture.service.getStatus("ws-dream")).latestHarvestRecord).toBeNull(); - - // Explicitly writable (and legacy records without the flag) harvest as before. - const allowed = await fixture.service.maybeHarvestThenSweep({ + // Recorded as terminal so recovery never retries it, with the reason. + const refusedRecord = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(refusedRecord?.status).toBe("failed"); + expect(refusedRecord?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect(refusedRecord?.error).toContain("read-only"); + + // Unknown policy (legacy record / no persisted value) fails closed too. + const unknown = await fixture.service.maybeHarvestThenSweep({ ...metadata, - workspaceMemoryWritable: true, + summaryMessageId: "summary-unknown", + workspaceMemoryWritable: undefined, }); + expect(unknown.success).toBe(false); + if (!unknown.success) expect(unknown.error).toContain("unknown"); + expect(fixture.modelCalls).toHaveLength(0); + + // Explicitly writable harvests as before. + const allowed = await fixture.service.maybeHarvestThenSweep(metadata); expect(allowed.success).toBe(true); - expect(fixture.modelCalls.length).toBeGreaterThan(0); + const harvested = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(harvested?.boundaryKey).toBe(metadata.summaryMessageId); + expect(harvested?.status).toBe("completed"); }); it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 5c4478959f8..40f3c8289ea 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -598,6 +598,38 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.runPromise(this.cancelInFlightConsolidationEffect(workspaceId)); } + /** Terminal failed record for a policy-refused harvest (see maybeHarvestThenSweep). */ + private async recordRefusedHarvest( + metadata: CompactionCompletionMetadata, + reason: string + ): Promise { + const sidecar = await this.load(); + const existing = sidecar.harvestsByWorkspace[metadata.workspaceId]?.[metadata.summaryMessageId]; + if (existing?.status === "completed") return; + const workspace = this.config.findWorkspace(metadata.workspaceId); + const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); + const now = Date.now(); + await Effect.runPromise( + this.saveHarvestRecordEffect( + metadata.workspaceId, + metadata.summaryMessageId, + { + status: "failed", + startedAt: existing?.startedAt ?? now, + completedAt: now, + attemptCount: HARVEST_MAX_ATTEMPTS, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + completionMetadata: metadata, + acceptedCandidates: 0, + skippedCandidates: 0, + error: reason, + }, + projectPath + ) + ); + } + /** * Removal teardown for harvest state: the workspace's transcript is about * to be deleted, so its failed/stale-pending harvest records can never be @@ -915,9 +947,17 @@ export class MemoryConsolidationService extends EventEmitter { } // The harvest writes /memories/workspace on the agent's behalf — for a // sub-agent, into the OWNER's shared notebook — and then sweeps it. A - // read-only (explore-like) agent's transcript must not reach either. - if (metadata.workspaceMemoryWritable === false) { - return Err("workspace memory is read-only for this agent; harvest and sweep refused"); + // read-only (explore-like) agent's transcript must not reach either, and + // an UNKNOWN policy (legacy record, no persisted value) fails closed. The + // refusal is recorded as a terminal harvest record so recovery does not + // retry it forever and the Memory tab shows why the epoch was skipped. + if (metadata.workspaceMemoryWritable !== true) { + const reason = + metadata.workspaceMemoryWritable === false + ? "workspace memory is read-only for this agent; harvest and sweep refused" + : "workspace memory write policy is unknown for this epoch; harvest refused (fail closed)"; + await this.recordRefusedHarvest(metadata, reason); + return Err(reason); } const boundaryRunKey = `${metadata.workspaceId}:${metadata.summaryMessageId}`; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index e112aa98c08..9f845f07d45 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -26,8 +26,10 @@ import { import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; +import { createRefinementRollbackTool } from "./tools/refinement_rollback"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; -import { TestTempDir } from "./tools/testHelpers"; +import { TestTempDir, mockToolCallOptions } from "./tools/testHelpers"; function pathExists(target: string): Promise { return fsPromises.access(target).then( @@ -1215,6 +1217,44 @@ describe("MemoryService", () => { expect(await pathExists(shared)).toBe(false); }); + it("the refinement_rollback tool refuses memory rollbacks into a read-only scope", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const [row] = await readRefinementEvents(childSessionDir); + const physical = path.join(ownerSessionDir, "memory", "n.md"); + + const makeTool = (access: MemoryScopeAccess) => + createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + memory: { service: fixture.service, ctx: fixture.ctx, access }, + }); + const run = async (access: MemoryScopeAccess) => + (await makeTool(access).execute!({ id: row.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + + // Explore-like agent: workspace scope is read-only → the rollback (a + // write into the owner's shared notebook) is refused before the engine. + const refused = await run({ global: "read", project: "read", workspace: "read" }); + expect(refused.success).toBe(false); + expect(refused.error).toContain("read-only"); + expect(await pathExists(physical)).toBe(true); + + const allowed = await run({ + global: "readwrite", + project: "readwrite", + workspace: "readwrite", + }); + expect(allowed.success).toBe(true); + expect(await pathExists(physical)).toBe(false); + }); + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1228,7 +1268,7 @@ describe("MemoryService", () => { path.join(fixture.xumHome, "elsewhere", "x.md"), ownerMemory, // the root itself is not a file inside the scope ]); - expect(events.map((event) => [event.scope, event.path, event.workspaceId])).toEqual([ + expect(events.map((event) => [event.scope, event.path, event.workspaceId]).sort()).toEqual([ ["global", "/memories/global", "ws-owner"], ["workspace", "/memories/workspace", "ws-owner"], ]); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 74417f02b2a..43c7d09053a 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1074,6 +1074,20 @@ export class MemoryService extends EventEmitter { */ notifyExternalMutation(ctx: MemoryScopeContext, physicalPaths: readonly string[]): void { const touched = new Set(); + for (const physicalPath of physicalPaths) { + const scope = this.scopeOfPhysicalPath(ctx, physicalPath); + if (scope !== null) touched.add(scope); + } + for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); + } + + /** + * The memory scope whose root (for this context) contains `physicalPath`, + * or null when the path lies outside every available scope root. Lets + * out-of-band writers (refinement rollback) apply the same per-scope write + * policy the memory tool enforces. + */ + scopeOfPhysicalPath(ctx: MemoryScopeContext, physicalPath: string): MemoryScope | null { for (const scope of MEMORY_SCOPES) { let root: string; try { @@ -1082,12 +1096,10 @@ export class MemoryService extends EventEmitter { if (error instanceof MemoryCommandError) continue; // scope unavailable in this context throw error; } - for (const physicalPath of physicalPaths) { - const rel = path.relative(root, path.resolve(physicalPath)); - if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) touched.add(scope); - } + const rel = path.relative(root, path.resolve(physicalPath)); + if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) return scope; } - for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); + return null; } /** diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index ec4cd7dd991..649fdc25022 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -12,17 +12,6 @@ import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJour import { appendRefinementEventOrThrow, type RefinementInverseDraft } from "./refinementJournal"; import { listRefinements } from "./refinementRollback"; -/** Removal must abort: a live shared-memory row could not be persisted in the owner's journal. */ -export class SharedMemoryRowMigrationError extends Error { - constructor(workspaceId: string, options?: ErrorOptions) { - super( - `Could not preserve ${workspaceId}'s shared-memory refinement rows in its owner's journal; removal aborted`, - options - ); - this.name = "SharedMemoryRowMigrationError"; - } -} - function inversePaths(inverse: RefinementInverse): string[] { switch (inverse.op) { case "delete-files": diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 58d229fb89a..36ff8c93e12 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -37,6 +37,7 @@ import type { PTCExecutionResult } from "@/node/services/ptc/types"; import { sandboxHostService, type SandboxMount } from "@/node/services/sandbox/sandboxHostService"; import { createRefinementRollbackTool } from "@/node/services/tools/refinement_rollback"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; import type { KernelFileLoader } from "@/node/services/tools/kernelFileLoad"; import { log } from "./log"; import type { MCPWorkspaceStats } from "@/node/services/mcpServerManager"; @@ -122,7 +123,7 @@ export interface ApplyToolPolicyAndExperimentsOptions { /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; /** Lets refinement_rollback announce its direct-to-disk memory writes. */ - memory?: { service: MemoryService; ctx: MemoryScopeContext }; + memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; kernelFileLoader?: KernelFileLoader; }; /** diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index ecafdb13f42..6d9a4a5c97f 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -2,7 +2,9 @@ import { tool, type Tool } from "ai"; import type { RefinementRollbackToolResult } from "@/common/types/tools"; import { TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; -import { rollbackRefinement } from "@/node/services/refinement/refinementRollback"; +import { listRefinements, rollbackRefinement } from "@/node/services/refinement/refinementRollback"; +import { RefinementInverseSchema } from "@/common/types/refinement"; +import type { MemoryScopeAccess } from "@/common/constants/memory"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; interface RefinementRollbackToolArgs { @@ -18,13 +20,48 @@ interface RefinementRollbackToolArgs { * No force parameter on purpose: divergence overrides are a human decision * (debug CLI --force). The model gets the refusal text and can report it. */ +/** + * Policy gate for memory rows: every path the target row's inverse would touch + * must lie in a scope this agent may write. Unknown rows/inverses fall through + * (null) so the engine produces its canonical refusal. + */ +async function refuseReadOnlyMemoryRollback( + sessionDir: string, + id: string, + memory: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess } +): Promise { + const row = (await listRefinements(sessionDir)).find((candidate) => candidate.id === id); + if (row?.data.kind !== "memory") return null; + const inverse = RefinementInverseSchema.safeParse(row.data.inverse); + if (!inverse.success) return null; + const paths = + inverse.data.op === "delete-files" + ? inverse.data.paths + : inverse.data.op === "rename" + ? [inverse.data.from, inverse.data.to] + : [...inverse.data.files.map((file) => file.path), ...(inverse.data.deletePaths ?? [])]; + for (const physicalPath of paths) { + const scope = memory.service.scopeOfPhysicalPath(memory.ctx, physicalPath); + if (scope !== null && memory.access[scope] !== "readwrite") { + return `The ${scope} memory scope is read-only for this agent; rolling back '${id}' would write into it.`; + } + } + return null; +} + export function createRefinementRollbackTool(ctx: { workspaceId: string; sessionDir: string; /** Owner session dir when this workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; - /** Announces rolled-back memory files so shared-store readers refresh. */ - memory?: { service: MemoryService; ctx: MemoryScopeContext }; + /** + * Memory integration: announces rolled-back memory files so shared-store + * readers refresh, and applies the agent's per-scope write policy — a + * rollback is a write into the scope, so a read-only scope (e.g. the shared + * workspace notebook for an explore-like sub-agent) refuses it, exactly as + * the memory tool would. + */ + memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; }): Tool { return tool({ description: TOOL_DEFINITIONS.refinement_rollback.description, @@ -33,6 +70,10 @@ export function createRefinementRollbackTool(ctx: { { id, reason }: RefinementRollbackToolArgs, { toolCallId } ): Promise => { + if (ctx.memory !== undefined) { + const refusal = await refuseReadOnlyMemoryRollback(ctx.sessionDir, id, ctx.memory); + if (refusal !== null) return { success: false, error: refusal }; + } const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, sharedWorkspaceMemorySessionDir: ctx.sharedWorkspaceMemorySessionDir, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 5b597b375f9..e9efc0b3e71 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2356,6 +2356,7 @@ export class TurnRequestBuilder { : { service: memoryService, ctx: memoryScopeContextFromToolConfig(toolsForModelConfig), + access: memoryAccess, }; const applyPolicyStartedAt = Date.now(); let attemptTools = await applyToolPolicyAndExperiments({ diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index aedc7163da7..13449ad3e43 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -138,10 +138,7 @@ import { TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; -import { - migrateSharedMemoryRefinementRows, - SharedMemoryRowMigrationError, -} from "@/node/services/refinement/sharedMemoryRowMigration"; +import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -4212,11 +4209,30 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } - /** TurnRequestBuilder → session: the agent's workspace-memory write policy for this turn. */ + /** + * TurnRequestBuilder → session: the agent's workspace-memory write policy + * for this turn. Also persisted on the workspace config entry (only when it + * changes) so a harvest completing in a fresh session after a restart can + * still be gated; see onCompactionComplete below. + */ recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): void { ( this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) )?.recordWorkspaceMemoryWritable(writable); + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + if (entry === null || entry.workspace.workspaceMemoryWritable === writable) return; + this.config + .editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, workspaceId); + if (current !== null) current.workspace.workspaceMemoryWritable = writable; + return cfg; + }) + .catch((error: unknown) => { + log.warn("Failed to persist workspace memory write policy", { + workspaceId, + error: getErrorMessage(error), + }); + }); } /** Transfer destructive cleanup out of a callback that still owns a session lease. */ @@ -4312,7 +4328,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.schedulePostCompactionMetadataRefresh(workspaceId); // Compaction marks a long session with accumulated learnings: harvest // the compacted epoch first, then let Dream sweep/merge the candidates. - this.memoryConsolidationService?.triggerHarvestThenSweepInBackground(metadata); + // The session knows the policy only if it built a normal turn; after a + // restart/recovery fall back to the persisted value. Still unknown → + // the harvest fails closed. + const persistedWritable = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId) + ?.workspace.workspaceMemoryWritable; + this.memoryConsolidationService?.triggerHarvestThenSweepInBackground({ + ...metadata, + workspaceMemoryWritable: metadata.workspaceMemoryWritable ?? persistedWritable, + }); }, onIdleCompactionOutcome: (success) => { // Reports the *persisted* idle-compaction outcome (success only after the summary @@ -5948,6 +5972,54 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await clearPendingBranchSummary(workspaceId); await this.refinePassCanceller?.cancelInFlightRefinePass(workspaceId); + // Shared workspace memory (sub-agents write into their task-tree + // owner's store): the fallible bookkeeping runs HERE, before any + // destructive step, so an abort leaves a fully intact, retryable + // workspace rather than a config entry whose checkout is gone. + // - pin the owner on surviving descendants (their parent chain is + // about to lose this node); + // - copy this workspace's live shared-memory refinement rows into + // the owner's journal (the only durable inverse once the session + // dir is deleted). + const sharedMemoryOwnerId = resolveWorkspaceMemoryOwnerId( + this.config.loadConfigOrDefault(), + workspaceId + ); + if (sharedMemoryOwnerId !== workspaceId) { + try { + await this.config.editConfig((cfg) => { + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if ( + workspace.parentWorkspaceId === workspaceId && + !workspace.memoryOwnerWorkspaceId + ) { + workspace.memoryOwnerWorkspaceId = sharedMemoryOwnerId; + } + } + } + return cfg; + }); + await migrateSharedMemoryRefinementRows({ + childSessionDir: path.join(this.config.sessionsDir, workspaceId), + childWorkspaceId: workspaceId, + ownerSessionDir: path.join(this.config.sessionsDir, sharedMemoryOwnerId), + ownerWorkspaceId: sharedMemoryOwnerId, + }); + } catch (error) { + if (!force) { + return Err( + `Failed to hand this sub-agent's shared workspace memory over to its owner (${getErrorMessage(error)}); the workspace was left intact — retry the removal` + ); + } + log.warn("Forced removal: shared-memory handover to the owner failed", { + workspaceId, + sharedMemoryOwnerId, + error: getErrorMessage(error), + }); + } + } + if (isMultiProject(metadata)) { const projects = getProjects(metadata); const deleteErrors: string[] = []; @@ -6255,7 +6327,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // already cancelled before the usage rollup above). Retryable harvest // records are finalized too: their transcript goes with the session. await this.memoryConsolidationService?.cancelInFlightConsolidation(workspaceId); - await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); // Cancel and drain any background branch-summary writer BEFORE deleting // the session directory: a mid-flight append could otherwise recreate @@ -6332,50 +6403,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // A sub-agent's workspace memory lives in its task-tree owner's // session dir; hold that store's lock as well so an admitted child // write cannot slip between this tombstone and its commit check. + // (Owner pinning and refinement-row migration ran before runtime + // deletion, where an abort still leaves an intact workspace.) const memoryOwnerId = resolveWorkspaceMemoryOwnerId( this.config.loadConfigOrDefault(), workspaceId ); - if (memoryOwnerId !== workspaceId) { - // Descendants that outlive this intermediate node would otherwise - // lose their path to the root: pin the owner on them first. - await this.config.editConfig((cfg) => { - for (const project of cfg.projects.values()) { - for (const workspace of project.workspaces) { - if ( - workspace.parentWorkspaceId === workspaceId && - !workspace.memoryOwnerWorkspaceId - ) { - workspace.memoryOwnerWorkspaceId = memoryOwnerId; - } - } - } - return cfg; - }); - // The child's memory edits live on in the owner's store; keep their - // audit trail / rollback IDs there too before the journal is deleted. - try { - await migrateSharedMemoryRefinementRows({ - childSessionDir: sessionDir, - childWorkspaceId: workspaceId, - ownerSessionDir: path.join(this.config.sessionsDir, memoryOwnerId), - ownerWorkspaceId: memoryOwnerId, - }); - } catch (error) { - // The child's journal is the only copy of these rows: do not - // delete it. Abort (workspace stays registered, retryable) unless - // the caller forces removal, in which case the audit trail is - // knowingly given up. - if (!force) { - throw new SharedMemoryRowMigrationError(workspaceId, { cause: error }); - } - log.warn("Forced removal: shared-memory refinement rows could not be migrated", { - workspaceId, - memoryOwnerId, - error: getErrorMessage(error), - }); - } - } await removeSessionDirUnderMemoryLocks({ rootDir: this.config.rootDir, sessionDir, @@ -6386,6 +6419,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ? undefined : path.join(this.config.sessionsDir, memoryOwnerId), }); + // Only once the session (and with it the transcript) is gone are the + // retryable harvest records truly unrecoverable; an aborted removal + // above must leave them retryable. + await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); } catch (error) { // r63: without a durable tombstone the retained orphan stays // writable by foreign backends forever — abort the removal (the @@ -6393,8 +6430,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // to deregistration below. if ( error instanceof TombstoneNotDurableError || - error instanceof SharedMemoryLockUnavailableError || - error instanceof SharedMemoryRowMigrationError + error instanceof SharedMemoryLockUnavailableError ) { throw error; } From 0cc6910a08fbb475f92667b4e15bb170b1a56228 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 17:06:53 +0000 Subject: [PATCH 15/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirteent?= =?UTF-8?q?h=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared-memory refinement rows are now copied to the owner INSIDE the removal target locks (beforeTombstone hook), so a child write landing after any earlier scan is still captured, nothing is committed to the owner before the point of no return, and the phantom (metadata-less) path is covered; a hook failure aborts removal (SharedMemoryRemovalAbortedError, renamed from the lock-only error). - Owner pinning on descendants stays before runtime deletion (safe, idempotent). - Workspace-memory write policy persistence is awaited by the request builder and verified by reading the config back; unconfirmed persistence is logged and the harvest gate keeps failing closed. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$9.50`_ --- src/node/services/turnRequestBuilder.ts | 29 +++++-- src/node/services/workspaceRemoval.test.ts | 59 +++++++++++++- src/node/services/workspaceRemoval.ts | 29 +++++-- src/node/services/workspaceService.ts | 94 ++++++++++++++-------- 4 files changed, 158 insertions(+), 53 deletions(-) diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e9efc0b3e71..490214d493c 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -536,9 +536,13 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { onWorkflowRunStatusChanged?: (event: WorkflowRunStatusChangedEvent) => Promise | void; workflowResultContinuationSender?: WorkflowResultContinuationSender; workspaceHeartbeatService?: ToolConfiguration["workspaceHeartbeatService"]; - /** Receives each normal turn's workspace-memory write policy (see recordWorkspaceMemoryWritable). */ + /** + * Receives each normal turn's workspace-memory write policy and persists it + * (see WorkspaceService.recordWorkspaceMemoryWritable); resolves false when + * the value could not be confirmed durable. + */ workspaceMemoryPolicySink?: { - recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): void; + recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; desktopSessionManager?: DesktopSessionManager; @@ -1424,11 +1428,22 @@ export class TurnRequestBuilder { // behalf; it must honor the same policy the memory tool enforces. The // compaction turn itself runs the "compact" agent, so record only normal // turns' policy (the session attaches it to the compaction completion). - if (!isCompactionRequest) { - this.dependencies.bindings.workspaceMemoryPolicySink?.recordWorkspaceMemoryWritable( - workspaceId, - memoryAccess.workspace === "readwrite" - ); + if (!isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink) { + // Awaited (a config write happens only when the value changes) so the + // durable policy is in place before this turn can produce a compaction. + const persisted = + await this.dependencies.bindings.workspaceMemoryPolicySink.recordWorkspaceMemoryWritable( + workspaceId, + memoryAccess.workspace === "readwrite" + ); + if (!persisted) { + log.warn( + "Workspace memory write policy could not be persisted; harvests will fail closed", + { + workspaceId, + } + ); + } } const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 771b287269f..7b1c7a8a63e 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -16,7 +16,7 @@ import { refineApplyLockPath, REMOVAL_TOMBSTONE_HEAL_MIN_AGE_MS, removeSessionDirUnderMemoryLocks, - SharedMemoryLockUnavailableError, + SharedMemoryRemovalAbortedError, rollbackRemovalTombstoneIfOwned, TombstoneNotDurableError, workspaceRemovalTombstonePath, @@ -138,6 +138,59 @@ describe("workspaceRemoval", () => { ).toBe(true); }); + test("runs beforeTombstone under the locks and aborts a shared-store removal when it throws", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const ownerSessionDir = path.join(rootDir, "sessions", "ws-owner"); + const childId = "ws-child-hook"; + const childSessionDir = path.join(rootDir, "sessions", childId); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory"), { recursive: true }); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + + // The hook observes the locked section: the owner store lock is held, so a + // concurrent writer cannot enter while it runs. + let writerRanDuringHook = false; + let thrown: unknown; + try { + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir: childSessionDir, + workspaceId: childId, + attemptId: "test-attempt", + sharedWorkspaceMemorySessionDir: ownerSessionDir, + beforeTombstone: async () => { + const writer = withTargetMutationLock( + rootDir, + path.join(ownerSessionDir, "memory"), + () => { + writerRanDuringHook = true; + return Promise.resolve(); + } + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(writerRanDuringHook).toBe(false); + // Let the writer settle later (after the locks release) and abort. + void writer; + throw new Error("migration failed"); + }, + }); + } catch (error) { + thrown = error; + } + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); + expect(String(thrown)).toContain("migration failed"); + expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); + expect( + await fsPromises.access(childSessionDir).then( + () => true, + () => false + ) + ).toBe(true); + // The queued writer runs once removal released the locks. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(writerRanDuringHook).toBe(true); + }); + test("sub-agent removal aborts (no tombstone) when the owner store lock cannot be acquired", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); @@ -170,7 +223,7 @@ describe("workspaceRemoval", () => { } catch (error) { thrown = error; } - expect(thrown).toBeInstanceOf(SharedMemoryLockUnavailableError); + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); // Unlike the own-store orphan path, NO tombstone is published: the // wedged holder targets the owner's live notebook, so the child must // stay registered and removal be retried. @@ -213,7 +266,7 @@ describe("workspaceRemoval", () => { } catch (error) { thrown = error; } - expect(thrown).toBeInstanceOf(SharedMemoryLockUnavailableError); + expect(thrown).toBeInstanceOf(SharedMemoryRemovalAbortedError); expect(await isWorkspaceRemovalTombstoned(rootDir, childId)).toBe(false); } finally { await historyLock[Symbol.asyncDispose](); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 5cc4b2eded1..29761d8af4c 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -50,18 +50,22 @@ import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; * Keeping the workspace registered keeps removal retryable instead. */ /** - * A sub-agent's removal could not take its memory owner's store lock (r61 - * for shared stores). The caller must ABORT the removal — no tombstone, no - * deregistration — because the orphan fallback would leave an admitted child - * write free to land in the owner's live notebook after removal. + * A sub-agent's removal failed before publishing its tombstone under its + * memory owner's store lock (lock unavailable, history lock timeout, or the + * in-lock shared-row migration failing). The caller must ABORT the removal — + * no tombstone, no deregistration — because the orphan fallback would leave + * an admitted child write free to land in the owner's live notebook after + * removal, or delete the only copy of a shared-memory refinement row. */ -export class SharedMemoryLockUnavailableError extends Error { +export class SharedMemoryRemovalAbortedError extends Error { constructor(workspaceId: string, options?: ErrorOptions) { super( - `Could not lock the shared workspace-memory store while removing ${workspaceId}; removal aborted`, + `Removing ${workspaceId} failed before its tombstone could be published under the shared workspace-memory store lock (${ + options?.cause instanceof Error ? options.cause.message : String(options?.cause) + }); removal aborted and can be retried`, options ); - this.name = "SharedMemoryLockUnavailableError"; + this.name = "SharedMemoryRemovalAbortedError"; } } @@ -165,6 +169,14 @@ export async function removeSessionDirUnderMemoryLocks(args: { * either commits before the tombstone or re-checks and refuses. */ sharedWorkspaceMemorySessionDir?: string; + /** + * Runs INSIDE the target locks, immediately before the tombstone is + * published: shared-memory refinement rows are copied to the owner here, + * so a child write that landed after any earlier scan (another backend) is + * still captured — the locks guarantee no further write can slip in. A + * throw aborts the removal (see SharedMemoryRemovalAbortedError). + */ + beforeTombstone?: () => Promise; }): Promise { assert(args.sessionDir.length > 0, "removeSessionDirUnderMemoryLocks requires a session dir"); // Crash clearly on a malformed config (test stubs, future refactors): an @@ -245,6 +257,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { timeoutMs: 10_000, label: "history write lock (removal)", }); + await args.beforeTombstone?.(); // Tombstone BEFORE rm: once the locks release, any waiting writer // re-checks it pre-commit (inside its own lock) and refuses, so the // deleted directory cannot be recreated by a late mutation or @@ -263,7 +276,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { // passed its commit check — or a new writer — finish after removal. // Abort instead: the workspace stays registered and removal is retried. if (ownerMemoryKeys.length > 0 && !tombstonePublishedUnderLocks) { - throw new SharedMemoryLockUnavailableError(args.workspaceId, { cause: error }); + throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); } // Fail-closed orphan path (r62): a wedged writer blocks the deletion, // but the caller proceeds to deregister the workspace regardless — so diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 13449ad3e43..2082c69613c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -131,7 +131,7 @@ import { import { healRemovalTombstonesForRegisteredWorkspaces, removeSessionDirUnderMemoryLocks, - SharedMemoryLockUnavailableError, + SharedMemoryRemovalAbortedError, refineApplyLockPath, rollbackRemovalTombstoneIfOwned, startRemovalTombstoneLease, @@ -4213,26 +4213,44 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * TurnRequestBuilder → session: the agent's workspace-memory write policy * for this turn. Also persisted on the workspace config entry (only when it * changes) so a harvest completing in a fresh session after a restart can - * still be gated; see onCompactionComplete below. + * still be gated; see onCompactionComplete. Awaited by the builder and + * VERIFIED by reading the config back: Config swallows write failures, and + * a stale persisted `true` would let a now read-only agent's transcript + * harvest into the shared notebook after a restart. Resolves false when the + * new value could not be confirmed durable. */ - recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): void { + async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { ( this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) )?.recordWorkspaceMemoryWritable(writable); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); - if (entry === null || entry.workspace.workspaceMemoryWritable === writable) return; - this.config - .editConfig((cfg) => { + if (entry === null) return false; + if (entry.workspace.workspaceMemoryWritable === writable) return true; + try { + await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); if (current !== null) current.workspace.workspaceMemoryWritable = writable; return cfg; - }) - .catch((error: unknown) => { - log.warn("Failed to persist workspace memory write policy", { - workspaceId, - error: getErrorMessage(error), - }); }); + } catch (error: unknown) { + log.error("Failed to persist workspace memory write policy", { + workspaceId, + writable, + error: getErrorMessage(error), + }); + return false; + } + const persisted = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace + .workspaceMemoryWritable; + if (persisted !== writable) { + log.error("Workspace memory write policy did not persist (config write swallowed?)", { + workspaceId, + writable, + persisted, + }); + return false; + } + return true; } /** Transfer destructive cleanup out of a callback that still owns a session lease. */ @@ -5973,14 +5991,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await this.refinePassCanceller?.cancelInFlightRefinePass(workspaceId); // Shared workspace memory (sub-agents write into their task-tree - // owner's store): the fallible bookkeeping runs HERE, before any - // destructive step, so an abort leaves a fully intact, retryable - // workspace rather than a config entry whose checkout is gone. - // - pin the owner on surviving descendants (their parent chain is - // about to lose this node); - // - copy this workspace's live shared-memory refinement rows into - // the owner's journal (the only durable inverse once the session - // dir is deleted). + // owner's store): pin the owner on surviving descendants HERE, before + // any destructive step — their parent chain is about to lose this + // node — so an abort leaves a fully intact, retryable workspace. The + // refinement-row handover itself happens under the removal locks + // below (the point of no return), where no further child write can + // slip past the scan. Idempotent on retry. const sharedMemoryOwnerId = resolveWorkspaceMemoryOwnerId( this.config.loadConfigOrDefault(), workspaceId @@ -6000,19 +6016,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } return cfg; }); - await migrateSharedMemoryRefinementRows({ - childSessionDir: path.join(this.config.sessionsDir, workspaceId), - childWorkspaceId: workspaceId, - ownerSessionDir: path.join(this.config.sessionsDir, sharedMemoryOwnerId), - ownerWorkspaceId: sharedMemoryOwnerId, - }); } catch (error) { if (!force) { return Err( - `Failed to hand this sub-agent's shared workspace memory over to its owner (${getErrorMessage(error)}); the workspace was left intact — retry the removal` + `Failed to pin the shared memory owner on this sub-agent's descendants (${getErrorMessage(error)}); the workspace was left intact — retry the removal` ); } - log.warn("Forced removal: shared-memory handover to the owner failed", { + log.warn("Forced removal: could not pin the shared memory owner on descendants", { workspaceId, sharedMemoryOwnerId, error: getErrorMessage(error), @@ -6402,22 +6412,36 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // from under a live commit. // A sub-agent's workspace memory lives in its task-tree owner's // session dir; hold that store's lock as well so an admitted child - // write cannot slip between this tombstone and its commit check. - // (Owner pinning and refinement-row migration ran before runtime - // deletion, where an abort still leaves an intact workspace.) + // write cannot slip between this tombstone and its commit check, and + // copy the child's live shared-memory refinement rows to the owner + // inside those locks (the only durable inverse once the session dir + // is deleted). Resolved from persisted config, so the phantom + // (metadata-less) path is covered too. const memoryOwnerId = resolveWorkspaceMemoryOwnerId( this.config.loadConfigOrDefault(), workspaceId ); + const ownerSessionDir = + memoryOwnerId === workspaceId + ? undefined + : path.join(this.config.sessionsDir, memoryOwnerId); await removeSessionDirUnderMemoryLocks({ rootDir: this.config.rootDir, sessionDir, workspaceId, attemptId: removalAttemptId, - sharedWorkspaceMemorySessionDir: - memoryOwnerId === workspaceId + sharedWorkspaceMemorySessionDir: ownerSessionDir, + beforeTombstone: + ownerSessionDir === undefined ? undefined - : path.join(this.config.sessionsDir, memoryOwnerId), + : async () => { + await migrateSharedMemoryRefinementRows({ + childSessionDir: sessionDir, + childWorkspaceId: workspaceId, + ownerSessionDir, + ownerWorkspaceId: memoryOwnerId, + }); + }, }); // Only once the session (and with it the transcript) is gone are the // retryable harvest records truly unrecoverable; an aborted removal @@ -6430,7 +6454,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // to deregistration below. if ( error instanceof TombstoneNotDurableError || - error instanceof SharedMemoryLockUnavailableError + error instanceof SharedMemoryRemovalAbortedError ) { throw error; } From 1a0234d68253da8558a86159fe7baa94ec1982e5 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 17:27:23 +0000 Subject: [PATCH 16/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fourteent?= =?UTF-8?q?h=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared-memory handover is two-pass: the bulk pass runs before any destructive teardown (abort leaves the workspace intact) and a delta pass runs under the removal locks; descendant owner pins are verified by reading the config back. - Inverse-blob quota retention orders rows by source time, so migrated (older) payloads cannot evict the owner's recent rollback data. - A turn is refused when a read-only workspace-memory policy cannot be persisted (a stale durable `true` would bypass the boundary after restart). --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$10.10`_ --- .../services/refinement/refinementJournal.ts | 34 ++++++++-- src/node/services/turnRequestBuilder.ts | 16 +++++ src/node/services/workspaceService.ts | 64 +++++++++++++------ 3 files changed, 90 insertions(+), 24 deletions(-) diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 84fe46f028a..6d6ceb221b2 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -175,7 +175,17 @@ const reclamationStates = new WeakMap { await journal.withBlobLock(async () => { let state = reclamationStates.get(journal); @@ -189,13 +199,25 @@ export async function reclaimExcessRefinementInverseBlobs( // foreign CLI appended and must be re-derived from the journal. const epoch = journal.blobIndexEpoch; let entries: BlobQuotaEntry[]; - if (state.retainedInverseBlobs !== null && state.retainedEpoch === epoch) { + if ( + state.retainedInverseBlobs !== null && + state.retainedEpoch === epoch && + options?.resweep !== true + ) { entries = [...published, ...state.retainedInverseBlobs]; } else { - // Recovery sweep: walk refinement rows newest-first and re-derive the + // Recovery sweep: walk refinement rows newest-first — by SOURCE time + // (`data.sourceTs ?? ts`, append sequence as tie-breaker), so migrated + // rows sit at their real chronological position — and re-derive the // retained set. Rows never recorded payload sizes, so stat the blobs; // a missing blob was already evicted (or never landed) — skip it. - const events = await journal.read(); + const events = (await journal.read()) + .filter((event) => event.kind === "refinement") + .sort((left, right) => { + const leftTs = left.data.sourceTs ?? left.ts; + const rightTs = right.data.sourceTs ?? right.ts; + return leftTs !== rightTs ? leftTs - rightTs : left.seq - right.seq; + }); entries = []; for (let i = events.length - 1; i >= 0; i--) { const event = events[i]; @@ -302,7 +324,9 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr // lock (reclaim takes it itself; the mutex is non-reentrant). Best-effort: // failure must never fail the mutation this row describes. try { - await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs); + await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs, { + resweep: args.sourceTs !== undefined, + }); } catch (error) { log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); } diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 490214d493c..11688d02038 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1437,6 +1437,22 @@ export class TurnRequestBuilder { memoryAccess.workspace === "readwrite" ); if (!persisted) { + if (memoryAccess.workspace !== "readwrite") { + // A stale persisted `true` would let this now read-only agent's + // transcript harvest into the (shared) workspace notebook after a + // restart. Refuse to run the turn until the deny is durable. + const errorMessage = + "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; + const errorEvent = createErrorEvent(workspaceId, { + messageId: createAssistantMessageId(), + error: errorMessage, + errorType: "unknown", + acpPromptId, + }); + if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); + onPreStartError?.(errorEvent); + return { type: "finished", result: Err({ type: "unknown", raw: errorMessage }) }; + } log.warn( "Workspace memory write policy could not be persisted; harvests will fail closed", { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 2082c69613c..83430e8e4d4 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4224,7 +4224,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) )?.recordWorkspaceMemoryWritable(writable); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); - if (entry === null) return false; + // Unregistered workspace: nothing durable to update and no stale + // permission to invalidate (harvests fail closed on the missing value). + if (entry === null) return true; if (entry.workspace.workspaceMemoryWritable === writable) return true; try { await this.config.editConfig((cfg) => { @@ -5991,38 +5993,62 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await this.refinePassCanceller?.cancelInFlightRefinePass(workspaceId); // Shared workspace memory (sub-agents write into their task-tree - // owner's store): pin the owner on surviving descendants HERE, before - // any destructive step — their parent chain is about to lose this - // node — so an abort leaves a fully intact, retryable workspace. The - // refinement-row handover itself happens under the removal locks - // below (the point of no return), where no further child write can - // slip past the scan. Idempotent on retry. + // owner's store). BEFORE any destructive step — so a failure leaves a + // fully intact, retryable workspace: + // - pin the owner on surviving descendants (their parent chain is + // about to lose this node), verified by reading the config back + // because Config swallows write failures; + // - hand this workspace's live shared-memory refinement rows over to + // the owner's journal (idempotent via `migratedFrom`). A second, + // delta pass runs under the removal locks below so a write that + // lands in between is captured too; that late pass only has the + // few rows appended since this one, keeping the fallible work at + // the point of no return minimal. const sharedMemoryOwnerId = resolveWorkspaceMemoryOwnerId( this.config.loadConfigOrDefault(), workspaceId ); if (sharedMemoryOwnerId !== workspaceId) { try { - await this.config.editConfig((cfg) => { + const pinOwner = (cfg: ReturnType): string[] => { + const pinned: string[] = []; for (const project of cfg.projects.values()) { for (const workspace of project.workspaces) { - if ( - workspace.parentWorkspaceId === workspaceId && - !workspace.memoryOwnerWorkspaceId - ) { - workspace.memoryOwnerWorkspaceId = sharedMemoryOwnerId; + if (workspace.parentWorkspaceId === workspaceId) { + if (!workspace.memoryOwnerWorkspaceId) { + workspace.memoryOwnerWorkspaceId = sharedMemoryOwnerId; + } + if (workspace.id !== undefined) pinned.push(workspace.id); } } } + return pinned; + }; + let pinnedIds: string[] = []; + await this.config.editConfig((cfg) => { + pinnedIds = pinOwner(cfg); return cfg; }); + const persisted = this.config.loadConfigOrDefault(); + for (const id of pinnedIds) { + const entry = findWorkspaceEntry(persisted, id); + if (entry === null || !entry.workspace.memoryOwnerWorkspaceId) { + throw new Error(`memory owner pin for descendant ${id} did not persist`); + } + } + await migrateSharedMemoryRefinementRows({ + childSessionDir: path.join(this.config.sessionsDir, workspaceId), + childWorkspaceId: workspaceId, + ownerSessionDir: path.join(this.config.sessionsDir, sharedMemoryOwnerId), + ownerWorkspaceId: sharedMemoryOwnerId, + }); } catch (error) { if (!force) { return Err( - `Failed to pin the shared memory owner on this sub-agent's descendants (${getErrorMessage(error)}); the workspace was left intact — retry the removal` + `Failed to hand this sub-agent's shared workspace memory over to its owner (${getErrorMessage(error)}); the workspace was left intact — retry the removal` ); } - log.warn("Forced removal: could not pin the shared memory owner on descendants", { + log.warn("Forced removal: shared-memory handover to the owner failed", { workspaceId, sharedMemoryOwnerId, error: getErrorMessage(error), @@ -6413,10 +6439,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // A sub-agent's workspace memory lives in its task-tree owner's // session dir; hold that store's lock as well so an admitted child // write cannot slip between this tombstone and its commit check, and - // copy the child's live shared-memory refinement rows to the owner - // inside those locks (the only durable inverse once the session dir - // is deleted). Resolved from persisted config, so the phantom - // (metadata-less) path is covered too. + // run the delta pass of the refinement-row handover inside those + // locks (rows appended since the pre-teardown pass; the phantom, + // metadata-less path gets its full pass here). Resolved from + // persisted config. const memoryOwnerId = resolveWorkspaceMemoryOwnerId( this.config.loadConfigOrDefault(), workspaceId From 802a4ca42f80f87e93a9564ec96ae54a321bfdc4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 17:28:51 +0000 Subject: [PATCH 17/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20lint=20(nullish=20a?= =?UTF-8?q?ssignment=20/=20optional=20chain=20in=20owner=20pinning)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$10.20`_ --- src/node/services/workspaceService.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 83430e8e4d4..e5864764748 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6015,9 +6015,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { for (const project of cfg.projects.values()) { for (const workspace of project.workspaces) { if (workspace.parentWorkspaceId === workspaceId) { - if (!workspace.memoryOwnerWorkspaceId) { - workspace.memoryOwnerWorkspaceId = sharedMemoryOwnerId; - } + workspace.memoryOwnerWorkspaceId ??= sharedMemoryOwnerId; if (workspace.id !== undefined) pinned.push(workspace.id); } } @@ -6032,7 +6030,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const persisted = this.config.loadConfigOrDefault(); for (const id of pinnedIds) { const entry = findWorkspaceEntry(persisted, id); - if (entry === null || !entry.workspace.memoryOwnerWorkspaceId) { + if (!entry?.workspace.memoryOwnerWorkspaceId) { throw new Error(`memory owner pin for descendant ${id} did not persist`); } } From 4292d171d87d1256a9368c84065e13735a535b4a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 17:56:02 +0000 Subject: [PATCH 18/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifteenth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared-memory row migration re-checks the owner journal for the row's source identity INSIDE the owner's publish lock (new RefinementEmitArgs skipIf), so two backends removing the same child concurrently cannot both copy one row; regression test races two migrations. - MemoryService.resolveWorkspaceMemoryOwnerId with a caller snapshot now resolves purely from that snapshot, skipping the per-call config stat and memo; launch sweep / recovery / change diffing no longer issue one synchronous statSync per workspace. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 45 ++++++++++++++++++- src/node/services/memoryService.ts | 12 ++--- .../services/refinement/refinementJournal.ts | 17 ++++++- .../refinement/sharedMemoryRowMigration.ts | 23 ++++++---- 4 files changed, 81 insertions(+), 16 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 9f845f07d45..a3645cf139a 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "bun:test"; +import { describe, it, expect, spyOn } from "bun:test"; import { MEMORY_MAX_FILES_PER_SCOPE, MEMORY_MAX_FILE_BYTES } from "@/common/constants/memory"; @@ -823,6 +823,24 @@ describe("MemoryService", () => { ); }); + it("resolves from a caller snapshot without touching the config file", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const cfg = fixture.config.loadConfigOrDefault(); + const stamp = spyOn(fixture.config, "configFileStamp"); + const load = spyOn(fixture.config, "loadConfigOrDefault"); + // Bulk passes (launch sweep over every recorded workspace) must not pay + // one synchronous stat per workspace on the main process. + for (const id of ["ws-owner", "ws-child", "ws-grandchild", "ws-solo"]) { + fixture.service.resolveWorkspaceMemoryOwnerId(id, () => cfg); + } + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-grandchild", () => cfg)).toBe( + "ws-owner" + ); + expect(stamp).not.toHaveBeenCalled(); + expect(load).not.toHaveBeenCalled(); + }); + it("keeps a grandchild on the root store via the pinned owner after its parent is removed", async () => { using fixture = await createFixture("ws-grandchild"); await registerTaskTree(fixture); @@ -1167,6 +1185,31 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); }); + it("concurrent migrations of the same child copy each row exactly once", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/a.md", "a", "agent"); + await fixture.service.create(fixture.ctx, "/memories/workspace/b.md", "b", "agent"); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + // Two removals of one child racing (two backends): both unlocked + // pre-filters see an empty owner journal, so only the in-lock check can + // keep the second from appending duplicate rows. + const counts = await Promise.all([migrate(), migrate()]); + expect(counts[0] + counts[1]).toBe(2); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect(ownerRows.map((row) => row.data.migratedFrom).sort()).toEqual( + (await readRefinementEvents(childSessionDir)).map((row) => `ws-child:${row.id}`).sort() + ); + }); + it("migrated rows keep their real order relative to the owner's own later edits", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 43c7d09053a..c538ea857d0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -616,15 +616,18 @@ export class MemoryService extends EventEmitter { /** * Memoized resolveWorkspaceMemoryOwnerId (see memoryWorkspaceOwner.ts). The * config is only loaded on a memo miss. Callers resolving many workspaces - * in one synchronous pass may supply a shared `snapshot`; results derived - * from a caller snapshot are NOT memoized, because the snapshot can predate - * the current file stamp (another backend rewriting config.json mid-pass) - * and would otherwise be cached under the newer stamp. + * in one synchronous pass (launch sweep, recovery, config-change diffing) + * supply a shared `snapshot`, which bypasses the memo entirely: neither the + * per-call config stat (O(n) synchronous statSync on the main process for + * n workspaces) nor memoization apply — the snapshot can predate the current + * file stamp (another backend rewriting config.json mid-pass) and would + * otherwise be cached under the newer stamp. */ resolveWorkspaceMemoryOwnerId( workspaceId: string, snapshot?: () => ReturnType ): string { + if (snapshot !== undefined) return resolveWorkspaceMemoryOwnerId(snapshot(), workspaceId); const stamp = this.config.configFileStamp(); if (stamp !== this.workspaceMemoryOwnerConfigStamp) { this.workspaceMemoryOwnerConfigStamp = stamp; @@ -632,7 +635,6 @@ export class MemoryService extends EventEmitter { } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; - if (snapshot !== undefined) return resolveWorkspaceMemoryOwnerId(snapshot(), workspaceId); const cfg = this.config.loadConfigOrDefault(); const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); // Only positive (registered) results are memoized; the workspace may diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 6d6ceb221b2..08bde764b22 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -86,6 +86,14 @@ export interface RefinementEmitArgs { * which only applies inverses to the host filesystem. */ runtime?: "remote"; + /** + * Runs INSIDE the journal's blob lock before anything is stored; returning + * true skips the append (no blob, no row). Every publisher into a journal + * serializes on that cross-process lock, so a duplicate check performed + * here has no check→append window — two backends removing the same + * sub-agent concurrently cannot both copy one row into the owner's journal. + */ + skipIf?: () => Promise; } /** Shared by the rollback engine to compare current files against `postState`. */ @@ -274,8 +282,9 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { +export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Promise { { assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); @@ -284,7 +293,8 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr // blob lock: a concurrent reclamation pass must never observe the // put→append window (see DurableEventJournal.withBlobLock). let publishedBlobs: BlobQuotaEntry[] = []; - await journal.withBlobLock(async () => { + const appended = await journal.withBlobLock(async () => { + if (args.skipIf !== undefined && (await args.skipIf())) return false; const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); const inverse = resolved.inverse; publishedBlobs = resolved.publishedBlobs; @@ -319,7 +329,9 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), }, }); + return true; }); + if (!appended) return false; // Bound retained inverse payloads per session AFTER releasing the publish // lock (reclaim takes it itself; the mutex is non-reentrant). Best-effort: // failure must never fail the mutation this row describes. @@ -330,6 +342,7 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr } catch (error) { log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); } + return true; } } diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 649fdc25022..9978dd7c116 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -96,12 +96,18 @@ export async function migrateSharedMemoryRefinementRows(args: { }; // Idempotent across retried removals (the child journal survives a // retryable removal failure or a crash before deletion): rows already - // copied are identified by their source identity on the owner side. - const alreadyMigrated = new Set( - (await listRefinements(args.ownerSessionDir)) - .map((row) => row.data.migratedFrom) - .filter((id): id is string => id !== undefined) - ); + // copied are identified by their source identity on the owner side. This + // unlocked read only pre-filters; the authoritative check re-runs inside + // the owner journal's publish lock per row (skipIf below), because two + // backends removing the same child concurrently can both pass this + // pre-filter before either append lands. + const migratedSourceIds = async () => + new Set( + (await listRefinements(args.ownerSessionDir)) + .map((row) => row.data.migratedFrom) + .filter((id): id is string => id !== undefined) + ); + const alreadyMigrated = await migratedSourceIds(); const childJournal = sharedDurableEventJournal(args.childSessionDir); let migrated = 0; for (const row of rows) { @@ -145,7 +151,8 @@ export async function migrateSharedMemoryRefinementRows(args: { const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); const postState = RefinementPostStateSchema.safeParse(row.data.postState); // Throws: this is the only durable copy once the child's journal goes. - await appendRefinementEventOrThrow({ + const appended = await appendRefinementEventOrThrow({ + skipIf: async () => (await migratedSourceIds()).has(migratedFrom), sessionDir: args.ownerSessionDir, workspaceId: args.ownerWorkspaceId, kind: "memory", @@ -165,7 +172,7 @@ export async function migrateSharedMemoryRefinementRows(args: { sourceTs: row.data.sourceTs ?? row.ts, ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); - migrated++; + if (appended) migrated++; } return migrated; } From 8500c5a88e0614ee584f6a805e023e92a7f300f8 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 18:18:04 +0000 Subject: [PATCH 19/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixteenth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared-memory row migration now runs as ONE locked batch on the owner journal: a single dedup read plus every append happen inside the same blob-lock hold (new appendRefinementEventUnderBlobLock / reclaimRefinementInverseBlobsBestEffort split), so migration is linear in journal size and the per-row skipIf rescans are gone. - Owner-memo invalidation on local config changes re-resolves the memoized shared children against the new config and emits ownersInvalidated only for mappings that actually changed; unrelated edits (titles, models, task status) no longer discard live children's memory contexts. The callback also adopts the new file stamp so the next resolve does not repeat the invalidation. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 20 +++ src/node/services/memoryService.ts | 43 +++-- .../services/refinement/refinementJournal.ts | 148 ++++++++-------- .../refinement/sharedMemoryRowMigration.ts | 159 ++++++++++-------- 4 files changed, 219 insertions(+), 151 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index a3645cf139a..9ab12c8e7b1 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1018,6 +1018,26 @@ describe("MemoryService", () => { ).toBe(true); }); + it("ignores config edits that leave the memory topology unchanged", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); + + // Ordinary churn (a retitle) must not make every live child rebuild its + // memory context, and the unchanged mapping stays memoized. + await fixture.config.editConfig((cfg) => { + const project = cfg.projects.get(FIXTURE_PROJECT_PATH)!; + project.workspaces.find((ws) => ws.id === "ws-child")!.title = "renamed"; + return cfg; + }); + expect(invalidated).toEqual([]); + const load = spyOn(fixture.config, "loadConfigOrDefault"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(load).not.toHaveBeenCalled(); + }); + it("re-resolves the owner after an EXTERNAL config rewrite (another backend removed it)", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index c538ea857d0..83f60ed1ee7 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -593,7 +593,12 @@ export class MemoryService extends EventEmitter { // its own store) instead of writing into the tombstoned owner forever. // Local edits notify here; edits by ANOTHER backend (multi-instance) are // caught by the config-file stamp check in resolveWorkspaceMemoryOwnerId. - this.config.onConfigChanged(() => this.invalidateWorkspaceMemoryOwnerMemo()); + // The notification fires after the file write, so adopting the new stamp + // here keeps the next resolve from repeating the invalidation. + this.config.onConfigChanged(() => { + this.workspaceMemoryOwnerConfigStamp = this.config.configFileStamp(); + this.invalidateWorkspaceMemoryOwnerMemo(); + }); } // ------------------------------------------------------------------------- @@ -646,18 +651,36 @@ export class MemoryService extends EventEmitter { } /** - * Drop the owner memo and tell listeners which workspaces were sharing a - * store: their live sessions hold a memory context built from an owner - * that may just have been removed, so core.ts invalidates those caches - * (there is no memory-change event for a removal to ride on). Idempotent - * and cheap when nothing was shared. + * Drop the owner memo and tell listeners which formerly-shared workspaces + * now resolve to a DIFFERENT owner: their live sessions hold a memory + * context built from an owner that was just removed, so core.ts invalidates + * those caches (there is no memory-change event for a removal to ride on). + * + * Most config edits (titles, models, task status) leave the topology alone; + * emitting for those would make every live child rebuild its index and hot + * set from disk on ordinary churn, so the memoized shared children are + * re-resolved against the new config first (one parse, only when something + * was shared) and only real owner changes are reported. Unchanged shared + * mappings are re-memoized on the spot. */ private invalidateWorkspaceMemoryOwnerMemo(): void { - const shared = [...this.workspaceMemoryOwnerById] - .filter(([workspaceId, owner]) => workspaceId !== owner) - .map(([workspaceId]) => workspaceId); + const shared = [...this.workspaceMemoryOwnerById].filter( + ([workspaceId, owner]) => workspaceId !== owner + ); this.workspaceMemoryOwnerById.clear(); - if (shared.length > 0) this.emit("ownersInvalidated", shared); + if (shared.length === 0) return; + const cfg = this.config.loadConfigOrDefault(); + const changed: string[] = []; + for (const [workspaceId, owner] of shared) { + // A non-self owner implies the child is registered (dangling roots + // resolve to self), so re-memoizing an unchanged mapping stays positive-only. + if (resolveWorkspaceMemoryOwnerId(cfg, workspaceId) === owner) { + this.workspaceMemoryOwnerById.set(workspaceId, owner); + } else { + changed.push(workspaceId); + } + } + if (changed.length > 0) this.emit("ownersInvalidated", changed); } /** diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 08bde764b22..1835b37969e 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -86,14 +86,6 @@ export interface RefinementEmitArgs { * which only applies inverses to the host filesystem. */ runtime?: "remote"; - /** - * Runs INSIDE the journal's blob lock before anything is stored; returning - * true skips the append (no blob, no row). Every publisher into a journal - * serializes on that cross-process lock, so a duplicate check performed - * here has no check→append window — two backends removing the same - * sub-agent concurrently cannot both copy one row into the owner's journal. - */ - skipIf?: () => Promise; } /** Shared by the rollback engine to compare current files against `postState`. */ @@ -282,67 +274,87 @@ export async function appendRefinementEvent(args: RefinementEmitArgs): Promise { - { - assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); - assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); - const journal = sharedDurableEventJournal(args.sessionDir); - // Inverse blob puts and the append referencing them run under the journal - // blob lock: a concurrent reclamation pass must never observe the - // put→append window (see DurableEventJournal.withBlobLock). - let publishedBlobs: BlobQuotaEntry[] = []; - const appended = await journal.withBlobLock(async () => { - if (args.skipIf !== undefined && (await args.skipIf())) return false; - const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); - const inverse = resolved.inverse; - publishedBlobs = resolved.publishedBlobs; - // Optional fields are spread conditionally: an explicit `undefined` value - // would fail the JsonValue schema validation on append and drop the row. - const evidence: RefinementEvidence = { - workspaceId: args.workspaceId, - toolName: args.evidence.toolName, - ...(args.evidence.toolCallId !== undefined ? { toolCallId: args.evidence.toolCallId } : {}), - ...(args.evidence.actor !== undefined ? { actor: args.evidence.actor } : {}), - }; - const postState: RefinementPostState | undefined = - args.postFiles !== undefined - ? { - files: args.postFiles.map((file) => ({ - path: file.path, - sha256: sha256Hex(file.content), - })), - } - : args.postState; - await journal.append({ - workspaceId: args.workspaceId, - kind: "refinement", - data: { - kind: args.kind, - action: args.action, - inverse, - evidence, - ...(postState !== undefined ? { postState } : {}), - ...(args.migratedFrom !== undefined ? { migratedFrom: args.migratedFrom } : {}), - ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), - ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), - }, - }); - return true; - }); - if (!appended) return false; - // Bound retained inverse payloads per session AFTER releasing the publish - // lock (reclaim takes it itself; the mutex is non-reentrant). Best-effort: - // failure must never fail the mutation this row describes. - try { - await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs, { - resweep: args.sourceTs !== undefined, - }); - } catch (error) { - log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); - } - return true; +export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Promise { + assert(args.sessionDir.length > 0, "refinement journal requires a session dir"); + const journal = sharedDurableEventJournal(args.sessionDir); + // Inverse blob puts and the append referencing them run under the journal + // blob lock: a concurrent reclamation pass must never observe the + // put→append window (see DurableEventJournal.withBlobLock). + const publishedBlobs = await journal.withBlobLock(() => + appendRefinementEventUnderBlobLock(journal, args) + ); + await reclaimRefinementInverseBlobsBestEffort(journal, publishedBlobs, { + resweep: args.sourceTs !== undefined, + }); +} + +/** + * The locked leg of appendRefinementEventOrThrow, for callers that batch + * several appends (plus their own journal-state checks) under ONE + * `journal.withBlobLock` section — shared-memory row migration dedups + * against the owner journal and copies every row inside the same hold, so + * neither a concurrent duplicate migration nor a reclamation pass can + * interleave. The caller MUST hold `journal`'s blob lock (asserted) and MUST + * hand the returned payload entries to + * reclaimRefinementInverseBlobsBestEffort after releasing it. + */ +export async function appendRefinementEventUnderBlobLock( + journal: DurableEventJournal, + args: RefinementEmitArgs +): Promise { + assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); + await journal.assertBlobLockOwned(); + const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); + const inverse = resolved.inverse; + // Optional fields are spread conditionally: an explicit `undefined` value + // would fail the JsonValue schema validation on append and drop the row. + const evidence: RefinementEvidence = { + workspaceId: args.workspaceId, + toolName: args.evidence.toolName, + ...(args.evidence.toolCallId !== undefined ? { toolCallId: args.evidence.toolCallId } : {}), + ...(args.evidence.actor !== undefined ? { actor: args.evidence.actor } : {}), + }; + const postState: RefinementPostState | undefined = + args.postFiles !== undefined + ? { + files: args.postFiles.map((file) => ({ + path: file.path, + sha256: sha256Hex(file.content), + })), + } + : args.postState; + await journal.append({ + workspaceId: args.workspaceId, + kind: "refinement", + data: { + kind: args.kind, + action: args.action, + inverse, + evidence, + ...(postState !== undefined ? { postState } : {}), + ...(args.migratedFrom !== undefined ? { migratedFrom: args.migratedFrom } : {}), + ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), + ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), + }, + }); + return resolved.publishedBlobs; +} + +/** + * Bound retained inverse payloads per session AFTER the publish lock is + * released (reclaim takes it itself; the mutex is non-reentrant). Best-effort: + * failure must never fail the mutation the row(s) describe. + */ +export async function reclaimRefinementInverseBlobsBestEffort( + journal: DurableEventJournal, + publishedBlobs: BlobQuotaEntry[], + options: { resweep: boolean } +): Promise { + try { + await reclaimExcessRefinementInverseBlobs(journal, publishedBlobs, options); + } catch (error) { + log.debug("[refinement] inverse blob reclamation failed; continuing", { error }); } } diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 9978dd7c116..3c7e8ff91f5 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -8,8 +8,13 @@ import { type RefinementInverse, } from "@/common/types/refinement"; import { log } from "@/node/services/log"; +import type { BlobQuotaEntry } from "@/node/utils/journal/blobReclamation"; import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; -import { appendRefinementEventOrThrow, type RefinementInverseDraft } from "./refinementJournal"; +import { + appendRefinementEventUnderBlobLock, + reclaimRefinementInverseBlobsBestEffort, + type RefinementInverseDraft, +} from "./refinementJournal"; import { listRefinements } from "./refinementRollback"; function inversePaths(inverse: RefinementInverse): string[] { @@ -94,85 +99,93 @@ export async function migrateSharedMemoryRefinementRows(args: { } return depth % 2 === 0; }; - // Idempotent across retried removals (the child journal survives a - // retryable removal failure or a crash before deletion): rows already - // copied are identified by their source identity on the owner side. This - // unlocked read only pre-filters; the authoritative check re-runs inside - // the owner journal's publish lock per row (skipIf below), because two - // backends removing the same child concurrently can both pass this - // pre-filter before either append lands. - const migratedSourceIds = async () => - new Set( + const childJournal = sharedDurableEventJournal(args.childSessionDir); + const ownerJournal = sharedDurableEventJournal(args.ownerSessionDir); + let migrated = 0; + const publishedBlobs: BlobQuotaEntry[] = []; + // One hold of the owner journal's publish lock for the whole batch: the + // dedup read and every append happen inside it, so a second backend + // removing the same child concurrently (or a retried removal — the child + // journal survives a retryable failure or a crash before deletion) sees the + // copied rows before it decides, and the owner journal is read once rather + // than once per row. Rows already copied are identified by their source + // identity on the owner side. + await ownerJournal.withBlobLock(async () => { + const alreadyMigrated = new Set( (await listRefinements(args.ownerSessionDir)) .map((row) => row.data.migratedFrom) .filter((id): id is string => id !== undefined) ); - const alreadyMigrated = await migratedSourceIds(); - const childJournal = sharedDurableEventJournal(args.childSessionDir); - let migrated = 0; - for (const row of rows) { - if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined) continue; - if (isLive(row.id) !== true) continue; - const migratedFrom = `${args.childWorkspaceId}:${row.id}`; - if (alreadyMigrated.has(migratedFrom)) continue; - const inverse = RefinementInverseSchema.safeParse(row.data.inverse); - const action = MemoryRefinementActionSchema.safeParse(row.data.action); - if (!inverse.success || !action.success) continue; - if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; + for (const row of rows) { + if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined) continue; + if (isLive(row.id) !== true) continue; + const migratedFrom = `${args.childWorkspaceId}:${row.id}`; + if (alreadyMigrated.has(migratedFrom)) continue; + const inverse = RefinementInverseSchema.safeParse(row.data.inverse); + const action = MemoryRefinementActionSchema.safeParse(row.data.action); + if (!inverse.success || !action.success) continue; + if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; - let draft: RefinementInverseDraft; - if (inverse.data.op === "restore-files") { - const files: Array<{ path: string; content: string }> = []; - for (const file of inverse.data.files) { - // Contents are blob-offloaded at append (resolveRefinementInverse); older - // rows may carry them inline. - const content = - file.text ?? - (file.blobRef === undefined ? null : await childJournal.blobs.getText(file.blobRef)); - if (content === null) break; - files.push({ path: file.path, content }); - } - if (files.length !== inverse.data.files.length) { - log.debug("[refinement] skipping shared-memory row migration: inverse payload missing", { - rowId: row.id, - }); - continue; + let draft: RefinementInverseDraft; + if (inverse.data.op === "restore-files") { + const files: Array<{ path: string; content: string }> = []; + for (const file of inverse.data.files) { + // Contents are blob-offloaded at append (resolveRefinementInverse); older + // rows may carry them inline. + const content = + file.text ?? + (file.blobRef === undefined ? null : await childJournal.blobs.getText(file.blobRef)); + if (content === null) break; + files.push({ path: file.path, content }); + } + if (files.length !== inverse.data.files.length) { + log.debug("[refinement] skipping shared-memory row migration: inverse payload missing", { + rowId: row.id, + }); + continue; + } + draft = { + op: "restore-files", + files, + ...(inverse.data.deletePaths !== undefined + ? { deletePaths: inverse.data.deletePaths } + : {}), + }; + } else { + draft = inverse.data; } - draft = { - op: "restore-files", - files, - ...(inverse.data.deletePaths !== undefined - ? { deletePaths: inverse.data.deletePaths } - : {}), - }; - } else { - draft = inverse.data; + const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); + const postState = RefinementPostStateSchema.safeParse(row.data.postState); + // Throws: this is the only durable copy once the child's journal goes. + publishedBlobs.push( + ...(await appendRefinementEventUnderBlobLock(ownerJournal, { + sessionDir: args.ownerSessionDir, + workspaceId: args.ownerWorkspaceId, + kind: "memory", + action: action.data, + inverse: draft, + evidence: { + toolName: evidence.success ? evidence.data.toolName : "memory", + ...(evidence.success && evidence.data.toolCallId !== undefined + ? { toolCallId: evidence.data.toolCallId } + : {}), + ...(evidence.success && evidence.data.actor !== undefined + ? { actor: evidence.data.actor } + : {}), + }, + ...(postState.success ? { postState: postState.data } : {}), + migratedFrom, + sourceTs: row.data.sourceTs ?? row.ts, + ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), + })) + ); + migrated++; } - const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); - const postState = RefinementPostStateSchema.safeParse(row.data.postState); - // Throws: this is the only durable copy once the child's journal goes. - const appended = await appendRefinementEventOrThrow({ - skipIf: async () => (await migratedSourceIds()).has(migratedFrom), - sessionDir: args.ownerSessionDir, - workspaceId: args.ownerWorkspaceId, - kind: "memory", - action: action.data, - inverse: draft, - evidence: { - toolName: evidence.success ? evidence.data.toolName : "memory", - ...(evidence.success && evidence.data.toolCallId !== undefined - ? { toolCallId: evidence.data.toolCallId } - : {}), - ...(evidence.success && evidence.data.actor !== undefined - ? { actor: evidence.data.actor } - : {}), - }, - ...(postState.success ? { postState: postState.data } : {}), - migratedFrom, - sourceTs: row.data.sourceTs ?? row.ts, - ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), - }); - if (appended) migrated++; + }); + // Migrated rows carry sourceTs (appended out of chronological order), so + // the quota pass re-derives retention in source order. + if (publishedBlobs.length > 0) { + await reclaimRefinementInverseBlobsBestEffort(ownerJournal, publishedBlobs, { resweep: true }); } return migrated; } From 55e23e33cc37852ecb9b7280c58621186ca333b9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 18:39:06 +0000 Subject: [PATCH 20/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventeen?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The owner memo now records every resolution for the current config stamp, including self-fallbacks (unregistered ID, config.json missing/malformed), and invalidation re-resolves each memoized workspace and emits ownersInvalidated for every mapping that changed. A child that built its memory context on its private store while config.json was unreadable is therefore told when ownership recovers, instead of staying on the wrong store until the next unrelated invalidation. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 21 ++++++++++ src/node/services/memoryService.ts | 56 ++++++++++--------------- 2 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 9ab12c8e7b1..7d6c7c373f8 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1064,6 +1064,27 @@ describe("MemoryService", () => { expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); }); + it("announces the self→shared transition when config.json recovers after being unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const invalidated: string[][] = []; + fixture.service.on("ownersInvalidated", (ids: string[]) => invalidated.push(ids)); + + // config.json vanishes (another backend mid-rewrite): the child cannot + // resolve its tree and falls back to its private store... + const configFile = path.join(fixture.xumHome, "config.json"); + const parked = `${configFile}.parked`; + await fsPromises.rename(configFile, parked); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + expect(invalidated).toEqual([]); + + // ...and once it is back, sessions that built a context on the fallback + // store must be told, even though no shared mapping was ever memoized. + await fsPromises.rename(parked, configFile); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + expect(invalidated).toEqual([["ws-child"]]); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 83f60ed1ee7..b46e09b8695 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -46,7 +46,6 @@ import { withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; -import { findWorkspaceEntry } from "@/node/services/taskUtils"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; import { REFINEMENT_CAPTURE_MAX_FILES, @@ -609,10 +608,13 @@ export class MemoryService extends EventEmitter { // ------------------------------------------------------------------------- /** - * Positive-only memo for resolveWorkspaceMemoryOwnerId: a workspace's - * parentWorkspaceId is fixed at creation and IDs are never reused, so a - * resolved chain stays valid for the process lifetime. Unknown IDs are not - * cached — the workspace may simply not be registered yet. + * Memo of every owner this process resolved (self-resolutions included), + * valid for one config-file stamp: a workspace's parentWorkspaceId is fixed + * at creation and IDs are never reused, so a mapping can only change through + * a config rewrite, which the stamp check / onConfigChanged catch. Fallback + * observations (unregistered ID, config missing or malformed → self) are + * memoized too, so their recovery to a shared owner is a visible transition + * (see invalidateWorkspaceMemoryOwnerMemo). */ private readonly workspaceMemoryOwnerById = new Map(); /** Config-file stamp (Config.configFileStamp) the memo was built against. */ @@ -640,45 +642,33 @@ export class MemoryService extends EventEmitter { } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; - const cfg = this.config.loadConfigOrDefault(); - const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); - // Only positive (registered) results are memoized; the workspace may - // simply not be registered yet. - if (findWorkspaceEntry(cfg, workspaceId) !== null) { - this.workspaceMemoryOwnerById.set(workspaceId, owner); - } + const owner = resolveWorkspaceMemoryOwnerId(this.config.loadConfigOrDefault(), workspaceId); + this.workspaceMemoryOwnerById.set(workspaceId, owner); return owner; } /** - * Drop the owner memo and tell listeners which formerly-shared workspaces - * now resolve to a DIFFERENT owner: their live sessions hold a memory - * context built from an owner that was just removed, so core.ts invalidates - * those caches (there is no memory-change event for a removal to ride on). + * Re-resolve every memoized workspace against the current config and tell + * listeners which ones now map to a DIFFERENT owner: their live sessions + * hold a memory context built from the previous store (an owner that was + * just removed — or the private fallback store used while config.json was + * missing/malformed and the tree could not be resolved), so core.ts + * invalidates those caches and the memory subscription refreshes. There is + * no memory-change event for either transition to ride on. * * Most config edits (titles, models, task status) leave the topology alone; * emitting for those would make every live child rebuild its index and hot - * set from disk on ordinary churn, so the memoized shared children are - * re-resolved against the new config first (one parse, only when something - * was shared) and only real owner changes are reported. Unchanged shared - * mappings are re-memoized on the spot. + * set from disk on ordinary churn, so only real owner changes are reported. + * One parse, only when something was memoized. */ private invalidateWorkspaceMemoryOwnerMemo(): void { - const shared = [...this.workspaceMemoryOwnerById].filter( - ([workspaceId, owner]) => workspaceId !== owner - ); - this.workspaceMemoryOwnerById.clear(); - if (shared.length === 0) return; + if (this.workspaceMemoryOwnerById.size === 0) return; const cfg = this.config.loadConfigOrDefault(); const changed: string[] = []; - for (const [workspaceId, owner] of shared) { - // A non-self owner implies the child is registered (dangling roots - // resolve to self), so re-memoizing an unchanged mapping stays positive-only. - if (resolveWorkspaceMemoryOwnerId(cfg, workspaceId) === owner) { - this.workspaceMemoryOwnerById.set(workspaceId, owner); - } else { - changed.push(workspaceId); - } + for (const [workspaceId, previousOwner] of this.workspaceMemoryOwnerById) { + const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + this.workspaceMemoryOwnerById.set(workspaceId, owner); + if (owner !== previousOwner) changed.push(workspaceId); } if (changed.length > 0) this.emit("ownersInvalidated", changed); } From df301b39939410d190d3f4278a192a08bc689ebf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 19:27:04 +0000 Subject: [PATCH 21/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighteent?= =?UTF-8?q?h=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - assertMutationCommittable compares the store's bound owner with a fresh resolution and refuses (recoverable) when they differ, so a command that resolved its store as a self-fallback while config.json was unreadable cannot commit into the private store after the file recovered. - Harvest finalization on removal is one read-check-write under the sidecar lock, and saveHarvestRecordEffect never lets a terminal record (completed, or failed with retries exhausted) be reopened by a residual pending/retryable write — only a genuine completion may replace it. - Cross-process cache coherence for the shared workspace store: every workspace-scope mutation rewrites //memory.revision; AgentSession records the token with its cached memory context and revalidates it before reuse (probeMemoryStore), and the Memory tab's existing 30s probe emits a root refresh when the token moved without an in-process change event. Scoped to the owner store; the metadata sidecar is not used so read/usage churn does not invalidate contexts. - The persisted workspaceMemoryWritable bit now mirrors the final toolset: readwrite access AND memory experiment + service AND a policy that keeps the memory tool (isMemoryToolDisabled mirrors applyToolPolicyToNames). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/constants/memory.ts | 8 ++ src/common/utils/tools/toolPolicy.ts | 10 ++ src/node/orpc/routerSubscriptions.test.ts | 1 + src/node/orpc/routerSubscriptions.ts | 47 ++++++-- .../agentSession.memoryContext.test.ts | 44 +++++++- src/node/services/agentSession.ts | 20 +++- src/node/services/aiService.ts | 19 ++-- .../memoryConsolidationService.test.ts | 51 ++++++++- .../services/memoryConsolidationService.ts | 79 ++++++++----- src/node/services/memoryOperations.ts | 12 +- src/node/services/memoryService.test.ts | 57 +++++++++- src/node/services/memoryService.ts | 104 +++++++++++++++--- src/node/services/toolAssembly.test.ts | 26 ++++- .../services/tools/refinement_rollback.ts | 2 +- src/node/services/turnRequestBuilder.ts | 17 ++- 15 files changed, 418 insertions(+), 79 deletions(-) diff --git a/src/common/constants/memory.ts b/src/common/constants/memory.ts index 35b1ad3f9f8..3b7082f0315 100644 --- a/src/common/constants/memory.ts +++ b/src/common/constants/memory.ts @@ -17,6 +17,14 @@ export const MEMORY_VIRTUAL_ROOT = "/memories"; export const MEMORY_SCOPES = ["global", "project", "workspace"] as const; + +/** + * `//memory.revision`: opaque token rewritten on every + * mutation of that owner's shared `/memories/workspace` store. Sessions and + * Memory tabs in OTHER backend processes (multi-instance) compare it before + * reusing a cached index/hot set — in-process consumers get change events. + */ +export const WORKSPACE_MEMORY_REVISION_FILE_NAME = "memory.revision"; export type MemoryScope = (typeof MEMORY_SCOPES)[number]; export type MemoryAccessLevel = "read" | "readwrite"; diff --git a/src/common/utils/tools/toolPolicy.ts b/src/common/utils/tools/toolPolicy.ts index 7368037a675..ef4d50e3fa8 100644 --- a/src/common/utils/tools/toolPolicy.ts +++ b/src/common/utils/tools/toolPolicy.ts @@ -82,3 +82,13 @@ export function applyToolPolicy( export function isSessionHistoryDisabled(policy?: ToolPolicy): boolean { return applyToolPolicyToNames(["session_history"], policy).length === 0; } + +/** + * Whether the effective policy strips the `memory` tool. The persisted + * post-compaction harvest permission must reflect the FINAL toolset, not just + * the agent class: an editing-capable sub-agent whose policy denies memory + * must not have its transcript harvested into the (shared) workspace notebook. + */ +export function isMemoryToolDisabled(policy?: ToolPolicy): boolean { + return applyToolPolicyToNames(["memory"], policy).length === 0; +} diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 8eb45a00f79..5afedd6e514 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -55,6 +55,7 @@ test("memory subscriptions match workspace-scope events on the shared memory own memoryService: Object.assign(memoryService, { resolveWorkspaceMemoryOwnerId: (workspaceId: string) => ownerOf.get(workspaceId) ?? workspaceId, + workspaceMemoryRevision: () => Promise.resolve("rev-1"), }), memoryConsolidationService, } as unknown as ORPCContext; diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 9dc55e2e7d6..90c4330c5b9 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -218,6 +218,23 @@ export function subscribeMemoryChanges( yield* runtimeSubscription(context, { signal, subscribe: (emit) => { + // Revision token of the displayed workspace store (see + // MemoryService.workspaceMemoryRevision), refreshed by every + // workspace event this subscription forwards so the probe below only + // fires for mutations this process never saw. Reads are async; a + // failed refresh leaves the old token, costing at most one redundant + // refresh on the next probe. + let storeRevision: string | null = null; + const refreshStoreRevision = () => { + if (!workspaceId) return; + context.memoryService.workspaceMemoryRevision(workspaceId).then( + (revision) => { + storeRevision = revision; + }, + () => undefined + ); + }; + refreshStoreRevision(); const onChange = (event: MemoryChangeEvent) => { if ( event.scope === "workspace" && @@ -228,6 +245,7 @@ export function subscribeMemoryChanges( ) return; if (event.scope === "project" && event.projectPath !== projectPath) return; + if (event.scope === "workspace") refreshStoreRevision(); emit.push(event); }; const onStatusChange = (event: MemoryConsolidationStatusChangeEventPayload) => @@ -248,14 +266,29 @@ export function subscribeMemoryChanges( emit.push({ kind: "consolidation_status", workspaceId, projectPath: projectPath ?? "" }); }; // ownersInvalidated is emitted lazily, when something probes ownership. - // Another backend (multi-instance) removing the owner leaves an idle - // tab with nothing to trigger that probe, so probe here: one stat of - // config.json per interval (see Config.configFileStamp). + // Another backend (multi-instance) removing the owner — or writing the + // shared store — leaves an idle tab with nothing to trigger that probe + // and no in-process change event, so probe here: one stat of + // config.json plus one small revision read per interval. A foreign + // write shows up as a changed token and synthesizes the same + // root-addressed refresh an ownership change does. const ownershipProbe = workspaceId - ? setInterval( - () => context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), - MEMORY_OWNERSHIP_PROBE_INTERVAL_MS - ).unref() + ? setInterval(() => { + context.memoryService.workspaceMemoryRevision(workspaceId).then( + (revision) => { + if (revision === storeRevision) return; + storeRevision = revision; + emit.push({ + scope: "workspace", + path: toVirtualPath("workspace", ""), + actor: "agent", + workspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), + projectPath: projectPath ?? "", + }); + }, + () => undefined + ); + }, MEMORY_OWNERSHIP_PROBE_INTERVAL_MS).unref() : undefined; context.memoryService.on("change", onChange); context.memoryService.on("ownersInvalidated", onOwnersInvalidated); diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 7313b91d163..153da6969c7 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -32,7 +32,7 @@ function createSession(args: { historyService: HistoryService; sessionDir: string; buildMemorySessionContext: AIService["buildMemorySessionContext"]; - probeMemoryOwnership?: AIService["probeMemoryOwnership"]; + probeMemoryStore?: AIService["probeMemoryStore"]; isExperimentEnabled?: AIService["isExperimentEnabled"]; }): AgentSession { const aiEmitter = new EventEmitter(); @@ -51,7 +51,7 @@ function createSession(args: { ), stopStream: mock(() => Promise.resolve({ success: true as const, data: undefined })), buildMemorySessionContext: args.buildMemorySessionContext, - probeMemoryOwnership: args.probeMemoryOwnership, + probeMemoryStore: args.probeMemoryStore, isExperimentEnabled: args.isExperimentEnabled ?? (() => false), } as unknown as AIService; @@ -165,14 +165,15 @@ describe("AgentSession memory context", () => { // synchronous ownersInvalidated → core.ts → invalidateMemoryContext chain. let ownerRemoved = false; const sessionRef: { current?: AgentSession } = {}; - const probeMemoryOwnership = mock(() => { + const probeMemoryStore = mock(() => { if (ownerRemoved) sessionRef.current?.invalidateMemoryContext(); + return Promise.resolve(undefined); }); const session = createSession({ historyService, sessionDir: path.join(sessionDir.path, WORKSPACE_ID), buildMemorySessionContext, - probeMemoryOwnership, + probeMemoryStore, isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, }); sessionRef.current = session; @@ -181,7 +182,7 @@ describe("AgentSession memory context", () => { await priv.resolveMemoryContext("test-model"); await priv.resolveMemoryContext("test-model"); expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); - expect(probeMemoryOwnership).toHaveBeenCalledTimes(2); + expect(probeMemoryStore).toHaveBeenCalledTimes(2); ownerRemoved = true; // The probe runs before the cache read, so THIS request rebuilds. @@ -192,6 +193,39 @@ describe("AgentSession memory context", () => { } }); + test("rebuilds when the owner store's revision advanced without an in-process change event", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-revision"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + const context: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => Promise.resolve(context)); + // Another backend process writing the shared notebook: no change event + // reaches this session, only the durable revision token differs. + let revision = "rev-1"; + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryStore: () => Promise.resolve(revision), + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + const priv = session as unknown as PrivateSessionAccess; + try { + await priv.resolveMemoryContext("test-model"); + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); + + revision = "rev-2"; + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + // Stable again: the rebuilt context is cached under the new token. + await priv.resolveMemoryContext("test-model"); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + } finally { + await session.dispose(); + } + }); + test("upgrades an index-only memory context when hot memories are requested", async () => { using sessionDir = new DisposableTempDir("agent-session-memory-context-upgrade"); const { historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 7a708a5d983..2cdd8348bb4 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -676,7 +676,7 @@ export interface AgentSessionAIService extends BranchSummaryAiService { replayStream?(workspaceId: string, options?: { afterTimestamp?: number }): Promise; getProvidersConfig(): ProvidersConfigMap | null; isExperimentEnabled(experimentId: ExperimentId): boolean; - probeMemoryOwnership?(workspaceId: string): void; + probeMemoryStore?(workspaceId: string): Promise; buildMemorySessionContext?( workspaceId: string, modelString: string, @@ -747,6 +747,8 @@ interface CachedMemoryContext { tokenBudgetActive: boolean; memoryEnabled: boolean; hotSetEnabled: boolean; + /** Owner store revision (AIService.probeMemoryStore) the context was built from. */ + storeRevision: string | undefined; } interface SendMessageInternalOptions { @@ -9841,11 +9843,15 @@ export class AgentSession { this.aiService.isExperimentEnabled(id); const memoryEnabled = enabled(EXPERIMENT_IDS.MEMORY); const hotSetEnabled = enabled(EXPERIMENT_IDS.MEMORY_HOT_SET); - // Ownership probe first: a removed owner invalidates this cache - // synchronously (see AIService.probeMemoryOwnership), so the lookup below - // never serves an index built from a store this workspace no longer reads. - if (memoryEnabled && typeof this.aiService.probeMemoryOwnership === "function") { - this.aiService.probeMemoryOwnership(this.workspaceId); + // Store probe first: a removed owner invalidates this cache synchronously + // (see AIService.probeMemoryStore), so the lookup below never serves an + // index built from a store this workspace no longer reads; and a store + // revision advanced by ANOTHER backend process (no in-process change + // event) fails the comparison below. Read before the build so a write + // racing the build is caught on the next probe. + let storeRevision: string | undefined; + if (memoryEnabled && typeof this.aiService.probeMemoryStore === "function") { + storeRevision = await this.aiService.probeMemoryStore(this.workspaceId); } const cached = cache.get(modelString); // Policy changes must not retain a previously injected extra (including index-only lookups). @@ -9853,6 +9859,7 @@ export class AgentSession { cached?.tokenBudgetActive === tokenBudgetActive && cached.memoryEnabled === memoryEnabled && cached.hotSetEnabled === hotSetEnabled && + cached.storeRevision === storeRevision && (cached.includesHotMemories || !includeHotMemories) ) { return cached.context ?? undefined; @@ -9875,6 +9882,7 @@ export class AgentSession { tokenBudgetActive, memoryEnabled, hotSetEnabled, + storeRevision, }); return context ?? undefined; } diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 9416f11890e..b43aecadbb8 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -247,15 +247,20 @@ export class AIService extends EventEmitter { } /** - * Re-check who owns this workspace's `/memories/workspace` store. Memoized - * and stamp-validated in MemoryService, so this is one stat per call; when - * another backend removed the owner since the last turn, the resulting + * Re-check who owns this workspace's `/memories/workspace` store and return + * that store's revision token. Ownership is memoized and stamp-validated in + * MemoryService (one stat), the token is one small read; when another + * backend removed the owner since the last turn, the resulting * `ownersInvalidated` event clears the affected sessions' cached context - * synchronously — AgentSession calls this BEFORE consulting its cache so the - * current request, not the next one, rebuilds from the right store. + * synchronously, and when another backend WROTE the shared store (no + * in-process change event) the token differs from the one recorded with the + * cached context. AgentSession calls this BEFORE consulting its cache so the + * current request, not the next one, rebuilds from the right, current store. */ - probeMemoryOwnership(workspaceId: string): void { - this.turnRequestBuilderBindings.memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId); + async probeMemoryStore(workspaceId: string): Promise { + return await this.turnRequestBuilderBindings.memoryService?.workspaceMemoryRevision( + workspaceId + ); } /** diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index f72a352efb3..9669716c68b 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { Effect } from "effect"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -7,7 +8,10 @@ import type { LanguageModelV3CallOptions, LanguageModelV3StreamPart } from "@ai- import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import { createMuxMessage } from "@/common/types/message"; -import type { MemoryConsolidationStatusChangeEventPayload } from "@/common/orpc/schemas/memory"; +import type { + MemoryConsolidationStatusChangeEventPayload, + MemoryHarvestRecordPayload, +} from "@/common/orpc/schemas/memory"; import { MULTI_PROJECT_CONFIG_KEY } from "@/common/constants/multiProject"; import { EXPERIMENT_IDS } from "@/common/constants/experiments"; import { @@ -1226,6 +1230,51 @@ describe("MemoryConsolidationService", () => { ); }); + it("keeps a removal-finalized harvest record terminal against residual retryable writes", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + const boundaryKey = metadata.summaryMessageId; + const base = { + startedAt: Date.now() - 10_000, + attemptCount: 1, + boundaryKey, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + completionMetadata: metadata, + }; + // Residual runs of the bounded cancellation drain record through the same + // path as the live harvest; reach it directly to interleave with finalization. + const save = (record: MemoryHarvestRecordPayload) => + Effect.runPromise( + ( + fixture.service as unknown as { + saveHarvestRecordEffect: ( + workspaceId: string, + boundaryKey: string, + record: MemoryHarvestRecordPayload, + projectPath: string + ) => Effect.Effect; + } + ).saveHarvestRecordEffect("ws-sub", boundaryKey, record, "") + ); + await save({ ...base, status: "pending" }); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + const latest = async () => (await fixture.service.getStatus("ws-sub")).latestHarvestRecord; + expect((await latest())?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + + // A residual retryable failure landing after finalization must not reopen the bucket... + await save({ ...base, status: "failed", completedAt: Date.now(), error: "residual failure" }); + expect((await latest())?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect((await latest())?.error).toContain("workspace removed"); + // ...while a residual completion (its writes really landed) is kept as the truth, + // and finalization never demotes a completed record. + await save({ ...base, status: "completed", completedAt: Date.now(), acceptedCandidates: 1 }); + await fixture.service.finalizeHarvestsForRemoval("ws-sub"); + expect((await latest())?.status).toBe("completed"); + }); + it("normalizes stale max-attempt pending harvest records to failed", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); const metadata = await seedCompactionEpoch(fixture); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 40f3c8289ea..f527888fea4 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -287,6 +287,25 @@ function pruneHarvestRecords(records: Record): void export const HARVEST_MAX_ATTEMPTS = 3; +/** Completed, or failed with retries exhausted: nothing may retry it. */ +function isTerminalHarvestRecord(record: MemoryHarvestRecord): boolean { + return ( + record.status === "completed" || + (record.status === "failed" && record.attemptCount >= HARVEST_MAX_ATTEMPTS) + ); +} + +/** Terminal marker for a bucket whose transcript is being deleted (see finalizeHarvestsForRemoval). */ +function finalizeHarvestRecordForRemoval(record: MemoryHarvestRecord): MemoryHarvestRecord { + return { + ...record, + status: "failed", + completedAt: record.completedAt ?? Date.now(), + attemptCount: HARVEST_MAX_ATTEMPTS, + error: "workspace removed before the harvest could be retried; transcript no longer available", + }; +} + export class MemoryConsolidationService extends EventEmitter { private readonly sidecarPath: string; /** Serializes sidecar read-modify-write cycles (journal persistence only). */ @@ -479,16 +498,27 @@ export class MemoryConsolidationService extends EventEmitter { const self = this; return Effect.uninterruptible( Effect.gen(function* () { - yield* Effect.promise(() => + const saved = yield* Effect.promise(() => self.locks.withLock(self.sidecarPath, async () => { const file = await self.load(); file.harvestsByWorkspace[workspaceId] ??= {}; + const existing = file.harvestsByWorkspace[workspaceId][boundaryKey]; + // A terminal record is never reopened: removal finalization + // (finalizeHarvestsForRemoval) races the bounded cancellation + // drain's residual harvest runs on this file, and a residual + // pending/retryable-failure write landing afterwards would turn + // a bucket whose transcript is gone back into a retry candidate. + // Only a genuine completion may replace it (the writes happened). + if (existing !== undefined && isTerminalHarvestRecord(existing)) { + if (record.status !== "completed") return false; + } file.harvestsByWorkspace[workspaceId][boundaryKey] = record; pruneHarvestRecords(file.harvestsByWorkspace[workspaceId]); await writeFileAtomic(self.sidecarPath, JSON.stringify(file, null, 2)); + return true; }) ); - self.emitStatusChange(workspaceId, projectPath); + if (saved) self.emitStatusChange(workspaceId, projectPath); }) ); } @@ -638,30 +668,29 @@ export class MemoryConsolidationService extends EventEmitter { * owner. Mark them terminal now so nothing lingers as "retryable". */ async finalizeHarvestsForRemoval(workspaceId: string): Promise { - const sidecar = await this.load(); - const records = sidecar.harvestsByWorkspace[workspaceId]; - if (records === undefined) return; + // One read-check-write under the sidecar lock: residual harvest runs + // (cancelInFlightConsolidation's drain is bounded) may still be recording + // outcomes, and a completion landing between an unlocked read and this + // write must not be overwritten with a failure. + const finalized = await this.locks.withLock(this.sidecarPath, async () => { + const file = await this.load(); + const records = file.harvestsByWorkspace[workspaceId]; + if (records === undefined) return false; + let changed = false; + for (const [boundaryKey, record] of Object.entries(records)) { + if (isTerminalHarvestRecord(record)) continue; + records[boundaryKey] = finalizeHarvestRecordForRemoval(record); + changed = true; + } + if (changed) await writeFileAtomic(this.sidecarPath, JSON.stringify(file, null, 2)); + return changed; + }); + if (!finalized) return; const workspace = this.config.findWorkspace(workspaceId); - const projectPath = workspace == null ? "" : resolveConsolidationProjectPath(workspace); - for (const [boundaryKey, record] of Object.entries(records)) { - if (record.status === "completed") continue; - if (record.status === "failed" && record.attemptCount >= HARVEST_MAX_ATTEMPTS) continue; - await Effect.runPromise( - this.saveHarvestRecordEffect( - workspaceId, - boundaryKey, - { - ...record, - status: "failed", - completedAt: record.completedAt ?? Date.now(), - attemptCount: HARVEST_MAX_ATTEMPTS, - error: - "workspace removed before the harvest could be retried; transcript no longer available", - }, - projectPath - ) - ); - } + this.emitStatusChange( + workspaceId, + workspace == null ? "" : resolveConsolidationProjectPath(workspace) + ); } /** diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index e892ca299f7..fd303d686dc 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -243,12 +243,12 @@ export function setMemoryPinnedEffect( input.pinned ) .pipe( - Effect.map(() => { - // Pins live in the sidecar, not the store, so nothing else emits a - // change: notify so the other tree members' tabs refetch too. - context.memoryService.notifyPinChange(resolved.scopeCtx, input.path); - return { success: true as const, data: undefined }; - }), + // Pins live in the sidecar, not the store, so nothing else emits a + // change: notify so the other tree members' tabs refetch too. + Effect.flatMap(() => + Effect.promise(() => context.memoryService.notifyPinChange(resolved.scopeCtx, input.path)) + ), + Effect.map(() => ({ success: true as const, data: undefined })), // Sidecar write failures (disk full, permissions) arrive as the typed // MemoryMetaWriteError and map onto the legacy string error channel // instead of escaping as an untyped INTERNAL_SERVER_ERROR rejection. diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 7d6c7c373f8..852ac251add 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1085,6 +1085,61 @@ describe("MemoryService", () => { expect(invalidated).toEqual([["ws-child"]]); }); + it("advances the owner store's revision token on shared writes, visible to another backend", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // A second MemoryService over the same Xum root stands in for another + // backend process: it receives none of this instance's change events. + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe("missing"); + + await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "v1", "agent"); + const afterCreate = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterCreate).not.toBe("missing"); + // Child and owner read the same (owner-keyed) token. + expect(await foreign.workspaceMemoryRevision("ws-child")).toBe(afterCreate); + + // Other scopes leave the workspace store's token alone... + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + // ...while every shared-store mutation advances it. + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/shared.md", + "v1", + "v2", + "agent" + ); + expect(await foreign.workspaceMemoryRevision("ws-owner")).not.toBe(afterCreate); + }); + + it("refuses to commit into a self-fallback store once config.json has recovered", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const configFile = path.join(fixture.xumHome, "config.json"); + const parked = `${configFile}.parked`; + // The command resolves its store while config.json is unreadable... + await fsPromises.rename(configFile, parked); + expect(fixture.service.ownerWorkspaceIdFor(fixture.ctx)).toBe("ws-child"); + // ...and the file recovers before it commits: the write must not land in + // the private store now that the tree is shared again. + await fsPromises.rename(parked, configFile); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/late.md", + "x", + "agent" + ); + expect(created.success).toBe(false); + if (!created.success) expect(created.error).toContain("Ownership of the workspace notebook"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "late.md")) + ).toBe(false); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-owner", "memory", "late.md")) + ).toBe(false); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1345,7 +1400,7 @@ describe("MemoryService", () => { const events: MemoryChangeEvent[] = []; fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); const ownerMemory = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); - fixture.service.notifyExternalMutation(fixture.ctx, [ + await fixture.service.notifyExternalMutation(fixture.ctx, [ path.join(ownerMemory, "a.md"), path.join(ownerMemory, "dir", "b.md"), path.join(fixture.xumHome, "memory", "global", "g.md"), diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index b46e09b8695..d5efd93d814 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -17,7 +17,7 @@ * documented limitation. */ import { EventEmitter } from "events"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -32,6 +32,7 @@ import { MEMORY_SCOPES, MEMORY_VIEW_MAX_DEPTH, MEMORY_VIRTUAL_ROOT, + WORKSPACE_MEMORY_REVISION_FILE_NAME, type MemoryScope, } from "@/common/constants/memory"; import { PlatformPaths } from "@/common/utils/paths"; @@ -925,6 +926,13 @@ export class MemoryService extends EventEmitter { * from the store the command already bound to — not re-resolved — so an * ownership change between resolution and lock acquisition cannot make the * check pass for the new owner while the write lands in the old one. + * + * The bound owner is then compared with a fresh resolution: the + * per-context cache (ownerWorkspaceIdFor) may hold a self-fallback taken + * while config.json was missing or malformed, and if the file recovers + * before this command commits, the write would land in the child's private + * store although the tree is shared again. Refused as a recoverable error; + * the retried command resolves the owner anew. */ private async assertMutationCommittable( ctx: MemoryScopeContext, @@ -942,7 +950,14 @@ export class MemoryService extends EventEmitter { // //memory → owner; global/project roots live elsewhere. const rel = path.relative(this.config.sessionsDir, store.physicalRoot); if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) { - guarded.add(rel.split(path.sep)[0]); + const boundOwner = rel.split(path.sep)[0]; + guarded.add(boundOwner); + const currentOwner = this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); + if (currentOwner !== boundOwner) { + throw new MemoryCommandError( + `Ownership of the workspace notebook changed while mutating ${virtualPath} (now ${currentOwner}); retry the command` + ); + } } for (const workspaceId of guarded) { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { @@ -1061,12 +1076,12 @@ export class MemoryService extends EventEmitter { } } - private emitChange( + private async emitChange( ctx: MemoryScopeContext, scope: MemoryScope, relPath: string, actor: MemoryActor - ) { + ): Promise { const event: MemoryChangeEvent = { scope, path: toVirtualPath(scope, relPath), @@ -1076,9 +1091,65 @@ export class MemoryService extends EventEmitter { workspaceId: this.ownerWorkspaceIdFor(ctx), projectPath: ctx.projectPath, }; + if (scope === "workspace" && event.workspaceId !== "") { + await this.bumpWorkspaceMemoryRevision(event.workspaceId); + } this.emit("change", event); } + private workspaceMemoryRevisionPath(ownerWorkspaceId: string): string { + return path.join( + this.config.sessionsDir, + ownerWorkspaceId, + WORKSPACE_MEMORY_REVISION_FILE_NAME + ); + } + + /** + * Rewrite the owner store's revision token (see + * WORKSPACE_MEMORY_REVISION_FILE_NAME) after a workspace-scope mutation. + * Runs before the in-process change event so an in-process rebuild + * triggered by the event records the new token. Best-effort and never + * creates the session directory: a pin toggle on a never-written store has + * no cache to invalidate, and a removed owner's directory must not be + * recreated (mutations are already refused pre-commit; see + * assertMutationCommittable). + */ + private async bumpWorkspaceMemoryRevision(ownerWorkspaceId: string): Promise { + try { + await fsPromises.writeFile( + this.workspaceMemoryRevisionPath(ownerWorkspaceId), + `${Date.now()}:${randomUUID()}` + ); + } catch (error) { + log.debug("[MemoryService] failed to write workspace memory revision", { + ownerWorkspaceId, + error, + }); + } + } + + /** + * Current revision token of the store backing `workspaceId`'s + * `/memories/workspace` (owner-resolved; one memoized ownership check plus + * one small read). Compared by consumers that cache a derived view of the + * store — AgentSession's memory context, the Memory tab subscription — so + * a mutation from ANOTHER backend process (multi-instance), which emits no + * change event here, still invalidates them on their next probe. A store + * never written or unreadable yields a fixed sentinel. + */ + async workspaceMemoryRevision(workspaceId: string): Promise { + assert(workspaceId.length > 0, "workspaceMemoryRevision requires a workspaceId"); + try { + return await fsPromises.readFile( + this.workspaceMemoryRevisionPath(this.resolveWorkspaceMemoryOwnerId(workspaceId)), + "utf-8" + ); + } catch { + return "missing"; + } + } + /** * Announces memory files mutated outside this service by a refinement * rollback (which applies inverses straight to disk). Physical paths are @@ -1087,13 +1158,16 @@ export class MemoryService extends EventEmitter { * shared workspace store — every task-tree session drops its cached * context (see the change listener wired in di/layers/core.ts). */ - notifyExternalMutation(ctx: MemoryScopeContext, physicalPaths: readonly string[]): void { + async notifyExternalMutation( + ctx: MemoryScopeContext, + physicalPaths: readonly string[] + ): Promise { const touched = new Set(); for (const physicalPath of physicalPaths) { const scope = this.scopeOfPhysicalPath(ctx, physicalPath); if (scope !== null) touched.add(scope); } - for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); + for (const scope of touched) await this.emitChange(ctx, scope, "", "agent"); } /** @@ -1122,10 +1196,10 @@ export class MemoryService extends EventEmitter { * subscribers of the same store — for the shared workspace notebook, every * task-tree member's tab — refetch their listing. */ - notifyPinChange(ctx: MemoryScopeContext, virtualPath: string): void { + async notifyPinChange(ctx: MemoryScopeContext, virtualPath: string): Promise { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); - this.emitChange(ctx, scope, parsed.relPath, "user"); + await this.emitChange(ctx, scope, parsed.relPath, "user"); } /** @@ -1251,7 +1325,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: fileText }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - this.emitChange(ctx, scope, parsed.relPath, actor); + await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Created ${toVirtualPath(scope, parsed.relPath)}`, @@ -1295,7 +1369,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: updated }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - this.emitChange(ctx, scope, parsed.relPath, actor); + await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Edited ${toVirtualPath(scope, parsed.relPath)}` }; }); }); @@ -1348,7 +1422,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: updated }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - this.emitChange(ctx, scope, parsed.relPath, actor); + await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Inserted ${insertedLineCount} line(s) into ${toVirtualPath(scope, parsed.relPath)} after line ${insertLine}`, @@ -1528,7 +1602,7 @@ export class MemoryService extends EventEmitter { ); } await this.recordDelete(ctx, scope, parsed.relPath); - this.emitChange(ctx, scope, parsed.relPath, actor); + await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Deleted ${toVirtualPath(scope, parsed.relPath)}`, @@ -1597,8 +1671,8 @@ export class MemoryService extends EventEmitter { toolCallId ); await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); - this.emitChange(ctx, scope, oldParsed.relPath, actor); - this.emitChange(ctx, scope, newParsed.relPath, actor); + await this.emitChange(ctx, scope, oldParsed.relPath, actor); + await this.emitChange(ctx, scope, newParsed.relPath, actor); return { success: true as const, output: `Renamed ${toVirtualPath(scope, oldParsed.relPath)} to ${toVirtualPath(scope, newParsed.relPath)}`, @@ -1732,7 +1806,7 @@ export class MemoryService extends EventEmitter { await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - this.emitChange(ctx, scope, parsed.relPath, actor); + await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, data: { sha256: sha256Hex(content) } }; } ); diff --git a/src/node/services/toolAssembly.test.ts b/src/node/services/toolAssembly.test.ts index 5629b449a25..52456cf455c 100644 --- a/src/node/services/toolAssembly.test.ts +++ b/src/node/services/toolAssembly.test.ts @@ -1,5 +1,5 @@ import { resolveToolPolicyForAgent } from "./agentDefinitions/resolveToolPolicy"; -import { isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; +import { isMemoryToolDisabled, isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; import { resolveAgentFrontmatter } from "./agentDefinitions/agentDefinitionsService"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { ToolBridge } from "./ptc/toolBridge"; @@ -549,6 +549,30 @@ describe("resolveBackendGatedPtcExperiments", () => { }); describe("token budget history policy", () => { + test.each([ + { add: [], allowed: false }, + { add: ["file_read"], allowed: false }, + { add: ["memory"], allowed: true }, + { add: ["mem.*"], allowed: true }, + { add: [".*"], allowed: true }, + ])("harvest permission mirrors the assembled memory tool: $add", async ({ add, allowed }) => { + // The persisted workspaceMemoryWritable bit is derived from the policy + // before tool assembly; it must agree with whether `memory` survives it. + const policy = resolveToolPolicyForAgent({ + agents: [{ tools: { add } }], + isSubagent: true, + disableTaskToolsForDepth: false, + }); + const memory = executableTool("Memory"); + const result = await applyToolPolicyAndExperiments({ + allTools: { memory, file_read: executableTool("Read") }, + effectiveToolPolicy: policy, + emitNestedToolEvent: () => undefined, + }); + expect(isMemoryToolDisabled(policy)).toBe(!allowed); + expect(result.memory === undefined).toBe(!allowed); + }); + test.each([ { add: [], allowed: false }, { add: ["file_read"], allowed: false }, diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 6d9a4a5c97f..33287564eb1 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -87,7 +87,7 @@ export function createRefinementRollbackTool(ctx: { // Rollback writes inverses straight to disk, bypassing MemoryService's // change events; announce them so the (possibly shared) store's other // readers — owner, siblings, open Memory tabs — do not keep stale context. - ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ + await ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ ...result.data.restored, ...result.data.deleted, ...(result.data.renamed === undefined diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 11688d02038..39997e672b1 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -235,7 +235,7 @@ export function resolveXumToolScope( import type { PostCompactionAttachment } from "@/common/types/attachment"; import type { ErrorEvent } from "@/common/types/stream"; -import type { ToolPolicy } from "@/common/utils/tools/toolPolicy"; +import { isMemoryToolDisabled, type ToolPolicy } from "@/common/utils/tools/toolPolicy"; import type { FileState } from "@/node/services/agentSession"; import type { ActiveTurnThinkingOverride } from "@/node/services/thinkingOverride"; import type { WorkspaceGoalService } from "@/node/services/workspaceGoalService"; @@ -1425,19 +1425,28 @@ export class TurnRequestBuilder { editingCapable: isExecLikeEditingCapableInResolvedChain(agentInheritanceChain), }); // Post-compaction harvest writes to /memories/workspace on the agent's - // behalf; it must honor the same policy the memory tool enforces. The + // behalf; it must honor exactly what the memory tool enforces: the scope + // access AND the tool's presence in the final toolset. The tool exists + // only with the experiment + service (tools.ts) and survives only if the + // effective policy keeps it (applied later in tool assembly, mirrored + // here) — tool-search deferral is not a permission decision. The // compaction turn itself runs the "compact" agent, so record only normal // turns' policy (the session attaches it to the compaction completion). + const workspaceMemoryWritable = + memoryAccess.workspace === "readwrite" && + memoryExperimentEnabled && + this.dependencies.bindings.memoryService !== undefined && + !isMemoryToolDisabled(effectiveToolPolicy); if (!isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink) { // Awaited (a config write happens only when the value changes) so the // durable policy is in place before this turn can produce a compaction. const persisted = await this.dependencies.bindings.workspaceMemoryPolicySink.recordWorkspaceMemoryWritable( workspaceId, - memoryAccess.workspace === "readwrite" + workspaceMemoryWritable ); if (!persisted) { - if (memoryAccess.workspace !== "readwrite") { + if (!workspaceMemoryWritable) { // A stale persisted `true` would let this now read-only agent's // transcript harvest into the (shared) workspace notebook after a // restart. Refuse to run the turn until the deny is durable. From 2383632dbdd63dc545a22a5291f862bbf6032310 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 20:00:12 +0000 Subject: [PATCH 22/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20nineteent?= =?UTF-8?q?h=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Harvest permission follows the FINAL toolset: after the primary request is prepared (request.assemble middleware included), an absent `memory` tool downgrades the persisted workspaceMemoryWritable bit to false (turn refused if the deny cannot be persisted). Built-in tools are never deferred by tool search, so absence means denied. - Consolidation sidecar read-modify-writes (run records, harvest records, removal finalization) take a cross-process lockfile alongside the in-process mutex, so a residual harvest in another backend cannot race finalization with a stale read. - refinement_rollback (RLM) refuses memory-row rollback when the memory tool is policy/grant-denied (view-only memory policy handed to the tool in tool assembly) or stripped by request.assemble middleware. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/constants/memory.ts | 2 +- .../memoryConsolidationService.test.ts | 49 ++++++++++ .../services/memoryConsolidationService.ts | 33 ++++++- src/node/services/toolAssembly.test.ts | 76 ++++++++++++++- src/node/services/toolAssembly.ts | 14 ++- src/node/services/tools/memory.ts | 2 +- src/node/services/turnRequestBuilder.ts | 93 ++++++++++++------- 7 files changed, 226 insertions(+), 43 deletions(-) diff --git a/src/common/constants/memory.ts b/src/common/constants/memory.ts index 3b7082f0315..f65757b305d 100644 --- a/src/common/constants/memory.ts +++ b/src/common/constants/memory.ts @@ -17,6 +17,7 @@ export const MEMORY_VIRTUAL_ROOT = "/memories"; export const MEMORY_SCOPES = ["global", "project", "workspace"] as const; +export type MemoryScope = (typeof MEMORY_SCOPES)[number]; /** * `//memory.revision`: opaque token rewritten on every @@ -25,7 +26,6 @@ export const MEMORY_SCOPES = ["global", "project", "workspace"] as const; * reusing a cached index/hot set — in-process consumers get change events. */ export const WORKSPACE_MEMORY_REVISION_FILE_NAME = "memory.revision"; -export type MemoryScope = (typeof MEMORY_SCOPES)[number]; export type MemoryAccessLevel = "read" | "readwrite"; diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 9669716c68b..87c3647fdba 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -36,6 +36,7 @@ import { MemoryService } from "./memoryService"; import { SessionUsageService } from "./sessionUsageService"; import { TestTempDir } from "./tools/testHelpers"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; /** * Behavior under test: the orchestration rails around the runner — @@ -1275,6 +1276,54 @@ describe("MemoryConsolidationService", () => { expect((await latest())?.status).toBe("completed"); }); + it("serializes sidecar harvest writes with other backends through the cross-process lock", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const metadata = await seedCompactionEpoch(fixture, "ws-sub"); + const sidecarPath = path.join(fixture.xumHome, "memory-consolidation.json"); + await fsPromises.writeFile( + sidecarPath, + JSON.stringify({ + workspaces: {}, + harvestsByWorkspace: { + "ws-sub": { + [metadata.summaryMessageId]: { + status: "failed", + startedAt: Date.now() - 10_000, + completedAt: Date.now() - 9_000, + attemptCount: 1, + boundaryKey: metadata.summaryMessageId, + compactionEpoch: metadata.compactionEpoch, + acceptedCandidates: 0, + skippedCandidates: 0, + error: "crashed mid-harvest", + completionMetadata: metadata, + }, + }, + }, + }) + ); + // Another backend mid read-modify-write: it holds the sidecar's file lock + // (the in-process MutexMap cannot see it), so this finalization must wait. + const foreignHold = await acquireProcessFileLock({ + lockPath: `${sidecarPath}.lock`, + timeoutMs: 1_000, + label: "test foreign backend", + }); + let finalized = false; + const finalizing = fixture.service.finalizeHarvestsForRemoval("ws-sub").then(() => { + finalized = true; + }); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(finalized).toBe(false); + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe(1); + await foreignHold[Symbol.asyncDispose](); + await finalizing; + expect((await fixture.service.getStatus("ws-sub")).latestHarvestRecord?.attemptCount).toBe( + HARVEST_MAX_ATTEMPTS + ); + }); + it("normalizes stale max-attempt pending harvest records to failed", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); const metadata = await seedCompactionEpoch(fixture); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index f527888fea4..c4205778a16 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -58,6 +58,7 @@ import { } from "@/node/services/branchSummary"; import { USAGE_WRITE_DRAIN_WINDOW_MS } from "@/constants/streamDrain"; import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { getErrorMessage } from "@/common/utils/errors"; import { Err, Ok } from "@/common/types/result"; @@ -287,6 +288,12 @@ function pruneHarvestRecords(records: Record): void export const HARVEST_MAX_ATTEMPTS = 3; +/** + * Bound on waiting for the cross-process sidecar lock; holders only do one + * small read-modify-write, so hitting it means another backend is wedged. + */ +const MEMORY_CONSOLIDATION_SIDECAR_LOCK_TIMEOUT_MS = 5_000; + /** Completed, or failed with retries exhausted: nothing may retry it. */ function isTerminalHarvestRecord(record: MemoryHarvestRecord): boolean { return ( @@ -310,6 +317,26 @@ export class MemoryConsolidationService extends EventEmitter { private readonly sidecarPath: string; /** Serializes sidecar read-modify-write cycles (journal persistence only). */ private readonly locks = new MutexMap(); + + /** + * Sidecar read-modify-write section. Two legs, like the durable journal's + * locks: the in-process MutexMap orders callers on this instance cheaply, + * and a cross-process lockfile (`.lock`) excludes OTHER backends + * over the same Xum root (multi-instance) — a residual harvest recording in + * one process must not read a stale record around another process's + * removal finalization, or the terminal-record guard in + * saveHarvestRecordEffect would be checked against a stale read. + */ + private withSidecarLock(fn: () => Promise): Promise { + return this.locks.withLock(this.sidecarPath, async () => { + await using _fileLock = await acquireProcessFileLock({ + lockPath: `${this.sidecarPath}.lock`, + timeoutMs: MEMORY_CONSOLIDATION_SIDECAR_LOCK_TIMEOUT_MS, + label: "memory consolidation sidecar", + }); + return await fn(); + }); + } /** * Per-workspace run lock holding the active run's promise. Reserved * SYNCHRONOUSLY in maybeRun before any await so two near-simultaneous @@ -472,7 +499,7 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.uninterruptible( Effect.gen(function* () { yield* Effect.promise(() => - self.locks.withLock(self.sidecarPath, async () => { + self.withSidecarLock(async () => { const file = await self.load(); file.workspaces[workspaceId] = record; if (projectPath !== "") { @@ -499,7 +526,7 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.uninterruptible( Effect.gen(function* () { const saved = yield* Effect.promise(() => - self.locks.withLock(self.sidecarPath, async () => { + self.withSidecarLock(async () => { const file = await self.load(); file.harvestsByWorkspace[workspaceId] ??= {}; const existing = file.harvestsByWorkspace[workspaceId][boundaryKey]; @@ -672,7 +699,7 @@ export class MemoryConsolidationService extends EventEmitter { // (cancelInFlightConsolidation's drain is bounded) may still be recording // outcomes, and a completion landing between an unlocked read and this // write must not be overwritten with a failure. - const finalized = await this.locks.withLock(this.sidecarPath, async () => { + const finalized = await this.withSidecarLock(async () => { const file = await this.load(); const records = file.harvestsByWorkspace[workspaceId]; if (records === undefined) return false; diff --git a/src/node/services/toolAssembly.test.ts b/src/node/services/toolAssembly.test.ts index 52456cf455c..ad9a507be1e 100644 --- a/src/node/services/toolAssembly.test.ts +++ b/src/node/services/toolAssembly.test.ts @@ -1,5 +1,9 @@ import { resolveToolPolicyForAgent } from "./agentDefinitions/resolveToolPolicy"; -import { isMemoryToolDisabled, isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; +import { + isMemoryToolDisabled, + isSessionHistoryDisabled, + type ToolPolicy, +} from "@/common/utils/tools/toolPolicy"; import { resolveAgentFrontmatter } from "./agentDefinitions/agentDefinitionsService"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { ToolBridge } from "./ptc/toolBridge"; @@ -15,6 +19,7 @@ import { sandboxHostService } from "@/node/services/sandbox/sandboxHostService"; import { DisposableTempDir } from "@/node/services/tempDir"; import { appendRefinementEvent } from "@/node/services/refinement/refinementJournal"; import { listRefinements } from "@/node/services/refinement/refinementRollback"; +import type { MemoryService } from "@/node/services/memoryService"; function executableTool(description: string): Tool { return { @@ -318,6 +323,75 @@ describe("persistent kernel graduation (RLM mode)", () => { } }); + test("refinement_rollback refuses memory rows when policy denies the memory tool", async () => { + using tmp = new DisposableTempDir("tool-assembly-rlm-rollback-memory-policy"); + const scopeKey = "ws-tool-assembly-rlm-rollback-memory-policy"; + const sessionDir = path.join(tmp.path, "sessions", scopeKey); + const noteFile = path.join(sessionDir, "memory", "note.md"); + const notified: string[][] = []; + // Stand-in MemoryService: classifies every path as the workspace scope and + // records rollback announcements; the row's inverse deletes a note. + const memoryService = { + scopeOfPhysicalPath: () => "workspace" as const, + notifyExternalMutation: (_ctx: unknown, paths: string[]) => { + notified.push(paths); + return Promise.resolve(); + }, + } as unknown as MemoryService; + const memory = { + service: memoryService, + ctx: { runtime: null, checkoutCwd: "", workspaceId: scopeKey, projectPath: "" }, + // Exec-like class: read-write everywhere — the class alone must not decide. + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" } as const, + }; + const assemble = (policy: ToolPolicy | undefined): Promise> => + applyToolPolicyAndExperiments({ + allTools: { memory: executableTool("Memory"), file_read: executableTool("Read a file") }, + effectiveToolPolicy: policy, + experiments: { programmaticToolCalling: true, rlm: true }, + emitNestedToolEvent: () => undefined, + sandbox: { workspaceId: scopeKey, sessionDir, memory }, + }); + const seedRow = async () => { + await fsPromises.mkdir(path.dirname(noteFile), { recursive: true }); + await fsPromises.writeFile(noteFile, "note", "utf-8"); + await appendRefinementEvent({ + sessionDir, + workspaceId: scopeKey, + kind: "memory", + action: { op: "create", path: "/memories/workspace/note.md" }, + inverse: { op: "delete-files", paths: [noteFile] }, + evidence: { toolName: "memory" }, + }); + return (await listRefinements(sessionDir)).at(-1)!.id; + }; + const rollback = async (tools: Record, id: string) => + (await tools.refinement_rollback.execute!( + { id, reason: "test" }, + { toolCallId: "test-call-id", messages: [], context: undefined } + )) as { success: boolean; error?: string }; + try { + // Policy strips `memory` but leaves the rollback surface: a memory-row + // rollback is a memory write, so it is refused like a read-only scope. + const denied = await assemble([{ regex_match: "memory", action: "disable" }]); + expect(denied.memory).toBeUndefined(); + expect(denied.refinement_rollback).toBeDefined(); + const refused = await rollback(denied, await seedRow()); + expect(refused.success).toBe(false); + expect(refused.error).toContain("read-only"); + expect(await fsPromises.readFile(noteFile, "utf-8")).toBe("note"); + expect(notified).toEqual([]); + + // With the memory tool allowed, the same rollback proceeds and announces. + const allowed = await assemble(undefined); + const rolledBack = await rollback(allowed, await seedRow()); + expect(rolledBack.success).toBe(true); + expect(notified).toEqual([[noteFile]]); + } finally { + await sandboxHostService.disposeScope(scopeKey); + } + }); + test("MUX_SANDBOX_PERSISTENT_MOUNTS=1 still opts in without the rlm experiment", async () => { using tmp = new DisposableTempDir("tool-assembly-env-mounts"); const scopeKey = "ws-tool-assembly-env-mounts"; diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 36ff8c93e12..9a4715b23a6 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -36,6 +36,7 @@ import type { ToolBridge } from "@/node/services/ptc/toolBridge"; import type { PTCExecutionResult } from "@/node/services/ptc/types"; import { sandboxHostService, type SandboxMount } from "@/node/services/sandbox/sandboxHostService"; import { createRefinementRollbackTool } from "@/node/services/tools/refinement_rollback"; +import { READ_ONLY_ACCESS } from "@/node/services/tools/memory"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; import type { MemoryScopeAccess } from "@/common/constants/memory"; import type { KernelFileLoader } from "@/node/services/tools/kernelFileLoad"; @@ -338,8 +339,19 @@ export async function applyToolPolicyAndExperiments( // disables the tool, e.g. a broad regex disable rule) must not gain a // harness-rollback surface. Unlike code_execution in exclusive mode, // rollback is never mandatory, so policy may freely remove it. + // A memory-row rollback is a memory WRITE. When policy or grants deny + // the memory tool itself (checked on the grant-and-policy-filtered base + // set: in PTC mode the tool may be bridged rather than model-visible), + // rollback must not become a side door into the (shared) notebook — + // hand it a view-only policy so every memory row is refused, exactly + // as an explore-like agent's would be. + const memoryToolAvailable = policyFilteredTools.memory !== undefined; let rollback: Record = { - refinement_rollback: createRefinementRollbackTool(sandbox), + refinement_rollback: createRefinementRollbackTool( + sandbox.memory === undefined || memoryToolAvailable + ? sandbox + : { ...sandbox, memory: { ...sandbox.memory, access: READ_ONLY_ACCESS } } + ), }; rollback = applyToolPolicy(rollback, effectiveToolPolicy); if (opts.capabilityGrants) { diff --git a/src/node/services/tools/memory.ts b/src/node/services/tools/memory.ts index addf62e2393..81f35517c50 100644 --- a/src/node/services/tools/memory.ts +++ b/src/node/services/tools/memory.ts @@ -14,7 +14,7 @@ import { } from "@/node/services/memoryService"; /** Safe default: without an explicit policy, every scope is read-only. */ -const READ_ONLY_ACCESS: MemoryScopeAccess = { +export const READ_ONLY_ACCESS: MemoryScopeAccess = { global: "read", project: "read", workspace: "read", diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 39997e672b1..664dc02bd1c 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -116,7 +116,7 @@ import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpSer import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; -import { resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; +import { READ_ONLY_ACCESS, resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; import { isWorkspaceTrustedForSharedExecution } from "@/node/services/utils/workspaceTrust"; import type { WorkspaceMcpOverridesService } from "./workspaceMcpOverridesService"; @@ -1429,46 +1429,47 @@ export class TurnRequestBuilder { // access AND the tool's presence in the final toolset. The tool exists // only with the experiment + service (tools.ts) and survives only if the // effective policy keeps it (applied later in tool assembly, mirrored - // here) — tool-search deferral is not a permission decision. The - // compaction turn itself runs the "compact" agent, so record only normal - // turns' policy (the session attaches it to the compaction completion). - const workspaceMemoryWritable = + // here). request.assemble middleware can still strip it — that is + // re-checked against the prepared request below. The compaction turn + // itself runs the "compact" agent, so record only normal turns' policy + // (the session attaches it to the compaction completion). + let workspaceMemoryWritable = memoryAccess.workspace === "readwrite" && memoryExperimentEnabled && this.dependencies.bindings.memoryService !== undefined && !isMemoryToolDisabled(effectiveToolPolicy); - if (!isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink) { - // Awaited (a config write happens only when the value changes) so the - // durable policy is in place before this turn can produce a compaction. - const persisted = - await this.dependencies.bindings.workspaceMemoryPolicySink.recordWorkspaceMemoryWritable( - workspaceId, - workspaceMemoryWritable - ); - if (!persisted) { - if (!workspaceMemoryWritable) { - // A stale persisted `true` would let this now read-only agent's - // transcript harvest into the (shared) workspace notebook after a - // restart. Refuse to run the turn until the deny is durable. - const errorMessage = - "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; - const errorEvent = createErrorEvent(workspaceId, { - messageId: createAssistantMessageId(), - error: errorMessage, - errorType: "unknown", - acpPromptId, - }); - if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); - onPreStartError?.(errorEvent); - return { type: "finished", result: Err({ type: "unknown", raw: errorMessage }) }; - } - log.warn( - "Workspace memory write policy could not be persisted; harvests will fail closed", - { - workspaceId, - } - ); + // Awaited (a config write happens only when the value changes) so the + // durable policy is in place before this turn can produce a compaction. + // Returns the turn-refusing outcome when a DENY could not be persisted: a + // stale persisted `true` would let this now read-only agent's transcript + // harvest into the (shared) workspace notebook after a restart. + const persistWorkspaceMemoryWritable = async ( + writable: boolean + ): Promise | null> => { + const sink = this.dependencies.bindings.workspaceMemoryPolicySink; + if (isCompactionRequest || !sink) return null; + if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable)) return null; + if (!writable) { + const errorMessage = + "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; + const errorEvent = createErrorEvent(workspaceId, { + messageId: createAssistantMessageId(), + error: errorMessage, + errorType: "unknown", + acpPromptId, + }); + if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); + onPreStartError?.(errorEvent); + return { type: "finished", result: Err({ type: "unknown", raw: errorMessage }) }; } + log.warn("Workspace memory write policy could not be persisted; harvests will fail closed", { + workspaceId, + }); + return null; + }; + { + const refused = await persistWorkspaceMemoryWritable(workspaceMemoryWritable); + if (refused) return refused; } const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic @@ -2511,6 +2512,13 @@ export class TurnRequestBuilder { if (!intuitionToolAvailable || attemptTools.memory === undefined) { delete attemptTools.intuition; } + // Nor may it leave refinement_rollback (RLM) as a side door into the + // notebook once `memory` is gone: the tool reads its memory policy + // from this per-attempt object by reference, so demote it to + // view-only the same way tool assembly does for policy-denied memory. + if (attemptTools.memory === undefined && sandboxMemory !== undefined) { + sandboxMemory.access = READ_ONLY_ACCESS; + } if (attemptTools.intuition === undefined) { assembleCtx.systemMessage = removeIntuitionGuidance( assembleCtx.systemMessage, @@ -2706,6 +2714,19 @@ export class TurnRequestBuilder { throw error; } const tools = primaryRequest.tools; + // request.assemble middleware ran inside prepareModelRequest and may have + // stripped `memory` from the final toolset; the persisted harvest + // permission must follow it. Tool search defers MCP tools only, so an + // absent built-in `memory` here means denied, not deferred. Fallback + // attempts run the same middleware and cannot widen the permission. + if (workspaceMemoryWritable && tools.memory === undefined) { + workspaceMemoryWritable = false; + const refused = await persistWorkspaceMemoryWritable(false); + if (refused) { + runLanguageModelCleanup(modelResult.data.model); + return refused; + } + } systemMessage = primaryRequest.system; systemMessageTokens = primaryRequest.systemMessageTokens; const finalMessages = primaryRequest.messages; From 78bedbe8ff3d05c8ff68e62c0d5dd098a9942054 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 21:02:53 +0000 Subject: [PATCH 23/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twentieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persist the workspace memory harvest permission for the request that actually streams, not at preparation time: the primary request records it inside `start` from its final toolset (so an admission-only candidate that is rejected or disposed leaves nothing behind), and the model fallback's `prepare` downgrades it when its own request.assemble pass strips `memory`. A deny that cannot be made durable still refuses the turn (or the fallback attempt). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/turnRequestBuilder.ts | 103 +++++++++++++----------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 664dc02bd1c..cdf4574f64e 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -522,6 +522,10 @@ export interface PreparedTurnRequest extends AsyncDisposable { start(thinkingOverride?: ActiveTurnThinkingOverride): Promise; } +/** Turn refused because a memory-policy DENY could not be made durable (see persistWorkspaceMemoryWritable). */ +const WORKSPACE_MEMORY_POLICY_PERSIST_ERROR = + "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; + type PreparedTurnRequestOutcome = | Extract | { type: "prepared"; request: PreparedTurnRequest }; @@ -1429,48 +1433,36 @@ export class TurnRequestBuilder { // access AND the tool's presence in the final toolset. The tool exists // only with the experiment + service (tools.ts) and survives only if the // effective policy keeps it (applied later in tool assembly, mirrored - // here). request.assemble middleware can still strip it — that is - // re-checked against the prepared request below. The compaction turn - // itself runs the "compact" agent, so record only normal turns' policy - // (the session attaches it to the compaction completion). - let workspaceMemoryWritable = + // here); request.assemble middleware can still strip it, so the FINAL + // check happens against the toolset of the request actually started (see + // persistWorkspaceMemoryWritable). The compaction turn itself runs the + // "compact" agent, so only normal turns record their policy (the session + // attaches it to the compaction completion). + const workspaceMemoryWritable = memoryAccess.workspace === "readwrite" && memoryExperimentEnabled && this.dependencies.bindings.memoryService !== undefined && !isMemoryToolDisabled(effectiveToolPolicy); - // Awaited (a config write happens only when the value changes) so the - // durable policy is in place before this turn can produce a compaction. - // Returns the turn-refusing outcome when a DENY could not be persisted: a - // stale persisted `true` would let this now read-only agent's transcript - // harvest into the (shared) workspace notebook after a restart. - const persistWorkspaceMemoryWritable = async ( - writable: boolean - ): Promise | null> => { + // Persist the harvest permission for a request that is about to stream: + // NOT at preparation time — an admission-only candidate + // (prepareStreamMessage) may be rejected or disposed without running and + // must not leave a writable bit behind for the preceding read-only + // transcript. Awaited (a config write happens only when the value + // changes) so the durable policy is in place before the turn can produce + // a compaction. Returns false when a DENY could not be persisted: a stale + // persisted `true` would let a now read-only agent's transcript harvest + // into the (shared) workspace notebook after a restart, so the turn must + // not start. + const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; - if (isCompactionRequest || !sink) return null; - if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable)) return null; - if (!writable) { - const errorMessage = - "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; - const errorEvent = createErrorEvent(workspaceId, { - messageId: createAssistantMessageId(), - error: errorMessage, - errorType: "unknown", - acpPromptId, - }); - if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); - onPreStartError?.(errorEvent); - return { type: "finished", result: Err({ type: "unknown", raw: errorMessage }) }; - } + if (isCompactionRequest || !sink) return true; + if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable)) return true; + if (!writable) return false; log.warn("Workspace memory write policy could not be persisted; harvests will fail closed", { workspaceId, }); - return null; + return true; }; - { - const refused = await persistWorkspaceMemoryWritable(workspaceMemoryWritable); - if (refused) return refused; - } const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic // repo hook execution (tool_env/tool_pre/tool_post) while keeping @@ -2714,19 +2706,6 @@ export class TurnRequestBuilder { throw error; } const tools = primaryRequest.tools; - // request.assemble middleware ran inside prepareModelRequest and may have - // stripped `memory` from the final toolset; the persisted harvest - // permission must follow it. Tool search defers MCP tools only, so an - // absent built-in `memory` here means denied, not deferred. Fallback - // attempts run the same middleware and cannot widen the permission. - if (workspaceMemoryWritable && tools.memory === undefined) { - workspaceMemoryWritable = false; - const refused = await persistWorkspaceMemoryWritable(false); - if (refused) { - runLanguageModelCleanup(modelResult.data.model); - return refused; - } - } systemMessage = primaryRequest.system; systemMessageTokens = primaryRequest.systemMessageTokens; const finalMessages = primaryRequest.messages; @@ -2780,6 +2759,27 @@ export class TurnRequestBuilder { this.dependencies.createAbortedTurnHandle(assistantMessageId, combinedAbortSignal) ), }; + // Final toolset of the request being started: request.assemble + // middleware ran inside prepareModelRequest, and `memory` is a built-in + // that tool search never defers, so its absence means denied. + if ( + !(await persistWorkspaceMemoryWritable( + workspaceMemoryWritable && tools.memory !== undefined + )) + ) { + const errorEvent = createErrorEvent(workspaceId, { + messageId: createAssistantMessageId(), + error: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR, + errorType: "unknown", + acpPromptId, + }); + if (!context.admissionOnly) this.dependencies.emit("error", errorEvent); + onPreStartError?.(errorEvent); + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } const assistantMessage = createMuxMessage(assistantMessageId, "assistant", "", { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), timestamp: Date.now(), @@ -3025,6 +3025,17 @@ export class TurnRequestBuilder { if (error instanceof ContextBudgetExceededError) return Err(error.details); throw error; } + // The fallback runs its own request.assemble pass, which may + // strip `memory`; the persisted harvest permission must follow + // the request actually streamed, not the refused primary. + if ( + workspaceMemoryWritable && + nextRequest.tools.memory === undefined && + !(await persistWorkspaceMemoryWritable(false)) + ) { + runLanguageModelCleanup(nextRequest.model); + return Err(WORKSPACE_MEMORY_POLICY_PERSIST_ERROR); + } let nextHeaders = nextRequest.headers; if (pendingRunMetadataId != null) { nextHeaders = { From ddceece12b1aacd1807ae1abf377e11da742c05e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 21:22:02 +0000 Subject: [PATCH 24/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-fi?= =?UTF-8?q?rst=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Persisting the memory harvest permission is now asymmetric by failure mode: a DENY lands at the top of `start` (and refuses the turn when it cannot be made durable — a stale deny only fails closed), while a GRANT is persisted through the new `onStreamStarted` hook that AIService awaits only after startStream succeeded, so append failures, late aborts, thinking-level rebuild rejections and stream start failures leave no stale grant behind. The model fallback persists `writable && nextRequest.tools.memory !== undefined` for every selected fallback, restoring the grant when only the primary's request.assemble pass had stripped `memory`. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/aiService.ts | 4 ++ src/node/services/turnRequestBuilder.ts | 61 ++++++++++++++++--------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index b43aecadbb8..a9079e0198a 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -1014,6 +1014,10 @@ export class AIService extends EventEmitter { startupState.pendingRunMetadataId = null; } await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); + } else { + // The stream is live: durable effects gated on "actually started" + // (memory harvest grant) may land now. + await buildOutcome.onStreamStarted?.(); } buildOutcome.logStartOutcome("started"); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index cdf4574f64e..a583a0458e1 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -512,6 +512,13 @@ type TurnRequestBuildOutcome = assistantMessageId: string; deleteAbortedPlaceholder: (messageId: string) => Promise; logStartOutcome: (outcome: "started" | "stream_start_failed", errorType?: string) => void; + /** + * Durable side effects that must only land once the stream has actually + * started (a granted memory harvest permission): every earlier exit — + * append failure, late abort, thinking rebuild, stream start failure — + * then leaves nothing behind. Awaited by the caller after startStream. + */ + onStreamStarted?: () => Promise; }; export interface PreparedStreamMessage extends AsyncDisposable { @@ -1443,16 +1450,17 @@ export class TurnRequestBuilder { memoryExperimentEnabled && this.dependencies.bindings.memoryService !== undefined && !isMemoryToolDisabled(effectiveToolPolicy); - // Persist the harvest permission for a request that is about to stream: - // NOT at preparation time — an admission-only candidate - // (prepareStreamMessage) may be rejected or disposed without running and - // must not leave a writable bit behind for the preceding read-only - // transcript. Awaited (a config write happens only when the value - // changes) so the durable policy is in place before the turn can produce - // a compaction. Returns false when a DENY could not be persisted: a stale - // persisted `true` would let a now read-only agent's transcript harvest - // into the (shared) workspace notebook after a restart, so the turn must - // not start. + // Persisting the harvest permission is asymmetric because the two values + // fail differently when the turn then never streams: a stale DENY only + // fails closed (the preceding transcript is not harvested), a stale GRANT + // would let the preceding read-only transcript harvest into the (shared) + // workspace notebook after a restart. So a deny is persisted inside + // `start` before anything else (and refuses the turn when it cannot be + // made durable), while a grant is persisted only once the stream has + // actually started (onStreamStarted) — never at preparation time, where + // an admission-only candidate (prepareStreamMessage) may be rejected or + // disposed without running. Awaited (a config write happens only when + // the value changes). Returns false when a deny could not be persisted. const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; @@ -2761,12 +2769,11 @@ export class TurnRequestBuilder { }; // Final toolset of the request being started: request.assemble // middleware ran inside prepareModelRequest, and `memory` is a built-in - // that tool search never defers, so its absence means denied. - if ( - !(await persistWorkspaceMemoryWritable( - workspaceMemoryWritable && tools.memory !== undefined - )) - ) { + // that tool search never defers, so its absence means denied. The deny + // lands now; the grant waits for onStreamStarted (see + // persistWorkspaceMemoryWritable). + const finalWorkspaceMemoryWritable = workspaceMemoryWritable && tools.memory !== undefined; + if (!finalWorkspaceMemoryWritable && !(await persistWorkspaceMemoryWritable(false))) { const errorEvent = createErrorEvent(workspaceId, { messageId: createAssistantMessageId(), error: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR, @@ -3026,12 +3033,14 @@ export class TurnRequestBuilder { throw error; } // The fallback runs its own request.assemble pass, which may - // strip `memory`; the persisted harvest permission must follow - // the request actually streamed, not the refused primary. + // strip or keep `memory` independently of the primary; the + // persisted harvest permission must follow the request + // actually streamed, in both directions. The stream is + // already running here, so a grant may land immediately. if ( - workspaceMemoryWritable && - nextRequest.tools.memory === undefined && - !(await persistWorkspaceMemoryWritable(false)) + !(await persistWorkspaceMemoryWritable( + workspaceMemoryWritable && nextRequest.tools.memory !== undefined + )) ) { runLanguageModelCleanup(nextRequest.model); return Err(WORKSPACE_MEMORY_POLICY_PERSIST_ERROR); @@ -3206,6 +3215,16 @@ export class TurnRequestBuilder { assistantMessageId, deleteAbortedPlaceholder, logStartOutcome, + ...(finalWorkspaceMemoryWritable + ? { + // Best-effort once streaming: the turn really runs with a + // writable memory tool, so an unpersisted grant only fails the + // harvest closed (persistWorkspaceMemoryWritable warns). + onStreamStarted: async () => { + await persistWorkspaceMemoryWritable(true); + }, + } + : {}), }; }; retained = true; From 8a6f2db7ef75032cdd4e1468f39bbe39fdce4065 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 21:55:41 +0000 Subject: [PATCH 25/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-se?= =?UTF-8?q?cond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Shared-store clock (refinement/workspaceMemoryRevision.ts): memory.revision now holds a strictly monotonic ms-domain value advanced under the store's target mutation lock by every workspace-scope mutation — memory commands AND rollbacks (including the debug CLI path) — and stamped as the row's sourceTs, giving owner and sub-agent journals one total order for rollback conflict detection and a cross-process change signal that rollbacks also publish. Quota resweep now keys on migratedFrom, not sourceTs. - refinement_rollback fails closed when a memory row's path cannot be classified against the context's scope roots. - MemoryService re-resolves the per-context owner at every command boundary (a stream-long context no longer pins a stale owner) and refuses workspace-scope reads once the acting workspace or the store's owner is tombstoned. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/durableEvent.ts | 8 +- src/node/services/memoryService.test.ts | 126 ++++++++++++++- src/node/services/memoryService.ts | 144 ++++++++++++------ .../services/refinement/refinementJournal.ts | 18 ++- .../services/refinement/refinementRollback.ts | 38 +++++ .../refinement/workspaceMemoryRevision.ts | 51 +++++++ .../services/tools/refinement_rollback.ts | 10 +- 7 files changed, 329 insertions(+), 66 deletions(-) create mode 100644 src/node/services/refinement/workspaceMemoryRevision.ts diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 01d23d9482e..4fcaaffc5be 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -93,7 +93,13 @@ export const RefinementDataSchema = z.object({ * (sharedMemoryRowMigration.ts); lets a retried migration skip it. */ migratedFrom: z.string().optional(), - /** Source row's `ts`, so rollback ordering keeps the mutation's real cross-session position. */ + /** + * Cross-session order key for rollback conflict detection (`sourceTs ?? ts`): + * a shared workspace store's monotonic clock, advanced under the store's + * mutation lock by every mutation (workspaceMemoryRevision.ts), so rows in + * an owner's and its sub-agents' journals — whose `ts`/`seq` are not + * comparable — still order totally; migrated rows keep their source value. + */ sourceTs: z.number().optional(), /** Expected post-action file hashes (RefinementPostStateSchema in refinement.ts). */ postState: JsonValueSchema.optional(), diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 852ac251add..0256ba2a3e4 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1116,14 +1116,13 @@ describe("MemoryService", () => { it("refuses to commit into a self-fallback store once config.json has recovered", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); - const configFile = path.join(fixture.xumHome, "config.json"); - const parked = `${configFile}.parked`; - // The command resolves its store while config.json is unreadable... - await fsPromises.rename(configFile, parked); - expect(fixture.service.ownerWorkspaceIdFor(fixture.ctx)).toBe("ws-child"); - // ...and the file recovers before it commits: the write must not land in - // the private store now that the tree is shared again. - await fsPromises.rename(parked, configFile); + // Mid-command race: the command resolves its store while config.json is + // unreadable (self-fallback) and the file recovers before the commit + // check inside the mutation lock. Only the command's FIRST resolution + // is faked; the pre-commit re-resolution sees the recovered tree. + spyOn(fixture.service, "resolveWorkspaceMemoryOwnerId").mockImplementationOnce( + () => "ws-child" + ); const created = await fixture.service.create( fixture.ctx, "/memories/workspace/late.md", @@ -1140,6 +1139,32 @@ describe("MemoryService", () => { ).toBe(false); }); + it("re-resolves the owner per command and refuses reads once the owner is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + // One context serves a whole stream (createMemoryTool): a cached owner + // must not outlive the command that resolved it. + expect(fixture.service.ownerWorkspaceIdFor(fixture.ctx)).toBe("ws-owner"); + const resolve = spyOn(fixture.service, "resolveWorkspaceMemoryOwnerId"); + expect((await fixture.service.view(fixture.ctx, "/memories/workspace/n.md")).success).toBe( + true + ); + expect(resolve).toHaveBeenCalled(); + + // Another backend removed the owner: its durable tombstone (no local + // event) must stop the child's reads of the shared notebook. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + const refused = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + const root = await fixture.service.view(fixture.ctx, "/memories"); + expect(root.success).toBe(true); + if (root.success) expect(root.output).toContain("unavailable"); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1356,6 +1381,70 @@ describe("MemoryService", () => { expect(await pathExists(shared)).toBe(false); }); + it("orders owner and child rows of the shared store by one store clock, advanced by rollbacks too", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // Interleaved edits from two journals: `ts`/`seq` are not comparable + // across them (and can tie within a millisecond), the store clock is. + await fixture.service.create(fixture.ctx, "/memories/workspace/s.md", "c1", "agent"); + await fixture.service.strReplace(ownerCtx, "/memories/workspace/s.md", "c1", "o1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/s.md", + "o1", + "c2", + "agent" + ); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + const [ownerEdit] = await readRefinementEvents(ownerSessionDir); + const clocks = [childCreate, ownerEdit, childEdit].map((row) => row.data.sourceTs); + expect(clocks.every((clock) => typeof clock === "number")).toBe(true); + expect(clocks[0]!).toBeLessThan(clocks[1]!); + expect(clocks[1]!).toBeLessThan(clocks[2]!); + // The published token never lags a row's clock (change events tick it once more). + expect( + Number(await fixture.service.workspaceMemoryRevision("ws-child")) + ).toBeGreaterThanOrEqual(Math.max(...(clocks as number[]))); + + // The owner's edit is not the newest for that path: refused without force. + const stale = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(stale.success).toBe(false); + // A rollback (here through the engine directly, as the debug CLI does) + // is a store mutation too: it advances the clock other backends watch... + const before = await fixture.service.workspaceMemoryRevision("ws-owner"); + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + const after = await fixture.service.workspaceMemoryRevision("ws-owner"); + expect(Number(after)).toBeGreaterThan(Number(before)); + // ...and its row takes the next clock value, so the owner's edit is now + // the newest and rolls back cleanly. + const rollbackRow = (await readRefinementEvents(childSessionDir)).find( + (row) => row.data.rollbackOf === childEdit.id + )!; + expect(rollbackRow.data.sourceTs).toBe(Number(after)); + expect( + ( + await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerEdit.id, + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + }); + it("the refinement_rollback tool refuses memory rollbacks into a read-only scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1385,6 +1474,27 @@ describe("MemoryService", () => { expect(refused.error).toContain("read-only"); expect(await pathExists(physical)).toBe(true); + // A context whose scope roots do not contain the row's paths (here an + // unrelated workspace's) cannot evaluate the policy: fail closed even + // with read-write access. + const foreignTool = createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + memory: { + service: fixture.service, + ctx: { ...fixture.ctx, workspaceId: "ws-solo" }, + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" }, + }, + }); + const unclassifiable = (await foreignTool.execute!( + { id: row.id, reason: "test" }, + mockToolCallOptions + )) as { success: boolean; error?: string }; + expect(unclassifiable.success).toBe(false); + expect(unclassifiable.error).toContain("Cannot classify"); + expect(await pathExists(physical)).toBe(true); + const allowed = await run({ global: "readwrite", project: "readwrite", diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index d5efd93d814..87f1b6ae7a0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -17,7 +17,7 @@ * documented limitation. */ import { EventEmitter } from "events"; -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -32,7 +32,6 @@ import { MEMORY_SCOPES, MEMORY_VIEW_MAX_DEPTH, MEMORY_VIRTUAL_ROOT, - WORKSPACE_MEMORY_REVISION_FILE_NAME, type MemoryScope, } from "@/common/constants/memory"; import { PlatformPaths } from "@/common/utils/paths"; @@ -48,6 +47,10 @@ import { } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { + advanceWorkspaceMemoryRevision, + readWorkspaceMemoryRevision, +} from "@/node/services/refinement/workspaceMemoryRevision"; import { REFINEMENT_CAPTURE_MAX_FILES, REFINEMENT_CAPTURE_MAX_TOTAL_BYTES, @@ -816,8 +819,15 @@ export class MemoryService extends EventEmitter { } private async runCommand( + ctx: MemoryScopeContext, operation: () => Promise ): Promise { + // The per-context owner cache is scoped to ONE command: createMemoryTool + // reuses a context for a whole stream, and a cached owner would otherwise + // let a child keep reading its parent's notebook after the tree changed + // (owner removed by another backend — no local event) for as long as the + // stream lives. Re-resolving costs one memoized, stamp-validated lookup. + this.ownerByContext.delete(ctx); try { return await operation(); } catch (error) { @@ -843,11 +853,51 @@ export class MemoryService extends EventEmitter { relPath: string ): Promise { const store = this.getStore(ctx, scope); + if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); await store.assertRootSafe(); await store.assertContained(relPath); return store; } + /** + * The workspace whose session dir physically holds `store` + * (//memory → owner), or null for global/project roots, + * which live elsewhere. + */ + private storeOwnerWorkspaceId(store: MemoryStore): string | null { + const rel = path.relative(this.config.sessionsDir, store.physicalRoot); + if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel)) return null; + return rel.split(path.sep)[0]; + } + + /** Acting workspace plus the store's owner: both must be alive to touch the store. */ + private guardedWorkspaceIds(ctx: MemoryScopeContext, store: MemoryStore): string[] { + const owner = this.storeOwnerWorkspaceId(store); + return [...new Set(owner === null ? [ctx.workspaceId] : [ctx.workspaceId, owner])]; + } + + /** + * Reads have no commit guard, so a removed child's stream in ANOTHER backend + * (which the remover cannot cancel) could keep viewing its former owner's + * notebook — including notes written after the removal — through the + * shared store. Refuse workspace-scope reads once the acting workspace or + * the store's owner is tombstoned (the tombstone is durable and + * cross-process; see workspaceRemoval.ts). + */ + private async assertWorkspaceStoreReadable( + ctx: MemoryScopeContext, + store: MemoryStore + ): Promise { + if (ctx.workspaceId === "") return; + for (const workspaceId of this.guardedWorkspaceIds(ctx, store)) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { + throw new MemoryCommandError( + `Workspace ${workspaceId} was removed; the workspace memory store is no longer available` + ); + } + } + } + private requireFilePath(parsed: ParsedMemoryPath, virtualPath: string): MemoryScope { if (parsed.scope === null || parsed.relPath === "") { throw new MemoryCommandError( @@ -872,6 +922,7 @@ export class MemoryService extends EventEmitter { */ private async journalRefinement( ctx: MemoryScopeContext, + store: MemoryStore, action: MemoryRefinementAction, inverse: RefinementInverseDraft, actor: MemoryActor, @@ -884,6 +935,9 @@ export class MemoryService extends EventEmitter { }); return; } + // Workspace-scope rows carry the owner store's clock so rows from every + // tree member's journal order consistently (see workspaceMemoryRevision.ts). + const sourceTs = await this.advanceStoreRevision(store); await appendRefinementEvent({ sessionDir: path.join(this.config.sessionsDir, ctx.workspaceId), workspaceId: ctx.workspaceId, @@ -896,6 +950,7 @@ export class MemoryService extends EventEmitter { ...(toolCallId !== undefined ? { toolCallId } : {}), }, ...(postFiles !== undefined ? { postFiles } : {}), + ...(sourceTs !== undefined ? { sourceTs } : {}), }); } @@ -946,12 +1001,8 @@ export class MemoryService extends EventEmitter { ); } if (ctx.workspaceId === "") return; - const guarded = new Set([ctx.workspaceId]); - // //memory → owner; global/project roots live elsewhere. - const rel = path.relative(this.config.sessionsDir, store.physicalRoot); - if (rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)) { - const boundOwner = rel.split(path.sep)[0]; - guarded.add(boundOwner); + const boundOwner = this.storeOwnerWorkspaceId(store); + if (boundOwner !== null) { const currentOwner = this.resolveWorkspaceMemoryOwnerId(ctx.workspaceId); if (currentOwner !== boundOwner) { throw new MemoryCommandError( @@ -959,7 +1010,7 @@ export class MemoryService extends EventEmitter { ); } } - for (const workspaceId of guarded) { + for (const workspaceId of this.guardedWorkspaceIds(ctx, store)) { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) { throw new MemoryCommandError( `Workspace ${workspaceId} was removed; refusing to commit the mutation of ${virtualPath}` @@ -1091,41 +1142,32 @@ export class MemoryService extends EventEmitter { workspaceId: this.ownerWorkspaceIdFor(ctx), projectPath: ctx.projectPath, }; + // Pin toggles and rollback announcements reach the store's other readers + // only through this event, so the cross-process token moves here as well + // (mutations already advanced it for their row's sourceTs; a second tick + // is harmless — the clock only ever grows). if (scope === "workspace" && event.workspaceId !== "") { - await this.bumpWorkspaceMemoryRevision(event.workspaceId); + await this.advanceStoreRevision(this.getStore(ctx, "workspace")); } this.emit("change", event); } - private workspaceMemoryRevisionPath(ownerWorkspaceId: string): string { - return path.join( - this.config.sessionsDir, - ownerWorkspaceId, - WORKSPACE_MEMORY_REVISION_FILE_NAME - ); - } - /** - * Rewrite the owner store's revision token (see - * WORKSPACE_MEMORY_REVISION_FILE_NAME) after a workspace-scope mutation. - * Runs before the in-process change event so an in-process rebuild - * triggered by the event records the new token. Best-effort and never - * creates the session directory: a pin toggle on a never-written store has - * no cache to invalidate, and a removed owner's directory must not be - * recreated (mutations are already refused pre-commit; see - * assertMutationCommittable). + * Advance the owner store's clock (see workspaceMemoryRevision.ts) for a + * workspace-scope mutation and return it for the row's `sourceTs`. Callers + * hold the store's target mutation lock. Best-effort: a failure (removed + * owner directory — mutations are already refused pre-commit, see + * assertMutationCommittable; a pin toggle on a never-written store) must + * not fail the command, so the row then falls back to its journal `ts`. */ - private async bumpWorkspaceMemoryRevision(ownerWorkspaceId: string): Promise { + private async advanceStoreRevision(store: MemoryStore): Promise { + const owner = this.storeOwnerWorkspaceId(store); + if (owner === null) return undefined; try { - await fsPromises.writeFile( - this.workspaceMemoryRevisionPath(ownerWorkspaceId), - `${Date.now()}:${randomUUID()}` - ); + return await advanceWorkspaceMemoryRevision(path.join(this.config.sessionsDir, owner)); } catch (error) { - log.debug("[MemoryService] failed to write workspace memory revision", { - ownerWorkspaceId, - error, - }); + log.debug("[MemoryService] failed to advance workspace memory revision", { owner, error }); + return undefined; } } @@ -1140,14 +1182,10 @@ export class MemoryService extends EventEmitter { */ async workspaceMemoryRevision(workspaceId: string): Promise { assert(workspaceId.length > 0, "workspaceMemoryRevision requires a workspaceId"); - try { - return await fsPromises.readFile( - this.workspaceMemoryRevisionPath(this.resolveWorkspaceMemoryOwnerId(workspaceId)), - "utf-8" - ); - } catch { - return "missing"; - } + const revision = await readWorkspaceMemoryRevision( + path.join(this.config.sessionsDir, this.resolveWorkspaceMemoryOwnerId(workspaceId)) + ); + return revision === null ? "missing" : String(revision); } /** @@ -1232,7 +1270,7 @@ export class MemoryService extends EventEmitter { virtualPath: string, options?: { offset?: number; limit?: number } ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); if (parsed.scope === null) { // Virtual root: list every scope. @@ -1241,6 +1279,7 @@ export class MemoryService extends EventEmitter { sections.push(`- ${scope}/`); try { const store = this.getStore(ctx, scope); + if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); // Read-only: never create roots just to list (missing ⇒ empty). await store.assertRootSafe(); const files = await store.listFiles(); @@ -1289,7 +1328,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); assertWithinFileSizeCap(fileText); @@ -1318,6 +1357,7 @@ export class MemoryService extends EventEmitter { // Row is written before the create is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + store, { op: "create", path: toVirtualPath(scope, parsed.relPath) }, { op: "delete-files", paths: [store.physicalPath(parsed.relPath)] }, actor, @@ -1343,7 +1383,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); if (oldStr.length === 0) { @@ -1359,6 +1399,7 @@ export class MemoryService extends EventEmitter { // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + store, { op: "str_replace", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1385,7 +1426,7 @@ export class MemoryService extends EventEmitter { expectedFingerprint?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); @@ -1412,6 +1453,7 @@ export class MemoryService extends EventEmitter { // Row is written before the edit is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + store, { op: "insert", path: toVirtualPath(scope, parsed.relPath) }, { op: "restore-files", @@ -1450,7 +1492,7 @@ export class MemoryService extends EventEmitter { | { command: "delete"; path: string } | { command: "rename"; path: string; new_path: string } ): Promise<{ ok: true } | { ok: false; error: string }> { - const result = await this.runCommand(async () => { + const result = await this.runCommand(ctx, async () => { const parsed = parseMemoryPath(command.path); const scope = this.requireFilePath(parsed, command.path); switch (command.command) { @@ -1565,7 +1607,7 @@ export class MemoryService extends EventEmitter { expectedFingerprint?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); @@ -1595,6 +1637,7 @@ export class MemoryService extends EventEmitter { if (inverse !== null) { await this.journalRefinement( ctx, + store, { op: "delete", path: toVirtualPath(scope, parsed.relPath) }, inverse, actor, @@ -1619,7 +1662,7 @@ export class MemoryService extends EventEmitter { toolCallId?: string, abortSignal?: AbortSignal ): Promise { - return this.runCommand(async () => { + return this.runCommand(ctx, async () => { const oldParsed = parseMemoryPath(oldVirtualPath); const newParsed = parseMemoryPath(newVirtualPath); const scope = this.requireFilePath(oldParsed, oldVirtualPath); @@ -1657,6 +1700,7 @@ export class MemoryService extends EventEmitter { // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, + store, { op: "rename", path: toVirtualPath(scope, oldParsed.relPath), diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 1835b37969e..bbbbfba037a 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -78,7 +78,11 @@ export interface RefinementEmitArgs { postState?: RefinementPostState; /** Source identity of a row copied from a removed sub-agent's journal (see durableEvent.ts). */ migratedFrom?: string; - /** Source row's `ts` for a migrated row (see durableEvent.ts). */ + /** + * Cross-journal order key (see durableEvent.ts): the shared store's clock + * for a workspace-scope mutation (workspaceMemoryRevision.ts), or the + * source row's value/`ts` for a migrated row. + */ sourceTs?: number; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). @@ -179,10 +183,10 @@ export async function reclaimExcessRefinementInverseBlobs( options?: { /** * The published payloads belong to rows appended out of chronological - * order (shared-memory row migration stamps `sourceTs`): re-derive the - * retained set from the journal in source order instead of treating them - * as the newest, so an old migrated inverse cannot evict the owner's - * genuinely recent rollback data. + * order (shared-memory row migration): re-derive the retained set from + * the journal in source order instead of treating them as the newest, so + * an old migrated inverse cannot evict the owner's genuinely recent + * rollback data. */ resweep?: boolean; } @@ -284,8 +288,10 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr const publishedBlobs = await journal.withBlobLock(() => appendRefinementEventUnderBlobLock(journal, args) ); + // Live rows may carry a store-clock `sourceTs` too (MemoryService); only + // MIGRATED rows are appended out of order and need the source-order resweep. await reclaimRefinementInverseBlobsBestEffort(journal, publishedBlobs, { - resweep: args.sourceTs !== undefined, + resweep: args.migratedFrom !== undefined, }); } diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index e3bb2d50062..0bd2458dfcc 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -53,6 +53,7 @@ import { type RefinementInverseDraft, } from "./refinementJournal"; import { withTargetMutationLocks } from "./targetMutationLocks"; +import { advanceWorkspaceMemoryRevision } from "./workspaceMemoryRevision"; import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; export type RefinementEvent = Extract; @@ -345,6 +346,24 @@ function resolveConfinementRoot( ); } +/** + * Whether `root` (a confinement root from resolveConfinementRoot) is a + * workspace memory root — this session's own or the sanctioned owner's. Its + * parent is then the session dir that carries the store's clock + * (workspaceMemoryRevision.ts). + */ +function isWorkspaceMemoryRoot( + sessionDir: string, + root: string, + sharedWorkspaceMemorySessionDir: string | undefined +): boolean { + const candidates = [path.join(path.resolve(sessionDir), "memory")]; + if (sharedWorkspaceMemorySessionDir !== undefined) { + candidates.push(path.join(path.resolve(sharedWorkspaceMemorySessionDir), "memory")); + } + return candidates.includes(path.resolve(root)); +} + /** * The components of a confinement root that repo (or harness-writable) * content controls and could substitute with a symlink: `.mux`/`.agents` and @@ -1020,6 +1039,24 @@ export async function rollbackRefinement( // Inverse blob puts + the append referencing them run under the // journal blob lock: a concurrent reclamation pass must never // observe the put→append window (see withBlobLock). + // A workspace-memory rollback is a store mutation like any other: + // advance the owner store's clock (under the target lock held here) + // for this row's cross-journal order AND as the cross-process change + // signal — the debug CLI reaches this engine without MemoryService, + // so nothing else would tell other backends' caches and Memory tabs. + const workspaceMemoryRoot = [...new Set(roots.values())].find((root) => + isWorkspaceMemoryRoot(opts.sessionDir, root, opts.sharedWorkspaceMemorySessionDir) + ); + let sourceTs: number | undefined; + if (kind === "memory" && workspaceMemoryRoot !== undefined) { + try { + sourceTs = await advanceWorkspaceMemoryRevision(path.dirname(workspaceMemoryRoot)); + } catch (error) { + // Best-effort like the rest of this block: the row must still be + // journaled (falling back to its own `ts` for ordering). + log.debug("[refinement] failed to advance workspace memory revision", { error }); + } + } let publishedBlobs: BlobQuotaEntry[] = []; const row = await journal.withBlobLock(async () => { const resolved = await resolveRefinementInverse(journal.blobs, newInverse); @@ -1040,6 +1077,7 @@ export async function rollbackRefinement( ...(opts.evidence.actor !== undefined ? { actor: opts.evidence.actor } : {}), }, rollbackOf: opts.id, + ...(sourceTs !== undefined ? { sourceTs } : {}), }, }); }); diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts new file mode 100644 index 00000000000..3ac23a79c82 --- /dev/null +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -0,0 +1,51 @@ +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; +import assert from "@/common/utils/assert"; +import { WORKSPACE_MEMORY_REVISION_FILE_NAME } from "@/common/constants/memory"; + +/** + * Per-owner clock for a shared `/memories/workspace` store, persisted at + * `/memory.revision`. Two jobs, one file: + * + * - Cross-journal order. Owner and sub-agent rows describing the same store + * live in DIFFERENT session journals, whose `seq`/`ts` are not comparable + * (a migrated child row gets a fresh owner sequence; same-millisecond + * edits tie on `ts`). Every store mutation — memory command or rollback — + * advances this clock while holding the store's target mutation lock and + * stamps the value as the row's `sourceTs`, so rollback's conflict + * detection sees one total order across the whole task tree. + * - Cross-process change signal. The value only ever grows, so consumers in + * other backends (AgentSession's cached memory context, the Memory tab) + * compare it before reusing a cached view (MemoryService.workspaceMemoryRevision). + * + * Millisecond domain: `max(now, previous + 1)` keeps it a real timestamp for + * ordinary spacing while guaranteeing strict monotonicity under the lock. + */ +export function workspaceMemoryRevisionPath(ownerSessionDir: string): string { + assert(ownerSessionDir.length > 0, "workspaceMemoryRevisionPath requires an owner session dir"); + return path.join(ownerSessionDir, WORKSPACE_MEMORY_REVISION_FILE_NAME); +} + +/** Current clock value, or null when the store was never written (or the file is unreadable). */ +export async function readWorkspaceMemoryRevision(ownerSessionDir: string): Promise { + try { + const raw = await fsPromises.readFile(workspaceMemoryRevisionPath(ownerSessionDir), "utf-8"); + const value = Number.parseInt(raw, 10); + return Number.isSafeInteger(value) && value > 0 ? value : null; + } catch { + return null; + } +} + +/** + * Advance and persist the clock; returns the new value. Callers MUST hold the + * store's target mutation lock (cross-process) — the read→write here is what + * that lock makes atomic. Throws when the owner session dir is missing: the + * file is never allowed to recreate a removed owner's directory. + */ +export async function advanceWorkspaceMemoryRevision(ownerSessionDir: string): Promise { + const previous = (await readWorkspaceMemoryRevision(ownerSessionDir)) ?? 0; + const next = Math.max(Date.now(), previous + 1); + await fsPromises.writeFile(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); + return next; +} diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 33287564eb1..449e4ef7f76 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -42,7 +42,15 @@ async function refuseReadOnlyMemoryRollback( : [...inverse.data.files.map((file) => file.path), ...(inverse.data.deletePaths ?? [])]; for (const physicalPath of paths) { const scope = memory.service.scopeOfPhysicalPath(memory.ctx, physicalPath); - if (scope !== null && memory.access[scope] !== "readwrite") { + // Fail closed: a memory row's paths always lie in some scope root, so + // "unclassifiable" means this context's roots no longer match the row + // (e.g. the owner root admitted at preparation time while the per-context + // resolver now falls back to self because config.json is unreadable) — + // the policy cannot be evaluated, so the write must not proceed. + if (scope === null) { + return `Cannot classify '${physicalPath}' against this agent's memory scopes; refusing to roll back '${id}'.`; + } + if (memory.access[scope] !== "readwrite") { return `The ${scope} memory scope is read-only for this agent; rolling back '${id}' would write into it.`; } } From 829666e7ad243d2904e442ddd865b36818dc8433 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 22:18:59 +0000 Subject: [PATCH 26/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-th?= =?UTF-8?q?ird=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AgentSession accumulates the workspace memory write policy fail-closed over the compaction epoch (one read-only turn denies the whole epoch's harvest even if a writable turn follows), restarting at the compaction boundary unless a preserved tail carries the epoch forward; WorkspaceService persists that effective value. - The store clock only advances under the store's mutation lock: mutations advance it when they journal (or, for UI saves, in-lock explicitly), notifyPinChange takes the lock for its advance, and the rollback announcement no longer re-advances (the engine did so under the lock). emitChange is a pure in-process signal again. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../agentSession.memoryContext.test.ts | 20 +++++++++++ src/node/services/agentSession.ts | 24 ++++++++++---- src/node/services/memoryService.test.ts | 8 ++++- src/node/services/memoryService.ts | 33 +++++++++++++------ src/node/services/workspaceService.ts | 16 +++++---- 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 153da6969c7..b4cfd79b686 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -226,6 +226,26 @@ describe("AgentSession memory context", () => { } }); + test("accumulates the workspace memory write policy fail-closed across an epoch", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-policy"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext: () => Promise.resolve(null), + }); + try { + // The harvest reads every message of the epoch: a read-only turn denies + // the epoch even when a writable turn follows. + expect(session.recordWorkspaceMemoryWritable(true)).toBe(true); + expect(session.recordWorkspaceMemoryWritable(false)).toBe(false); + expect(session.recordWorkspaceMemoryWritable(true)).toBe(false); + } finally { + await session.dispose(); + } + }); + test("upgrades an index-only memory context when hot memories are requested", async () => { using sessionDir = new DisposableTempDir("agent-session-memory-context-upgrade"); const { historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 2cdd8348bb4..90a18004745 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1054,15 +1054,21 @@ export class AgentSession { private memoryContextGeneration = 0; /** - * Workspace-memory write policy of the last normal turn (TurnRequestBuilder - * via WorkspaceService). Attached to compaction completions so the memory - * harvest — which writes to the (possibly shared) workspace store on this - * agent's behalf — can honor a read-only agent's policy. + * Workspace-memory write policy accumulated over the current compaction + * epoch (TurnRequestBuilder via WorkspaceService). Attached to compaction + * completions so the memory harvest — which writes to the (possibly shared) + * workspace store on this agent's behalf — can honor a read-only agent's + * policy. Fail-closed across turns: the harvest reads EVERY message of the + * epoch, so one read-only turn denies the whole epoch even if a writable + * turn follows; the accumulator restarts at the compaction boundary + * (unless a preserved tail carries the epoch's messages forward). */ private workspaceMemoryWritable: boolean | undefined; - recordWorkspaceMemoryWritable(writable: boolean): void { - this.workspaceMemoryWritable = writable; + /** Returns the effective (accumulated) value, which is what gets persisted. */ + recordWorkspaceMemoryWritable(writable: boolean): boolean { + this.workspaceMemoryWritable = (this.workspaceMemoryWritable ?? true) && writable; + return this.workspaceMemoryWritable; } /** @@ -1233,6 +1239,12 @@ export class AgentSession { ? { workspaceMemoryWritable: this.workspaceMemoryWritable } : {}), }); + // New epoch. A preserved tail copies messages produced under this + // epoch's policy into the next one, so the fail-closed accumulator + // carries over with them; otherwise the next normal turn restarts it. + if ((metadata.preservedTailMessageCount ?? 0) === 0) { + this.workspaceMemoryWritable = undefined; + } }, onIdleCompactionOutcome, }); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 0256ba2a3e4..795be64a81c 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1102,6 +1102,10 @@ describe("MemoryService", () => { // Other scopes leave the workspace store's token alone... await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + // ...a pin toggle (hot-set input, no store write) advances it... + await fixture.service.notifyPinChange(fixture.ctx, "/memories/workspace/shared.md"); + const afterPin = await foreign.workspaceMemoryRevision("ws-owner"); + expect(Number(afterPin)).toBeGreaterThan(Number(afterCreate)); // ...while every shared-store mutation advances it. await fixture.service.strReplace( fixture.ctx, @@ -1110,7 +1114,9 @@ describe("MemoryService", () => { "v2", "agent" ); - expect(await foreign.workspaceMemoryRevision("ws-owner")).not.toBe(afterCreate); + expect(Number(await foreign.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + Number(afterPin) + ); }); it("refuses to commit into a self-fallback store once config.json has recovered", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 87f1b6ae7a0..8704a1b3850 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1142,21 +1142,20 @@ export class MemoryService extends EventEmitter { workspaceId: this.ownerWorkspaceIdFor(ctx), projectPath: ctx.projectPath, }; - // Pin toggles and rollback announcements reach the store's other readers - // only through this event, so the cross-process token moves here as well - // (mutations already advanced it for their row's sourceTs; a second tick - // is harmless — the clock only ever grows). - if (scope === "workspace" && event.workspaceId !== "") { - await this.advanceStoreRevision(this.getStore(ctx, "workspace")); - } + // Pure in-process signal. The cross-process token (store clock) is NOT + // advanced here: it must move under the store's mutation lock, which + // mutations hold when they journal (journalRefinement / saveFile) and the + // rollback engine holds when it appends its row; notifyPinChange takes it + // explicitly. this.emit("change", event); } /** * Advance the owner store's clock (see workspaceMemoryRevision.ts) for a * workspace-scope mutation and return it for the row's `sourceTs`. Callers - * hold the store's target mutation lock. Best-effort: a failure (removed - * owner directory — mutations are already refused pre-commit, see + * MUST hold the store's target mutation lock (the clock's monotonicity is + * only as good as the lock around its read→write). Best-effort: a failure + * (removed owner directory — mutations are already refused pre-commit, see * assertMutationCommittable; a pin toggle on a never-written store) must * not fail the command, so the row then falls back to its journal `ts`. */ @@ -1194,7 +1193,9 @@ export class MemoryService extends EventEmitter { * classified against this context's scope roots and one root-addressed * event per touched scope is emitted, so Memory tabs refresh and — for the * shared workspace store — every task-tree session drops its cached - * context (see the change listener wired in di/layers/core.ts). + * context (see the change listener wired in di/layers/core.ts). The store + * clock is not touched: the rollback engine advanced it under the target + * lock when it journaled its row (refinementRollback.ts). */ async notifyExternalMutation( ctx: MemoryScopeContext, @@ -1237,6 +1238,16 @@ export class MemoryService extends EventEmitter { async notifyPinChange(ctx: MemoryScopeContext, virtualPath: string): Promise { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); + if (scope === "workspace") { + // A pin changes the hot set other backends derive from this store, so + // the store clock must move — under the same mutation lock writers + // hold, or an unlocked read→write could overwrite a concurrent + // mutation's higher value and break the clock's monotonicity. + const store = this.getStore(ctx, scope); + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), () => + this.advanceStoreRevision(store) + ); + } await this.emitChange(ctx, scope, parsed.relPath, "user"); } @@ -1849,6 +1860,8 @@ export class MemoryService extends EventEmitter { } await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); + // UI saves are not journaled, so advance the store clock here (in-lock). + await this.advanceStoreRevision(store); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, data: { sha256: sha256Hex(content) } }; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e5864764748..e223bf20c16 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4220,18 +4220,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * new value could not be confirmed durable. */ async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { - ( - this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) - )?.recordWorkspaceMemoryWritable(writable); + // The session accumulates the policy over the compaction epoch + // (fail-closed); the durable copy mirrors that effective value so a + // restart-time fallback cannot be more permissive than the live session. + const effective = + ( + this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) + )?.recordWorkspaceMemoryWritable(writable) ?? writable; const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); // Unregistered workspace: nothing durable to update and no stale // permission to invalidate (harvests fail closed on the missing value). if (entry === null) return true; - if (entry.workspace.workspaceMemoryWritable === writable) return true; + if (entry.workspace.workspaceMemoryWritable === effective) return true; try { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); - if (current !== null) current.workspace.workspaceMemoryWritable = writable; + if (current !== null) current.workspace.workspaceMemoryWritable = effective; return cfg; }); } catch (error: unknown) { @@ -4244,7 +4248,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } const persisted = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace .workspaceMemoryWritable; - if (persisted !== writable) { + if (persisted !== effective) { log.error("Workspace memory write policy did not persist (config write swallowed?)", { workspaceId, writable, From 6cf04ee2fee216f1410ec12c325b76eb0607ecb9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 22:22:06 +0000 Subject: [PATCH 27/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20lint=20(emitChange?= =?UTF-8?q?=20/=20notifyExternalMutation=20are=20synchronous=20again)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 2 +- src/node/services/memoryService.ts | 27 +++++++++---------- .../services/tools/refinement_rollback.ts | 2 +- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 795be64a81c..0aa33734bd9 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1516,7 +1516,7 @@ describe("MemoryService", () => { const events: MemoryChangeEvent[] = []; fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); const ownerMemory = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); - await fixture.service.notifyExternalMutation(fixture.ctx, [ + fixture.service.notifyExternalMutation(fixture.ctx, [ path.join(ownerMemory, "a.md"), path.join(ownerMemory, "dir", "b.md"), path.join(fixture.xumHome, "memory", "global", "g.md"), diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8704a1b3850..4410b073749 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1127,12 +1127,12 @@ export class MemoryService extends EventEmitter { } } - private async emitChange( + private emitChange( ctx: MemoryScopeContext, scope: MemoryScope, relPath: string, actor: MemoryActor - ): Promise { + ): void { const event: MemoryChangeEvent = { scope, path: toVirtualPath(scope, relPath), @@ -1197,16 +1197,13 @@ export class MemoryService extends EventEmitter { * clock is not touched: the rollback engine advanced it under the target * lock when it journaled its row (refinementRollback.ts). */ - async notifyExternalMutation( - ctx: MemoryScopeContext, - physicalPaths: readonly string[] - ): Promise { + notifyExternalMutation(ctx: MemoryScopeContext, physicalPaths: readonly string[]): void { const touched = new Set(); for (const physicalPath of physicalPaths) { const scope = this.scopeOfPhysicalPath(ctx, physicalPath); if (scope !== null) touched.add(scope); } - for (const scope of touched) await this.emitChange(ctx, scope, "", "agent"); + for (const scope of touched) this.emitChange(ctx, scope, "", "agent"); } /** @@ -1248,7 +1245,7 @@ export class MemoryService extends EventEmitter { this.advanceStoreRevision(store) ); } - await this.emitChange(ctx, scope, parsed.relPath, "user"); + this.emitChange(ctx, scope, parsed.relPath, "user"); } /** @@ -1376,7 +1373,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: fileText }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - await this.emitChange(ctx, scope, parsed.relPath, actor); + this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Created ${toVirtualPath(scope, parsed.relPath)}`, @@ -1421,7 +1418,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: updated }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - await this.emitChange(ctx, scope, parsed.relPath, actor); + this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Edited ${toVirtualPath(scope, parsed.relPath)}` }; }); }); @@ -1475,7 +1472,7 @@ export class MemoryService extends EventEmitter { [{ path: store.physicalPath(parsed.relPath), content: updated }] ); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - await this.emitChange(ctx, scope, parsed.relPath, actor); + this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Inserted ${insertedLineCount} line(s) into ${toVirtualPath(scope, parsed.relPath)} after line ${insertLine}`, @@ -1656,7 +1653,7 @@ export class MemoryService extends EventEmitter { ); } await this.recordDelete(ctx, scope, parsed.relPath); - await this.emitChange(ctx, scope, parsed.relPath, actor); + this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Deleted ${toVirtualPath(scope, parsed.relPath)}`, @@ -1726,8 +1723,8 @@ export class MemoryService extends EventEmitter { toolCallId ); await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); - await this.emitChange(ctx, scope, oldParsed.relPath, actor); - await this.emitChange(ctx, scope, newParsed.relPath, actor); + this.emitChange(ctx, scope, oldParsed.relPath, actor); + this.emitChange(ctx, scope, newParsed.relPath, actor); return { success: true as const, output: `Renamed ${toVirtualPath(scope, oldParsed.relPath)} to ${toVirtualPath(scope, newParsed.relPath)}`, @@ -1863,7 +1860,7 @@ export class MemoryService extends EventEmitter { // UI saves are not journaled, so advance the store clock here (in-lock). await this.advanceStoreRevision(store); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); - await this.emitChange(ctx, scope, parsed.relPath, actor); + this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, data: { sha256: sha256Hex(content) } }; } ); diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 449e4ef7f76..e5e31636039 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -95,7 +95,7 @@ export function createRefinementRollbackTool(ctx: { // Rollback writes inverses straight to disk, bypassing MemoryService's // change events; announce them so the (possibly shared) store's other // readers — owner, siblings, open Memory tabs — do not keep stale context. - await ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ + ctx.memory?.service.notifyExternalMutation(ctx.memory.ctx, [ ...result.data.restored, ...result.data.deleted, ...(result.data.renamed === undefined From 0adbd6d45c64c3157d26c91a398b4ddb3e82fc7c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 22:45:16 +0000 Subject: [PATCH 28/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-fo?= =?UTF-8?q?urth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The persisted workspaceMemoryWritable bit is now THE epoch accumulator: WorkspaceService ANDs each turn's policy into the durable value (fail-closed), so a restart mid-epoch and backends sharing one chat.jsonl (multi-instance) all contribute to one conjunction; the session only mirrors it for the compaction completion. - Both copies restart at context boundaries: compaction without a preserved tail and destructive boundaries (/clear, reset, history replace via clearPostCompactionState) delete the durable field, so a stale deny cannot refuse an all-writable new epoch. - Unjournaled deletes (unrepresentable subtree) still advance the store revision so other backends drop cached views of the removed entry. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../agentSession.memoryContext.test.ts | 20 ------- src/node/services/agentSession.ts | 57 +++++++++++++++---- src/node/services/memoryService.ts | 4 ++ src/node/services/workspaceService.test.ts | 51 +++++++++++++++++ src/node/services/workspaceService.ts | 17 +++--- 5 files changed, 110 insertions(+), 39 deletions(-) diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index b4cfd79b686..153da6969c7 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -226,26 +226,6 @@ describe("AgentSession memory context", () => { } }); - test("accumulates the workspace memory write policy fail-closed across an epoch", async () => { - using sessionDir = new DisposableTempDir("agent-session-memory-policy"); - const { historyService, cleanup } = await createTestHistoryService(); - historyCleanup = cleanup; - const session = createSession({ - historyService, - sessionDir: path.join(sessionDir.path, WORKSPACE_ID), - buildMemorySessionContext: () => Promise.resolve(null), - }); - try { - // The harvest reads every message of the epoch: a read-only turn denies - // the epoch even when a writable turn follows. - expect(session.recordWorkspaceMemoryWritable(true)).toBe(true); - expect(session.recordWorkspaceMemoryWritable(false)).toBe(false); - expect(session.recordWorkspaceMemoryWritable(true)).toBe(false); - } finally { - await session.dispose(); - } - }); - test("upgrades an index-only memory context when hot memories are requested", async () => { using sessionDir = new DisposableTempDir("agent-session-memory-context-upgrade"); const { historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 90a18004745..4c571935e44 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1055,20 +1055,40 @@ export class AgentSession { /** * Workspace-memory write policy accumulated over the current compaction - * epoch (TurnRequestBuilder via WorkspaceService). Attached to compaction - * completions so the memory harvest — which writes to the (possibly shared) - * workspace store on this agent's behalf — can honor a read-only agent's - * policy. Fail-closed across turns: the harvest reads EVERY message of the - * epoch, so one read-only turn denies the whole epoch even if a writable - * turn follows; the accumulator restarts at the compaction boundary - * (unless a preserved tail carries the epoch's messages forward). + * epoch, mirrored from the DURABLE accumulator in config.json + * (WorkspaceService.recordWorkspaceMemoryWritable performs the fail-closed + * AND against the persisted value, so backends sharing one chat.jsonl — + * XUM_ALLOW_MULTIPLE_INSTANCES — and a restarted process all contribute to + * one conjunction). Attached to compaction completions so the memory + * harvest — which writes to the (possibly shared) workspace store on this + * agent's behalf — honors a read-only turn anywhere in the epoch: the + * harvest reads EVERY message of the epoch. Both copies restart at a + * context boundary (compaction without a preserved tail, /clear, context + * reset, destructive history replace) via resetWorkspaceMemoryWritable. */ private workspaceMemoryWritable: boolean | undefined; - /** Returns the effective (accumulated) value, which is what gets persisted. */ - recordWorkspaceMemoryWritable(writable: boolean): boolean { - this.workspaceMemoryWritable = (this.workspaceMemoryWritable ?? true) && writable; - return this.workspaceMemoryWritable; + recordWorkspaceMemoryWritable(effective: boolean): void { + this.workspaceMemoryWritable = effective; + } + + /** + * Start a fresh policy epoch: the in-memory mirror and the durable + * accumulator both forget the previous epoch's turns. Durable-or-throw like + * the other boundary invalidations: a stale persisted deny would refuse + * harvests of the new, possibly all-writable epoch forever (and a stale + * grant is never left behind by this path — grants are re-recorded per + * turn). Nothing to do when the field is already absent. + */ + private async resetWorkspaceMemoryWritable(): Promise { + this.workspaceMemoryWritable = undefined; + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); + if (entry?.workspace.workspaceMemoryWritable === undefined) return; + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, this.workspaceId); + if (current !== null) delete current.workspace.workspaceMemoryWritable; + return cfg; + }); } /** @@ -1242,8 +1262,17 @@ export class AgentSession { // New epoch. A preserved tail copies messages produced under this // epoch's policy into the next one, so the fail-closed accumulator // carries over with them; otherwise the next normal turn restarts it. + // The completion callback is synchronous; the durable reset runs + // detached and is logged on failure (the next turn re-records the + // policy anyway, so a failed reset can only delay a grant, never + // widen one). if ((metadata.preservedTailMessageCount ?? 0) === 0) { - this.workspaceMemoryWritable = undefined; + this.resetWorkspaceMemoryWritable().catch((error: unknown) => { + log.warn("Failed to reset the workspace memory policy epoch", { + workspaceId: this.workspaceId, + error, + }); + }); } }, onIdleCompactionOutcome, @@ -9821,6 +9850,10 @@ export class AgentSession { this.postCompactionLoadedSkills = []; this.postCompactionReadFilePaths = []; this.pendingPostCompactionStateToAcknowledge = null; + // A destructive context boundary (/clear, reset, history replace) starts + // a new harvest epoch: the fail-closed policy of the discarded transcript + // must not keep denying the new one. + await this.resetWorkspaceMemoryWritable(); // Durable-or-throw: a swallowed unlink failure would leave the stale // post-compaction.json to re-inject pre-boundary carryover after a // restart while the boundary caller reports success — the same diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 4410b073749..ae421b95b72 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1651,6 +1651,10 @@ export class MemoryService extends EventEmitter { actor, toolCallId ); + } else { + // Unjournaled delete (unrepresentable subtree): the store still + // changed, so other backends' cached views must still see it. + await this.advanceStoreRevision(store); } await this.recordDelete(ctx, scope, parsed.relPath); this.emitChange(ctx, scope, parsed.relPath, actor); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 50c00b58900..0737e71ec85 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,3 +1,4 @@ +import { findWorkspaceEntry } from "@/node/services/taskUtils"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -9274,6 +9275,56 @@ describe("WorkspaceService initialize", () => { } }); + test("accumulates the workspace memory write policy fail-closed in config across an epoch", async () => { + const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); + const scratchDir = path.join(realConfig.rootDir, "scratch", "policy-scratch"); + await fsPromises.mkdir(scratchDir, { recursive: true }); + await realConfig.editConfig((cfg) => { + cfg.projects.set(SCRATCH_PROJECT_CONFIG_KEY, { + workspaces: [ + { + kind: "scratch", + path: scratchDir, + id: "policy-scratch", + name: "scratch-policy-scratch", + runtimeConfig: { type: "local" }, + }, + ], + projectKind: "system", + trusted: true, + }); + return cfg; + }); + const aiService = { + ...createStreamLifecycleMocks(), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + const service = createWorkspaceServiceForTest({ + config: realConfig, + historyService, + aiService, + initStateManager: mockInitStateManager as InitStateManager, + }); + const persisted = () => + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace + .workspaceMemoryWritable; + try { + // The durable bit is the epoch accumulator: the harvest reads every + // message of the epoch, so a read-only turn denies the epoch even when + // a writable turn follows — across restarts and backends, since the + // conjunction lives in config.json rather than in one process. + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(persisted()).toBe(false); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(false); + } finally { + await cleanup(); + } + }); + test("removes stale orphaned scratch workdirs but keeps referenced and recent ones", async () => { const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); const scratchDirFor = (id: string) => path.join(realConfig.rootDir, "scratch", id); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e223bf20c16..ae8ea08b265 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4220,14 +4220,17 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * new value could not be confirmed durable. */ async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { - // The session accumulates the policy over the compaction epoch - // (fail-closed); the durable copy mirrors that effective value so a - // restart-time fallback cannot be more permissive than the live session. - const effective = - ( - this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) - )?.recordWorkspaceMemoryWritable(writable) ?? writable; const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + // The persisted bit is the epoch accumulator, fail-closed: the harvest + // reads every message of the compaction epoch, so one read-only turn + // denies the whole epoch even if writable turns follow. Durable so it + // survives a restart mid-epoch AND so backends sharing one chat.jsonl + // (multi-instance) contribute to the same conjunction; it restarts at + // context boundaries (AgentSession.resetWorkspaceMemoryWritable). + const effective = (entry?.workspace.workspaceMemoryWritable ?? true) && writable; + ( + this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) + )?.recordWorkspaceMemoryWritable(effective); // Unregistered workspace: nothing durable to update and no stale // permission to invalidate (harvests fail closed on the missing value). if (entry === null) return true; From 029e9e28b5829c481ef10006018e7973df5f94fd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 23:09:19 +0000 Subject: [PATCH 29/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-fi?= =?UTF-8?q?fth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The epoch conjunction is computed inside the config transaction from the value current at write time, ANDed with the session's own mirror, so two backends racing on an absent bit cannot publish false→true and a deny a session observed survives another backend's boundary reset. - The compaction-boundary reset is fenced to the value the session observed at the boundary; a newer epoch's write from another backend survives. - At compaction, either observed deny (session mirror or durable bit) is authoritative instead of the mirror overriding a durable false. - Memory mutations record usage/pin sidecar state before the row advances the store clock, so a foreign rebuild on the new revision already sees it. - Owner resolution indexes the config snapshot once (WeakMap per snapshot), making memo revalidation on config edits O(n) instead of O(n²). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 34 ++++++-- src/node/services/memoryService.ts | 35 +++++--- src/node/services/memoryWorkspaceOwner.ts | 98 ++++++++++++++-------- src/node/services/workspaceService.test.ts | 17 ++++ src/node/services/workspaceService.ts | 56 +++++++++---- 5 files changed, 169 insertions(+), 71 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 4c571935e44..87aadc081b5 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1072,6 +1072,11 @@ export class AgentSession { this.workspaceMemoryWritable = effective; } + /** The mirror, for WorkspaceService's conjunction (undefined until a turn recorded). */ + workspaceMemoryWritableMirror(): boolean | undefined { + return this.workspaceMemoryWritable; + } + /** * Start a fresh policy epoch: the in-memory mirror and the durable * accumulator both forget the previous epoch's turns. Durable-or-throw like @@ -1080,13 +1085,24 @@ export class AgentSession { * grant is never left behind by this path — grants are re-recorded per * turn). Nothing to do when the field is already absent. */ - private async resetWorkspaceMemoryWritable(): Promise { + private async resetWorkspaceMemoryWritable(options?: { + closing: boolean | undefined; + }): Promise { this.workspaceMemoryWritable = undefined; const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); if (entry?.workspace.workspaceMemoryWritable === undefined) return; await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, this.workspaceId); - if (current !== null) delete current.workspace.workspaceMemoryWritable; + if (current === null) return cfg; + // Fenced to the epoch being closed: another backend may already have + // recorded the first turn of the NEW epoch between the completion and + // this locked write; a value different from the one this session + // observed at the boundary is that newer epoch's and must survive + // (a same-valued newer deny survives through that backend's mirror). + if (options !== undefined && current.workspace.workspaceMemoryWritable !== options.closing) { + return cfg; + } + delete current.workspace.workspaceMemoryWritable; return cfg; }); } @@ -1267,12 +1283,14 @@ export class AgentSession { // policy anyway, so a failed reset can only delay a grant, never // widen one). if ((metadata.preservedTailMessageCount ?? 0) === 0) { - this.resetWorkspaceMemoryWritable().catch((error: unknown) => { - log.warn("Failed to reset the workspace memory policy epoch", { - workspaceId: this.workspaceId, - error, - }); - }); + this.resetWorkspaceMemoryWritable({ closing: this.workspaceMemoryWritable }).catch( + (error: unknown) => { + log.warn("Failed to reset the workspace memory policy epoch", { + workspaceId: this.workspaceId, + error, + }); + } + ); } }, onIdleCompactionOutcome, diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index ae421b95b72..8d77dce4493 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -46,7 +46,10 @@ import { withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; -import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { + resolveWorkspaceMemoryOwnerId, + workspaceMemoryOwnerResolver, +} from "@/node/services/memoryWorkspaceOwner"; import { advanceWorkspaceMemoryRevision, readWorkspaceMemoryRevision, @@ -667,10 +670,11 @@ export class MemoryService extends EventEmitter { */ private invalidateWorkspaceMemoryOwnerMemo(): void { if (this.workspaceMemoryOwnerById.size === 0) return; - const cfg = this.config.loadConfigOrDefault(); + // One parse and one ID index for the whole pass (O(n), not O(n²)). + const resolve = workspaceMemoryOwnerResolver(this.config.loadConfigOrDefault()); const changed: string[] = []; for (const [workspaceId, previousOwner] of this.workspaceMemoryOwnerById) { - const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + const owner = resolve(workspaceId); this.workspaceMemoryOwnerById.set(workspaceId, owner); if (owner !== previousOwner) changed.push(workspaceId); } @@ -1362,7 +1366,10 @@ export class MemoryService extends EventEmitter { } await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, fileText); - // Row is written before the create is acknowledged (mutation → row → ack). + // Usage stats land BEFORE the row advances the store clock: a foreign + // backend rebuilding its hot set on the new revision must already see + // them (see workspaceMemoryRevision.ts). Row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, store, @@ -1372,7 +1379,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: store.physicalPath(parsed.relPath), content: fileText }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1404,7 +1410,8 @@ export class MemoryService extends EventEmitter { assertWithinFileSizeCap(updated); await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); - // Row is written before the edit is acknowledged (mutation → row → ack). + // Usage stats before the row (store clock); row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, store, @@ -1417,7 +1424,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: store.physicalPath(parsed.relPath), content: updated }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, output: `Edited ${toVirtualPath(scope, parsed.relPath)}` }; }); @@ -1458,7 +1464,8 @@ export class MemoryService extends EventEmitter { assertWithinFileSizeCap(updated); await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, updated); - // Row is written before the edit is acknowledged (mutation → row → ack). + // Usage stats before the row (store clock); row before ack. + await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); await this.journalRefinement( ctx, store, @@ -1471,7 +1478,6 @@ export class MemoryService extends EventEmitter { toolCallId, [{ path: store.physicalPath(parsed.relPath), content: updated }] ); - await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1642,6 +1648,8 @@ export class MemoryService extends EventEmitter { const inverse = await this.captureDeleteInverse(store, parsed.relPath, kind); await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.remove(parsed.relPath); + // Sidecar first, then the row advances the store clock (see create). + await this.recordDelete(ctx, scope, parsed.relPath); if (inverse !== null) { await this.journalRefinement( ctx, @@ -1656,7 +1664,6 @@ export class MemoryService extends EventEmitter { // changed, so other backends' cached views must still see it. await this.advanceStoreRevision(store); } - await this.recordDelete(ctx, scope, parsed.relPath); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, @@ -1709,6 +1716,8 @@ export class MemoryService extends EventEmitter { } await this.assertMutationCommittable(ctx, store, abortSignal, oldVirtualPath); await store.rename(oldParsed.relPath, newParsed.relPath); + // Sidecar first, then the row advances the store clock (see create). + await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); // Row is written before the rename is acknowledged (mutation → row → ack). await this.journalRefinement( ctx, @@ -1726,7 +1735,6 @@ export class MemoryService extends EventEmitter { actor, toolCallId ); - await this.recordRename(ctx, scope, oldParsed.relPath, newParsed.relPath); this.emitChange(ctx, scope, oldParsed.relPath, actor); this.emitChange(ctx, scope, newParsed.relPath, actor); return { @@ -1861,9 +1869,10 @@ export class MemoryService extends EventEmitter { } await this.assertMutationCommittable(ctx, store, abortSignal, virtualPath); await store.writeFile(parsed.relPath, content); - // UI saves are not journaled, so advance the store clock here (in-lock). - await this.advanceStoreRevision(store); await this.recordUsage(ctx, scope, parsed.relPath, { write: true }); + // UI saves are not journaled, so advance the store clock here + // (in-lock, after the sidecar so foreign rebuilds see the stats). + await this.advanceStoreRevision(store); this.emitChange(ctx, scope, parsed.relPath, actor); return { success: true as const, data: { sha256: sha256Hex(content) } }; } diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index e1a1952d83c..20c9ce2f6dc 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -1,7 +1,6 @@ import assert from "@/common/utils/assert"; -import type { Config } from "@/node/config"; +import type { Config, Workspace as WorkspaceConfigEntry } from "@/node/config"; import { log } from "@/node/services/log"; -import { findWorkspaceEntry } from "@/node/services/taskUtils"; type ProjectsConfig = ReturnType; @@ -19,46 +18,73 @@ type ProjectsConfig = ReturnType; * child?" must compare the result to the input rather than test * parentWorkspaceId, so those fallbacks keep their private store usable. * - * Pure over one config snapshot; MemoryService memoizes it, removal and the + * Pure over one config snapshot (indexed once per snapshot, see + * workspaceMemoryOwnerResolver); MemoryService memoizes it, removal and the * rollback tooling call it directly. */ export function resolveWorkspaceMemoryOwnerId(cfg: ProjectsConfig, workspaceId: string): string { - assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); - let current = workspaceId; - const visited = new Set(); - for (let depth = 0; depth < 32; depth++) { - if (visited.has(current)) { - log.warn("[memory] parentWorkspaceId cycle; using acting workspace as memory owner", { - workspaceId, - }); - return workspaceId; + return workspaceMemoryOwnerResolver(cfg)(workspaceId); +} + +/** + * Per-snapshot resolvers keyed by the config object: bulk passes (memo + * revalidation on every local config edit, launch sweep, change diffing) + * resolve many workspaces against one snapshot, and a linear + * findWorkspaceEntry per chain hop would make them O(n²) on the main process. + * The index is built once per snapshot and the snapshot is never mutated + * after load, so a WeakMap keyed by identity is safe. + */ +const resolversBySnapshot = new WeakMap string>(); + +export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: string) => string { + const cached = resolversBySnapshot.get(cfg); + if (cached !== undefined) return cached; + const byId = new Map(); + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.id !== undefined) byId.set(workspace.id, workspace); } - visited.add(current); - const entry = findWorkspaceEntry(cfg, current); - if (entry === null) { - // Only the chain root may be unknown without invalidating the walk: an - // unregistered starting workspace simply resolves to itself. - if (current !== workspaceId) { - log.warn("[memory] parentWorkspaceId points at an unknown workspace", { + } + const resolver = (workspaceId: string): string => { + assert(workspaceId.length > 0, "resolveWorkspaceMemoryOwnerId requires a workspaceId"); + let current = workspaceId; + const visited = new Set(); + for (let depth = 0; depth < 32; depth++) { + if (visited.has(current)) { + log.warn("[memory] parentWorkspaceId cycle; using acting workspace as memory owner", { workspaceId, - parentWorkspaceId: current, }); + return workspaceId; } - return workspaceId; - } - // A pinned owner (recorded when an intermediate ancestor was removed) - // short-circuits the walk; if that owner is itself gone, fall through to - // the parent chain, which then dangles and resolves to self. - const pinned = entry.workspace.memoryOwnerWorkspaceId; - if (pinned !== undefined && pinned !== "" && findWorkspaceEntry(cfg, pinned) !== null) { - return pinned; + visited.add(current); + const entry = byId.get(current); + if (entry === undefined) { + // Only the chain root may be unknown without invalidating the walk: an + // unregistered starting workspace simply resolves to itself. + if (current !== workspaceId) { + log.warn("[memory] parentWorkspaceId points at an unknown workspace", { + workspaceId, + parentWorkspaceId: current, + }); + } + return workspaceId; + } + // A pinned owner (recorded when an intermediate ancestor was removed) + // short-circuits the walk; if that owner is itself gone, fall through to + // the parent chain, which then dangles and resolves to self. + const pinned = entry.memoryOwnerWorkspaceId; + if (pinned !== undefined && pinned !== "" && byId.has(pinned)) { + return pinned; + } + const parentWorkspaceId = entry.parentWorkspaceId; + if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; + current = parentWorkspaceId; } - const parentWorkspaceId = entry.workspace.parentWorkspaceId; - if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; - current = parentWorkspaceId; - } - log.warn("[memory] parentWorkspaceId chain too deep; using acting workspace as memory owner", { - workspaceId, - }); - return workspaceId; + log.warn("[memory] parentWorkspaceId chain too deep; using acting workspace as memory owner", { + workspaceId, + }); + return workspaceId; + }; + resolversBySnapshot.set(cfg, resolver); + return resolver; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0737e71ec85..943d5b6f725 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9320,6 +9320,23 @@ describe("WorkspaceService initialize", () => { expect(persisted()).toBe(false); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); expect(persisted()).toBe(false); + + // New epoch (field cleared at the boundary), then ANOTHER backend records + // a read-only turn straight into config: this backend's next grant must + // not publish false→true. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(true); + await realConfig.editConfig((cfg) => { + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable = false; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(false); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ae8ea08b265..909f3b2f19e 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4220,25 +4220,45 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * new value could not be confirmed durable. */ async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { - const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + const session = + this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); + const mirror = session?.workspaceMemoryWritableMirror(); + const before = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + // Unregistered workspace: nothing durable to update and no stale + // permission to invalidate (harvests fail closed on the missing value). + if (before === null) { + session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable); + return true; + } // The persisted bit is the epoch accumulator, fail-closed: the harvest // reads every message of the compaction epoch, so one read-only turn // denies the whole epoch even if writable turns follow. Durable so it // survives a restart mid-epoch AND so backends sharing one chat.jsonl // (multi-instance) contribute to the same conjunction; it restarts at - // context boundaries (AgentSession.resetWorkspaceMemoryWritable). - const effective = (entry?.workspace.workspaceMemoryWritable ?? true) && writable; - ( - this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId) - )?.recordWorkspaceMemoryWritable(effective); - // Unregistered workspace: nothing durable to update and no stale - // permission to invalidate (harvests fail closed on the missing value). - if (entry === null) return true; - if (entry.workspace.workspaceMemoryWritable === effective) return true; + // context boundaries (AgentSession.resetWorkspaceMemoryWritable). The + // conjunction is computed INSIDE the config transaction (registration + // lock, cross-process) from the value current at write time — two + // backends reading an absent bit concurrently could otherwise publish + // false→true. The session additionally contributes its own mirror: a + // deny it observed survives even if another backend's boundary reset + // removed the durable field underneath it. + const conjunction = (durable: boolean | undefined): boolean => + (durable ?? true) && (mirror ?? true) && writable; + // Fast path (no write): the outcome cannot differ from the stored value — + // it is already false, or already true and this turn grants. + const stored = before.workspace.workspaceMemoryWritable; + if (stored === false || (stored === true && conjunction(stored))) { + session?.recordWorkspaceMemoryWritable(stored); + return true; + } + let effective = conjunction(stored); try { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); - if (current !== null) current.workspace.workspaceMemoryWritable = effective; + if (current !== null) { + effective = conjunction(current.workspace.workspaceMemoryWritable); + current.workspace.workspaceMemoryWritable = effective; + } return cfg; }); } catch (error: unknown) { @@ -4249,6 +4269,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); return false; } + session?.recordWorkspaceMemoryWritable(effective); const persisted = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace .workspaceMemoryWritable; if (persisted !== effective) { @@ -4355,14 +4376,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.schedulePostCompactionMetadataRefresh(workspaceId); // Compaction marks a long session with accumulated learnings: harvest // the compacted epoch first, then let Dream sweep/merge the candidates. - // The session knows the policy only if it built a normal turn; after a - // restart/recovery fall back to the persisted value. Still unknown → + // Two observations of the epoch policy — the session's mirror (attached + // to the completion) and the durable accumulator (which other backends + // sharing this chat.jsonl also write) — and either deny is + // authoritative; both unknown (fresh recovery session, field absent) → // the harvest fails closed. const persistedWritable = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId) ?.workspace.workspaceMemoryWritable; + const observed = [metadata.workspaceMemoryWritable, persistedWritable].filter( + (value): value is boolean => value !== undefined + ); this.memoryConsolidationService?.triggerHarvestThenSweepInBackground({ ...metadata, - workspaceMemoryWritable: metadata.workspaceMemoryWritable ?? persistedWritable, + ...(observed.length > 0 + ? { workspaceMemoryWritable: observed.every((value) => value) } + : {}), }); }, onIdleCompactionOutcome: (success) => { From 6cbdd84bcca338bec28f882e36e2feeef2b5330d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 23:37:47 +0000 Subject: [PATCH 30/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-si?= =?UTF-8?q?xth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MemoryMetaService stamp-validates its sidecar cache (re-stat per load, reparse on change) and serializes read-modify-writes across processes with a sidecar lockfile, so a backend rebuilding a hot set on another backend's store revision sees that backend's pins/usage stats and never rewrites the file from a stale cache. - The memory subscription announces its store-revision baseline with one root-addressed refresh, so a foreign write landing between the client's initial listing and the subscription start is not silently adopted as the baseline. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/orpc/routerSubscriptions.test.ts | 10 +++++ src/node/orpc/routerSubscriptions.ts | 27 ++++++++---- src/node/services/memoryMeta.test.ts | 16 ++++++++ src/node/services/memoryMeta.ts | 48 ++++++++++++++++++++-- src/node/services/memoryOperations.test.ts | 12 ++++++ 5 files changed, 101 insertions(+), 12 deletions(-) diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 5afedd6e514..7cb2d69c02a 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -61,6 +61,16 @@ test("memory subscriptions match workspace-scope events on the shared memory own } as unknown as ORPCContext; const stream = subscribeMemoryChanges(context, "ws-child", controller.signal); try { + // Baseline handshake: the store revision is announced once as a + // root-addressed refresh so the client's listing catches up with a write + // that landed between its initial fetch and this subscription. + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: "", + }); const first = stream.next(); // The listener attaches once the generator has started running. while (memoryService.listenerCount("change") === 0) { diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 90c4330c5b9..93cebf8d494 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -225,16 +225,31 @@ export function subscribeMemoryChanges( // failed refresh leaves the old token, costing at most one redundant // refresh on the next probe. let storeRevision: string | null = null; - const refreshStoreRevision = () => { + const rootRefresh = (): MemoryChangeEventPayload => ({ + scope: "workspace", + path: toVirtualPath("workspace", ""), + actor: "agent", + workspaceId: workspaceId + ? context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId) + : "", + projectPath: projectPath ?? "", + }); + const refreshStoreRevision = (options?: { announce: boolean }) => { if (!workspaceId) return; context.memoryService.workspaceMemoryRevision(workspaceId).then( (revision) => { storeRevision = revision; + if (options?.announce) emit.push(rootRefresh()); }, () => undefined ); }; - refreshStoreRevision(); + // Baseline handshake: the client's initial listing and this + // subscription start independently, so a foreign write landing between + // them would be adopted here as the baseline while the listing shows + // the old files. Announce the baseline once it is read: the client + // refetches, and the listing is then at least as new as the token. + refreshStoreRevision({ announce: true }); const onChange = (event: MemoryChangeEvent) => { if ( event.scope === "workspace" && @@ -278,13 +293,7 @@ export function subscribeMemoryChanges( (revision) => { if (revision === storeRevision) return; storeRevision = revision; - emit.push({ - scope: "workspace", - path: toVirtualPath("workspace", ""), - actor: "agent", - workspaceId: context.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId), - projectPath: projectPath ?? "", - }); + emit.push(rootRefresh()); }, () => undefined ); diff --git a/src/node/services/memoryMeta.test.ts b/src/node/services/memoryMeta.test.ts index b39581fb4b8..ee2e03868bb 100644 --- a/src/node/services/memoryMeta.test.ts +++ b/src/node/services/memoryMeta.test.ts @@ -64,6 +64,22 @@ describe("MemoryMetaService", () => { ); }); + it("sees another backend's sidecar writes and never overwrites them from a stale cache", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + // Two services over one Xum root stand in for two backend processes + // (XUM_ALLOW_MULTIPLE_INSTANCES): neither sees the other's in-memory cache. + const a = new MemoryMetaService(tempDir.path); + const b = new MemoryMetaService(tempDir.path); + await b.getEntries(); // B caches the (empty) sidecar + await a.setPinned("workspace:ws-owner:notes.md", true); + // B's stamp-validated load picks up A's write... + expect(await b.getPinnedKeys()).toEqual(new Set(["workspace:ws-owner:notes.md"])); + // ...and B's own mutation starts from the current file, keeping A's pin. + await b.recordAccess("global:other.md", { write: false }); + expect(await a.getPinnedKeys()).toEqual(new Set(["workspace:ws-owner:notes.md"])); + expect((await a.getEntries()).has("global:other.md")).toBe(true); + }); + it("unpinning removes the key", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index 137e650e6c3..58927983a2d 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -13,6 +13,7 @@ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; +import { acquireProcessFileLock } from "@/node/utils/concurrency/fileLock"; import { Effect, Schema, Semaphore } from "effect"; import type { MemoryScope } from "@/common/constants/memory"; import { getErrorMessage } from "@/common/utils/errors"; @@ -136,6 +137,9 @@ export class MemoryMetaWriteError extends Schema.TaggedError self.fileStamp()); + if (self.cache !== null && stamp === self.cacheStamp) return self.cache; const parsed = yield* Effect.tryPromise({ try: async (): Promise => JSON.parse(await fsPromises.readFile(self.metaPath, "utf-8")), @@ -265,12 +282,24 @@ export class MemoryMetaService { }) ); self.cache = sanitizeMetaFile(parsed); + self.cacheStamp = stamp; return self.cache; }); } + /** Cheap change signal for the sidecar (same scheme as Config.configFileStamp). */ + private async fileStamp(): Promise { + try { + const st = await fsPromises.stat(this.metaPath, { bigint: true }); + return `${st.dev}:${st.ino}:${st.size}:${st.mtimeNs}`; + } catch { + return "missing"; + } + } + /** - * Read-modify-write cycle under the sidecar semaphore. Persists before + * Read-modify-write cycle under the sidecar semaphore (in-process) and the + * sidecar lockfile (other backends over the same Xum root). Persists before * updating the in-memory cache so observers never see state that didn't make * it to disk. Entries that end up entirely default are dropped. */ @@ -281,6 +310,18 @@ export class MemoryMetaService { const self = this; return this.writeLock.withPermit( Effect.gen(function* () { + const fileLock = yield* Effect.tryPromise({ + try: () => + acquireProcessFileLock({ + lockPath: `${self.metaPath}.lock`, + timeoutMs: MEMORY_META_LOCK_TIMEOUT_MS, + label: "memory metadata sidecar", + }), + catch: (cause) => + new MemoryMetaWriteError({ metaPath: self.metaPath, reason: getErrorMessage(cause) }), + }); + yield* Effect.addFinalizer(() => Effect.promise(() => fileLock[Symbol.asyncDispose]())); + // Stamp-validated: sees a foreign backend's write that landed since. const meta = yield* self.load(); const entries = { ...meta.entries }; update(entries); @@ -307,9 +348,10 @@ export class MemoryMetaService { }), }); self.cache = next; + self.cacheStamp = yield* Effect.promise(() => self.fileStamp()); }) ); - }) + }).pipe(Effect.scoped) ); } diff --git a/src/node/services/memoryOperations.test.ts b/src/node/services/memoryOperations.test.ts index 20ad73f028f..cdc15b4ab33 100644 --- a/src/node/services/memoryOperations.test.ts +++ b/src/node/services/memoryOperations.test.ts @@ -273,6 +273,15 @@ describe("memory operations", () => { expect(await memoryMetaService.getPinnedKeys()).toEqual(new Set()); }); + // The subscription announces its store-revision baseline once with a + // root-addressed workspace refresh (see subscribeMemoryChanges); tests + // about forwarded events skip it. + const isBaselineRefresh = (event: MemoryChangeEventPayload): boolean => + !("kind" in event) && + event.scope === "workspace" && + event.path === "/memories/workspace" && + event.actor === "agent"; + test("onChange streams change events from UI saves", async () => { const client = createClient({ enabled: true }); const controller = new AbortController(); @@ -283,6 +292,7 @@ describe("memory operations", () => { const firstEvent = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; return event; } return null; @@ -436,6 +446,7 @@ describe("memory operations", () => { const received: MemoryChangeEventPayload[] = []; const consumer = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; received.push(event); if (received.length >= 3) break; } @@ -508,6 +519,7 @@ describe("memory operations", () => { const received: MemoryChangeEventPayload[] = []; const consumer = (async () => { for await (const event of iterator) { + if (isBaselineRefresh(event)) continue; received.push(event); break; } From 25a06cf725bd7051bddc2e13f2b8007cb1792eb7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 8 Sep 2026 23:56:44 +0000 Subject: [PATCH 31/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-se?= =?UTF-8?q?venth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pin toggles go through MemoryService.setPinned: for the workspace scope the sidecar write and the store-clock advance both happen under the store's mutation lock, so a lock timeout fails before anything commits (no durable pin behind a failed route and a stale foreign hot set). - The memory-context probe reports "revoked" once the acting workspace or the store's owner is tombstoned, invalidating a cached context, and listIndexEntries (which also feeds the hot set) refuses the workspace store under the same tombstone guard as reads. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryOperations.ts | 46 +++++++++++------------- src/node/services/memoryService.test.ts | 10 +++++- src/node/services/memoryService.ts | 48 +++++++++++++++++-------- 3 files changed, 64 insertions(+), 40 deletions(-) diff --git a/src/node/services/memoryOperations.ts b/src/node/services/memoryOperations.ts index fd303d686dc..603ae34344d 100644 --- a/src/node/services/memoryOperations.ts +++ b/src/node/services/memoryOperations.ts @@ -23,7 +23,7 @@ import { resolveMemoryProjectIdentity, type MemoryScopeContext, } from "./memoryService"; -import { memoryLogicalKey } from "./memoryMeta"; +import { MemoryMetaWriteError, memoryLogicalKey } from "./memoryMeta"; type MemoryContext = Pick< ORPCContext, @@ -234,31 +234,27 @@ export function setMemoryPinnedEffect( success: false as const, error: "Project memory is unavailable: no project is associated with this session", }; - return yield* context.memoryMetaService.effects - .setPinned( - memoryLogicalKey(scope, relPath, { - projectPath: resolved.projectPath, - workspaceId: resolved.ownerWorkspaceId, - }), - input.pinned + // Sidecar write + (workspace scope) store-clock advance + change event, + // committed under the store's mutation lock by MemoryService. + return yield* Effect.tryPromise({ + try: () => context.memoryService.setPinned(resolved.scopeCtx, input.path, input.pinned), + catch: (error: unknown) => error, + }).pipe( + Effect.map(() => ({ success: true as const, data: undefined })), + // Sidecar write failures (disk full, permissions) arrive as the typed + // MemoryMetaWriteError; lock timeouts and command errors map onto the + // same legacy string error channel instead of escaping as an untyped + // INTERNAL_SERVER_ERROR rejection. + Effect.catch((error) => + Effect.succeed({ + success: false as const, + error: + error instanceof MemoryMetaWriteError + ? `Failed to persist pin state: ${error.reason}` + : `Failed to update pin: ${getErrorMessage(error)}`, + }) ) - .pipe( - // Pins live in the sidecar, not the store, so nothing else emits a - // change: notify so the other tree members' tabs refetch too. - Effect.flatMap(() => - Effect.promise(() => context.memoryService.notifyPinChange(resolved.scopeCtx, input.path)) - ), - Effect.map(() => ({ success: true as const, data: undefined })), - // Sidecar write failures (disk full, permissions) arrive as the typed - // MemoryMetaWriteError and map onto the legacy string error channel - // instead of escaping as an untyped INTERNAL_SERVER_ERROR rejection. - Effect.catchTag("MemoryMetaWriteError", (error) => - Effect.succeed({ - success: false as const, - error: `Failed to persist pin state: ${error.reason}`, - }) - ) - ); + ); }).pipe(Effect.catchTag("MemoryWorkspaceNotFoundError", workspaceNotFoundAsStringError)); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 0aa33734bd9..b0dd1c6344b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1103,7 +1103,7 @@ describe("MemoryService", () => { await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); // ...a pin toggle (hot-set input, no store write) advances it... - await fixture.service.notifyPinChange(fixture.ctx, "/memories/workspace/shared.md"); + await fixture.service.setPinned(fixture.ctx, "/memories/workspace/shared.md", true); const afterPin = await foreign.workspaceMemoryRevision("ws-owner"); expect(Number(afterPin)).toBeGreaterThan(Number(afterCreate)); // ...while every shared-store mutation advances it. @@ -1169,6 +1169,14 @@ describe("MemoryService", () => { const root = await fixture.service.view(fixture.ctx, "/memories"); expect(root.success).toBe(true); if (root.success) expect(root.output).toContain("unavailable"); + // The prompt-context path is guarded too: the probe reports revocation + // (invalidating a cached context) and the index no longer lists the store. + expect(await fixture.service.workspaceMemoryRevision("ws-child")).toBe("revoked"); + expect( + (await fixture.service.listIndexEntries(fixture.ctx)).some( + (entry) => entry.scope === "workspace" + ) + ).toBe(false); }); it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8d77dce4493..527946fa085 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1185,9 +1185,14 @@ export class MemoryService extends EventEmitter { */ async workspaceMemoryRevision(workspaceId: string): Promise { assert(workspaceId.length > 0, "workspaceMemoryRevision requires a workspaceId"); - const revision = await readWorkspaceMemoryRevision( - path.join(this.config.sessionsDir, this.resolveWorkspaceMemoryOwnerId(workspaceId)) - ); + const owner = this.resolveWorkspaceMemoryOwnerId(workspaceId); + // Access revoked (acting workspace or owner tombstoned by any backend): + // a distinct token so a cached context built from the owner's notes is + // invalidated and the rebuild (listIndexEntries) then excludes the store. + for (const guarded of new Set([workspaceId, owner])) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return "revoked"; + } + const revision = await readWorkspaceMemoryRevision(path.join(this.config.sessionsDir, owner)); return revision === null ? "missing" : String(revision); } @@ -1232,22 +1237,32 @@ export class MemoryService extends EventEmitter { } /** - * Announces a sidecar-only change (pin toggled from the Memory tab) so other - * subscribers of the same store — for the shared workspace notebook, every - * task-tree member's tab — refetch their listing. + * Toggle a pin (Memory tab). Pins live in the sidecar, not the store, so + * nothing else emits a change or moves the store clock: for the workspace + * scope the sidecar write AND the clock advance happen under the store's + * mutation lock, so a lock timeout fails BEFORE anything is committed (no + * durable pin with a stale foreign hot set and a failed route), and the + * other tree members' tabs are told afterwards. Sidecar write failures + * surface as MemoryMetaWriteError. */ - async notifyPinChange(ctx: MemoryScopeContext, virtualPath: string): Promise { + async setPinned(ctx: MemoryScopeContext, virtualPath: string, pinned: boolean): Promise { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); + const key = this.logicalKeyFor(ctx, scope, parsed.relPath); + if (key === null) { + throw new MemoryCommandError( + "Project memory is unavailable: no project is associated with this session" + ); + } if (scope === "workspace") { - // A pin changes the hot set other backends derive from this store, so - // the store clock must move — under the same mutation lock writers - // hold, or an unlocked read→write could overwrite a concurrent - // mutation's higher value and break the clock's monotonicity. const store = this.getStore(ctx, scope); - await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), () => - this.advanceStoreRevision(store) - ); + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + await this.metaService.setPinned(key, pinned); + // A pin changes the hot set other backends derive from this store. + await this.advanceStoreRevision(store); + }); + } else { + await this.metaService.setPinned(key, pinned); } this.emitChange(ctx, scope, parsed.relPath, "user"); } @@ -1900,6 +1915,11 @@ export class MemoryService extends EventEmitter { for (const scope of MEMORY_SCOPES) { try { const store = this.getStore(ctx, scope); + // Prompt context is a read of the (possibly shared) store: a removed + // child's stream in another backend must not keep indexing / hot-set + // reading its former owner's notes. Refused here (skipped below) like + // any other scope failure. + if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); // Read-only enumeration (stream startup, Memory tab) must not create // scope roots unnecessarily. Missing roots list as empty. await store.assertRootSafe(); From 5489cf0a6f3e5a27b5da224ed73e9fa94cce5d2c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 00:44:42 +0000 Subject: [PATCH 32/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-ei?= =?UTF-8?q?ghth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - setPinned re-runs the file-mutation commit guard inside the store lock so a pin bound to a since-removed (or moved) owner is refused instead of landing under a dead logical key while the route reports success. - Sub-agents created before sharing kept /memories/workspace in their own session dir; on first shared-store access their legacy notebook is now imported into the owner store (same relPath when free/identical, else imported//), pins/stats follow, and the legacy directory is renamed aside so nothing is discarded at removal. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 100 +++++++++++++ src/node/services/memoryService.ts | 179 +++++++++++++++++++++++- 2 files changed, 276 insertions(+), 3 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index b0dd1c6344b..88624100a0f 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -6,6 +6,7 @@ import { createHash } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import { Config } from "@/node/config"; +import { getErrorMessage } from "@/common/utils/errors"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; import { extractMemoryDescription, @@ -1179,6 +1180,105 @@ describe("MemoryService", () => { ).toBe(false); }); + it("refuses a pin toggle once the owner it was bound to is tombstoned", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + // Owner removed by another backend between the tab's owner resolution + // and the pin's lock acquisition: the pin must not be committed under + // the dead owner's logical key while the route reports success. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-owner"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-owner" })); + const refused = await fixture.service + .setPinned(fixture.ctx, "/memories/workspace/n.md", true) + .then( + () => null, + (error: unknown) => error + ); + expect(refused).toBeInstanceOf(Error); + expect(getErrorMessage(refused)).toContain("was removed"); + expect((await fixture.metaService.getPinnedKeys()).size).toBe(0); + expect(events).toEqual([]); + }); + + it("adopts a sub-agent's pre-sharing private notebook into the shared store on first access", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // Notes written by a build that kept the child's workspace scope in its + // own session dir, plus a pin recorded under the child's logical key. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "only-child.md"), "child notes"); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "same.md"), "identical"); + await fsPromises.writeFile(path.join(legacyRoot, "clash.md"), "child version"); + await fixture.metaService.setPinned("workspace:ws-child:only-child.md", true); + // The owner already holds one identical and one conflicting file. + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + await fixture.service.create( + ownerCtx, + "/memories/workspace/sub/same.md", + "identical", + "agent" + ); + await fixture.service.create( + ownerCtx, + "/memories/workspace/clash.md", + "owner version", + "agent" + ); + const events: unknown[] = []; + fixture.service.on("change", (event) => events.push(event)); + + const listed = await fixture.service.listIndexEntries(fixture.ctx); + expect(listed.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "clash.md", + "imported/ws-child/clash.md", + "only-child.md", + "sub/same.md", + ]); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "child notes" + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "clash.md"), "utf-8")).toBe( + "owner version" + ); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "clash.md"), "utf-8") + ).toBe("child version"); + // The pin followed the file to the owner-keyed logical key. + expect([...(await fixture.metaService.getPinnedKeys())]).toEqual([ + "workspace:ws-owner:only-child.md", + ]); + // Legacy directory is moved aside (never deleted), so a second access + // is a no-op and the tree's tabs were told once. + expect(await pathExists(legacyRoot)).toBe(false); + const sessionEntries = await fsPromises.readdir( + path.join(fixture.config.sessionsDir, "ws-child") + ); + expect(sessionEntries.some((name) => name.startsWith("memory.migrated-"))).toBe(true); + expect(events).toEqual([ + { + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: FIXTURE_PROJECT_PATH, + }, + ]); + await fixture.service.listIndexEntries(fixture.ctx); + expect(events).toHaveLength(1); + // A workspace that is its own owner keeps its private store untouched. + const solo = { ...fixture.ctx, workspaceId: "ws-solo" }; + await fixture.service.create(solo, "/memories/workspace/mine.md", "solo", "agent"); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-solo", "memory", "mine.md")) + ).toBe(true); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 527946fa085..ab171d02598 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -389,6 +389,20 @@ interface MemoryStore { assertContained(relPath: string): Promise; } +/** + * Owner-store directory that receives a sub-agent's legacy private notes whose + * relPath the owner already uses with different content (adoptLegacyPrivateStore). + */ +const LEGACY_IMPORT_DIR = "imported"; + +async function isDirectory(absPath: string): Promise { + try { + return (await fsPromises.stat(absPath)).isDirectory(); + } catch { + return false; + } +} + function isPathWithinRoot( realRoot: string, candidate: string, @@ -690,6 +704,12 @@ export class MemoryService extends EventEmitter { */ private readonly ownerByContext = new WeakMap(); + /** + * Sub-agents whose pre-sharing private notebook was found absent or already + * adopted during this process lifetime (see adoptLegacyPrivateStore). + */ + private readonly legacyStoreChecked = new Set(); + /** * Owner of the workspace scope for this context ("" when there is no * workspace). Public so callers that key sidecar metadata for the same @@ -857,12 +877,159 @@ export class MemoryService extends EventEmitter { relPath: string ): Promise { const store = this.getStore(ctx, scope); - if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); await store.assertRootSafe(); await store.assertContained(relPath); return store; } + /** + * Every workspace-scope entry point (commands, root listing, index build) + * goes through here: refuse revoked access, then fold a sub-agent's + * pre-sharing private notebook into the shared store it now resolves to. + */ + private async openWorkspaceStore(ctx: MemoryScopeContext, store: MemoryStore): Promise { + await this.assertWorkspaceStoreReadable(ctx, store); + await this.adoptLegacyPrivateStore(ctx, store); + } + + /** + * Upgrade compatibility for the shared task-tree notebook. Sub-agents + * created by builds before sharing kept `/memories/workspace` in their OWN + * session dir (//memory). getStore now redirects them to + * the owner's root, which would make those notes invisible — and removal + * later deletes the child's session dir, discarding them for good. On the + * child's first shared-store access, import every legacy file into the + * owner store (same relPath when free or identical; otherwise under + * imported//), move pins/stats with them, and rename the legacy + * directory aside so the import is one-shot and nothing is lost even for + * files the import skips (binary/oversize). Downgrading afterwards shows + * the notes in the parent's notebook rather than the child's — no loss. + * + * Runs under the owner store's mutation lock with the same commit guard as + * file mutations, and never throws: an import failure (lock timeout, disk) + * leaves the legacy directory intact for the next access, while the caller + * proceeds with the shared store. Not journaled: this is a mechanical + * relocation, not an agent edit; pre-upgrade child journal rows keep + * targeting the legacy physical paths (rollback restores there and the next + * access re-imports). + */ + private async adoptLegacyPrivateStore( + ctx: MemoryScopeContext, + store: MemoryStore + ): Promise { + const childId = ctx.workspaceId; + if (childId === "" || this.legacyStoreChecked.has(childId)) return; + const owner = this.storeOwnerWorkspaceId(store); + assert(owner !== null, "workspace-scope stores live under sessionsDir"); + if (owner === childId) return; // not redirected: the private store IS the store + const legacyRoot = path.join(this.config.sessionsDir, childId, "memory"); + if (!(await isDirectory(legacyRoot))) { + // One stat per child per process; a legacy dir can only reappear via a + // downgrade/upgrade cycle, which restarts the backend. + this.legacyStoreChecked.add(childId); + return; + } + try { + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + if (!(await isDirectory(legacyRoot))) return; // adopted by a concurrent command + await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); + const legacy = new LocalMemoryStore(legacyRoot); + const files = await legacy.listFiles(); + let imported = 0; + let skipped = 0; + for (const relPath of files) { + // Oversize or binary (lossy utf-8 decode) files cannot be carried + // by a text write; they stay in the renamed-aside directory. + const content = await this.readBoundedTextFile(legacy, relPath, relPath).catch( + () => null + ); + if (content === null || content.includes("\uFFFD")) { + skipped++; + continue; + } + const target = await this.legacyImportTarget(store, childId, relPath, content); + if (target === null) { + skipped++; + continue; + } + if (target.write) await store.writeFile(target.relPath, content); + imported++; + // Pins/stats were keyed by the child; they follow a written file. + // For an identical file the owner already has, the owner's own + // stats stand and the child's stale key is dropped. + const childKey = memoryLogicalKey("workspace", relPath, { + projectPath: ctx.projectPath, + workspaceId: childId, + }); + try { + if (target.write) { + await this.metaService.renameKeys( + childKey, + memoryLogicalKey("workspace", target.relPath, { + projectPath: ctx.projectPath, + workspaceId: owner, + }) + ); + } else { + await this.metaService.removeKeys(childKey); + } + } catch (error) { + log.debug("[MemoryService] failed to move legacy memory stats", { relPath, error }); + } + } + await fsPromises.rename(legacyRoot, `${legacyRoot}.migrated-${Date.now()}`); + await this.advanceStoreRevision(store); + log.info( + "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", + { + childId, + owner, + imported, + skipped, + } + ); + }); + this.legacyStoreChecked.add(childId); + } catch (error) { + log.warn( + "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", + { + childId, + owner, + error, + } + ); + return; + } + this.emitChange(ctx, "workspace", "", "agent"); + } + + /** + * Where a legacy file lands in the owner store: its own relPath when free + * (write) or already identical (no write); the per-child import directory + * when the owner has different content there; null when even that slot is + * taken by different content (left in the renamed-aside legacy directory). + */ + private async legacyImportTarget( + store: MemoryStore, + childId: string, + relPath: string, + content: string + ): Promise<{ relPath: string; write: boolean } | null> { + for (const candidate of [relPath, `${LEGACY_IMPORT_DIR}/${childId}/${relPath}`]) { + const kind = await store.kind(candidate); + if (kind === null) return { relPath: candidate, write: true }; + if (kind === "file") { + const existing = await this.readBoundedTextFile(store, candidate, candidate).catch( + () => null + ); + if (existing === content) return { relPath: candidate, write: false }; + } + } + return null; + } + /** * The workspace whose session dir physically holds `store` * (//memory → owner), or null for global/project roots, @@ -1257,6 +1424,12 @@ export class MemoryService extends EventEmitter { if (scope === "workspace") { const store = this.getStore(ctx, scope); await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + // Same commit guard as file mutations: `key` and `store` were bound to + // the owner the context resolved BEFORE the lock. If that owner was + // removed meanwhile (removal publishes its tombstone under this lock) + // or ownership moved, the pin would land under a dead logical key and + // the route would still report success. Refuse instead. + await this.assertMutationCommittable(ctx, store, undefined, virtualPath); await this.metaService.setPinned(key, pinned); // A pin changes the hot set other backends derive from this store. await this.advanceStoreRevision(store); @@ -1306,7 +1479,7 @@ export class MemoryService extends EventEmitter { sections.push(`- ${scope}/`); try { const store = this.getStore(ctx, scope); - if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); // Read-only: never create roots just to list (missing ⇒ empty). await store.assertRootSafe(); const files = await store.listFiles(); @@ -1919,7 +2092,7 @@ export class MemoryService extends EventEmitter { // child's stream in another backend must not keep indexing / hot-set // reading its former owner's notes. Refused here (skipped below) like // any other scope failure. - if (scope === "workspace") await this.assertWorkspaceStoreReadable(ctx, store); + if (scope === "workspace") await this.openWorkspaceStore(ctx, store); // Read-only enumeration (stream startup, Memory tab) must not create // scope roots unnecessarily. Missing roots list as empty. await store.assertRootSafe(); From 99f20a1a2498786a56a98a5844e58b33d6acde69 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 01:15:15 +0000 Subject: [PATCH 33/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20twenty-ni?= =?UTF-8?q?nth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy notebook adoption keeps the child's `memory` dir in place (where a downgraded build reads and writes it) instead of renaming it aside; the copy is idempotent via a dotfile manifest of adopted content hashes, so shared edits never resurface as stale duplicates and downgrade-time notes are folded in on the next upgrade. - The legacy root is lstat-checked (symlinked roots are ignored) and every file passes the store's containment check before it is read. - Rollback divergence detection merges the shared-store rows of every live task-tree member's journal (new `listSharedWorkspaceMemoryPeerSessionDirs` option, resolved per rollback from config), so a child's later edit under a path the owner renamed refuses the owner's rollback. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/cli/debug/refinements.ts | 13 +- src/node/services/memoryService.test.ts | 156 +++++++++++++++++- src/node/services/memoryService.ts | 135 ++++++++++----- src/node/services/memoryWorkspaceOwner.ts | 27 +++ .../services/refinement/refinementRollback.ts | 51 +++++- src/node/services/toolAssembly.ts | 2 + .../services/tools/refinement_rollback.ts | 3 + src/node/services/turnRequestBuilder.ts | 13 ++ 8 files changed, 345 insertions(+), 55 deletions(-) diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index 512504d9ec9..c737ee8059b 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -1,6 +1,9 @@ import * as path from "path"; import { defaultConfig } from "@/node/config"; -import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { + resolveWorkspaceMemoryOwnerId, + sharedWorkspaceMemoryPeerSessionDirs, +} from "@/node/services/memoryWorkspaceOwner"; import { MemoryRefinementActionSchema, RollbackRefinementActionSchema, @@ -53,16 +56,16 @@ export async function refinementsCommand( if (opts.rollback !== undefined) { // Sub-agents journal workspace-scope rows that point into the owner's // session dir; admit that root the same way the in-app tool does. - const memoryOwnerId = resolveWorkspaceMemoryOwnerId( - defaultConfig.loadConfigOrDefault(), - workspaceId - ); + const cfg = defaultConfig.loadConfigOrDefault(); + const memoryOwnerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); const result = await rollbackRefinement({ sessionDir, sharedWorkspaceMemorySessionDir: memoryOwnerId === workspaceId ? undefined : path.join(defaultConfig.sessionsDir, memoryOwnerId), + listSharedWorkspaceMemoryPeerSessionDirs: () => + sharedWorkspaceMemoryPeerSessionDirs(cfg, defaultConfig.sessionsDir, workspaceId), id: opts.rollback, force: opts.force, evidence: { toolName: "debug-cli", actor: "user" }, diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 88624100a0f..dd7aa6c2a47 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -27,6 +27,7 @@ import { import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; +import { sharedWorkspaceMemoryPeerSessionDirs } from "./memoryWorkspaceOwner"; import { createRefinementRollbackTool } from "./tools/refinement_rollback"; import type { MemoryScopeAccess } from "@/common/constants/memory"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; @@ -1204,7 +1205,7 @@ describe("MemoryService", () => { expect(events).toEqual([]); }); - it("adopts a sub-agent's pre-sharing private notebook into the shared store on first access", async () => { + it("adopts a sub-agent's pre-sharing private notebook into the shared store, keeping the legacy copy downgrade-readable", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); // Notes written by a build that kept the child's workspace scope in its @@ -1253,13 +1254,11 @@ describe("MemoryService", () => { expect([...(await fixture.metaService.getPinnedKeys())]).toEqual([ "workspace:ws-owner:only-child.md", ]); - // Legacy directory is moved aside (never deleted), so a second access - // is a no-op and the tree's tabs were told once. - expect(await pathExists(legacyRoot)).toBe(false); - const sessionEntries = await fsPromises.readdir( - path.join(fixture.config.sessionsDir, "ws-child") + // The legacy copy stays where a downgraded build reads it; the tree's + // tabs were told once and a second access is a no-op. + expect(await fsPromises.readFile(path.join(legacyRoot, "only-child.md"), "utf-8")).toBe( + "child notes" ); - expect(sessionEntries.some((name) => name.startsWith("memory.migrated-"))).toBe(true); expect(events).toEqual([ { scope: "workspace", @@ -1271,6 +1270,32 @@ describe("MemoryService", () => { ]); await fixture.service.listIndexEntries(fixture.ctx); expect(events).toHaveLength(1); + + // Edited through the shared store, then a backend restart: the legacy + // copy is known to be folded in already and must not resurface as a + // stale duplicate. + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/only-child.md", + "child notes", + "shared edit", + "agent" + ); + // A downgraded build wrote a new note into the legacy dir meanwhile. + await fsPromises.writeFile(path.join(legacyRoot, "downgrade.md"), "written on old build"); + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const relisted = await restarted.listIndexEntries(fixture.ctx); + expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "clash.md", + "downgrade.md", + "imported/ws-child/clash.md", + "only-child.md", + "sub/same.md", + ]); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "shared edit" + ); + // A workspace that is its own owner keeps its private store untouched. const solo = { ...fixture.ctx, workspaceId: "ws-solo" }; await fixture.service.create(solo, "/memories/workspace/mine.md", "solo", "agent"); @@ -1279,6 +1304,37 @@ describe("MemoryService", () => { ).toBe(true); }); + it("never imports through a symlinked legacy notebook root or escaped files", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const outside = path.join(fixture.xumHome, "outside"); + await fsPromises.mkdir(outside, { recursive: true }); + await fsPromises.writeFile(path.join(outside, "secret.md"), "host file"); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + await fsPromises.mkdir(childSessionDir, { recursive: true }); + // Root itself is a symlink: refused outright (lstat, never followed). + await fsPromises.symlink(outside, path.join(childSessionDir, "memory")); + expect( + (await fixture.service.listIndexEntries(fixture.ctx)).filter((e) => e.scope === "workspace") + ).toEqual([]); + + // Real root whose entries point outside: symlinked entries are not + // regular files to the walk, and a symlinked subdirectory is never + // descended into. + await fsPromises.unlink(path.join(childSessionDir, "memory")); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot); + await fsPromises.symlink(path.join(outside, "secret.md"), path.join(legacyRoot, "link.md")); + await fsPromises.symlink(outside, path.join(legacyRoot, "linked-dir")); + await fsPromises.writeFile(path.join(legacyRoot, "real.md"), "real note"); + const fresh = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect( + (await fresh.listIndexEntries(fixture.ctx)) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath) + ).toEqual(["real.md"]); + }); + it("refuses a child's rollback into the shared store once the owner is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1328,6 +1384,92 @@ describe("MemoryService", () => { expect(await readRefinementEvents(childSessionDir)).toHaveLength(1); }); + it("sees a live tree member's later shared-store edit as divergence when rolling back", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const peersOf = (workspaceId: string) => () => + sharedWorkspaceMemoryPeerSessionDirs( + fixture.config.loadConfigOrDefault(), + fixture.config.sessionsDir, + workspaceId + ); + expect(peersOf("ws-owner")()).toEqual([ + childSessionDir, + path.join(fixture.config.sessionsDir, "ws-grandchild"), + ]); + expect(peersOf("ws-solo")()).toEqual([]); + + // Owner renames a directory; the child then edits a file beneath the + // destination. That edit lives only in the child's journal. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "v1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "child edit", + "agent" + ); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const renameRow = ownerRows.find( + (row) => (row.data.action as { op: string }).op === "rename" + )!; + + // Own journal only: the rename looks cleanly undoable and would move + // the child's newer content back without a word. + const blind = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [], + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeTargetLock: () => Promise.reject(new Error("would have applied")), + }); + expect(blind.success).toBe(false); + if (!blind.success) expect(blind.error).toContain("would have applied"); + + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("touched the same paths"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("child edit"); + + // Rolling the child's edit back first (its journal sees the owner's + // rename as EARLIER, not a conflict) unblocks the owner's rollback. + const [childRow] = await readRefinementEvents(childSessionDir); + const childUndo = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-child"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(childUndo.success).toBe(true); + const ownerUndo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(ownerUndo.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "notes", "a.md"), "utf-8") + ).toBe("v1"); + }); + it("migrates a removed sub-agent's live shared-memory rows into the owner's journal, rollbackable there", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index ab171d02598..7fe5e03f193 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -395,11 +395,35 @@ interface MemoryStore { */ const LEGACY_IMPORT_DIR = "imported"; -async function isDirectory(absPath: string): Promise { +/** + * Dotfile inside a sub-agent's legacy `memory` dir recording, per relPath, the + * sha256 of the content already copied into the shared store + * (adoptLegacyPrivateStore). Dotfiles are invisible to every build's listing. + */ +const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; + +/** Self-healing read of the adoption manifest: anything malformed reads as empty. */ +async function readLegacyAdoptionManifest(manifestPath: string): Promise> { + try { + const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {}; + return Object.fromEntries( + Object.entries(parsed).filter( + (entry): entry is [string, string] => typeof entry[1] === "string" + ) + ); + } catch { + return {}; + } +} + +/** Link-aware kind of a path: symlinks are reported as such, never followed. */ +async function lstatKind(absPath: string): Promise<"dir" | "symlink" | "other" | "missing"> { try { - return (await fsPromises.stat(absPath)).isDirectory(); + const stat = await fsPromises.lstat(absPath); + return stat.isSymbolicLink() ? "symlink" : stat.isDirectory() ? "dir" : "other"; } catch { - return false; + return "missing"; } } @@ -899,20 +923,28 @@ export class MemoryService extends EventEmitter { * session dir (//memory). getStore now redirects them to * the owner's root, which would make those notes invisible — and removal * later deletes the child's session dir, discarding them for good. On the - * child's first shared-store access, import every legacy file into the - * owner store (same relPath when free or identical; otherwise under - * imported//), move pins/stats with them, and rename the legacy - * directory aside so the import is one-shot and nothing is lost even for - * files the import skips (binary/oversize). Downgrading afterwards shows - * the notes in the parent's notebook rather than the child's — no loss. + * child's first shared-store access per process, copy every legacy file + * into the owner store (same relPath when free or identical; otherwise + * under imported//) and move pins/stats with them. * - * Runs under the owner store's mutation lock with the same commit guard as - * file mutations, and never throws: an import failure (lock timeout, disk) - * leaves the legacy directory intact for the next access, while the caller - * proceeds with the shared store. Not journaled: this is a mechanical - * relocation, not an agent edit; pre-upgrade child journal rows keep - * targeting the legacy physical paths (rollback restores there and the next - * access re-imports). + * The legacy directory is left in place, untouched: it is exactly where a + * DOWNGRADED build reads (and writes) this child's notebook, so the notes + * stay visible across upgrade↔downgrade and files the import cannot carry + * (binary/oversize, dotfiles, doubly conflicting) are never moved anywhere. + * The copy is idempotent — identical files are skipped, differing ones land + * under imported// — so notes edited during a downgrade are folded in + * again on the next upgrade. Writes made through the shared store meanwhile + * live in the owner's notebook, which the downgraded build shows there. + * + * Security: the legacy root must be a real directory (a symlinked root + * would let an index build copy arbitrary host text into the shared + * notebook and the model's context), and every file passes the store's + * containment check before it is read. Runs under the owner store's + * mutation lock with the same commit guard as file mutations, and never + * throws: a failure (lock timeout, disk) is retried on the next access, + * while the caller proceeds with the shared store. Not journaled: this is + * a mechanical copy, not an agent edit; pre-upgrade child journal rows keep + * targeting the legacy physical paths. */ private async adoptLegacyPrivateStore( ctx: MemoryScopeContext, @@ -924,37 +956,59 @@ export class MemoryService extends EventEmitter { assert(owner !== null, "workspace-scope stores live under sessionsDir"); if (owner === childId) return; // not redirected: the private store IS the store const legacyRoot = path.join(this.config.sessionsDir, childId, "memory"); - if (!(await isDirectory(legacyRoot))) { - // One stat per child per process; a legacy dir can only reappear via a - // downgrade/upgrade cycle, which restarts the backend. + // One lstat per child per process (the copy below is idempotent, so a + // repeat after restart only re-reads unchanged files). + const legacyRootKind = await lstatKind(legacyRoot); + if (legacyRootKind !== "dir") { + if (legacyRootKind === "symlink") { + log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { + childId, + legacyRoot, + }); + } this.legacyStoreChecked.add(childId); return; } + let imported = 0; try { await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { - if (!(await isDirectory(legacyRoot))) return; // adopted by a concurrent command await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); + if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock const legacy = new LocalMemoryStore(legacyRoot); const files = await legacy.listFiles(); - let imported = 0; + // Content hashes already folded in, kept beside the legacy files (a + // dotfile, so neither build lists it). Without it, an imported note + // later edited through the shared store would be re-imported as a + // stale duplicate under imported// on every backend start. + const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); + const adopted = await readLegacyAdoptionManifest(manifestPath); + let manifestDirty = false; let skipped = 0; for (const relPath of files) { - // Oversize or binary (lossy utf-8 decode) files cannot be carried - // by a text write; they stay in the renamed-aside directory. - const content = await this.readBoundedTextFile(legacy, relPath, relPath).catch( - () => null - ); + // Same read gates as a memory command: containment (no symlink + // escape), size cap, and text-only (a lossy utf-8 decode cannot be + // carried by a text write). + const content = await legacy + .assertContained(relPath) + .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) + .catch(() => null); if (content === null || content.includes("\uFFFD")) { skipped++; continue; } + const contentHash = sha256Hex(content); + if (adopted[relPath] === contentHash) continue; // folded in earlier, unchanged since const target = await this.legacyImportTarget(store, childId, relPath, content); if (target === null) { skipped++; continue; } - if (target.write) await store.writeFile(target.relPath, content); - imported++; + if (target.write) { + await store.writeFile(target.relPath, content); + imported++; + } + adopted[relPath] = contentHash; + manifestDirty = true; // Pins/stats were keyed by the child; they follow a written file. // For an identical file the owner already has, the owner's own // stats stand and the child's stale key is dropped. @@ -978,17 +1032,16 @@ export class MemoryService extends EventEmitter { log.debug("[MemoryService] failed to move legacy memory stats", { relPath, error }); } } - await fsPromises.rename(legacyRoot, `${legacyRoot}.migrated-${Date.now()}`); - await this.advanceStoreRevision(store); - log.info( - "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", - { - childId, - owner, - imported, - skipped, - } - ); + if (manifestDirty) { + await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); + } + if (imported > 0) { + await this.advanceStoreRevision(store); + log.info( + "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", + { childId, owner, imported, skipped } + ); + } }); this.legacyStoreChecked.add(childId); } catch (error) { @@ -1002,14 +1055,14 @@ export class MemoryService extends EventEmitter { ); return; } - this.emitChange(ctx, "workspace", "", "agent"); + if (imported > 0) this.emitChange(ctx, "workspace", "", "agent"); } /** * Where a legacy file lands in the owner store: its own relPath when free * (write) or already identical (no write); the per-child import directory * when the owner has different content there; null when even that slot is - * taken by different content (left in the renamed-aside legacy directory). + * taken by different content (the file stays only in the legacy directory). */ private async legacyImportTarget( store: MemoryStore, diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index 20c9ce2f6dc..1996ca62b7b 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -1,3 +1,4 @@ +import * as path from "node:path"; import assert from "@/common/utils/assert"; import type { Config, Workspace as WorkspaceConfigEntry } from "@/node/config"; import { log } from "@/node/services/log"; @@ -88,3 +89,29 @@ export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: resolversBySnapshot.set(cfg, resolver); return resolver; } + +/** + * Session dirs of the OTHER registered members of `workspaceId`'s task tree — + * every workspace resolving to the same owner (the owner itself, siblings, + * descendants). They journal their own mutations of the shared + * `/memories/workspace` store, so a rollback in one member must consult all + * of them for later conflicting rows (refinementRollback.ts). Empty for a + * workspace that owns its store alone. + */ +export function sharedWorkspaceMemoryPeerSessionDirs( + cfg: ProjectsConfig, + sessionsDir: string, + workspaceId: string +): string[] { + assert(sessionsDir.length > 0, "sharedWorkspaceMemoryPeerSessionDirs requires sessionsDir"); + const resolve = workspaceMemoryOwnerResolver(cfg); + const owner = resolve(workspaceId); + const peers: string[] = []; + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.id === undefined || workspace.id === workspaceId) continue; + if (resolve(workspace.id) === owner) peers.push(path.join(sessionsDir, workspace.id)); + } + } + return peers; +} diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 0bd2458dfcc..0bef55c2cad 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -77,6 +77,15 @@ export interface RollbackRefinementOptions { * owns its store). Admits that one extra memory root for confinement. */ sharedWorkspaceMemorySessionDir?: string; + /** + * Session dirs of the OTHER live task-tree members sharing this session's + * `/memories/workspace` store (owner, siblings, descendants), resolved at + * rollback time. Each member journals only its own mutations of the shared + * store, so their rows are merged into divergence detection: a child's + * later edit under a path the owner renamed must surface as a conflict + * when the owner rolls the rename back. Omit when the store is private. + */ + listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; /** Attribution for the emitted rollback row. */ evidence: { toolName: string; toolCallId?: string; actor?: string }; /** Caller-supplied justification, recorded in the rollback row's action. */ @@ -525,6 +534,38 @@ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { return rowTs > otherTs || (rowTs === otherTs && row.seq > other.seq); } +/** + * Memory rows from the other task-tree members' journals that touched the + * shared workspace store (owner's /memory). Read without their + * session locks: journals are append-only and self-healing on read, and a + * row landing after this read is caught by the fs-level checks like any + * other concurrent writer. + */ +async function readSharedMemoryPeerRows( + opts: RollbackRefinementOptions +): Promise { + const peerDirs = opts.listSharedWorkspaceMemoryPeerSessionDirs?.() ?? []; + if (peerDirs.length === 0) return []; + const sharedRoot = path.join( + path.resolve(opts.sharedWorkspaceMemorySessionDir ?? opts.sessionDir), + "memory" + ); + const peerRows: RefinementEvent[] = []; + for (const peerDir of peerDirs) { + assert( + path.resolve(peerDir) !== path.resolve(opts.sessionDir), + "peer session dirs exclude the acting session" + ); + for (const row of await listRefinements(peerDir)) { + if (row.data.kind !== "memory") continue; + const parsed = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsed.success) continue; + if (inversePaths(parsed.data).some((p) => pathsOverlap(p, sharedRoot))) peerRows.push(row); + } + } + return peerRows; +} + async function collectDivergence( rows: RefinementEvent[], target: RefinementEvent, @@ -856,7 +897,13 @@ export async function rollbackRefinement( }, }; - const divergence = await collectDivergence(rows, target, inverse, readContent); + // Conflict detection sees this journal plus every live tree member's + // shared-store rows (see listSharedWorkspaceMemoryPeerSessionDirs); the + // store clock (`sourceTs`) orders rows across journals. Target lookup, + // rollbackOf checks and the appended row stay on this session's journal. + const divergenceRows = + kind === "memory" ? [...rows, ...(await readSharedMemoryPeerRows(opts))] : rows; + const divergence = await collectDivergence(divergenceRows, target, inverse, readContent); if (divergence.length > 0 && opts.force !== true) { throw new RollbackError( `Refusing rollback of '${opts.id}': current state diverges from what the inverse expects:\n` + @@ -917,7 +964,7 @@ export async function rollbackRefinement( // (live app vs. debug CLI) does not contend on this in-process lock, so // this re-verify narrows but cannot fully close that window. if (opts.force !== true) { - const raced = await collectDivergence(rows, target, inverse, readContent); + const raced = await collectDivergence(divergenceRows, target, inverse, readContent); if (raced.length > 0) { throw new RollbackError( `Refusing rollback of '${opts.id}': a concurrent mutation landed before the apply:\n` + diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 9a4715b23a6..0eefa39b351 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -123,6 +123,8 @@ export interface ApplyToolPolicyAndExperimentsOptions { sessionDir: string; /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; + /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ + listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; /** Lets refinement_rollback announce its direct-to-disk memory writes. */ memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; kernelFileLoader?: KernelFileLoader; diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index e5e31636039..751adb8dd55 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -62,6 +62,8 @@ export function createRefinementRollbackTool(ctx: { sessionDir: string; /** Owner session dir when this workspace is a sub-agent sharing its notebook. */ sharedWorkspaceMemorySessionDir?: string; + /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ + listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; /** * Memory integration: announces rolled-back memory files so shared-store * readers refresh, and applies the agent's per-scope write policy — a @@ -85,6 +87,7 @@ export function createRefinementRollbackTool(ctx: { const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, sharedWorkspaceMemorySessionDir: ctx.sharedWorkspaceMemorySessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: ctx.listSharedWorkspaceMemoryPeerSessionDirs, id, reason, evidence: { toolName: "refinement_rollback", toolCallId, actor: "agent" }, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index a583a0458e1..5c6d4289fe0 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -114,6 +114,7 @@ import { } from "@/common/utils/providers/customProviders"; import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpServerManager"; import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; +import { sharedWorkspaceMemoryPeerSessionDirs } from "@/node/services/memoryWorkspaceOwner"; import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; import { READ_ONLY_ACCESS, resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; @@ -2391,6 +2392,17 @@ export class TurnRequestBuilder { memoryOwnerId === workspaceId ? undefined : path.join(this.dependencies.config.sessionsDir, memoryOwnerId); + // Resolved per rollback (not per turn): tree membership changes as + // sub-agents are spawned and removed while the tool instance lives. + const listSharedWorkspaceMemoryPeerSessionDirs = + memoryService === undefined + ? undefined + : () => + sharedWorkspaceMemoryPeerSessionDirs( + this.dependencies.config.loadConfigOrDefault(), + this.dependencies.config.sessionsDir, + workspaceId + ); const sandboxMemory = memoryService === undefined ? undefined @@ -2414,6 +2426,7 @@ export class TurnRequestBuilder { workspaceId, sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), sharedWorkspaceMemorySessionDir, + listSharedWorkspaceMemoryPeerSessionDirs, memory: sandboxMemory, kernelFileLoader, }, From c4e9c4b5dda7b9c961bbfa43a52fd208bcc1ecf5 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 01:37:44 +0000 Subject: [PATCH 34/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirtieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rollback re-reads the acting and peer journals inside the target locks before the final divergence check, so a tree member's edit that journaled between the plan-time scan and the lock (invisible to fs checks for a rename without post-state hash) refuses the apply. - Legacy adoption copies (MemoryMetaService.copyKeys) the child-keyed pin/usage entries to the owner key instead of moving them; the child key stays for a downgraded build like the legacy file does. - The rollback tool's memory-access holder is named per attempt with its per-attempt construction documented (fallbacks start from the policy). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryMeta.ts | 21 +++++++++ src/node/services/memoryService.test.ts | 44 +++++++++++++++++-- src/node/services/memoryService.ts | 40 ++++++++--------- .../services/refinement/refinementRollback.ts | 34 +++++++++----- src/node/services/turnRequestBuilder.ts | 13 ++++-- 5 files changed, 114 insertions(+), 38 deletions(-) diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index 58927983a2d..d4e00261aa0 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -240,6 +240,22 @@ export class MemoryMetaService { } }), + /** + * Duplicate a subtree's entries under a second key, keeping the source: + * a legacy sub-agent note copied into the shared store stays readable by + * a downgraded build under its child key, so its pin/stats must too. + */ + copyKeys: ( + sourceLogicalKey: string, + targetLogicalKey: string + ): Effect.Effect => + this.mutate((entries) => { + for (const [key, entry] of Object.entries(entries)) { + if (!keyInSubtree(key, sourceLogicalKey)) continue; + entries[`${targetLogicalKey}${key.slice(sourceLogicalKey.length)}`] = { ...entry }; + } + }), + /** * Drop all entries for a deleted file or directory subtree so a future file * at the same path never resurrects stale pins or stats. @@ -386,6 +402,11 @@ export class MemoryMetaService { await Effect.runPromise(this.effects.renameKeys(oldLogicalKey, newLogicalKey)); } + /** Duplicate a subtree's entries under `targetLogicalKey`, keeping the source (see effects). */ + async copyKeys(sourceLogicalKey: string, targetLogicalKey: string): Promise { + await Effect.runPromise(this.effects.copyKeys(sourceLogicalKey, targetLogicalKey)); + } + /** * Drop all entries for a deleted file or directory subtree so a future file * at the same path never resurrects stale pins or stats. diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index dd7aa6c2a47..037868e0433 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1250,8 +1250,10 @@ describe("MemoryService", () => { expect( await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "clash.md"), "utf-8") ).toBe("child version"); - // The pin followed the file to the owner-keyed logical key. - expect([...(await fixture.metaService.getPinnedKeys())]).toEqual([ + // The pin is copied to the owner-keyed logical key; the child-keyed + // entry stays for a downgraded build, which keys by the child id. + expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ + "workspace:ws-child:only-child.md", "workspace:ws-owner:only-child.md", ]); // The legacy copy stays where a downgraded build reads it; the tree's @@ -1448,7 +1450,9 @@ describe("MemoryService", () => { ).toBe("child edit"); // Rolling the child's edit back first (its journal sees the owner's - // rename as EARLIER, not a conflict) unblocks the owner's rollback. + // rename as EARLIER, not a conflict) unblocks the owner's rollback — + // unless another child edit lands between the owner's plan-time scan + // and its target lock: the journals are re-read under the lock. const [childRow] = await readRefinementEvents(childSessionDir); const childUndo = await rollbackRefinement({ sessionDir: childSessionDir, @@ -1458,6 +1462,40 @@ describe("MemoryService", () => { evidence: { toolName: "test", actor: "user" }, }); expect(childUndo.success).toBe(true); + const raced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-owner"), + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeTargetLock: async () => { + const late = await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "late child edit", + "agent" + ); + expect(late.success).toBe(true); + }, + }); + expect(raced.success).toBe(false); + if (!raced.success) expect(raced.error).toContain("a concurrent mutation landed"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("late child edit"); + // LIFO: undo the late edit (its own journal, netted out) and the + // owner's rollback goes through. + const childRows = await readRefinementEvents(childSessionDir); + const lateRow = childRows[childRows.length - 1]; + expect(lateRow.data.rollbackOf).toBeUndefined(); + const lateUndo = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: lateRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: peersOf("ws-child"), + evidence: { toolName: "test", actor: "user" }, + }); + expect(lateUndo.success).toBe(true); const ownerUndo = await rollbackRefinement({ sessionDir: ownerSessionDir, id: renameRow.id, diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 7fe5e03f193..294a2f6afdb 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -925,11 +925,12 @@ export class MemoryService extends EventEmitter { * later deletes the child's session dir, discarding them for good. On the * child's first shared-store access per process, copy every legacy file * into the owner store (same relPath when free or identical; otherwise - * under imported//) and move pins/stats with them. + * under imported//) and copy pins/stats to the owner key. * * The legacy directory is left in place, untouched: it is exactly where a * DOWNGRADED build reads (and writes) this child's notebook, so the notes - * stay visible across upgrade↔downgrade and files the import cannot carry + * stay visible across upgrade↔downgrade (the child-keyed sidecar entries + * stay for the same reason) and files the import cannot carry * (binary/oversize, dotfiles, doubly conflicting) are never moved anywhere. * The copy is idempotent — identical files are skipped, differing ones land * under imported// — so notes edited during a downgrade are folded in @@ -1009,27 +1010,24 @@ export class MemoryService extends EventEmitter { } adopted[relPath] = contentHash; manifestDirty = true; - // Pins/stats were keyed by the child; they follow a written file. - // For an identical file the owner already has, the owner's own - // stats stand and the child's stale key is dropped. - const childKey = memoryLogicalKey("workspace", relPath, { - projectPath: ctx.projectPath, - workspaceId: childId, - }); + // Pins/stats were keyed by the child: a written copy gets them under + // the owner key too. The child-keyed entry stays — like the legacy + // file, it is what a downgraded build reads. For an identical file + // the owner already has, the owner's own stats stand. + if (!target.write) continue; try { - if (target.write) { - await this.metaService.renameKeys( - childKey, - memoryLogicalKey("workspace", target.relPath, { - projectPath: ctx.projectPath, - workspaceId: owner, - }) - ); - } else { - await this.metaService.removeKeys(childKey); - } + await this.metaService.copyKeys( + memoryLogicalKey("workspace", relPath, { + projectPath: ctx.projectPath, + workspaceId: childId, + }), + memoryLogicalKey("workspace", target.relPath, { + projectPath: ctx.projectPath, + workspaceId: owner, + }) + ); } catch (error) { - log.debug("[MemoryService] failed to move legacy memory stats", { relPath, error }); + log.debug("[MemoryService] failed to copy legacy memory stats", { relPath, error }); } } if (manifestDirty) { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 0bef55c2cad..43137f85e06 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -901,9 +901,13 @@ export async function rollbackRefinement( // shared-store rows (see listSharedWorkspaceMemoryPeerSessionDirs); the // store clock (`sourceTs`) orders rows across journals. Target lookup, // rollbackOf checks and the appended row stay on this session's journal. - const divergenceRows = - kind === "memory" ? [...rows, ...(await readSharedMemoryPeerRows(opts))] : rows; - const divergence = await collectDivergence(divergenceRows, target, inverse, readContent); + // Re-read under the target locks below before the apply. + const divergence = await collectDivergence( + kind === "memory" ? [...rows, ...(await readSharedMemoryPeerRows(opts))] : rows, + target, + inverse, + readContent + ); if (divergence.length > 0 && opts.force !== true) { throw new RollbackError( `Refusing rollback of '${opts.id}': current state diverges from what the inverse expects:\n` + @@ -957,14 +961,24 @@ export async function rollbackRefinement( } // Re-verify INSIDE the lock, immediately before mutating: a writer that // won the lock first has already landed, and its change must surface as - // divergence rather than be overwritten. `rows` is intentionally the - // pre-lock read — the fs-level checks (postState hashes, presence) are - // what detect concurrent mutations; force skips this exactly like the - // plan-time check. Cross-process residual: a writer in ANOTHER process - // (live app vs. debug CLI) does not contend on this in-process lock, so - // this re-verify narrows but cannot fully close that window. + // divergence rather than be overwritten. The journals are re-read here + // too: a rename row carries no post-state hash, so a tree member's edit + // beneath the renamed destination that journaled between the plan-time + // scan and this lock is visible only as its (now committed) row. Force + // skips this exactly like the plan-time check. Cross-process residual: + // a writer in ANOTHER process (live app vs. debug CLI) does not contend + // on this in-process lock, so this re-verify narrows but cannot fully + // close that window. if (opts.force !== true) { - const raced = await collectDivergence(divergenceRows, target, inverse, readContent); + const lockedRows = await listRefinements(opts.sessionDir); + const raced = await collectDivergence( + kind === "memory" + ? [...lockedRows, ...(await readSharedMemoryPeerRows(opts))] + : lockedRows, + target, + inverse, + readContent + ); if (raced.length > 0) { throw new RollbackError( `Refusing rollback of '${opts.id}': a concurrent mutation landed before the apply:\n` + diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 5c6d4289fe0..9bba0b4900d 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2403,7 +2403,12 @@ export class TurnRequestBuilder { this.dependencies.config.sessionsDir, workspaceId ); - const sandboxMemory = + // Built anew for EVERY attempt (prepareModelRequest runs per primary / + // fallback request) from the never-mutated policy: refinement_rollback + // reads it by reference, and the request.assemble demotion below only + // touches this attempt's object, so a fallback whose middleware keeps + // `memory` starts writable again. + const attemptSandboxMemory = memoryService === undefined ? undefined : { @@ -2427,7 +2432,7 @@ export class TurnRequestBuilder { sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), sharedWorkspaceMemorySessionDir, listSharedWorkspaceMemoryPeerSessionDirs, - memory: sandboxMemory, + memory: attemptSandboxMemory, kernelFileLoader, }, }); @@ -2529,8 +2534,8 @@ export class TurnRequestBuilder { // notebook once `memory` is gone: the tool reads its memory policy // from this per-attempt object by reference, so demote it to // view-only the same way tool assembly does for policy-denied memory. - if (attemptTools.memory === undefined && sandboxMemory !== undefined) { - sandboxMemory.access = READ_ONLY_ACCESS; + if (attemptTools.memory === undefined && attemptSandboxMemory !== undefined) { + attemptSandboxMemory.access = READ_ONLY_ACCESS; } if (attemptTools.intuition === undefined) { assembleCtx.systemMessage = removeIntuitionGuidance( From 291886519b2effe296cbb2f5ef91a469f920899e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 02:03:35 +0000 Subject: [PATCH 35/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-fi?= =?UTF-8?q?rst=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption copies child-keyed pin/usage entries to the owner key even when the target bytes already exist, and records a file in the adoption manifest only after that sidecar copy succeeded, so an adoption interrupted after writeFile (or a failing sidecar write) completes on the next access. copyKeys fills only missing target entries, keeping an owner's independently tracked history. - Document that `memory` is ptcExcluded (never bridged), so its top-level presence remains the writable-bit signal under PTC. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryMeta.ts | 16 +++++++++++----- src/node/services/memoryService.test.ts | 13 +++++++++++++ src/node/services/memoryService.ts | 23 +++++++++++++++-------- src/node/services/turnRequestBuilder.ts | 7 ++++--- 4 files changed, 43 insertions(+), 16 deletions(-) diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index d4e00261aa0..6beaa0d73a9 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -241,9 +241,13 @@ export class MemoryMetaService { }), /** - * Duplicate a subtree's entries under a second key, keeping the source: - * a legacy sub-agent note copied into the shared store stays readable by - * a downgraded build under its child key, so its pin/stats must too. + * Duplicate a subtree's entries under a second key, keeping the source + * and never overwriting an entry the target already has: a legacy + * sub-agent note copied into the shared store stays readable by a + * downgraded build under its child key (so its pin/stats must too), while + * a note the owner already tracked keeps the owner's own history. + * Idempotent, so a retried adoption fills in what an interrupted one + * missed without disturbing anything recorded since. */ copyKeys: ( sourceLogicalKey: string, @@ -252,7 +256,9 @@ export class MemoryMetaService { this.mutate((entries) => { for (const [key, entry] of Object.entries(entries)) { if (!keyInSubtree(key, sourceLogicalKey)) continue; - entries[`${targetLogicalKey}${key.slice(sourceLogicalKey.length)}`] = { ...entry }; + const targetKey = `${targetLogicalKey}${key.slice(sourceLogicalKey.length)}`; + if (targetKey in entries) continue; + entries[targetKey] = { ...entry }; } }), @@ -402,7 +408,7 @@ export class MemoryMetaService { await Effect.runPromise(this.effects.renameKeys(oldLogicalKey, newLogicalKey)); } - /** Duplicate a subtree's entries under `targetLogicalKey`, keeping the source (see effects). */ + /** Fill in a subtree's entries under `targetLogicalKey`, keeping the source and existing targets (see effects). */ async copyKeys(sourceLogicalKey: string, targetLogicalKey: string): Promise { await Effect.runPromise(this.effects.copyKeys(sourceLogicalKey, targetLogicalKey)); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 037868e0433..dfde692810a 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1285,11 +1285,18 @@ describe("MemoryService", () => { ); // A downgraded build wrote a new note into the legacy dir meanwhile. await fsPromises.writeFile(path.join(legacyRoot, "downgrade.md"), "written on old build"); + // An earlier adoption was interrupted right after writing this file's + // bytes: identical bytes in the owner store, pin only under the child + // key, nothing in the manifest. The retry must still copy the pin. + await fsPromises.writeFile(path.join(legacyRoot, "half.md"), "half adopted"); + await fsPromises.writeFile(path.join(ownerRoot, "half.md"), "half adopted"); + await fixture.metaService.setPinned("workspace:ws-child:half.md", true); const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); const relisted = await restarted.listIndexEntries(fixture.ctx); expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ "clash.md", "downgrade.md", + "half.md", "imported/ws-child/clash.md", "only-child.md", "sub/same.md", @@ -1297,6 +1304,12 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( "shared edit" ); + expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ + "workspace:ws-child:half.md", + "workspace:ws-child:only-child.md", + "workspace:ws-owner:half.md", + "workspace:ws-owner:only-child.md", + ]); // A workspace that is its own owner keeps its private store untouched. const solo = { ...fixture.ctx, workspaceId: "ws-solo" }; diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 294a2f6afdb..9f612886a00 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1008,13 +1008,14 @@ export class MemoryService extends EventEmitter { await store.writeFile(target.relPath, content); imported++; } - adopted[relPath] = contentHash; - manifestDirty = true; - // Pins/stats were keyed by the child: a written copy gets them under - // the owner key too. The child-keyed entry stays — like the legacy - // file, it is what a downgraded build reads. For an identical file - // the owner already has, the owner's own stats stand. - if (!target.write) continue; + // Pins/stats were keyed by the child: fill them in under the owner + // key too. The child-keyed entry stays — like the legacy file, it is + // what a downgraded build reads. Done even when the bytes were + // already there, and recorded in the manifest only once it + // succeeded: an adoption interrupted after its writeFile (or a + // failing sidecar write) retries this step on the next access. + // copyKeys never overwrites an entry the owner already has, so a + // note the owner tracked independently keeps the owner's history. try { await this.metaService.copyKeys( memoryLogicalKey("workspace", relPath, { @@ -1027,8 +1028,14 @@ export class MemoryService extends EventEmitter { }) ); } catch (error) { - log.debug("[MemoryService] failed to copy legacy memory stats", { relPath, error }); + log.warn("[MemoryService] failed to copy legacy memory stats; retrying on next access", { + relPath, + error, + }); + continue; } + adopted[relPath] = contentHash; + manifestDirty = true; } if (manifestDirty) { await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 9bba0b4900d..00e7bbf8d82 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2787,9 +2787,10 @@ export class TurnRequestBuilder { }; // Final toolset of the request being started: request.assemble // middleware ran inside prepareModelRequest, and `memory` is a built-in - // that tool search never defers, so its absence means denied. The deny - // lands now; the grant waits for onStreamStarted (see - // persistWorkspaceMemoryWritable). + // that tool search never defers and PTC never bridges (ptcExcluded in + // toolDefinitions: its top-level presence keys the memory index / hot + // set), so its absence means denied. The deny lands now; the grant + // waits for onStreamStarted (see persistWorkspaceMemoryWritable). const finalWorkspaceMemoryWritable = workspaceMemoryWritable && tools.memory !== undefined; if (!finalWorkspaceMemoryWritable && !(await persistWorkspaceMemoryWritable(false))) { const errorEvent = createErrorEvent(workspaceId, { From ef11a3b60063621afc4af0a5def6de6144b2ac45 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 02:04:39 +0000 Subject: [PATCH 36/98] =?UTF-8?q?=F0=9F=A4=96=20style:=20format=20memorySe?= =?UTF-8?q?rvice.ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 9f612886a00..f2f4559c557 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1028,10 +1028,13 @@ export class MemoryService extends EventEmitter { }) ); } catch (error) { - log.warn("[MemoryService] failed to copy legacy memory stats; retrying on next access", { - relPath, - error, - }); + log.warn( + "[MemoryService] failed to copy legacy memory stats; retrying on next access", + { + relPath, + error, + } + ); continue; } adopted[relPath] = contentHash; From 8038ae4ae8bae1d964b059ecbf11d82e9ce30337 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 02:27:54 +0000 Subject: [PATCH 37/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-se?= =?UTF-8?q?cond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A legacy adoption pass that only copied sidecar entries (identical bytes already present) now advances the store revision and emits the change event like a written file, so other backends' cached hot sets pick up the adopted pin. - The debug CLI's peer-journal thunk reloads config per check instead of reusing the pre-rollback snapshot, so a tree member registered while the CLI waits for the shared-store lock is included in the in-lock reread. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/cli/debug/refinements.ts | 9 ++++++++- src/node/services/memoryService.test.ts | 27 +++++++++++++++++++++++++ src/node/services/memoryService.ts | 12 ++++++++--- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index c737ee8059b..92cd3210c56 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -64,8 +64,15 @@ export async function refinementsCommand( memoryOwnerId === workspaceId ? undefined : path.join(defaultConfig.sessionsDir, memoryOwnerId), + // Reloaded per check (plan-time and in-lock), not from the snapshot + // above: a live backend may register a new tree member while this + // process waits for the shared-store lock, and its rows must count. listSharedWorkspaceMemoryPeerSessionDirs: () => - sharedWorkspaceMemoryPeerSessionDirs(cfg, defaultConfig.sessionsDir, workspaceId), + sharedWorkspaceMemoryPeerSessionDirs( + defaultConfig.loadConfigOrDefault(), + defaultConfig.sessionsDir, + workspaceId + ), id: opts.rollback, force: opts.force, evidence: { toolName: "debug-cli", actor: "user" }, diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index dfde692810a..74070e7dccd 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1292,7 +1292,32 @@ describe("MemoryService", () => { await fsPromises.writeFile(path.join(ownerRoot, "half.md"), "half adopted"); await fixture.metaService.setPinned("workspace:ws-child:half.md", true); const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const restartedEvents: unknown[] = []; + restarted.on("change", (event) => restartedEvents.push(event)); + const revisionBefore = Number(await fixture.service.workspaceMemoryRevision("ws-owner")); const relisted = await restarted.listIndexEntries(fixture.ctx); + // The pass wrote one file and copied one pin: both change what other + // backends derive from the store, so the clock moved and the tabs heard. + expect(Number(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + revisionBefore + ); + expect(restartedEvents).toHaveLength(1); + // Metadata-only pass (nothing to write, one pin to copy): same signals. + await fsPromises.writeFile(path.join(legacyRoot, "meta-only.md"), "same bytes"); + await fsPromises.writeFile(path.join(ownerRoot, "meta-only.md"), "same bytes"); + await fixture.metaService.setPinned("workspace:ws-child:meta-only.md", true); + const metaOnly = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const metaOnlyEvents: unknown[] = []; + metaOnly.on("change", (event) => metaOnlyEvents.push(event)); + const revisionMid = Number(await fixture.service.workspaceMemoryRevision("ws-owner")); + await metaOnly.listIndexEntries(fixture.ctx); + expect(Number(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + revisionMid + ); + expect(metaOnlyEvents).toHaveLength(1); + expect(await fixture.metaService.getPinnedKeys()).toContain( + "workspace:ws-owner:meta-only.md" + ); expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ "clash.md", "downgrade.md", @@ -1306,8 +1331,10 @@ describe("MemoryService", () => { ); expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ "workspace:ws-child:half.md", + "workspace:ws-child:meta-only.md", "workspace:ws-child:only-child.md", "workspace:ws-owner:half.md", + "workspace:ws-owner:meta-only.md", "workspace:ws-owner:only-child.md", ]); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index f2f4559c557..850674e57ef 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -970,7 +970,9 @@ export class MemoryService extends EventEmitter { this.legacyStoreChecked.add(childId); return; } - let imported = 0; + // Files adopted this pass (bytes written OR only their sidecar entries + // copied): either changes what the shared store's readers derive from it. + let adoptedCount = 0; try { await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); @@ -984,6 +986,7 @@ export class MemoryService extends EventEmitter { const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); const adopted = await readLegacyAdoptionManifest(manifestPath); let manifestDirty = false; + let imported = 0; let skipped = 0; for (const relPath of files) { // Same read gates as a memory command: containment (no symlink @@ -1039,11 +1042,14 @@ export class MemoryService extends EventEmitter { } adopted[relPath] = contentHash; manifestDirty = true; + adoptedCount++; } if (manifestDirty) { await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); } - if (imported > 0) { + if (adoptedCount > 0) { + // A metadata-only adoption (identical bytes, child pin copied) still + // changes the hot set other backends derive, so the clock moves too. await this.advanceStoreRevision(store); log.info( "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", @@ -1063,7 +1069,7 @@ export class MemoryService extends EventEmitter { ); return; } - if (imported > 0) this.emitChange(ctx, "workspace", "", "agent"); + if (adoptedCount > 0) this.emitChange(ctx, "workspace", "", "agent"); } /** From 76794a30e7b750f7a674685586eca92611e42e97 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 03:17:28 +0000 Subject: [PATCH 38/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-th?= =?UTF-8?q?ird=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Memory context: a snapshot whose store changed during its build (generation bump or a differing post-build probe, e.g. a foreign-backend removal) is discarded and rebuilt (bounded); never served. - Write-policy deny that config.json cannot hold falls back to a durable session-dir marker (workspaceMemoryDenyMarker.ts) ANDed into the epoch accumulator and the compaction observation, cleared (fenced) at the epoch boundary; the turn no longer has to be refused with its rows left durable. - Legacy adoption manifest records the child sidecar fingerprint and the target relPath; MemoryMetaService.mergeKeys folds pin/usage changes made on a downgraded build into the owner key without undoing the owner's own later choices. - Legacy-store check is keyed to the resolved owner, so a note written under a self-fallback is adopted once config recovers. - Removal reuses the owner verified by the pre-teardown pass; without one it loads config strictly and aborts (retryable) instead of guessing. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../agentSession.memoryContext.test.ts | 58 +++++- src/node/services/agentSession.ts | 105 ++++++----- src/node/services/memoryMeta.ts | 50 ++++-- src/node/services/memoryService.test.ts | 58 +++++- src/node/services/memoryService.ts | 165 ++++++++++++------ .../services/workspaceMemoryDenyMarker.ts | 85 +++++++++ src/node/services/workspaceService.test.ts | 48 +++++ src/node/services/workspaceService.ts | 109 +++++++++--- 8 files changed, 543 insertions(+), 135 deletions(-) create mode 100644 src/node/services/workspaceMemoryDenyMarker.ts diff --git a/src/node/services/agentSession.memoryContext.test.ts b/src/node/services/agentSession.memoryContext.test.ts index 153da6969c7..641432c29ab 100644 --- a/src/node/services/agentSession.memoryContext.test.ts +++ b/src/node/services/agentSession.memoryContext.test.ts @@ -166,7 +166,11 @@ describe("AgentSession memory context", () => { let ownerRemoved = false; const sessionRef: { current?: AgentSession } = {}; const probeMemoryStore = mock(() => { - if (ownerRemoved) sessionRef.current?.invalidateMemoryContext(); + // One-shot like the real memo invalidation: the stamp check fires once. + if (ownerRemoved) { + ownerRemoved = false; + sessionRef.current?.invalidateMemoryContext(); + } return Promise.resolve(undefined); }); const session = createSession({ @@ -182,7 +186,9 @@ describe("AgentSession memory context", () => { await priv.resolveMemoryContext("test-model"); await priv.resolveMemoryContext("test-model"); expect(buildMemorySessionContext).toHaveBeenCalledTimes(1); - expect(probeMemoryStore).toHaveBeenCalledTimes(2); + // A build is bracketed by two probes (before the cache read, after the + // build); a cache hit costs one. + expect(probeMemoryStore).toHaveBeenCalledTimes(3); ownerRemoved = true; // The probe runs before the cache read, so THIS request rebuilds. @@ -226,6 +232,54 @@ describe("AgentSession memory context", () => { } }); + test("discards a snapshot whose store changed during the build instead of serving it", async () => { + using sessionDir = new DisposableTempDir("agent-session-memory-context-midbuild"); + const { historyService, cleanup } = await createTestHistoryService(); + historyCleanup = cleanup; + // The owner is removed by ANOTHER backend while the build reads hot files: + // no in-process event, the probe flips to "revoked" only after the build + // started, and the snapshot built from the former owner's notes must + // never reach the provider. + let revision = "rev-1"; + let flipDuringBuild = true; + const stale: MemorySessionContext = { + indexEntries: [{ path: "/memories/workspace/x.md", description: "" }], + hotMemoriesBlock: null, + }; + const fresh: MemorySessionContext = { indexEntries: [], hotMemoriesBlock: null }; + const buildMemorySessionContext = mock(() => { + if (flipDuringBuild) { + flipDuringBuild = false; + revision = "revoked"; + return Promise.resolve(stale); + } + return Promise.resolve(fresh); + }); + const session = createSession({ + historyService, + sessionDir: path.join(sessionDir.path, WORKSPACE_ID), + buildMemorySessionContext, + probeMemoryStore: () => Promise.resolve(revision), + isExperimentEnabled: (id) => id === EXPERIMENT_IDS.MEMORY, + }); + const priv = session as unknown as PrivateSessionAccess; + try { + expect(await priv.resolveMemoryContext("test-model")).toBe(fresh); + expect(buildMemorySessionContext).toHaveBeenCalledTimes(2); + + // A store that will not hold still yields no context at all (bounded + // retries) rather than a snapshot of unknown provenance. + let tick = 0; + buildMemorySessionContext.mockImplementation(() => { + revision = `rev-${++tick}`; + return Promise.resolve(stale); + }); + expect(await priv.resolveMemoryContext("other-model")).toBeUndefined(); + } finally { + await session.dispose(); + } + }); + test("upgrades an index-only memory context when hot memories are requested", async () => { using sessionDir = new DisposableTempDir("agent-session-memory-context-upgrade"); const { historyService, cleanup } = await createTestHistoryService(); diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 87aadc081b5..631a03814ae 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -92,6 +92,7 @@ import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { clearWorkspaceMemoryDenyMarker } from "@/node/services/workspaceMemoryDenyMarker"; import { buildStreamErrorEventData, createStreamErrorMessage, @@ -1088,7 +1089,15 @@ export class AgentSession { private async resetWorkspaceMemoryWritable(options?: { closing: boolean | undefined; }): Promise { + const boundaryAt = Date.now(); this.workspaceMemoryWritable = undefined; + // The session-dir deny marker (fallback for an unwritable config.json) + // belongs to the closing epoch too. Same fence idea as below: a deny + // recorded after this boundary is the new epoch's and survives. + await clearWorkspaceMemoryDenyMarker( + path.join(this.config.sessionsDir, this.workspaceId), + options !== undefined ? { notAfter: boundaryAt } : undefined + ); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); if (entry?.workspace.workspaceMemoryWritable === undefined) return; await this.config.editConfig((cfg) => { @@ -9906,48 +9915,64 @@ export class AgentSession { this.aiService.isExperimentEnabled(id); const memoryEnabled = enabled(EXPERIMENT_IDS.MEMORY); const hotSetEnabled = enabled(EXPERIMENT_IDS.MEMORY_HOT_SET); - // Store probe first: a removed owner invalidates this cache synchronously - // (see AIService.probeMemoryStore), so the lookup below never serves an - // index built from a store this workspace no longer reads; and a store - // revision advanced by ANOTHER backend process (no in-process change - // event) fails the comparison below. Read before the build so a write - // racing the build is caught on the next probe. - let storeRevision: string | undefined; - if (memoryEnabled && typeof this.aiService.probeMemoryStore === "function") { - storeRevision = await this.aiService.probeMemoryStore(this.workspaceId); - } - const cached = cache.get(modelString); - // Policy changes must not retain a previously injected extra (including index-only lookups). - if ( - cached?.tokenBudgetActive === tokenBudgetActive && - cached.memoryEnabled === memoryEnabled && - cached.hotSetEnabled === hotSetEnabled && - cached.storeRevision === storeRevision && - (cached.includesHotMemories || !includeHotMemories) - ) { - return cached.context ?? undefined; - } + const probe = async (): Promise => + memoryEnabled && typeof this.aiService.probeMemoryStore === "function" + ? await this.aiService.probeMemoryStore(this.workspaceId) + : undefined; + // Bounded: each retry means the store changed underneath the build; a + // store that will not hold still yields no memory context for this + // request rather than a snapshot of unknown provenance. + for (let attempt = 0; attempt < 3; attempt++) { + // Store probe first: a removed owner invalidates this cache synchronously + // (see AIService.probeMemoryStore), so the lookup below never serves an + // index built from a store this workspace no longer reads; and a store + // revision advanced by ANOTHER backend process (no in-process change + // event) fails the comparison below. + const storeRevision = await probe(); + const cached = cache.get(modelString); + // Policy changes must not retain a previously injected extra (including index-only lookups). + if ( + cached?.tokenBudgetActive === tokenBudgetActive && + cached.memoryEnabled === memoryEnabled && + cached.hotSetEnabled === hotSetEnabled && + cached.storeRevision === storeRevision && + (cached.includesHotMemories || !includeHotMemories) + ) { + return cached.context ?? undefined; + } - const generation = this.memoryContextGeneration; - // Guard for test mocks that may not implement buildMemorySessionContext. - const context = - typeof this.aiService.buildMemorySessionContext === "function" - ? await this.aiService.buildMemorySessionContext(this.workspaceId, modelString, { - includeHotMemories, - tokenBudgetActive, - }) - : null; - // Invalidated mid-build: serve this snapshot once but do not cache it. - if (generation !== this.memoryContextGeneration) return context ?? undefined; - cache.set(modelString, { - context, - includesHotMemories: includeHotMemories, - tokenBudgetActive, - memoryEnabled, - hotSetEnabled, - storeRevision, + const generation = this.memoryContextGeneration; + // Guard for test mocks that may not implement buildMemorySessionContext. + const context = + typeof this.aiService.buildMemorySessionContext === "function" + ? await this.aiService.buildMemorySessionContext(this.workspaceId, modelString, { + includeHotMemories, + tokenBudgetActive, + }) + : null; + // Invalidated mid-build — by an in-process change (generation) or by a + // store the build cannot observe changing under it: a removal in + // ANOTHER backend revokes access ("revoked") without any local event, + // and the readability check ran before the hot-file reads. Such a + // snapshot is never served: the next attempt re-probes and rebuilds + // (a revoked store then lists nothing). + if (generation !== this.memoryContextGeneration || (await probe()) !== storeRevision) { + continue; + } + cache.set(modelString, { + context, + includesHotMemories: includeHotMemories, + tokenBudgetActive, + memoryEnabled, + hotSetEnabled, + storeRevision, + }); + return context ?? undefined; + } + log.debug("[AgentSession] memory context kept changing during its build; omitting it", { + workspaceId: this.workspaceId, }); - return context ?? undefined; + return undefined; } /** diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index 6beaa0d73a9..bee2454557f 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -66,6 +66,12 @@ export interface MemoryMetaEntry { lastWriteAt: number | null; } +function maxTimestamp(a: number | null, b: number | null): number | null { + if (a === null) return b; + if (b === null) return a; + return Math.max(a, b); +} + const EMPTY_ENTRY: MemoryMetaEntry = { pinned: false, accessCount: 0, @@ -241,24 +247,34 @@ export class MemoryMetaService { }), /** - * Duplicate a subtree's entries under a second key, keeping the source - * and never overwriting an entry the target already has: a legacy - * sub-agent note copied into the shared store stays readable by a - * downgraded build under its child key (so its pin/stats must too), while - * a note the owner already tracked keeps the owner's own history. - * Idempotent, so a retried adoption fills in what an interrupted one - * missed without disturbing anything recorded since. + * Fold a subtree's entries into a second key, keeping the source: a + * legacy sub-agent note copied into the shared store stays readable by a + * downgraded build under its child key, so its pin/stats must too. A + * missing target entry is copied; an existing one keeps the larger + * counters/timestamps, and its pin either stands (`pinned: "target"`, a + * first adoption must not override the owner's own choice) or follows the + * source (`pinned: "source"`, the child changed it since the last + * adoption — see MemoryService.adoptLegacyPrivateStore). Idempotent. */ - copyKeys: ( + mergeKeys: ( sourceLogicalKey: string, - targetLogicalKey: string + targetLogicalKey: string, + options: { pinned: "target" | "source" } ): Effect.Effect => this.mutate((entries) => { - for (const [key, entry] of Object.entries(entries)) { + for (const [key, source] of Object.entries(entries)) { if (!keyInSubtree(key, sourceLogicalKey)) continue; const targetKey = `${targetLogicalKey}${key.slice(sourceLogicalKey.length)}`; - if (targetKey in entries) continue; - entries[targetKey] = { ...entry }; + const target = entries[targetKey]; + entries[targetKey] = + target === undefined + ? { ...source } + : { + pinned: options.pinned === "source" ? source.pinned : target.pinned, + accessCount: Math.max(target.accessCount, source.accessCount), + lastAccessedAt: maxTimestamp(target.lastAccessedAt, source.lastAccessedAt), + lastWriteAt: maxTimestamp(target.lastWriteAt, source.lastWriteAt), + }; } }), @@ -408,9 +424,13 @@ export class MemoryMetaService { await Effect.runPromise(this.effects.renameKeys(oldLogicalKey, newLogicalKey)); } - /** Fill in a subtree's entries under `targetLogicalKey`, keeping the source and existing targets (see effects). */ - async copyKeys(sourceLogicalKey: string, targetLogicalKey: string): Promise { - await Effect.runPromise(this.effects.copyKeys(sourceLogicalKey, targetLogicalKey)); + /** Fold a subtree's entries into `targetLogicalKey`, keeping the source (see effects). */ + async mergeKeys( + sourceLogicalKey: string, + targetLogicalKey: string, + options: { pinned: "target" | "source" } + ): Promise { + await Effect.runPromise(this.effects.mergeKeys(sourceLogicalKey, targetLogicalKey, options)); } /** diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 74070e7dccd..07047e3b1da 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1318,6 +1318,24 @@ describe("MemoryService", () => { expect(await fixture.metaService.getPinnedKeys()).toContain( "workspace:ws-owner:meta-only.md" ); + + // Sidecar-only changes made on a downgraded build (bytes untouched) are + // folded in on the next upgrade: an unpin of only-child.md under the + // child key reaches the owner key... + const freshService = () => + new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await fixture.metaService.setPinned("workspace:ws-child:only-child.md", false); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).not.toContain( + "workspace:ws-owner:only-child.md" + ); + // ...while the owner's OWN later choice is not undone by an unchanged + // child entry on every restart. + await fixture.metaService.setPinned("workspace:ws-owner:only-child.md", true); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).toContain( + "workspace:ws-owner:only-child.md" + ); expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ "clash.md", "downgrade.md", @@ -1332,7 +1350,6 @@ describe("MemoryService", () => { expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ "workspace:ws-child:half.md", "workspace:ws-child:meta-only.md", - "workspace:ws-child:only-child.md", "workspace:ws-owner:half.md", "workspace:ws-owner:meta-only.md", "workspace:ws-owner:only-child.md", @@ -1346,6 +1363,45 @@ describe("MemoryService", () => { ).toBe(true); }); + it("folds in a note written under a self-fallback once ownership resolves to the tree root again", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // Shared access first: the (absent) legacy store is checked against ws-owner. + expect((await fixture.service.listIndexEntries(fixture.ctx)).length).toBe(0); + // config.json goes missing: the child resolves to itself and writes a + // note into its private dir. + const configPath = path.join(fixture.xumHome, "config.json"); + const savedConfig = await fsPromises.readFile(configPath); + await fsPromises.rm(configPath); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + const created = await fixture.service.create( + fixture.ctx, + "/memories/workspace/fallback.md", + "written while config was gone", + "agent" + ); + expect(created.success).toBe(true); + expect( + await pathExists(path.join(fixture.config.sessionsDir, "ws-child", "memory", "fallback.md")) + ).toBe(true); + // Config recovers: the same process must fold that note into the + // shared store instead of trusting its earlier "nothing to adopt". + await fsPromises.writeFile(configPath, savedConfig); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // Index builds get their own context object in production (the owner + // cache is per context); mirror that instead of reusing the command's. + const listed = await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(listed.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "fallback.md", + ]); + expect( + await fsPromises.readFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "fallback.md"), + "utf-8" + ) + ).toBe("written while config was gone"); + }); + it("never imports through a symlinked legacy notebook root or escaped files", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 850674e57ef..14cd34ac5ea 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -402,14 +402,33 @@ const LEGACY_IMPORT_DIR = "imported"; */ const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; -/** Self-healing read of the adoption manifest: anything malformed reads as empty. */ -async function readLegacyAdoptionManifest(manifestPath: string): Promise> { +/** One adopted legacy file: content hash, child sidecar fingerprint, owner-store relPath. */ +interface LegacyAdoptionRecord { + content: string; + sidecar: string; + target: string; +} + +function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { + if (typeof value !== "object" || value === null) return false; + const record = value as Record; + return ( + typeof record.content === "string" && + typeof record.sidecar === "string" && + typeof record.target === "string" + ); +} + +/** Self-healing read of the adoption manifest: anything malformed reads as not adopted. */ +async function readLegacyAdoptionManifest( + manifestPath: string +): Promise> { try { const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {}; return Object.fromEntries( - Object.entries(parsed).filter( - (entry): entry is [string, string] => typeof entry[1] === "string" + Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => + isLegacyAdoptionRecord(entry[1]) ) ); } catch { @@ -730,9 +749,10 @@ export class MemoryService extends EventEmitter { /** * Sub-agents whose pre-sharing private notebook was found absent or already - * adopted during this process lifetime (see adoptLegacyPrivateStore). + * adopted during this process lifetime, keyed to the owner they resolved to + * at the time (see adoptLegacyPrivateStore). */ - private readonly legacyStoreChecked = new Set(); + private readonly legacyStoreCheckedAgainst = new Map(); /** * Owner of the workspace scope for this context ("" when there is no @@ -952,13 +972,23 @@ export class MemoryService extends EventEmitter { store: MemoryStore ): Promise { const childId = ctx.workspaceId; - if (childId === "" || this.legacyStoreChecked.has(childId)) return; + if (childId === "") return; const owner = this.storeOwnerWorkspaceId(store); assert(owner !== null, "workspace-scope stores live under sessionsDir"); - if (owner === childId) return; // not redirected: the private store IS the store + // Checked once per (child, resolved owner) per process. Keyed by the owner + // because ownership can move: a command served while config.json was + // missing/malformed resolves the child to itself and writes into the + // legacy dir; once config recovers, the owner differs from the one this + // marker recorded and the fallback write is folded in on the next access. + if (this.legacyStoreCheckedAgainst.get(childId) === owner) return; + if (owner === childId) { + // Not redirected: the private store IS the store. Recorded so a later + // redirect (config recovered) is seen as an ownership change above. + this.legacyStoreCheckedAgainst.set(childId, owner); + return; + } const legacyRoot = path.join(this.config.sessionsDir, childId, "memory"); - // One lstat per child per process (the copy below is idempotent, so a - // repeat after restart only re-reads unchanged files). + // One lstat per (child, owner) per process; the pass below is idempotent. const legacyRootKind = await lstatKind(legacyRoot); if (legacyRootKind !== "dir") { if (legacyRootKind === "symlink") { @@ -967,11 +997,11 @@ export class MemoryService extends EventEmitter { legacyRoot, }); } - this.legacyStoreChecked.add(childId); + this.legacyStoreCheckedAgainst.set(childId, owner); return; } // Files adopted this pass (bytes written OR only their sidecar entries - // copied): either changes what the shared store's readers derive from it. + // folded in): either changes what the shared store's readers derive from it. let adoptedCount = 0; try { await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { @@ -979,12 +1009,16 @@ export class MemoryService extends EventEmitter { if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock const legacy = new LocalMemoryStore(legacyRoot); const files = await legacy.listFiles(); - // Content hashes already folded in, kept beside the legacy files (a - // dotfile, so neither build lists it). Without it, an imported note - // later edited through the shared store would be re-imported as a - // stale duplicate under imported// on every backend start. + // What was already folded in, kept beside the legacy files (a dotfile, + // so neither build lists it): per relPath the content hash, the + // fingerprint of the child-keyed sidecar entry, and where the copy + // landed. Content: without it, a note later edited through the shared + // store would be re-imported as a stale duplicate on every backend + // start. Sidecar: a downgraded build can change only a pin or usage + // stats, which must reach the owner key without the bytes changing. const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); const adopted = await readLegacyAdoptionManifest(manifestPath); + const sidecarEntries = await this.metaService.getEntries(); let manifestDirty = false; let imported = 0; let skipped = 0; @@ -1000,47 +1034,63 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - const contentHash = sha256Hex(content); - if (adopted[relPath] === contentHash) continue; // folded in earlier, unchanged since - const target = await this.legacyImportTarget(store, childId, relPath, content); - if (target === null) { - skipped++; - continue; + const childKey = memoryLogicalKey("workspace", relPath, { + projectPath: ctx.projectPath, + workspaceId: childId, + }); + const childEntry = sidecarEntries.get(childKey); + const record: LegacyAdoptionRecord = { + content: sha256Hex(content), + sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), + target: "", + }; + const previous = adopted[relPath]; + if (previous?.content === record.content && previous.sidecar === record.sidecar) { + continue; // folded in earlier, nothing changed since } - if (target.write) { - await store.writeFile(target.relPath, content); - imported++; + let target: { relPath: string; write: boolean } | null; + if (previous?.content === record.content) { + // Bytes already adopted: only the sidecar changed; same target. + target = { relPath: previous.target, write: false }; + } else { + target = await this.legacyImportTarget(store, childId, relPath, content); + if (target === null) { + skipped++; + continue; + } + if (target.write) { + await store.writeFile(target.relPath, content); + imported++; + } } - // Pins/stats were keyed by the child: fill them in under the owner - // key too. The child-keyed entry stays — like the legacy file, it is - // what a downgraded build reads. Done even when the bytes were - // already there, and recorded in the manifest only once it - // succeeded: an adoption interrupted after its writeFile (or a - // failing sidecar write) retries this step on the next access. - // copyKeys never overwrites an entry the owner already has, so a - // note the owner tracked independently keeps the owner's history. - try { - await this.metaService.copyKeys( - memoryLogicalKey("workspace", relPath, { - projectPath: ctx.projectPath, - workspaceId: childId, - }), - memoryLogicalKey("workspace", target.relPath, { - projectPath: ctx.projectPath, - workspaceId: owner, - }) - ); - } catch (error) { - log.warn( - "[MemoryService] failed to copy legacy memory stats; retrying on next access", - { - relPath, - error, - } - ); - continue; + record.target = target.relPath; + // Pins/stats were keyed by the child: fold them into the owner key. + // The child-keyed entry stays — like the legacy file, it is what a + // downgraded build reads. Recorded in the manifest only once this + // succeeded, so an adoption interrupted after its writeFile (or a + // failing sidecar write) retries this step on the next access. A + // first adoption keeps the owner's own pin (a note the owner tracked + // independently); a sidecar the CHILD changed since its last + // adoption (downgrade-time pin/unpin) is the newer intent and wins. + if (childEntry !== undefined) { + try { + await this.metaService.mergeKeys( + childKey, + memoryLogicalKey("workspace", target.relPath, { + projectPath: ctx.projectPath, + workspaceId: owner, + }), + { pinned: previous === undefined ? "target" : "source" } + ); + } catch (error) { + log.warn( + "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", + { relPath, error } + ); + continue; + } } - adopted[relPath] = contentHash; + adopted[relPath] = record; manifestDirty = true; adoptedCount++; } @@ -1048,8 +1098,9 @@ export class MemoryService extends EventEmitter { await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); } if (adoptedCount > 0) { - // A metadata-only adoption (identical bytes, child pin copied) still - // changes the hot set other backends derive, so the clock moves too. + // A metadata-only adoption (identical bytes, child pin folded in) + // still changes the hot set other backends derive, so the clock + // moves too. await this.advanceStoreRevision(store); log.info( "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", @@ -1057,7 +1108,7 @@ export class MemoryService extends EventEmitter { ); } }); - this.legacyStoreChecked.add(childId); + this.legacyStoreCheckedAgainst.set(childId, owner); } catch (error) { log.warn( "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts new file mode 100644 index 00000000000..3cd966a93ae --- /dev/null +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -0,0 +1,85 @@ +/** + * Durable fallback for the workspace-memory write-policy DENY. + * + * The epoch accumulator normally lives on the workspace's config.json entry + * (`workspaceMemoryWritable`, see WorkspaceService.recordWorkspaceMemoryWritable). + * By the time a turn learns its final tool set has no `memory` tool, its user + * row is already durable in chat.jsonl — so if the config write fails, refusing + * the turn is not enough: after a restart the accumulator would be absent, a + * later writable turn would publish `true`, and the compaction harvest (which + * reads EVERY message of the epoch) would carry the refused turn's rows into + * the shared notebook. This marker lives in the session dir — the same + * durability domain as chat.jsonl — and is ANDed into the accumulator wherever + * it is consulted; it is cleared only at the epoch boundary that clears the + * config bit. + * + * Fail-closed by construction: a missing marker is "no deny"; a present, + * malformed, or unreadable marker is a deny. + */ +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; +import writeFileAtomic from "write-file-atomic"; +import assert from "@/common/utils/assert"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; + +export const WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME = "memory-policy-deny.json"; + +export function workspaceMemoryDenyMarkerPath(sessionDir: string): string { + assert(sessionDir.length > 0, "workspaceMemoryDenyMarkerPath requires a session dir"); + return path.join(sessionDir, WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME); +} + +/** Durable-or-throw: verified by reading the marker back. */ +export async function writeWorkspaceMemoryDenyMarker(sessionDir: string): Promise { + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + await fsPromises.mkdir(sessionDir, { recursive: true }); + await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now() }), { + encoding: "utf-8", + }); + if (!(await readWorkspaceMemoryDenyMarker(sessionDir))) { + throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); + } +} + +/** True when a deny is recorded (or the marker cannot be trusted); false only when absent. */ +export async function readWorkspaceMemoryDenyMarker(sessionDir: string): Promise { + try { + await fsPromises.access(workspaceMemoryDenyMarkerPath(sessionDir)); + return true; + } catch (error) { + return !hasErrorCode(error, "ENOENT"); + } +} + +/** + * Epoch boundary: remove the marker, durable-or-throw (verified absent). + * `notAfter` fences the clear to denies recorded up to the boundary: a deny + * another backend recorded for the NEW epoch in the meantime must survive. + * A marker whose timestamp cannot be read is treated as current (kept). + */ +export async function clearWorkspaceMemoryDenyMarker( + sessionDir: string, + options?: { notAfter: number } +): Promise { + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + if (options !== undefined) { + let deniedAt: number; + try { + const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); + deniedAt = + typeof parsed === "object" && + parsed !== null && + typeof (parsed as { deniedAt?: unknown }).deniedAt === "number" + ? (parsed as { deniedAt: number }).deniedAt + : Number.POSITIVE_INFINITY; + } catch (error) { + if (hasErrorCode(error, "ENOENT")) return; + deniedAt = Number.POSITIVE_INFINITY; + } + if (deniedAt > options.notAfter) return; + } + await fsPromises.rm(markerPath, { force: true }); + if (await readWorkspaceMemoryDenyMarker(sessionDir)) { + throw new Error(`Workspace memory deny marker could not be removed at ${markerPath}`); + } +} diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 943d5b6f725..84bb8fc877b 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -1,4 +1,10 @@ import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + clearWorkspaceMemoryDenyMarker, + readWorkspaceMemoryDenyMarker, + workspaceMemoryDenyMarkerPath, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -9337,6 +9343,45 @@ describe("WorkspaceService initialize", () => { }); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); expect(persisted()).toBe(false); + + // New epoch again, and now config.json cannot take the deny: the turn's + // user row is already durable, so the deny falls back to the session + // dir (same durability domain) and the record still succeeds. A later + // writable turn — even from a fresh process with no mirror — stays + // denied by that marker, and it is ANDed into the compaction + // observation as well. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + const sessionDir = path.join(realConfig.sessionsDir, "policy-scratch"); + spyOn(realConfig, "editConfig").mockImplementationOnce(() => + Promise.reject(new Error("disk full")) + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(false); + // Malformed marker still denies; an epoch boundary clears it and the + // next epoch can become writable again. + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + await clearWorkspaceMemoryDenyMarker(sessionDir); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(persisted()).toBe(true); + // The fenced clear (compaction boundary) keeps a deny recorded after it. + await writeWorkspaceMemoryDenyMarker(sessionDir); + await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() - 60_000 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() + 60_000 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); } finally { await cleanup(); } @@ -18638,6 +18683,9 @@ describe("WorkspaceService init cancellation", () => { sessionsDir: tempRoot, removeWorkspace: mock(() => Promise.resolve()), findWorkspace: mock(() => null), + // The metadata-less removal path resolves the shared-memory owner + // strictly from config; an unreadable config aborts the removal. + loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( mockConfig as Config, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 909f3b2f19e..a28e94a7075 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -138,6 +138,10 @@ import { TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { + readWorkspaceMemoryDenyMarker, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { @@ -4209,6 +4213,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } } + /** + * Config snapshot for a removal's destructive step: strict load, so a + * transiently unreadable config.json aborts the removal (the workspace stays + * registered and retryable) rather than yielding the fresh-install default + * whose empty topology would resolve every sub-agent to itself. + */ + private loadConfigForRemovalOrAbort(workspaceId: string): ProjectsConfig { + try { + return this.config.loadConfigOrDefault({ throwOnError: true }); + } catch (error: unknown) { + throw new SharedMemoryRemovalAbortedError(workspaceId, { cause: error }); + } + } + /** * TurnRequestBuilder → session: the agent's workspace-memory write policy * for this turn. Also persisted on the workspace config entry (only when it @@ -4216,8 +4234,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * still be gated; see onCompactionComplete. Awaited by the builder and * VERIFIED by reading the config back: Config swallows write failures, and * a stale persisted `true` would let a now read-only agent's transcript - * harvest into the shared notebook after a restart. Resolves false when the - * new value could not be confirmed durable. + * harvest into the shared notebook after a restart. A deny that config.json + * cannot hold is recorded in the session dir instead + * (workspaceMemoryDenyMarker.ts); resolves false only when the new value + * could not be confirmed durable anywhere. */ async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { const session = @@ -4242,8 +4262,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // false→true. The session additionally contributes its own mirror: a // deny it observed survives even if another backend's boundary reset // removed the durable field underneath it. + // A fourth input: the session-dir deny marker, the durable fallback taken + // when config.json could not record a deny (see below). + const sessionDir = path.join(this.config.sessionsDir, workspaceId); + const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir); const conjunction = (durable: boolean | undefined): boolean => - (durable ?? true) && (mirror ?? true) && writable; + !denyMarker && (durable ?? true) && (mirror ?? true) && writable; // Fast path (no write): the outcome cannot differ from the stored value — // it is already false, or already true and this turn grants. const stored = before.workspace.workspaceMemoryWritable; @@ -4252,6 +4276,28 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return true; } let effective = conjunction(stored); + // A deny that config.json cannot hold falls back to the session dir: the + // turn's user row is already durable in chat.jsonl there, so the deny + // must become durable in the same place or the epoch could later be + // harvested as writable after a restart (the mirror is process-local). + const denyDurableFallback = async (cause: string): Promise => { + if (effective) return false; + try { + await writeWorkspaceMemoryDenyMarker(sessionDir); + } catch (markerError: unknown) { + log.error("Workspace memory deny could not be made durable anywhere", { + workspaceId, + cause, + error: getErrorMessage(markerError), + }); + // Process-local floor: this session at least keeps refusing until + // the next successful persist writes the false it now mirrors. + session?.recordWorkspaceMemoryWritable(false); + return false; + } + session?.recordWorkspaceMemoryWritable(false); + return true; + }; try { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); @@ -4267,7 +4313,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { writable, error: getErrorMessage(error), }); - return false; + return denyDurableFallback(getErrorMessage(error)); } session?.recordWorkspaceMemoryWritable(effective); const persisted = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace @@ -4278,7 +4324,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { writable, persisted, }); - return false; + return denyDurableFallback("config write swallowed"); } return true; } @@ -4383,15 +4429,30 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // the harvest fails closed. const persistedWritable = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId) ?.workspace.workspaceMemoryWritable; - const observed = [metadata.workspaceMemoryWritable, persistedWritable].filter( - (value): value is boolean => value !== undefined - ); - this.memoryConsolidationService?.triggerHarvestThenSweepInBackground({ - ...metadata, - ...(observed.length > 0 - ? { workspaceMemoryWritable: observed.every((value) => value) } - : {}), - }); + // Third observation: the session-dir deny marker (fallback taken when + // config.json could not record a deny; see recordWorkspaceMemoryWritable). + // The completion callback is synchronous and the harvest runs in the + // background anyway, so the marker read simply precedes the trigger. + readWorkspaceMemoryDenyMarker(path.join(this.config.sessionsDir, workspaceId)) + .then((denyMarker) => { + const observed = [ + metadata.workspaceMemoryWritable, + persistedWritable, + ...(denyMarker ? [false] : []), + ].filter((value): value is boolean => value !== undefined); + this.memoryConsolidationService?.triggerHarvestThenSweepInBackground({ + ...metadata, + ...(observed.length > 0 + ? { workspaceMemoryWritable: observed.every((value) => value) } + : {}), + }); + }) + .catch((error: unknown) => { + log.warn("Skipping post-compaction memory harvest: deny marker unreadable", { + workspaceId, + error: getErrorMessage(error), + }); + }); }, onIdleCompactionOutcome: (success) => { // Reports the *persisted* idle-compaction outcome (success only after the summary @@ -5960,6 +6021,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let parentWorkspaceId: string | null = null; let childTaskModelString: string | undefined; let childTaskThinkingLevel: ThinkingLevel | undefined; + // Shared-memory owner as verified by the pre-teardown handover below; + // the destructive step reuses it rather than re-resolving (see there). + let verifiedSharedMemoryOwnerId: string | null = null; const metadataResult = await this.aiService.getWorkspaceMetadata(workspaceId); if (metadataResult.success) { @@ -6043,6 +6107,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.config.loadConfigOrDefault(), workspaceId ); + verifiedSharedMemoryOwnerId = sharedMemoryOwnerId; if (sharedMemoryOwnerId !== workspaceId) { try { const pinOwner = (cfg: ReturnType): string[] => { @@ -6474,12 +6539,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // write cannot slip between this tombstone and its commit check, and // run the delta pass of the refinement-row handover inside those // locks (rows appended since the pre-teardown pass; the phantom, - // metadata-less path gets its full pass here). Resolved from - // persisted config. - const memoryOwnerId = resolveWorkspaceMemoryOwnerId( - this.config.loadConfigOrDefault(), - workspaceId - ); + // metadata-less path gets its full pass here). The owner verified by + // that earlier pass is retained: a config.json that turns missing or + // malformed in between would otherwise resolve the child to ITSELF + // here and drop the owner-store lock, letting a foreign child write + // admitted under the real owner lock recreate the deleted session dir. + // Without an earlier pass, resolve strictly — an unreadable config + // aborts the removal (retryable) instead of guessing the topology. + const memoryOwnerId = + verifiedSharedMemoryOwnerId ?? + resolveWorkspaceMemoryOwnerId(this.loadConfigForRemovalOrAbort(workspaceId), workspaceId); const ownerSessionDir = memoryOwnerId === workspaceId ? undefined From 0af1ff6729c6bab78fae7019759f5fb29aba66b4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 03:51:39 +0000 Subject: [PATCH 39/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-fo?= =?UTF-8?q?urth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Unknown history fails closed: when neither the durable accumulator, the deny marker nor the session mirror knows the epoch's policy but the active context already holds turns, recordWorkspaceMemoryWritable publishes a deny for the epoch (the record was lost — a deny that could not be made durable anywhere, an upgrade mid-epoch) instead of letting the next writable turn grant it. The builder derives epochHasPriorTurns from the active context minus the turn being started and compaction request rows. - Legacy adoption stops at the shared store's remaining file capacity; files left behind stay unrecorded and are retried once space frees up. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 51 ++++++++++++++++++++++ src/node/services/memoryService.ts | 19 ++++++++ src/node/services/turnRequestBuilder.ts | 28 +++++++++++- src/node/services/workspaceService.test.ts | 28 ++++++++++++ src/node/services/workspaceService.ts | 20 +++++++-- 5 files changed, 141 insertions(+), 5 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 07047e3b1da..db210cba3f7 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1363,6 +1363,57 @@ describe("MemoryService", () => { ).toBe(true); }); + it("stops adopting legacy notes at the shared store's remaining file capacity", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner two below the cap; child brings five (one identical to an owner + // file, which needs no slot). + await Promise.all( + Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE - 2 }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ) + ); + await fsPromises.writeFile(path.join(ownerRoot, "shared.md"), "same"); + for (const name of ["a.md", "b.md", "c.md", "d.md"]) { + await fsPromises.writeFile(path.join(legacyRoot, name), `child ${name}`); + } + await fsPromises.writeFile(path.join(legacyRoot, "shared.md"), "same"); + + const listed = await fixture.service.listIndexEntries(fixture.ctx); + const workspaceFiles = listed.filter((e) => e.scope === "workspace").map((e) => e.relPath); + // Exactly at the cap, never above: one slot was already taken by + // shared.md's owner copy, so only one of the four new notes fit. + expect(workspaceFiles).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); + expect(workspaceFiles.filter((f) => ["a.md", "b.md", "c.md", "d.md"].includes(f))).toEqual([ + "a.md", + ]); + // A create into the full scope is refused like before, so the invariant holds. + const full = await fixture.service.create( + fixture.ctx, + "/memories/workspace/new.md", + "x", + "agent" + ); + expect(full.success).toBe(false); + // Freed capacity lets a later pass (fresh process) fold in the rest. + await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0000.md", "agent"); + await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0001.md", "agent"); + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const relisted = (await restarted.listIndexEntries({ ...fixture.ctx })) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath); + expect(relisted).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); + expect(relisted.filter((f) => ["a.md", "b.md", "c.md", "d.md"].includes(f))).toEqual([ + "a.md", + "b.md", + "c.md", + ]); + }); + it("folds in a note written under a self-fallback once ownership resolves to the tree root again", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 14cd34ac5ea..75665556745 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1019,6 +1019,13 @@ export class MemoryService extends EventEmitter { const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); const adopted = await readLegacyAdoptionManifest(manifestPath); const sidecarEntries = await this.metaService.getEntries(); + // The per-scope file cap is a store invariant (create/rename enforce + // it): the copy stops at the owner store's remaining capacity so a + // combined notebook cannot exceed it — an over-full scope is silently + // truncated by the index and refuses every later create. Files left + // behind stay unrecorded and are retried once space frees up. + let remainingCapacity = MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles()).length; + let capacityExhausted = false; let manifestDirty = false; let imported = 0; let skipped = 0; @@ -1059,7 +1066,13 @@ export class MemoryService extends EventEmitter { continue; } if (target.write) { + if (remainingCapacity <= 0) { + capacityExhausted = true; + skipped++; + continue; + } await store.writeFile(target.relPath, content); + remainingCapacity--; imported++; } } @@ -1097,6 +1110,12 @@ export class MemoryService extends EventEmitter { if (manifestDirty) { await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); } + if (capacityExhausted) { + log.warn( + "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", + { childId, owner, cap: MEMORY_MAX_FILES_PER_SCOPE } + ); + } if (adoptedCount > 0) { // A metadata-only adoption (identical bytes, child pin folded in) // still changes the hot set other backends derive, so the clock diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 00e7bbf8d82..7b024f31110 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -554,7 +554,11 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { * the value could not be confirmed durable. */ workspaceMemoryPolicySink?: { - recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise; + recordWorkspaceMemoryWritable( + workspaceId: string, + writable: boolean, + options: { epochHasPriorTurns: boolean } + ): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; desktopSessionManager?: DesktopSessionManager; @@ -1462,10 +1466,30 @@ export class TurnRequestBuilder { // an admission-only candidate (prepareStreamMessage) may be rejected or // disposed without running. Awaited (a config write happens only when // the value changes). Returns false when a deny could not be persisted. + // Whether the active context already holds turns whose policy this + // process never recorded (see the unknown-history rule in + // WorkspaceService.recordWorkspaceMemoryWritable). The turn being started + // is its last user row plus that row's prelude snapshots; compaction + // request rows open an epoch rather than belong to one. + const epochHasPriorTurns = ((): boolean => { + const currentBatch = new Set( + latestUserMessage === undefined + ? [] + : [latestUserMessage.id, ...(latestUserMessage.metadata?.requestPreludeMessageIds ?? [])] + ); + return activeContextMessages.some( + (message) => + message.role === "user" && + !currentBatch.has(message.id) && + message.metadata?.muxMetadata?.type !== "compaction-request" + ); + })(); const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; - if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable)) return true; + if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { epochHasPriorTurns })) { + return true; + } if (!writable) return false; log.warn("Workspace memory write policy could not be persisted; harvests will fail closed", { workspaceId, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 84bb8fc877b..ac2540a5944 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9382,6 +9382,34 @@ describe("WorkspaceService initialize", () => { expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() + 60_000 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + + // Unknown history fails closed: no accumulator, no marker, no mirror + // (this service never recorded this epoch), yet the epoch already holds + // turns — the record was lost (a deny that could not be made durable + // anywhere before a restart, an upgrade mid-epoch). A writable turn + // must not grant the whole epoch; the first turn of a fresh epoch does. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: true, + }) + ).toBe(true); + expect(persisted()).toBe(false); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + }) + ).toBe(true); + expect(persisted()).toBe(true); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index a28e94a7075..8b866123cff 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4239,7 +4239,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { * (workspaceMemoryDenyMarker.ts); resolves false only when the new value * could not be confirmed durable anywhere. */ - async recordWorkspaceMemoryWritable(workspaceId: string, writable: boolean): Promise { + async recordWorkspaceMemoryWritable( + workspaceId: string, + writable: boolean, + options?: { epochHasPriorTurns: boolean } + ): Promise { const session = this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); const mirror = session?.workspaceMemoryWritableMirror(); @@ -4266,11 +4270,21 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // when config.json could not record a deny (see below). const sessionDir = path.join(this.config.sessionsDir, workspaceId); const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir); + // Unknown history fails closed, like the harvest's own unknown → closed + // rule: with no durable accumulator, no marker and no mirror, an epoch + // that already holds turns has a policy nobody recorded — the record was + // lost (a deny that could not be made durable anywhere before this + // process died, an upgrade mid-epoch) — so this epoch's harvest is denied + // until the next boundary rather than granted by whichever turn comes + // first. The first turn of a fresh epoch has no prior turns and grants + // normally. + const stored = before.workspace.workspaceMemoryWritable; + const unknownHistory = + stored === undefined && mirror === undefined && options?.epochHasPriorTurns === true; const conjunction = (durable: boolean | undefined): boolean => - !denyMarker && (durable ?? true) && (mirror ?? true) && writable; + !denyMarker && !unknownHistory && (durable ?? true) && (mirror ?? true) && writable; // Fast path (no write): the outcome cannot differ from the stored value — // it is already false, or already true and this turn grants. - const stored = before.workspace.workspaceMemoryWritable; if (stored === false || (stored === true && conjunction(stored))) { session?.recordWorkspaceMemoryWritable(stored); return true; From e1a7c5e8ed43bbfa732950f1a51a594aa4f424fd Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 04:13:30 +0000 Subject: [PATCH 40/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-fi?= =?UTF-8?q?fth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removal resolves the shared-memory owner from a strict config load in the pre-teardown pass too; a missing or malformed config.json aborts before anything is torn down instead of retaining a self-owner that would drop the owner-store lock at the destructive step. - Legacy adoption containment-checks the selected owner-store destination (candidate selection and immediately before the write), so a symlinked component such as imported/ can never route the copy outside the memory root. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 28 +++++++++++++++++++++++-- src/node/services/memoryService.ts | 22 +++++++++++++++++++ src/node/services/workspaceService.ts | 16 +++++++++----- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index db210cba3f7..004658a7044 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1467,11 +1467,35 @@ describe("MemoryService", () => { (await fixture.service.listIndexEntries(fixture.ctx)).filter((e) => e.scope === "workspace") ).toEqual([]); + // Destination side: the owner store's imported/ component is a + // symlink out of the root. A conflicting legacy note would land there; + // the write is refused (and nothing is written outside), the note stays + // in the legacy dir unrecorded. + await fsPromises.unlink(path.join(childSessionDir, "memory")); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot); + await fsPromises.writeFile(path.join(legacyRoot, "clash.md"), "child version"); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + await fsPromises.mkdir(path.join(ownerRoot, "imported"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "clash.md"), "owner version"); + await fsPromises.symlink(outside, path.join(ownerRoot, "imported", "ws-child")); + const escaped = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + expect( + (await escaped.listIndexEntries(fixture.ctx)) + .filter((e) => e.scope === "workspace") + .map((e) => e.relPath) + ).toEqual(["clash.md"]); + expect(await pathExists(path.join(outside, "clash.md"))).toBe(false); + expect(await pathExists(path.join(legacyRoot, ".adopted-into-shared-store.json"))).toBe( + false + ); + await fsPromises.unlink(path.join(ownerRoot, "imported", "ws-child")); + await fsPromises.rm(legacyRoot, { recursive: true }); + await fsPromises.rm(path.join(ownerRoot, "clash.md")); + // Real root whose entries point outside: symlinked entries are not // regular files to the walk, and a symlinked subdirectory is never // descended into. - await fsPromises.unlink(path.join(childSessionDir, "memory")); - const legacyRoot = path.join(childSessionDir, "memory"); await fsPromises.mkdir(legacyRoot); await fsPromises.symlink(path.join(outside, "secret.md"), path.join(legacyRoot, "link.md")); await fsPromises.symlink(outside, path.join(legacyRoot, "linked-dir")); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 75665556745..b0d66666a2a 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1071,6 +1071,22 @@ export class MemoryService extends EventEmitter { skipped++; continue; } + // Destination containment immediately before the write (the + // same check a memory create runs): a symlinked component under + // the owner root — e.g. imported/ pointing elsewhere — + // must never let the copy land outside the store. + try { + await store.assertContained(target.relPath); + } catch (error) { + log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { + childId, + relPath, + target: target.relPath, + error, + }); + skipped++; + continue; + } await store.writeFile(target.relPath, content); remainingCapacity--; imported++; @@ -1155,6 +1171,12 @@ export class MemoryService extends EventEmitter { content: string ): Promise<{ relPath: string; write: boolean } | null> { for (const candidate of [relPath, `${LEGACY_IMPORT_DIR}/${childId}/${relPath}`]) { + // Never even compare through an escaping path (the write site re-checks). + const contained = await store.assertContained(candidate).then( + () => true, + () => false + ); + if (!contained) continue; const kind = await store.kind(candidate); if (kind === null) return { relPath: candidate, write: true }; if (kind === "file") { diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 8b866123cff..189e6a650ae 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4214,10 +4214,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } /** - * Config snapshot for a removal's destructive step: strict load, so a - * transiently unreadable config.json aborts the removal (the workspace stays - * registered and retryable) rather than yielding the fresh-install default - * whose empty topology would resolve every sub-agent to itself. + * Config snapshot for resolving a removal's shared-memory owner: strict + * load, so a transiently unreadable config.json aborts the removal (the + * workspace stays registered and retryable) rather than yielding the + * fresh-install default whose empty topology would resolve every sub-agent + * to itself. */ private loadConfigForRemovalOrAbort(workspaceId: string): ProjectsConfig { try { @@ -6117,8 +6118,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // lands in between is captured too; that late pass only has the // few rows appended since this one, keeping the fallible work at // the point of no return minimal. + // Strict load: a config.json that is missing or malformed right now + // would read as the fresh-install default and resolve this child to + // ITSELF — dropping the owner-store lock and the row handover at the + // destructive step below, which reuses this value. Nothing has been + // torn down yet, so aborting here leaves the workspace intact. const sharedMemoryOwnerId = resolveWorkspaceMemoryOwnerId( - this.config.loadConfigOrDefault(), + this.loadConfigForRemovalOrAbort(workspaceId), workspaceId ); verifiedSharedMemoryOwnerId = sharedMemoryOwnerId; From 369f9e3da8538754a088a21d1f33fc252b5f8c07 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 04:41:20 +0000 Subject: [PATCH 41/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-si?= =?UTF-8?q?xth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The deny-marker fallback writes under the session-dir target mutation lock with a removal-tombstone check inside it (like headless usage), so a late deny reaching the fallback after removal cannot recreate the deleted session dir; a removed workspace is recorded as done. - The legacy-store shortcut is keyed on the resolved owner AND the legacy store's change stamp (its store clock plus root mtime), so a self-fallback write by ANOTHER backend re-runs the idempotent adoption pass in this process instead of waiting for a restart. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 18 ++++++++ src/node/services/memoryService.ts | 50 ++++++++++++++++------ src/node/services/workspaceService.test.ts | 24 +++++++++++ src/node/services/workspaceService.ts | 20 ++++++++- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 004658a7044..ab7225718f7 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1451,6 +1451,24 @@ describe("MemoryService", () => { "utf-8" ) ).toBe("written while config was gone"); + + // ANOTHER backend hits the same fallback while this process's resolution + // never changes: its write advances the child's store clock, which this + // process notices on its next access and folds the note in. + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + spyOn(foreign, "resolveWorkspaceMemoryOwnerId").mockReturnValue("ws-child"); + const foreignWrite = await foreign.create( + { ...fixture.ctx }, + "/memories/workspace/foreign.md", + "written by another backend's fallback", + "agent" + ); + expect(foreignWrite.success).toBe(true); + const afterForeign = await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(afterForeign.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ + "fallback.md", + "foreign.md", + ]); }); it("never imports through a symlinked legacy notebook root or escaped files", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index b0d66666a2a..5bfce8284c2 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -436,6 +436,21 @@ async function readLegacyAdoptionManifest( } } +/** + * Change stamp of a sub-agent's legacy private store: its store clock (any + * MemoryService write there advances it, including a foreign backend's + * self-fallback write) plus the root directory's mtime (top-level entry + * changes made outside MemoryService). Missing pieces read as fixed tokens. + */ +async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Promise { + const revision = await readWorkspaceMemoryRevision(childSessionDir).catch(() => null); + const rootMtime = await fsPromises + .stat(legacyRoot) + .then((stat) => String(stat.mtimeMs)) + .catch(() => "missing"); + return `${revision ?? "none"}:${rootMtime}`; +} + /** Link-aware kind of a path: symlinks are reported as such, never followed. */ async function lstatKind(absPath: string): Promise<"dir" | "symlink" | "other" | "missing"> { try { @@ -749,8 +764,8 @@ export class MemoryService extends EventEmitter { /** * Sub-agents whose pre-sharing private notebook was found absent or already - * adopted during this process lifetime, keyed to the owner they resolved to - * at the time (see adoptLegacyPrivateStore). + * adopted during this process lifetime, keyed to the owner and legacy-store + * state observed at the time (see adoptLegacyPrivateStore). */ private readonly legacyStoreCheckedAgainst = new Map(); @@ -975,21 +990,28 @@ export class MemoryService extends EventEmitter { if (childId === "") return; const owner = this.storeOwnerWorkspaceId(store); assert(owner !== null, "workspace-scope stores live under sessionsDir"); - // Checked once per (child, resolved owner) per process. Keyed by the owner - // because ownership can move: a command served while config.json was - // missing/malformed resolves the child to itself and writes into the - // legacy dir; once config recovers, the owner differs from the one this - // marker recorded and the fallback write is folded in on the next access. - if (this.legacyStoreCheckedAgainst.get(childId) === owner) return; if (owner === childId) { // Not redirected: the private store IS the store. Recorded so a later - // redirect (config recovered) is seen as an ownership change above. + // redirect (config recovered) is seen as a change of the key below. this.legacyStoreCheckedAgainst.set(childId, owner); return; } - const legacyRoot = path.join(this.config.sessionsDir, childId, "memory"); - // One lstat per (child, owner) per process; the pass below is idempotent. + // Checked once per (child, owner, legacy-store state) per process. The + // owner is part of the key because ownership can move: a command served + // while config.json was missing/malformed resolves the child to itself + // and writes into the legacy dir. The legacy store's own state is part of + // it because ANOTHER backend can do the same while this process's + // resolution never changes: its self-fallback write advances the child's + // store clock (memory.revision in the child's session dir) and replaces a + // root entry, so either signal re-runs the pass. Two small stats per + // workspace-scope access; the pass itself is idempotent. + const childSessionDir = path.join(this.config.sessionsDir, childId); + const legacyRoot = path.join(childSessionDir, "memory"); const legacyRootKind = await lstatKind(legacyRoot); + const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ + legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" + }`; + if (this.legacyStoreCheckedAgainst.get(childId) === checkKey) return; if (legacyRootKind !== "dir") { if (legacyRootKind === "symlink") { log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { @@ -997,7 +1019,7 @@ export class MemoryService extends EventEmitter { legacyRoot, }); } - this.legacyStoreCheckedAgainst.set(childId, owner); + this.legacyStoreCheckedAgainst.set(childId, checkKey); return; } // Files adopted this pass (bytes written OR only their sidecar entries @@ -1143,7 +1165,9 @@ export class MemoryService extends EventEmitter { ); } }); - this.legacyStoreCheckedAgainst.set(childId, owner); + // Recorded against the state observed BEFORE the pass: a foreign write + // landing during it changes the stamp and re-runs the (idempotent) pass. + this.legacyStoreCheckedAgainst.set(childId, checkKey); } catch (error) { log.warn( "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ac2540a5944..e092649e758 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5,6 +5,7 @@ import { workspaceMemoryDenyMarkerPath, writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; +import { workspaceRemovalTombstonePath } from "@/node/services/workspaceRemoval"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -9410,6 +9411,29 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persisted()).toBe(true); + + // A late deny reaching the marker fallback after the workspace was + // removed (tombstoned, session dir deleted) must not recreate the + // session dir as an orphan; nothing is left to harvest, so it is done. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!; + delete entry.workspace.workspaceMemoryWritable; + return cfg; + }); + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + const tombstonePath = workspaceRemovalTombstonePath(realConfig.rootDir, "policy-scratch"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "policy-scratch" })); + spyOn(realConfig, "editConfig").mockImplementationOnce(() => + Promise.reject(new Error("disk full")) + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect( + await fsPromises.stat(sessionDir).then( + () => true, + () => false + ) + ).toBe(false); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 189e6a650ae..08c2ad5776a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -130,6 +130,7 @@ import { } from "@/node/services/branchSummary"; import { healRemovalTombstonesForRegisteredWorkspaces, + isWorkspaceRemovalTombstoned, removeSessionDirUnderMemoryLocks, SharedMemoryRemovalAbortedError, refineApplyLockPath, @@ -143,6 +144,7 @@ import { writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { ADDITIONAL_SYSTEM_CONTEXT_DISABLED_FILENAME, @@ -4298,7 +4300,23 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const denyDurableFallback = async (cause: string): Promise => { if (effective) return false; try { - await writeWorkspaceMemoryDenyMarker(sessionDir); + // Late session-dir writer (like headless usage): with several + // backends, this turn may reach the fallback after a remover has + // tombstoned, deleted and deregistered the workspace — the marker's + // mkdir would recreate the session dir as an orphan. Gate + write run + // inside the session-dir target lock that removal's tombstone+delete + // critical section also holds, so the check cannot go stale. A + // removed workspace has nothing left to harvest: recorded as done. + const written = await withTargetMutationLock(this.config.rootDir, sessionDir, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return false; + await writeWorkspaceMemoryDenyMarker(sessionDir); + return true; + }); + if (!written) { + log.debug("Skipping workspace memory deny marker for removed workspace", { workspaceId }); + session?.recordWorkspaceMemoryWritable(false); + return true; + } } catch (markerError: unknown) { log.error("Workspace memory deny could not be made durable anywhere", { workspaceId, From 7dba2c9abb3e3ad482a91d07893961537eb42096 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 05:13:34 +0000 Subject: [PATCH 42/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-se?= =?UTF-8?q?venth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Harvest refuses (terminally) an epoch whose tail is user rows no turn ever answered: a turn records its memory policy in start(), before its assistant row, so such rows belong to a turn whose policy nobody recorded (another backend's in-flight batch, a pre-start refusal) and the grant evaluated at completion could not account for them. - The no-tail epoch reset runs after the completion observation settled and is awaited by the next policy record (settleWorkspaceMemoryPolicyEpoch), so no turn of the new epoch ANDs itself with the closing epoch's deny. - A malformed deny marker cannot claim to be a newer deny: the fenced boundary clear heals it instead of keeping it forever. - Legacy adoption reuses a recorded import target for sidecar-only changes only while it still holds the adopted bytes; otherwise the note is placed anew so the child's pin never lands on replaced content or a missing file. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 52 +++++++++++++------ .../memoryConsolidationService.test.ts | 51 ++++++++++++++++++ .../services/memoryConsolidationService.ts | 40 ++++++++++++++ src/node/services/memoryService.test.ts | 28 +++++++--- src/node/services/memoryService.ts | 22 ++++++-- .../services/workspaceMemoryDenyMarker.ts | 23 ++++---- src/node/services/workspaceService.test.ts | 45 +++++++++++++++- src/node/services/workspaceService.ts | 9 ++-- 8 files changed, 228 insertions(+), 42 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 631a03814ae..f1a324f3caa 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -726,7 +726,7 @@ interface AgentSessionOptions { runtimeConfig: RuntimeConfig | undefined; }) => Promise; /** Called when compaction completes (e.g., to clear idle compaction pending state) */ - onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void; + onCompactionComplete?: (metadata: CompactionCompletionMetadata) => void | Promise; /** Called with the terminal outcome of an idle compaction (persisted success / post-stream failure) */ onIdleCompactionOutcome?: (success: boolean) => void; /** Called when post-compaction context state may have changed (plan/file edits) */ @@ -1078,6 +1078,18 @@ export class AgentSession { return this.workspaceMemoryWritable; } + /** In-flight durable epoch reset started by a no-tail compaction (see the completion callback). */ + private workspaceMemoryEpochReset: Promise | undefined; + + /** + * Resolves once the durable epoch reset of the last no-tail compaction (if + * any is still running) has settled, so a policy record for the new epoch + * never reads the closing epoch's accumulator or deny marker. + */ + async settleWorkspaceMemoryPolicyEpoch(): Promise { + await this.workspaceMemoryEpochReset; + } + /** * Start a fresh policy epoch: the in-memory mirror and the durable * accumulator both forget the previous epoch's turns. Durable-or-throw like @@ -1278,28 +1290,38 @@ export class AgentSession { this.coordinator.recordCompactionSummary( (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null ); - onCompactionComplete?.({ - ...metadata, - ...(this.workspaceMemoryWritable !== undefined - ? { workspaceMemoryWritable: this.workspaceMemoryWritable } - : {}), - }); + const closing = this.workspaceMemoryWritable; + const observed = Promise.resolve( + onCompactionComplete?.({ + ...metadata, + ...(closing !== undefined ? { workspaceMemoryWritable: closing } : {}), + }) + ); // New epoch. A preserved tail copies messages produced under this // epoch's policy into the next one, so the fail-closed accumulator // carries over with them; otherwise the next normal turn restarts it. - // The completion callback is synchronous; the durable reset runs - // detached and is logged on failure (the next turn re-records the - // policy anyway, so a failed reset can only delay a grant, never - // widen one). + // The mirror forgets the closing epoch right here, synchronously; the + // durable reset runs AFTER the completion observation settled (it + // reads the closing epoch's marker/config) and is awaited by the next + // turn's policy record (settleWorkspaceMemoryPolicyEpoch), so no turn + // of the new epoch can AND itself with the closing epoch's stale deny. if ((metadata.preservedTailMessageCount ?? 0) === 0) { - this.resetWorkspaceMemoryWritable({ closing: this.workspaceMemoryWritable }).catch( - (error: unknown) => { + this.workspaceMemoryWritable = undefined; + const reset = observed + .catch(() => undefined) + .then(() => this.resetWorkspaceMemoryWritable({ closing })) + .catch((error: unknown) => { log.warn("Failed to reset the workspace memory policy epoch", { workspaceId: this.workspaceId, error, }); - } - ); + }) + .finally(() => { + if (this.workspaceMemoryEpochReset === reset) { + this.workspaceMemoryEpochReset = undefined; + } + }); + this.workspaceMemoryEpochReset = reset; } }, onIdleCompactionOutcome, diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 87c3647fdba..11b19d97a9c 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -331,6 +331,12 @@ async function seedCompactionEpoch( workspaceId, createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests.") ); + // The turn ran: its policy was recorded before this reply was appended + // (an epoch ending in unanswered user rows is refused; see below). + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("reply-1", "assistant", "Noted.") + ); await fixture.historyService.appendToHistory( workspaceId, createMuxMessage("compact-request", "user", "Please compact", { @@ -1194,6 +1200,51 @@ describe("MemoryConsolidationService", () => { expect(harvested?.status).toBe("completed"); }); + it("refuses to harvest an epoch whose tail is a user batch no turn ever answered", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Another backend appended a turn's user rows; before that turn recorded + // its policy (start()), this backend compacted above them with a grant. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests.") + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.") + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + // The sweep still runs (success) while the harvest record is terminal + // (never completed, never retried), so recovery cannot replay the grant. + expect(result.success).toBe(true); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); + expect(record?.error).toContain("never recorded"); + }); + it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index c4205778a16..8c312e93423 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -67,6 +67,7 @@ import { resolveHeadlessAgentDefinition } from "@/node/services/agentDefinitions import type { AgentDefinitionPackage } from "@/common/types/agentDefinition"; import { log } from "@/node/services/log"; import type { HistoryService } from "@/node/services/historyService"; +import type { MuxMessage } from "@/common/types/message"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; @@ -313,6 +314,28 @@ function finalizeHarvestRecordForRemoval(record: MemoryHarvestRecord): MemoryHar }; } +/** Harvest refused with a terminal record already journaled (see runHarvestAttemptEffect). */ +class HarvestRefusedError extends Error { + constructor(reason: string) { + super(reason); + this.name = "HarvestRefusedError"; + } +} + +/** + * Whether the epoch's tail is a user batch with no assistant reply: rows of a + * turn that never started (its policy record happens in start(), before the + * assistant row is appended), so nothing accounted for them. + */ +function epochEndsWithUnansweredUserRows(messages: readonly MuxMessage[]): boolean { + for (let index = messages.length - 1; index >= 0; index -= 1) { + const role = messages[index].role; + if (role === "assistant") return false; + if (role === "user") return true; + } + return false; +} + export class MemoryConsolidationService extends EventEmitter { private readonly sidecarPath: string; /** Serializes sidecar read-modify-write cycles (journal persistence only). */ @@ -1102,6 +1125,9 @@ export class MemoryConsolidationService extends EventEmitter { // channel AND defects identically. const journalHarvestFailure = (error: unknown): Effect.Effect => Effect.gen(function* () { + // A refusal already journaled its terminal record; re-journaling + // would make it retryable again. + if (error instanceof HarvestRefusedError) return; yield* self.saveHarvestRecordEffect( metadata.workspaceId, boundaryKey, @@ -1166,6 +1192,20 @@ export class MemoryConsolidationService extends EventEmitter { catch: (error) => error, }); if (!epoch.success) return yield* Effect.fail(new Error(epoch.error)); + // Every scanned row must belong to a turn whose write policy was + // recorded. A turn records its policy in start(), BEFORE its assistant + // row is appended, so user rows after the epoch's last assistant reply + // are a turn that had not started when the summary landed above them — + // another backend's in-flight batch (multi-instance), or a turn refused + // pre-start. Their policy is unknown and the grant evaluated at + // completion could not have accounted for them. Terminal refusal: a + // retry would replay the same recorded grant. + if (epochEndsWithUnansweredUserRows(epoch.data.messages)) { + const reason = + "the compacted epoch ends with user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)"; + yield* Effect.promise(() => self.recordRefusedHarvest(metadata, reason)); + return yield* Effect.fail(new HarvestRefusedError(reason)); + } const modelString = resolveDreamModelString(self.config, metadata.workspaceId); const modelResult = yield* Effect.tryPromise({ diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index ab7225718f7..93f5770fadb 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1320,22 +1320,35 @@ describe("MemoryService", () => { ); // Sidecar-only changes made on a downgraded build (bytes untouched) are - // folded in on the next upgrade: an unpin of only-child.md under the - // child key reaches the owner key... + // folded in on the next upgrade: an unpin of half.md under the child key + // reaches the owner key (its recorded target still holds the bytes)... const freshService = () => new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); - await fixture.metaService.setPinned("workspace:ws-child:only-child.md", false); + await fixture.metaService.setPinned("workspace:ws-child:half.md", false); await freshService().listIndexEntries(fixture.ctx); - expect(await fixture.metaService.getPinnedKeys()).not.toContain( - "workspace:ws-owner:only-child.md" - ); + expect(await fixture.metaService.getPinnedKeys()).not.toContain("workspace:ws-owner:half.md"); // ...while the owner's OWN later choice is not undone by an unchanged // child entry on every restart. - await fixture.metaService.setPinned("workspace:ws-owner:only-child.md", true); + await fixture.metaService.setPinned("workspace:ws-owner:half.md", true); + await freshService().listIndexEntries(fixture.ctx); + expect(await fixture.metaService.getPinnedKeys()).toContain("workspace:ws-owner:half.md"); + // only-child.md's recorded target was replaced by the shared edit: the + // child's pin change must not land on the owner's new content. The + // legacy note is placed anew (imported/) and carries the child's state. + await fixture.metaService.setPinned("workspace:ws-child:only-child.md", false); await freshService().listIndexEntries(fixture.ctx); expect(await fixture.metaService.getPinnedKeys()).toContain( "workspace:ws-owner:only-child.md" ); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "only-child.md"), + "utf-8" + ) + ).toBe("child notes"); + expect(await fixture.metaService.getPinnedKeys()).not.toContain( + "workspace:ws-owner:imported/ws-child/only-child.md" + ); expect(relisted.filter((e) => e.scope === "workspace").map((e) => e.relPath)).toEqual([ "clash.md", "downgrade.md", @@ -1348,7 +1361,6 @@ describe("MemoryService", () => { "shared edit" ); expect([...(await fixture.metaService.getPinnedKeys())].sort()).toEqual([ - "workspace:ws-child:half.md", "workspace:ws-child:meta-only.md", "workspace:ws-owner:half.md", "workspace:ws-owner:meta-only.md", diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 5bfce8284c2..370bd8ae412 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1077,11 +1077,25 @@ export class MemoryService extends EventEmitter { if (previous?.content === record.content && previous.sidecar === record.sidecar) { continue; // folded in earlier, nothing changed since } - let target: { relPath: string; write: boolean } | null; + let target: { relPath: string; write: boolean } | null = null; if (previous?.content === record.content) { - // Bytes already adopted: only the sidecar changed; same target. - target = { relPath: previous.target, write: false }; - } else { + // Bytes already adopted and only the sidecar changed: the recorded + // target is reused only while it still holds the adopted bytes — + // the owner may have edited, replaced or deleted it since, and the + // child's pin must not land on unrelated content or a missing + // file. Otherwise the note is placed anew like a fresh adoption. + const stillAdopted = + (await store.assertContained(previous.target).then( + () => true, + () => false + )) && + (await store.kind(previous.target)) === "file" && + (await this.readBoundedTextFile(store, previous.target, previous.target).catch( + () => null + )) === content; + if (stillAdopted) target = { relPath: previous.target, write: false }; + } + if (target === null) { target = await this.legacyImportTarget(store, childId, relPath, content); if (target === null) { skipped++; diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 3cd966a93ae..55dfa44cee7 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -55,7 +55,11 @@ export async function readWorkspaceMemoryDenyMarker(sessionDir: string): Promise * Epoch boundary: remove the marker, durable-or-throw (verified absent). * `notAfter` fences the clear to denies recorded up to the boundary: a deny * another backend recorded for the NEW epoch in the meantime must survive. - * A marker whose timestamp cannot be read is treated as current (kept). + * Only a well-formed marker can claim to be newer; a truncated or malformed + * one is stale state from some earlier epoch (every reader already treated + * it as a deny for as long as it existed) and is healed here — otherwise one + * corrupt file would force every later epoch's accumulator to false until a + * destructive history clear. */ export async function clearWorkspaceMemoryDenyMarker( sessionDir: string, @@ -63,20 +67,19 @@ export async function clearWorkspaceMemoryDenyMarker( ): Promise { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); if (options !== undefined) { - let deniedAt: number; + let deniedAt: number | null; try { const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); - deniedAt = - typeof parsed === "object" && - parsed !== null && - typeof (parsed as { deniedAt?: unknown }).deniedAt === "number" - ? (parsed as { deniedAt: number }).deniedAt - : Number.POSITIVE_INFINITY; + const candidate = + typeof parsed === "object" && parsed !== null + ? (parsed as { deniedAt?: unknown }).deniedAt + : undefined; + deniedAt = typeof candidate === "number" && Number.isFinite(candidate) ? candidate : null; } catch (error) { if (hasErrorCode(error, "ENOENT")) return; - deniedAt = Number.POSITIVE_INFINITY; + deniedAt = null; // unreadable/malformed: cannot be a newer deny } - if (deniedAt > options.notAfter) return; + if (deniedAt !== null && deniedAt > options.notAfter) return; } await fsPromises.rm(markerPath, { force: true }); if (await readWorkspaceMemoryDenyMarker(sessionDir)) { diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e092649e758..212dab808e6 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9365,10 +9365,14 @@ describe("WorkspaceService initialize", () => { expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); expect(persisted()).toBe(false); - // Malformed marker still denies; an epoch boundary clears it and the - // next epoch can become writable again. + // Malformed marker still denies; an epoch boundary — even the fenced + // compaction one, since a malformed file cannot claim to be a newer + // deny — heals it and the next epoch can become writable again. await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() - 60_000 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); await clearWorkspaceMemoryDenyMarker(sessionDir); await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; @@ -9412,6 +9416,43 @@ describe("WorkspaceService initialize", () => { ).toBe(true); expect(persisted()).toBe(true); + // A record must wait for the session's in-flight epoch reset (a no-tail + // compaction clearing the closing epoch's accumulator/marker), or the + // first turn of the new epoch would AND itself with the stale deny. + let releaseReset!: () => void; + const resetInFlight = new Promise((resolve) => (releaseReset = resolve)); + const fakeSession = { + settleWorkspaceMemoryPolicyEpoch: () => resetInFlight, + workspaceMemoryWritableMirror: () => undefined, + recordWorkspaceMemoryWritable: () => undefined, + }; + (service as unknown as { sessions: Map }).sessions.set( + "policy-scratch", + fakeSession + ); + await realConfig.editConfig((cfg) => { + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable = false; + return cfg; + }); + let settled = false; + const pendingRecord = service + .recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false }) + .then((ok) => { + settled = true; + return ok; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); + // The reset finishes (field cleared) and only then does the record read. + await realConfig.editConfig((cfg) => { + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable; + return cfg; + }); + releaseReset(); + expect(await pendingRecord).toBe(true); + expect(persisted()).toBe(true); + (service as unknown as { sessions: Map }).sessions.delete("policy-scratch"); + // A late deny reaching the marker fallback after the workspace was // removed (tombstoned, session dir deleted) must not recreate the // session dir as an orphan; nothing is left to harvest, so it is done. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 08c2ad5776a..ddb38fb2a4c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4249,6 +4249,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ): Promise { const session = this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); + // A no-tail compaction's durable epoch reset may still be in flight: read + // nothing of the closing epoch (accumulator, marker) before it settled. + await session?.settleWorkspaceMemoryPolicyEpoch(); const mirror = session?.workspaceMemoryWritableMirror(); const before = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); // Unregistered workspace: nothing durable to update and no stale @@ -4464,9 +4467,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ?.workspace.workspaceMemoryWritable; // Third observation: the session-dir deny marker (fallback taken when // config.json could not record a deny; see recordWorkspaceMemoryWritable). - // The completion callback is synchronous and the harvest runs in the - // background anyway, so the marker read simply precedes the trigger. - readWorkspaceMemoryDenyMarker(path.join(this.config.sessionsDir, workspaceId)) + // Returned so the session orders its epoch reset (which clears the + // marker) after this observation; the harvest runs in the background. + return readWorkspaceMemoryDenyMarker(path.join(this.config.sessionsDir, workspaceId)) .then((denyMarker) => { const observed = [ metadata.workspaceMemoryWritable, From 94d719f743006feca300c4cfdef2a163fe9c5a9b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 05:40:28 +0000 Subject: [PATCH 43/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-ei?= =?UTF-8?q?ghth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Harvest coverage is bound to request snapshots: a user row is accounted for only when some assistant row's requestHistorySequence reaches it (that turn recorded its policy in start() before the row landed). A later assistant row is no proof by itself, so a foreign batch appended between a turn's snapshot and its assistant row refuses the harvest terminally. - Rollback peer topology is loaded strictly (builder and debug CLI); a resolver that cannot read config.json makes the rollback refuse instead of proceeding with no peer journals. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/cli/debug/refinements.ts | 3 +- .../memoryConsolidationService.test.ts | 37 ++++++++------- .../services/memoryConsolidationService.ts | 45 +++++++++++-------- src/node/services/memoryService.test.ts | 13 ++++++ .../services/refinement/refinementRollback.ts | 15 ++++++- src/node/services/turnRequestBuilder.ts | 4 +- 6 files changed, 80 insertions(+), 37 deletions(-) diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index 92cd3210c56..e58e3d04609 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -67,9 +67,10 @@ export async function refinementsCommand( // Reloaded per check (plan-time and in-lock), not from the snapshot // above: a live backend may register a new tree member while this // process waits for the shared-store lock, and its rows must count. + // Strict: an unreadable config refuses the rollback (empty tree = guess). listSharedWorkspaceMemoryPeerSessionDirs: () => sharedWorkspaceMemoryPeerSessionDirs( - defaultConfig.loadConfigOrDefault(), + defaultConfig.loadConfigOrDefault({ throwOnError: true }), defaultConfig.sessionsDir, workspaceId ), diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 11b19d97a9c..621441924a3 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -327,15 +327,16 @@ async function seedCompactionEpoch( fixture: Fixture, workspaceId = "ws-dream" ): Promise { + const prompt = createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests."); + await fixture.historyService.appendToHistory(workspaceId, prompt); + // The turn ran: its policy was recorded before this reply was appended, and + // the reply's request snapshot covers the prompt (uncovered user rows make + // the harvest refuse; see below). await fixture.historyService.appendToHistory( workspaceId, - createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests.") - ); - // The turn ran: its policy was recorded before this reply was appended - // (an epoch ending in unanswered user rows is refused; see below). - await fixture.historyService.appendToHistory( - workspaceId, - createMuxMessage("reply-1", "assistant", "Noted.") + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + }) ); await fixture.historyService.appendToHistory( workspaceId, @@ -1200,21 +1201,27 @@ describe("MemoryConsolidationService", () => { expect(harvested?.status).toBe("completed"); }); - it("refuses to harvest an epoch whose tail is a user batch no turn ever answered", async () => { + it("refuses to harvest an epoch holding user rows no turn's request snapshot covers", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); - // Another backend appended a turn's user rows; before that turn recorded - // its policy (start()), this backend compacted above them with a grant. - await fixture.historyService.appendToHistory( - "ws-dream", - createMuxMessage("pref-1", "user", "Please remember that I prefer concise tests.") + // Backend A snapshots its request through pref-1; backend B appends a + // read-only turn's user row BEFORE that turn records its policy; A's + // assistant row lands above it carrying its snapshot bound; A compacts + // with a grant. The later assistant is no proof for B's row. + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." ); + await fixture.historyService.appendToHistory("ws-dream", prompt); await fixture.historyService.appendToHistory( "ws-dream", - createMuxMessage("reply-1", "assistant", "Noted.") + createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") ); await fixture.historyService.appendToHistory( "ws-dream", - createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + }) ); await fixture.historyService.appendToHistory( "ws-dream", diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 8c312e93423..ffacdaa4902 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -323,17 +323,28 @@ class HarvestRefusedError extends Error { } /** - * Whether the epoch's tail is a user batch with no assistant reply: rows of a - * turn that never started (its policy record happens in start(), before the - * assistant row is appended), so nothing accounted for them. + * Whether some user row of the epoch is covered by no assistant row's request + * snapshot. An assistant row records `requestHistorySequence` — the last + * history sequence its turn's request was built from — and that turn recorded + * its write policy in start(), before the row was appended. A user row above + * every such snapshot belongs to a turn nobody accounted for: another + * backend's batch appended between a turn's snapshot and its assistant row + * (multi-instance), or a turn refused pre-start. A later assistant row is no + * proof by itself; only its snapshot bound is. Assistant rows without the + * field cover nothing (fail closed). */ -function epochEndsWithUnansweredUserRows(messages: readonly MuxMessage[]): boolean { - for (let index = messages.length - 1; index >= 0; index -= 1) { - const role = messages[index].role; - if (role === "assistant") return false; - if (role === "user") return true; +function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { + let coverage = -1; + for (const message of messages) { + if (message.role !== "assistant") continue; + const bound = message.metadata?.requestHistorySequence; + if (typeof bound === "number" && bound > coverage) coverage = bound; } - return false; + return messages.some((message) => { + if (message.role !== "user") return false; + const sequence = message.metadata?.historySequence; + return typeof sequence !== "number" || sequence > coverage; + }); } export class MemoryConsolidationService extends EventEmitter { @@ -1192,17 +1203,13 @@ export class MemoryConsolidationService extends EventEmitter { catch: (error) => error, }); if (!epoch.success) return yield* Effect.fail(new Error(epoch.error)); - // Every scanned row must belong to a turn whose write policy was - // recorded. A turn records its policy in start(), BEFORE its assistant - // row is appended, so user rows after the epoch's last assistant reply - // are a turn that had not started when the summary landed above them — - // another backend's in-flight batch (multi-instance), or a turn refused - // pre-start. Their policy is unknown and the grant evaluated at - // completion could not have accounted for them. Terminal refusal: a - // retry would replay the same recorded grant. - if (epochEndsWithUnansweredUserRows(epoch.data.messages)) { + // Every scanned user row must be covered by a turn whose write policy + // was recorded (see epochHasUncoveredUserRows). Uncovered rows have an + // unknown policy the grant evaluated at completion could not have + // accounted for. Terminal refusal: a retry would replay that grant. + if (epochHasUncoveredUserRows(epoch.data.messages)) { const reason = - "the compacted epoch ends with user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)"; + "the compacted epoch holds user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)"; yield* Effect.promise(() => self.recordRefusedHarvest(metadata, reason)); return yield* Effect.fail(new HarvestRefusedError(reason)); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 93f5770fadb..b7a7c3fe005 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1626,6 +1626,19 @@ describe("MemoryService", () => { (row) => (row.data.action as { op: string }).op === "rename" )!; + // Membership that cannot be established (unreadable config) refuses the + // rollback instead of guessing an empty tree. + const unresolvable = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => { + throw new Error("config.json unreadable"); + }, + evidence: { toolName: "test", actor: "user" }, + }); + expect(unresolvable.success).toBe(false); + if (!unresolvable.success) expect(unresolvable.error).toContain("could not be resolved"); + // Own journal only: the rename looks cleanly undoable and would move // the child's newer content back without a word. const blind = await rollbackRefinement({ diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 43137f85e06..00a56b1e583 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -84,6 +84,8 @@ export interface RollbackRefinementOptions { * store, so their rows are merged into divergence detection: a child's * later edit under a path the owner renamed must surface as a conflict * when the owner rolls the rename back. Omit when the store is private. + * MUST throw when membership cannot be established (unreadable config); + * the rollback is then refused instead of assuming an empty tree. */ listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; /** Attribution for the emitted rollback row. */ @@ -544,7 +546,18 @@ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { async function readSharedMemoryPeerRows( opts: RollbackRefinementOptions ): Promise { - const peerDirs = opts.listSharedWorkspaceMemoryPeerSessionDirs?.() ?? []; + // Membership must be established, not guessed: a resolver that cannot read + // the topology (config.json missing/malformed) throws, and the rollback is + // refused rather than proceeding with no peer journals — an owner would + // otherwise move a child's later edit without warning. + let peerDirs: string[]; + try { + peerDirs = opts.listSharedWorkspaceMemoryPeerSessionDirs?.() ?? []; + } catch (error) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the task tree sharing this workspace's memory store could not be resolved (${getErrorMessage(error)})` + ); + } if (peerDirs.length === 0) return []; const sharedRoot = path.join( path.resolve(opts.sharedWorkspaceMemorySessionDir ?? opts.sessionDir), diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 7b024f31110..d298e6a69bb 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2418,12 +2418,14 @@ export class TurnRequestBuilder { : path.join(this.dependencies.config.sessionsDir, memoryOwnerId); // Resolved per rollback (not per turn): tree membership changes as // sub-agents are spawned and removed while the tool instance lives. + // Strict load: an unreadable config must refuse the rollback (see + // RollbackRefinementOptions), not read as an empty tree. const listSharedWorkspaceMemoryPeerSessionDirs = memoryService === undefined ? undefined : () => sharedWorkspaceMemoryPeerSessionDirs( - this.dependencies.config.loadConfigOrDefault(), + this.dependencies.config.loadConfigOrDefault({ throwOnError: true }), this.dependencies.config.sessionsDir, workspaceId ); From 435dfbd3497f4212d5acc684b45bdc4fd503f022 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 07:42:56 +0000 Subject: [PATCH 44/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20thirty-ni?= =?UTF-8?q?nth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Harvest coverage is bound to each turn's own batch: an assistant row covers the LAST user row at or below its requestHistorySequence plus that row's requestPreludeMessageIds (exact ids, never adjacency), so a foreign backend's row inside the snapshot leaves the turn's own row uncovered and the harvest refuses. Ordinary sends now record requestPreludeMessageIds too; token-budget control rows are exempt. - recordWorkspaceMemoryWritable loads config strictly: an unreadable config.json routes a deny to the session-dir marker fallback and reports a grant unpersisted instead of taking the unregistered path. - Removal adopts a sub-agent's legacy private notebook into the owner store before teardown (adoptLegacyPrivateStoreForRemoval, throwing). - Shared-memory row migration copies rollback rows with rollbackOf remapped to the owner copy, so a row rolled back between the two handover passes is not left live in the owner journal. - Workspace-scope read-side usage (view, recall) advances the store clock under the store lock and emits a change event, like a pin. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 61 ++- src/node/services/di/layers/core.ts | 1 + .../memoryConsolidationService.test.ts | 130 ++++++ .../services/memoryConsolidationService.ts | 55 ++- src/node/services/memoryService.test.ts | 150 ++++++- src/node/services/memoryService.ts | 394 +++++++++++------- .../services/refinement/refinementJournal.ts | 19 +- .../refinement/sharedMemoryRowMigration.ts | 102 +++-- src/node/services/workspaceService.test.ts | 22 + src/node/services/workspaceService.ts | 130 ++++-- 10 files changed, 781 insertions(+), 283 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index f1a324f3caa..8bae162a270 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -4481,32 +4481,49 @@ export class AgentSession { ); } if (await cancelBeforeAcceptance()) return Ok(undefined); - } else if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { - const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ - ...internal.preTurnMessages, - userMessage, - ]); - if (!batchAppendResult.success) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(batchAppendResult.error)); - } - persistedCancelableMessageIds.push( - ...internal.preTurnMessages.map((message) => message.id), - userMessage.id - ); - if (await cancelBeforeAcceptance()) { - return Ok(undefined); - } } else if (!autoCompactionMessage) { // When on-send compaction triggers, the user message is NOT persisted to // history (it's sent as follow-up after compaction). Otherwise, persist - // normally. - const appendResult = await this.historyService.appendToHistory(this.workspaceId, userMessage); - if (!appendResult.success) { - await rollbackPersistedTurnRows(); - return Err(createUnknownSendMessageError(appendResult.error)); + // normally. The snapshot rows appended above and the pre-turn payloads + // are this turn's request prelude; recorded on the user row exactly as + // the token-budget path does, so the post-compaction harvest gate can + // tell the turn's own batch from a row another backend interleaved + // (epochHasUncoveredUserRows matches prelude rows by id, never by + // adjacency) and the builder's unknown-history rule does not count the + // turn's own snapshots as turns nobody recorded. + const requestPreludeMessageIds = [ + ...(snapshotResult?.snapshotMessage ? [snapshotResult.snapshotMessage] : []), + ...skillSnapshotMessages, + ...mcpPromptSnapshotMessages, + ...(internal?.preTurnMessages ?? []), + ].map((row) => row.id); + if (requestPreludeMessageIds.length > 0) { + userMessage.metadata = { ...userMessage.metadata, requestPreludeMessageIds }; + } + if (internal?.preTurnMessages != null && internal.preTurnMessages.length > 0) { + const batchAppendResult = await this.historyService.appendManyToHistory(this.workspaceId, [ + ...internal.preTurnMessages, + userMessage, + ]); + if (!batchAppendResult.success) { + await rollbackPersistedTurnRows(); + return Err(createUnknownSendMessageError(batchAppendResult.error)); + } + persistedCancelableMessageIds.push( + ...internal.preTurnMessages.map((message) => message.id), + userMessage.id + ); + } else { + const appendResult = await this.historyService.appendToHistory( + this.workspaceId, + userMessage + ); + if (!appendResult.success) { + await rollbackPersistedTurnRows(); + return Err(createUnknownSendMessageError(appendResult.error)); + } + persistedCancelableMessageIds.push(userMessage.id); } - persistedCancelableMessageIds.push(userMessage.id); if (await cancelBeforeAcceptance()) { return Ok(undefined); } diff --git a/src/node/services/di/layers/core.ts b/src/node/services/di/layers/core.ts index ab1f4e70d25..d4fa0210889 100644 --- a/src/node/services/di/layers/core.ts +++ b/src/node/services/di/layers/core.ts @@ -544,6 +544,7 @@ export const CoreWiringLive: Layer.Layer< workspaceService.emitWorkflowRunActivity(event); turnRequestBuilderBindings.workflowResultContinuationSender = workspaceService; workspaceService.setMemoryConsolidationService(memoryConsolidationService); + workspaceService.setSharedWorkspaceMemoryStore(memoryService); // Workspace-scope change events carry the memory OWNER (task-tree root); // every live session resolving to that owner reads the same notebook. memoryService.on("change", (event: MemoryChangeEvent) => { diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 621441924a3..d067eb7e8fc 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1252,6 +1252,136 @@ describe("MemoryConsolidationService", () => { expect(record?.error).toContain("never recorded"); }); + it("covers only a turn's own batch: a foreign row inside the request snapshot leaves the turn's own row uncovered", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Backend B (read-only) appends late-1 and pauses before its deny lands; + // backend A then snapshots THROUGH late-1 (its request's latest user row + // is now B's), records writable and replies. A's bound covers late-1 as + // A's batch — so pref-1, A's own row, belongs to no turn at all. + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + const foreign = createMuxMessage( + "late-1", + "user", + "Read-only agent's prompt, deny not recorded" + ); + await fixture.historyService.appendToHistory("ws-dream", foreign); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: foreign.metadata?.historySequence, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(true); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + + it("covers a turn's request prelude rows by id, not by adjacency", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + let previousBoundaryHistorySequence: number | undefined; + const harvest = async (ids: { listed: string[]; summary: string; leadIn?: boolean }) => { + // Two synthetic snapshot rows precede the user row; only the listed + // ones are the turn's own prelude. The reply snapshots through the + // user row, so every user row lies below its bound. + if (ids.leadIn) { + // Token-budget control row (backend template text, same durable batch + // as the turn): needs no listing. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-lead-in`, "user", "A context window rollover started.", { + synthetic: true, + muxMetadata: { type: "context-window-lead-in", rolloverId: "r1" }, + }) + ); + } + const snapshots = [`${ids.summary}-snap-a`, `${ids.summary}-snap-b`].map((id) => + createMuxMessage(id, "user", "Snapshot content", { synthetic: true }) + ); + for (const snapshot of snapshots) { + await fixture.historyService.appendToHistory("ws-dream", snapshot); + } + const prompt = createMuxMessage( + `${ids.summary}-pref`, + "user", + "Remember I prefer concise tests.", + { + requestPreludeMessageIds: ids.listed, + } + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-reply`, "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage(`${ids.summary}-compact`, "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage(ids.summary, "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: ids.summary, + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: `${ids.summary}-compact`, + ...(previousBoundaryHistorySequence !== undefined + ? { previousBoundaryHistorySequence } + : {}), + }); + expect(result.success).toBe(true); + previousBoundaryHistorySequence = summary.metadata?.historySequence; + return (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + }; + // Only one of the two adjacent snapshot rows is listed: the other is a + // foreign row that merely sits next to the batch. + const partial = await harvest({ listed: ["s1-snap-a"], summary: "s1" }); + expect(partial?.status).toBe("failed"); + expect(partial?.error).toContain("never recorded"); + // Both listed: the whole batch is the turn's own and harvests. + const complete = await harvest({ + listed: ["s2-snap-a", "s2-snap-b"], + summary: "s2", + leadIn: true, + }); + expect(complete?.status).toBe("completed"); + }); + it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index ffacdaa4902..2d5d000f4d5 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -67,7 +67,8 @@ import { resolveHeadlessAgentDefinition } from "@/node/services/agentDefinitions import type { AgentDefinitionPackage } from "@/common/types/agentDefinition"; import { log } from "@/node/services/log"; import type { HistoryService } from "@/node/services/historyService"; -import type { MuxMessage } from "@/common/types/message"; +import { isTokenBudgetInternalMessage, type MuxMessage } from "@/common/types/message"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; @@ -323,28 +324,48 @@ class HarvestRefusedError extends Error { } /** - * Whether some user row of the epoch is covered by no assistant row's request - * snapshot. An assistant row records `requestHistorySequence` — the last - * history sequence its turn's request was built from — and that turn recorded - * its write policy in start(), before the row was appended. A user row above - * every such snapshot belongs to a turn nobody accounted for: another - * backend's batch appended between a turn's snapshot and its assistant row - * (multi-instance), or a turn refused pre-start. A later assistant row is no - * proof by itself; only its snapshot bound is. Assistant rows without the - * field cover nothing (fail closed). + * Whether some user row of the epoch belongs to no turn that recorded its + * write policy. A turn records that policy in start(), before its assistant + * row is appended, and the assistant row carries `requestHistorySequence` — + * the last history sequence its request was built from. The turn's own batch + * is the LAST user row at or below that bound (the request's latest user + * message) plus the snapshot/payload rows that row lists in + * `requestPreludeMessageIds`. Only that batch is covered — not every user row + * below the bound: with several backends on one chat.jsonl, another + * backend's read-only batch can land between this turn's user row and its + * request snapshot while that backend's deny has not been recorded yet; the + * snapshot would then include the foreign row (making it this turn's latest + * user message) and this turn's own row would be left without a turn of its + * own — which is exactly what surfaces here as uncovered. Rows are matched by + * exact id, never by adjacency, so no interleaved foreign row can ride along. + * Assistant rows without the bound cover nothing (fail closed). Token-budget + * control rows (rollover lead-in, budget warning) need no turn: backend + * template text appended in the same durable batch as the turn they precede, + * carrying neither agent nor repository content. */ function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { - let coverage = -1; + const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + for (const message of messages) { + const sequence = message.metadata?.historySequence; + if (message.role === "user" && typeof sequence === "number") + userRows.push({ message, sequence }); + } + const covered = new Set(); for (const message of messages) { if (message.role !== "assistant") continue; const bound = message.metadata?.requestHistorySequence; - if (typeof bound === "number" && bound > coverage) coverage = bound; + if (typeof bound !== "number") continue; + const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; + if (anchor === undefined) continue; + covered.add(anchor.id); + for (const id of getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds)) { + covered.add(id); + } } - return messages.some((message) => { - if (message.role !== "user") return false; - const sequence = message.metadata?.historySequence; - return typeof sequence !== "number" || sequence > coverage; - }); + return messages.some( + (message) => + message.role === "user" && !covered.has(message.id) && !isTokenBudgetInternalMessage(message) + ); } export class MemoryConsolidationService extends EventEmitter { diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index b7a7c3fe005..74ab1886869 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -921,16 +921,18 @@ describe("MemoryService", () => { expect(solo.success).toBe(false); // Change events name the owner so the owner's Memory tab (and every - // tree member's) refreshes; sidecar stats are keyed by the owner too. - expect(events).toEqual([ - { + // tree member's) refreshes — the create and each shared read (the two + // views re-rank the shared hot set); sidecar stats are keyed by the + // owner too. + expect(events).toEqual( + Array.from({ length: 3 }, () => ({ scope: "workspace", path: "/memories/workspace/context-notes.md", actor: "agent", workspaceId: "ws-owner", projectPath: FIXTURE_PROJECT_PATH, - }, - ]); + })) + ); const meta = await fixture.metaService.getEntries(); expect( meta.get( @@ -1116,9 +1118,33 @@ describe("MemoryService", () => { "v2", "agent" ); + const afterEdit = await foreign.workspaceMemoryRevision("ws-owner"); + expect(Number(afterEdit)).toBeGreaterThan(Number(afterPin)); + // A read-side access (view / recall) re-ranks the shared hot set through + // the owner-keyed usage stats: it advances the clock and announces the + // owner's store like a pin does, so the rest of the tree (and other + // backends) drop their cached hot set too. + const events: MemoryChangeEvent[] = []; + fixture.service.on("change", (event: MemoryChangeEvent) => events.push(event)); + expect( + (await fixture.service.view(fixture.ctx, "/memories/workspace/shared.md")).success + ).toBe(true); + const afterView = await foreign.workspaceMemoryRevision("ws-owner"); + expect(Number(afterView)).toBeGreaterThan(Number(afterEdit)); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + scope: "workspace", + path: "/memories/workspace/shared.md", + workspaceId: "ws-owner", + }); + await fixture.service.recordRecall(fixture.ctx, "/memories/workspace/shared.md"); expect(Number(await foreign.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( - Number(afterPin) + Number(afterView) ); + expect(events).toHaveLength(2); + // Global reads have no store clock and stay silent. + expect((await fixture.service.view(fixture.ctx, "/memories/global/g.md")).success).toBe(true); + expect(events).toHaveLength(2); }); it("refuses to commit into a self-fallback store once config.json has recovered", async () => { @@ -1426,6 +1452,42 @@ describe("MemoryService", () => { ]); }); + it("adopts the legacy notebook for removal without any prior access, and throws instead of deferring", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "only-child.md"), "child notes"); + // No workspace-memory entry point ever served ws-child in this process: + // removal's handover must fold the notes in by itself. + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "only-child.md"), "utf-8")).toBe( + "child notes" + ); + // Idempotent: a retried removal re-runs the pass (the per-process memo + // is bypassed) and finds nothing new. + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + // A removal that verified a different owner than the store now resolves + // to must not adopt into the wrong notebook. + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-other") + .then(() => null, getErrorMessage) + ).toMatch(/resolved to ws-owner/); + // Failures surface (the access-time pass only logs and retries later). + spyOn(fixture.metaService, "getEntries").mockImplementationOnce(() => + Promise.reject(new Error("sidecar unreadable")) + ); + await fsPromises.writeFile(path.join(legacyRoot, "late.md"), "written later"); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/sidecar unreadable/); + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + }); + it("folds in a note written under a self-fallback once ownership resolves to the tree root again", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1784,22 +1846,31 @@ describe("MemoryService", () => { ownerSessionDir, ownerWorkspaceId: "ws-owner", }); - expect(await migrate()).toBe(3); + // Three live edits plus redone.md's full rollback lineage (rollback and + // its re-apply); undone.md's dead lineage stays behind. + expect(await migrate()).toBe(5); // Idempotent: a retried removal migrates nothing twice. expect(await migrate()).toBe(0); const ownerRows = await readRefinementEvents(ownerSessionDir); expect( ownerRows.map((row) => [ (row.data.action as { op: string }).op, - (row.data.action as { path: string }).path, + (row.data.action as { path?: string }).path, (row.data.evidence as { workspaceId: string }).workspaceId, ]) ).toEqual([ ["create", "/memories/workspace/keep.md", "ws-owner"], ["str_replace", "/memories/workspace/keep.md", "ws-owner"], ["create", "/memories/workspace/redone.md", "ws-owner"], + ["rollback", undefined, "ws-owner"], + ["rollback", undefined, "ws-owner"], ]); expect(ownerRows.every((row) => row.data.migratedFrom?.startsWith("ws-child:"))).toBe(true); + // The copied rollback rows point at the owner-side copies, not at + // child ids that no longer exist anywhere. + expect(ownerRows[3].data.rollbackOf).toBe(ownerRows[2].id); + expect((ownerRows[3].data.action as { of: string }).of).toBe(ownerRows[2].id); + expect(ownerRows[4].data.rollbackOf).toBe(ownerRows[3].id); // The child is gone; the owner rolls the edit back from its own journal // (payload blobs were copied, postState hashes preserved). @@ -1814,6 +1885,69 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); }); + it("follows a row rolled back between the two handover passes with its rollback row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/keep.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/keep.md", + "v1", + "v2", + "agent" + ); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + // Pre-teardown pass copies the live edit... + expect(await migrate()).toBe(2); + // ...then another backend rolls it back in the child journal before the + // in-lock delta pass runs. + const edit = (await readRefinementEvents(childSessionDir)).find( + (row) => (row.data.action as { op: string }).op === "str_replace" + )!; + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: edit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + const keep = path.join(ownerSessionDir, "memory", "keep.md"); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + // The delta pass copies exactly the rollback row, remapped onto the + // owner-side copy of the edit. + expect(await migrate()).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const editCopy = ownerRows.find((row) => row.data.migratedFrom === `ws-child:${edit.id}`)!; + const rollbackCopy = ownerRows.find((row) => row.data.rollbackOf !== undefined)!; + expect(rollbackCopy.data.rollbackOf).toBe(editCopy.id); + // The owner journal knows the edit is no longer live: rolling it back + // again is refused instead of re-applying an inverse that already ran. + const again = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(again.success).toBe(false); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); + // Rolling back the copied rollback (re-apply) works from the owner journal. + const redo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: rollbackCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(redo.success).toBe(true); + expect(await fsPromises.readFile(keep, "utf-8")).toBe("v2"); + }); + it("concurrent migrations of the same child copy each row exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 370bd8ae412..a0cd41003be 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -804,6 +804,29 @@ export class MemoryService extends EventEmitter { try { const key = this.logicalKeyFor(ctx, scope, relPath); if (key === null) return; + if (scope === "workspace" && !options.write) { + // A read-side access (view, recall) re-ranks the shared hot set the + // whole task tree derives from the owner's sidecar entries — like a + // pin does — so it is published the same way: store clock under the + // store's mutation lock (other backends' probes), then the change + // event (this backend's other sessions and Memory tabs). Writes are + // already inside their mutation's lock and publish with it; a + // read holds no lock yet, hence the explicit one here, with the same + // commit guard so a removed owner's directory is never recreated. + const store = this.getStore(ctx, scope); + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + await this.assertMutationCommittable( + ctx, + store, + undefined, + toVirtualPath(scope, relPath) + ); + await this.metaService.recordAccess(key, options); + await this.advanceStoreRevision(store); + }); + this.emitChange(ctx, scope, relPath, "agent"); + return; + } await this.metaService.recordAccess(key, options); } catch (error) { log.debug("[MemoryService] failed to record memory usage", { scope, relPath, error }); @@ -996,6 +1019,77 @@ export class MemoryService extends EventEmitter { this.legacyStoreCheckedAgainst.set(childId, owner); return; } + try { + await this.adoptLegacyPrivateStoreOrThrow(ctx, store, owner); + } catch (error) { + log.warn( + "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", + { + childId, + owner, + error, + } + ); + } + } + + /** + * Removal handover: before a sub-agent's session directory is deleted, + * fold its legacy private notebook (if any) into the owner store. The + * access-time adoption above only runs when some workspace-memory entry + * point serves the child; a child removed right after an upgrade (an + * inactive-descendant deletion cascade, say) may never have had one, and + * the deletion would discard its notes for good. Runs BEFORE any teardown + * step (removal reuses the owner it verified with a strict config load), so + * a failure aborts the removal with the workspace intact: this variant + * THROWS instead of deferring to a next access that will never come. Not + * run again under the removal locks — the owner-store lock this takes is + * the one removal's critical section holds, and the legacy directory is + * written only by downgraded builds or a self-fallback backend, neither of + * which the in-lock delta pass could fence anyway. + */ + async adoptLegacyPrivateStoreForRemoval( + childWorkspaceId: string, + ownerWorkspaceId: string + ): Promise { + assert(childWorkspaceId.length > 0, "adoptLegacyPrivateStoreForRemoval requires a child id"); + assert( + ownerWorkspaceId.length > 0 && ownerWorkspaceId !== childWorkspaceId, + "adoptLegacyPrivateStoreForRemoval requires a distinct owner id" + ); + // Workspace-scope keys and roots embed only the workspace id (see + // logicalKeyFor / getStore), so no project identity is needed here. + const ctx: MemoryScopeContext = { + runtime: null, + checkoutCwd: "", + workspaceId: childWorkspaceId, + projectPath: "", + }; + const store = this.getStore(ctx, "workspace"); + // The store resolved from current config must be the owner removal + // verified; a disagreement means the topology changed under removal's + // feet, and adopting into the wrong notebook would be worse than aborting. + const resolvedOwner = this.storeOwnerWorkspaceId(store); + if (resolvedOwner !== ownerWorkspaceId) { + throw new Error( + `shared memory owner of ${childWorkspaceId} resolved to ${String(resolvedOwner)} while removal verified ${ownerWorkspaceId}` + ); + } + await this.adoptLegacyPrivateStoreOrThrow(ctx, store, ownerWorkspaceId, { force: true }); + } + + /** + * The adoption pass (see adoptLegacyPrivateStore). `force` skips the + * per-process "already checked" memo: removal wants the pass to run against + * the current legacy directory regardless of what an earlier access saw. + */ + private async adoptLegacyPrivateStoreOrThrow( + ctx: MemoryScopeContext, + store: MemoryStore, + owner: string, + options?: { force: boolean } + ): Promise { + const childId = ctx.workspaceId; // Checked once per (child, owner, legacy-store state) per process. The // owner is part of the key because ownership can move: a command served // while config.json was missing/malformed resolves the child to itself @@ -1011,7 +1105,7 @@ export class MemoryService extends EventEmitter { const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" }`; - if (this.legacyStoreCheckedAgainst.get(childId) === checkKey) return; + if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) return; if (legacyRootKind !== "dir") { if (legacyRootKind === "symlink") { log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { @@ -1025,174 +1119,162 @@ export class MemoryService extends EventEmitter { // Files adopted this pass (bytes written OR only their sidecar entries // folded in): either changes what the shared store's readers derive from it. let adoptedCount = 0; - try { - await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { - await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); - if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock - const legacy = new LocalMemoryStore(legacyRoot); - const files = await legacy.listFiles(); - // What was already folded in, kept beside the legacy files (a dotfile, - // so neither build lists it): per relPath the content hash, the - // fingerprint of the child-keyed sidecar entry, and where the copy - // landed. Content: without it, a note later edited through the shared - // store would be re-imported as a stale duplicate on every backend - // start. Sidecar: a downgraded build can change only a pin or usage - // stats, which must reach the owner key without the bytes changing. - const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); - const adopted = await readLegacyAdoptionManifest(manifestPath); - const sidecarEntries = await this.metaService.getEntries(); - // The per-scope file cap is a store invariant (create/rename enforce - // it): the copy stops at the owner store's remaining capacity so a - // combined notebook cannot exceed it — an over-full scope is silently - // truncated by the index and refuses every later create. Files left - // behind stay unrecorded and are retried once space frees up. - let remainingCapacity = MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles()).length; - let capacityExhausted = false; - let manifestDirty = false; - let imported = 0; - let skipped = 0; - for (const relPath of files) { - // Same read gates as a memory command: containment (no symlink - // escape), size cap, and text-only (a lossy utf-8 decode cannot be - // carried by a text write). - const content = await legacy - .assertContained(relPath) - .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) - .catch(() => null); - if (content === null || content.includes("\uFFFD")) { + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); + if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock + const legacy = new LocalMemoryStore(legacyRoot); + const files = await legacy.listFiles(); + // What was already folded in, kept beside the legacy files (a dotfile, + // so neither build lists it): per relPath the content hash, the + // fingerprint of the child-keyed sidecar entry, and where the copy + // landed. Content: without it, a note later edited through the shared + // store would be re-imported as a stale duplicate on every backend + // start. Sidecar: a downgraded build can change only a pin or usage + // stats, which must reach the owner key without the bytes changing. + const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); + const adopted = await readLegacyAdoptionManifest(manifestPath); + const sidecarEntries = await this.metaService.getEntries(); + // The per-scope file cap is a store invariant (create/rename enforce + // it): the copy stops at the owner store's remaining capacity so a + // combined notebook cannot exceed it — an over-full scope is silently + // truncated by the index and refuses every later create. Files left + // behind stay unrecorded and are retried once space frees up. + let remainingCapacity = MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles()).length; + let capacityExhausted = false; + let manifestDirty = false; + let imported = 0; + let skipped = 0; + for (const relPath of files) { + // Same read gates as a memory command: containment (no symlink + // escape), size cap, and text-only (a lossy utf-8 decode cannot be + // carried by a text write). + const content = await legacy + .assertContained(relPath) + .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) + .catch(() => null); + if (content === null || content.includes("\uFFFD")) { + skipped++; + continue; + } + const childKey = memoryLogicalKey("workspace", relPath, { + projectPath: ctx.projectPath, + workspaceId: childId, + }); + const childEntry = sidecarEntries.get(childKey); + const record: LegacyAdoptionRecord = { + content: sha256Hex(content), + sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), + target: "", + }; + const previous = adopted[relPath]; + if (previous?.content === record.content && previous.sidecar === record.sidecar) { + continue; // folded in earlier, nothing changed since + } + let target: { relPath: string; write: boolean } | null = null; + if (previous?.content === record.content) { + // Bytes already adopted and only the sidecar changed: the recorded + // target is reused only while it still holds the adopted bytes — + // the owner may have edited, replaced or deleted it since, and the + // child's pin must not land on unrelated content or a missing + // file. Otherwise the note is placed anew like a fresh adoption. + const stillAdopted = + (await store.assertContained(previous.target).then( + () => true, + () => false + )) && + (await store.kind(previous.target)) === "file" && + (await this.readBoundedTextFile(store, previous.target, previous.target).catch( + () => null + )) === content; + if (stillAdopted) target = { relPath: previous.target, write: false }; + } + if (target === null) { + target = await this.legacyImportTarget(store, childId, relPath, content); + if (target === null) { skipped++; continue; } - const childKey = memoryLogicalKey("workspace", relPath, { - projectPath: ctx.projectPath, - workspaceId: childId, - }); - const childEntry = sidecarEntries.get(childKey); - const record: LegacyAdoptionRecord = { - content: sha256Hex(content), - sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), - target: "", - }; - const previous = adopted[relPath]; - if (previous?.content === record.content && previous.sidecar === record.sidecar) { - continue; // folded in earlier, nothing changed since - } - let target: { relPath: string; write: boolean } | null = null; - if (previous?.content === record.content) { - // Bytes already adopted and only the sidecar changed: the recorded - // target is reused only while it still holds the adopted bytes — - // the owner may have edited, replaced or deleted it since, and the - // child's pin must not land on unrelated content or a missing - // file. Otherwise the note is placed anew like a fresh adoption. - const stillAdopted = - (await store.assertContained(previous.target).then( - () => true, - () => false - )) && - (await store.kind(previous.target)) === "file" && - (await this.readBoundedTextFile(store, previous.target, previous.target).catch( - () => null - )) === content; - if (stillAdopted) target = { relPath: previous.target, write: false }; - } - if (target === null) { - target = await this.legacyImportTarget(store, childId, relPath, content); - if (target === null) { + if (target.write) { + if (remainingCapacity <= 0) { + capacityExhausted = true; skipped++; continue; } - if (target.write) { - if (remainingCapacity <= 0) { - capacityExhausted = true; - skipped++; - continue; - } - // Destination containment immediately before the write (the - // same check a memory create runs): a symlinked component under - // the owner root — e.g. imported/ pointing elsewhere — - // must never let the copy land outside the store. - try { - await store.assertContained(target.relPath); - } catch (error) { - log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { - childId, - relPath, - target: target.relPath, - error, - }); - skipped++; - continue; - } - await store.writeFile(target.relPath, content); - remainingCapacity--; - imported++; - } - } - record.target = target.relPath; - // Pins/stats were keyed by the child: fold them into the owner key. - // The child-keyed entry stays — like the legacy file, it is what a - // downgraded build reads. Recorded in the manifest only once this - // succeeded, so an adoption interrupted after its writeFile (or a - // failing sidecar write) retries this step on the next access. A - // first adoption keeps the owner's own pin (a note the owner tracked - // independently); a sidecar the CHILD changed since its last - // adoption (downgrade-time pin/unpin) is the newer intent and wins. - if (childEntry !== undefined) { + // Destination containment immediately before the write (the + // same check a memory create runs): a symlinked component under + // the owner root — e.g. imported/ pointing elsewhere — + // must never let the copy land outside the store. try { - await this.metaService.mergeKeys( - childKey, - memoryLogicalKey("workspace", target.relPath, { - projectPath: ctx.projectPath, - workspaceId: owner, - }), - { pinned: previous === undefined ? "target" : "source" } - ); + await store.assertContained(target.relPath); } catch (error) { - log.warn( - "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", - { relPath, error } - ); + log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { + childId, + relPath, + target: target.relPath, + error, + }); + skipped++; continue; } + await store.writeFile(target.relPath, content); + remainingCapacity--; + imported++; } - adopted[relPath] = record; - manifestDirty = true; - adoptedCount++; } - if (manifestDirty) { - await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); - } - if (capacityExhausted) { - log.warn( - "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", - { childId, owner, cap: MEMORY_MAX_FILES_PER_SCOPE } - ); - } - if (adoptedCount > 0) { - // A metadata-only adoption (identical bytes, child pin folded in) - // still changes the hot set other backends derive, so the clock - // moves too. - await this.advanceStoreRevision(store); - log.info( - "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", - { childId, owner, imported, skipped } - ); - } - }); - // Recorded against the state observed BEFORE the pass: a foreign write - // landing during it changes the stamp and re-runs the (idempotent) pass. - this.legacyStoreCheckedAgainst.set(childId, checkKey); - } catch (error) { - log.warn( - "[MemoryService] failed to adopt a sub-agent's legacy workspace notebook; retrying on next access", - { - childId, - owner, - error, + record.target = target.relPath; + // Pins/stats were keyed by the child: fold them into the owner key. + // The child-keyed entry stays — like the legacy file, it is what a + // downgraded build reads. Recorded in the manifest only once this + // succeeded, so an adoption interrupted after its writeFile (or a + // failing sidecar write) retries this step on the next access. A + // first adoption keeps the owner's own pin (a note the owner tracked + // independently); a sidecar the CHILD changed since its last + // adoption (downgrade-time pin/unpin) is the newer intent and wins. + if (childEntry !== undefined) { + try { + await this.metaService.mergeKeys( + childKey, + memoryLogicalKey("workspace", target.relPath, { + projectPath: ctx.projectPath, + workspaceId: owner, + }), + { pinned: previous === undefined ? "target" : "source" } + ); + } catch (error) { + log.warn( + "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", + { relPath, error } + ); + continue; + } } - ); - return; - } + adopted[relPath] = record; + manifestDirty = true; + adoptedCount++; + } + if (manifestDirty) { + await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); + } + if (capacityExhausted) { + log.warn( + "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", + { childId, owner, cap: MEMORY_MAX_FILES_PER_SCOPE } + ); + } + if (adoptedCount > 0) { + // A metadata-only adoption (identical bytes, child pin folded in) + // still changes the hot set other backends derive, so the clock + // moves too. + await this.advanceStoreRevision(store); + log.info( + "[MemoryService] adopted a sub-agent's legacy workspace notebook into the shared store", + { childId, owner, imported, skipped } + ); + } + }); + // Recorded against the state observed BEFORE the pass: a foreign write + // landing during it changes the stamp and re-runs the (idempotent) pass. + this.legacyStoreCheckedAgainst.set(childId, checkKey); if (adoptedCount > 0) this.emitChange(ctx, "workspace", "", "agent"); } diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index bbbbfba037a..9eddc5843cc 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -26,6 +26,7 @@ import { type RefinementEvidence, type RefinementInverse, type RefinementPostState, + type RollbackRefinementAction, type SkillRefinementAction, } from "@/common/types/refinement"; import type { BlobStore } from "@/node/utils/journal/blobStore"; @@ -61,7 +62,7 @@ export interface RefinementEmitArgs { sessionDir: string; workspaceId: string; kind: "memory" | "skill"; - action: MemoryRefinementAction | SkillRefinementAction; + action: MemoryRefinementAction | SkillRefinementAction | RollbackRefinementAction; inverse: RefinementInverseDraft; evidence: { toolName: string; toolCallId?: string; actor?: string }; /** @@ -78,6 +79,13 @@ export interface RefinementEmitArgs { postState?: RefinementPostState; /** Source identity of a row copied from a removed sub-agent's journal (see durableEvent.ts). */ migratedFrom?: string; + /** + * For a migrated ROLLBACK row: the owner-journal id of the row it rolled + * back (the copy of its source target), so the lineage stays intact on the + * owner side. Only shared-memory row migration sets this; the rollback + * engine appends its own rows directly. + */ + rollbackOf?: string; /** * Cross-journal order key (see durableEvent.ts): the shared store's clock * for a workspace-scope mutation (workspaceMemoryRevision.ts), or the @@ -285,7 +293,7 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr // Inverse blob puts and the append referencing them run under the journal // blob lock: a concurrent reclamation pass must never observe the // put→append window (see DurableEventJournal.withBlobLock). - const publishedBlobs = await journal.withBlobLock(() => + const { publishedBlobs } = await journal.withBlobLock(() => appendRefinementEventUnderBlobLock(journal, args) ); // Live rows may carry a store-clock `sourceTs` too (MemoryService); only @@ -308,7 +316,7 @@ export async function appendRefinementEventOrThrow(args: RefinementEmitArgs): Pr export async function appendRefinementEventUnderBlobLock( journal: DurableEventJournal, args: RefinementEmitArgs -): Promise { +): Promise<{ rowId: string; publishedBlobs: BlobQuotaEntry[] }> { assert(args.workspaceId.length > 0, "refinement journal requires a workspace id"); await journal.assertBlobLockOwned(); const resolved = await resolveRefinementInverse(journal.blobs, args.inverse); @@ -330,7 +338,7 @@ export async function appendRefinementEventUnderBlobLock( })), } : args.postState; - await journal.append({ + const row = await journal.append({ workspaceId: args.workspaceId, kind: "refinement", data: { @@ -340,11 +348,12 @@ export async function appendRefinementEventUnderBlobLock( evidence, ...(postState !== undefined ? { postState } : {}), ...(args.migratedFrom !== undefined ? { migratedFrom: args.migratedFrom } : {}), + ...(args.rollbackOf !== undefined ? { rollbackOf: args.rollbackOf } : {}), ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), }, }); - return resolved.publishedBlobs; + return { rowId: row.id, publishedBlobs: resolved.publishedBlobs }; } /** diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 3c7e8ff91f5..d910a90df61 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -5,7 +5,10 @@ import { RefinementEvidenceSchema, RefinementInverseSchema, RefinementPostStateSchema, + RollbackRefinementActionSchema, + type MemoryRefinementAction, type RefinementInverse, + type RollbackRefinementAction, } from "@/common/types/refinement"; import { log } from "@/node/services/log"; import type { BlobQuotaEntry } from "@/node/utils/journal/blobReclamation"; @@ -39,8 +42,17 @@ function isInside(root: string, filePath: string): boolean { * store into the OWNER's journal, copying the inverse blob payloads. The * edits themselves already live in the owner's store; without this their * audit trail and rollback IDs would vanish with the child's journal. Rows - * already rolled back (or rollback rows themselves) and rows targeting other - * roots (global/project) are left alone — they die with the child as before. + * already rolled back and rows targeting other roots (global/project) are + * left alone — they die with the child as before. + * + * Rollback rows travel too, but only when their target has an owner copy, + * with `rollbackOf` remapped to that copy: removal runs this in two passes + * (pre-teardown, then a delta pass under the removal locks), and another + * backend can roll a row back in between. Its copy is already in the owner + * journal by then; without the rollback row following it, the owner journal + * would claim an edit whose inverse was already applied is still live and + * rollbackable. A row rolled back before its FIRST copy is simply dead and + * stays behind with its whole lineage. * * A row whose payload cannot be reconstructed (evicted blob, unparseable * action) is skipped with a log line — nothing durable exists to preserve. @@ -111,19 +123,41 @@ export async function migrateSharedMemoryRefinementRows(args: { // than once per row. Rows already copied are identified by their source // identity on the owner side. await ownerJournal.withBlobLock(async () => { - const alreadyMigrated = new Set( - (await listRefinements(args.ownerSessionDir)) - .map((row) => row.data.migratedFrom) - .filter((id): id is string => id !== undefined) + const ownerRows = await listRefinements(args.ownerSessionDir); + // Source identity → owner-journal id of its copy (earlier passes and this one). + const ownerIdBySource = new Map(); + for (const ownerRow of ownerRows) { + if (ownerRow.data.migratedFrom !== undefined) { + ownerIdBySource.set(ownerRow.data.migratedFrom, ownerRow.id); + } + } + // Owner rows already rolled back (by anyone): a second rollback row for + // the same target would corrupt the lineage the rollback engine walks. + const ownerRollbackTargets = new Set( + ownerRows.map((ownerRow) => ownerRow.data.rollbackOf).filter((id) => id !== undefined) ); for (const row of rows) { - if (row.data.kind !== "memory" || row.data.rollbackOf !== undefined) continue; - if (isLive(row.id) !== true) continue; + if (row.data.kind !== "memory") continue; const migratedFrom = `${args.childWorkspaceId}:${row.id}`; - if (alreadyMigrated.has(migratedFrom)) continue; + if (ownerIdBySource.has(migratedFrom)) continue; + let action: MemoryRefinementAction | RollbackRefinementAction; + let rollbackOf: string | undefined; + if (row.data.rollbackOf === undefined) { + if (isLive(row.id) !== true) continue; + const parsed = MemoryRefinementActionSchema.safeParse(row.data.action); + if (!parsed.success) continue; + action = parsed.data; + } else { + // Child journal order puts a rollback row after its target, so the + // target's copy (from an earlier pass or this loop) is known here. + rollbackOf = ownerIdBySource.get(`${args.childWorkspaceId}:${row.data.rollbackOf}`); + if (rollbackOf === undefined || ownerRollbackTargets.has(rollbackOf)) continue; + const parsed = RollbackRefinementActionSchema.safeParse(row.data.action); + if (!parsed.success) continue; + action = { ...parsed.data, of: rollbackOf }; + } const inverse = RefinementInverseSchema.safeParse(row.data.inverse); - const action = MemoryRefinementActionSchema.safeParse(row.data.action); - if (!inverse.success || !action.success) continue; + if (!inverse.success) continue; if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; let draft: RefinementInverseDraft; @@ -157,28 +191,30 @@ export async function migrateSharedMemoryRefinementRows(args: { const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); const postState = RefinementPostStateSchema.safeParse(row.data.postState); // Throws: this is the only durable copy once the child's journal goes. - publishedBlobs.push( - ...(await appendRefinementEventUnderBlobLock(ownerJournal, { - sessionDir: args.ownerSessionDir, - workspaceId: args.ownerWorkspaceId, - kind: "memory", - action: action.data, - inverse: draft, - evidence: { - toolName: evidence.success ? evidence.data.toolName : "memory", - ...(evidence.success && evidence.data.toolCallId !== undefined - ? { toolCallId: evidence.data.toolCallId } - : {}), - ...(evidence.success && evidence.data.actor !== undefined - ? { actor: evidence.data.actor } - : {}), - }, - ...(postState.success ? { postState: postState.data } : {}), - migratedFrom, - sourceTs: row.data.sourceTs ?? row.ts, - ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), - })) - ); + const appended = await appendRefinementEventUnderBlobLock(ownerJournal, { + sessionDir: args.ownerSessionDir, + workspaceId: args.ownerWorkspaceId, + kind: "memory", + action, + inverse: draft, + evidence: { + toolName: evidence.success ? evidence.data.toolName : "memory", + ...(evidence.success && evidence.data.toolCallId !== undefined + ? { toolCallId: evidence.data.toolCallId } + : {}), + ...(evidence.success && evidence.data.actor !== undefined + ? { actor: evidence.data.actor } + : {}), + }, + ...(postState.success ? { postState: postState.data } : {}), + migratedFrom, + ...(rollbackOf !== undefined ? { rollbackOf } : {}), + sourceTs: row.data.sourceTs ?? row.ts, + ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), + }); + publishedBlobs.push(...appended.publishedBlobs); + ownerIdBySource.set(migratedFrom, appended.rowId); + if (rollbackOf !== undefined) ownerRollbackTargets.add(rollbackOf); migrated++; } }); diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 212dab808e6..7188c379b01 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9453,6 +9453,28 @@ describe("WorkspaceService initialize", () => { expect(persisted()).toBe(true); (service as unknown as { sessions: Map }).sessions.delete("policy-scratch"); + // An unreadable config.json (missing/malformed while the turn starts) + // must not make the still-registered workspace look unregistered: the + // deny takes the session-dir fallback instead of being reported durable + // without a record anywhere, and a grant is reported unpersisted (the + // harvest stays closed) rather than "done". + await realConfig.editConfig((cfg) => { + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable; + return cfg; + }); + const unreadable = () => { + throw new Error("config.json: unexpected token"); + }; + spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(false); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(persisted()).toBeUndefined(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + await clearWorkspaceMemoryDenyMarker(sessionDir); + // A late deny reaching the marker fallback after the workspace was // removed (tombstoned, session dir deleted) must not recreate the // session dir as an orphan; nothing is left to harvest, so it is done. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index ddb38fb2a4c..e5ed46f53cd 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -144,6 +144,13 @@ import { writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; import { migrateSharedMemoryRefinementRows } from "@/node/services/refinement/sharedMemoryRowMigration"; +import type { MemoryService } from "@/node/services/memoryService"; + +/** MemoryService methods removal needs (see MemoryService.adoptLegacyPrivateStoreForRemoval). */ +type SharedWorkspaceMemoryStoreForRemoval = Pick< + MemoryService, + "adoptLegacyPrivateStoreForRemoval" +>; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import { orchestrateFork } from "@/node/services/utils/forkOrchestrator"; import { @@ -2742,6 +2749,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { cancelInFlightConsolidation(workspaceId: string): Promise; finalizeHarvestsForRemoval(workspaceId: string): Promise; }; + /** Narrow MemoryService surface for removal's shared-memory handover; wired by coreServices. */ + private sharedWorkspaceMemoryStore?: SharedWorkspaceMemoryStoreForRemoval; private worktreeArchiveSnapshotService?: WorktreeArchiveSnapshotLifecycleService; private agentTaskIntegration?: AgentTaskIntegration; private workspaceGoalService?: WorkspaceGoalService; @@ -3089,6 +3098,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { this.memoryConsolidationService = service; } + setSharedWorkspaceMemoryStore(store: SharedWorkspaceMemoryStoreForRemoval): void { + this.sharedWorkspaceMemoryStore = store; + } + setWorkspaceLifecycleHooks(hooks: WorkspaceLifecycleHooks): void { this.workspaceLifecycleHooks = hooks; } @@ -4253,53 +4266,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // nothing of the closing epoch (accumulator, marker) before it settled. await session?.settleWorkspaceMemoryPolicyEpoch(); const mirror = session?.workspaceMemoryWritableMirror(); - const before = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); - // Unregistered workspace: nothing durable to update and no stale - // permission to invalidate (harvests fail closed on the missing value). - if (before === null) { - session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable); - return true; - } - // The persisted bit is the epoch accumulator, fail-closed: the harvest - // reads every message of the compaction epoch, so one read-only turn - // denies the whole epoch even if writable turns follow. Durable so it - // survives a restart mid-epoch AND so backends sharing one chat.jsonl - // (multi-instance) contribute to the same conjunction; it restarts at - // context boundaries (AgentSession.resetWorkspaceMemoryWritable). The - // conjunction is computed INSIDE the config transaction (registration - // lock, cross-process) from the value current at write time — two - // backends reading an absent bit concurrently could otherwise publish - // false→true. The session additionally contributes its own mirror: a - // deny it observed survives even if another backend's boundary reset - // removed the durable field underneath it. - // A fourth input: the session-dir deny marker, the durable fallback taken - // when config.json could not record a deny (see below). const sessionDir = path.join(this.config.sessionsDir, workspaceId); - const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir); - // Unknown history fails closed, like the harvest's own unknown → closed - // rule: with no durable accumulator, no marker and no mirror, an epoch - // that already holds turns has a policy nobody recorded — the record was - // lost (a deny that could not be made durable anywhere before this - // process died, an upgrade mid-epoch) — so this epoch's harvest is denied - // until the next boundary rather than granted by whichever turn comes - // first. The first turn of a fresh epoch has no prior turns and grants - // normally. - const stored = before.workspace.workspaceMemoryWritable; - const unknownHistory = - stored === undefined && mirror === undefined && options?.epochHasPriorTurns === true; - const conjunction = (durable: boolean | undefined): boolean => - !denyMarker && !unknownHistory && (durable ?? true) && (mirror ?? true) && writable; - // Fast path (no write): the outcome cannot differ from the stored value — - // it is already false, or already true and this turn grants. - if (stored === false || (stored === true && conjunction(stored))) { - session?.recordWorkspaceMemoryWritable(stored); - return true; - } - let effective = conjunction(stored); // A deny that config.json cannot hold falls back to the session dir: the // turn's user row is already durable in chat.jsonl there, so the deny // must become durable in the same place or the epoch could later be // harvested as writable after a restart (the mirror is process-local). + // `effective` is what the caller computed for the epoch so far; a grant + // never takes the fallback (harvest stays closed on the unpersisted bit). + let effective = false; const denyDurableFallback = async (cause: string): Promise => { if (effective) return false; try { @@ -4334,6 +4308,70 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { session?.recordWorkspaceMemoryWritable(false); return true; }; + // Strict load: a config.json that is missing or malformed right now reads + // as the fresh-install default, in which this still-registered workspace + // is absent — the "unregistered" shortcut below would then report a deny + // as durable without persisting anything, while another backend keeps a + // prior durable grant. Treat an unreadable config like a failed config + // write: the deny takes the session-dir fallback (tombstone-gated, so a + // genuinely deregistered workspace still skips), a grant leaves the + // harvest closed. + let before: ReturnType; + try { + before = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + workspaceId + ); + } catch (error: unknown) { + log.error("Workspace memory write policy: config unreadable", { + workspaceId, + writable, + error: getErrorMessage(error), + }); + effective = writable; + return denyDurableFallback(`config unreadable: ${getErrorMessage(error)}`); + } + // Unregistered workspace: nothing durable to update and no stale + // permission to invalidate (harvests fail closed on the missing value). + if (before === null) { + session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable); + return true; + } + // The persisted bit is the epoch accumulator, fail-closed: the harvest + // reads every message of the compaction epoch, so one read-only turn + // denies the whole epoch even if writable turns follow. Durable so it + // survives a restart mid-epoch AND so backends sharing one chat.jsonl + // (multi-instance) contribute to the same conjunction; it restarts at + // context boundaries (AgentSession.resetWorkspaceMemoryWritable). The + // conjunction is computed INSIDE the config transaction (registration + // lock, cross-process) from the value current at write time — two + // backends reading an absent bit concurrently could otherwise publish + // false→true. The session additionally contributes its own mirror: a + // deny it observed survives even if another backend's boundary reset + // removed the durable field underneath it. + // A fourth input: the session-dir deny marker, the durable fallback taken + // when config.json could not record a deny (denyDurableFallback above). + const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir); + // Unknown history fails closed, like the harvest's own unknown → closed + // rule: with no durable accumulator, no marker and no mirror, an epoch + // that already holds turns has a policy nobody recorded — the record was + // lost (a deny that could not be made durable anywhere before this + // process died, an upgrade mid-epoch) — so this epoch's harvest is denied + // until the next boundary rather than granted by whichever turn comes + // first. The first turn of a fresh epoch has no prior turns and grants + // normally. + const stored = before.workspace.workspaceMemoryWritable; + const unknownHistory = + stored === undefined && mirror === undefined && options?.epochHasPriorTurns === true; + const conjunction = (durable: boolean | undefined): boolean => + !denyMarker && !unknownHistory && (durable ?? true) && (mirror ?? true) && writable; + // Fast path (no write): the outcome cannot differ from the stored value — + // it is already false, or already true and this turn grants. + if (stored === false || (stored === true && conjunction(stored))) { + session?.recordWorkspaceMemoryWritable(stored); + return true; + } + effective = conjunction(stored); try { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); @@ -6175,6 +6213,14 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { throw new Error(`memory owner pin for descendant ${id} did not persist`); } } + // A pre-sharing build kept this child's notebook in its OWN + // session dir (//memory); access-time adoption + // may never have run for a child removed right after the upgrade, + // and the deletion below would take those notes with it. + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + sharedMemoryOwnerId + ); await migrateSharedMemoryRefinementRows({ childSessionDir: path.join(this.config.sessionsDir, workspaceId), childWorkspaceId: workspaceId, From 9d62467e1cb2a11d2f88f2d2b541592dfd51daa7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 08:16:51 +0000 Subject: [PATCH 45/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fortieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Assistant rows of turns that went through the memory policy sink are stamped workspaceMemoryPolicyRecorded; the harvest gate takes coverage only from such rows, so turns run by a downgraded build mid-epoch cannot ride a stale durable grant. - openWorkspaceStore re-checks acting/owner tombstones after the legacy adoption pass (its lock wait is a removal window). - Shared-memory row migration copies rows whose inverse payload was reclaimed as audit-only records (bare blob reference), keeping them visible to conflict detection while rollback refuses them. - resolveWorkspaceMemoryOwnerId memoizes only successful strict loads; a readable-but-unparseable config.json yields an uncached self fallback. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/message.ts | 8 ++ .../memoryConsolidationService.test.ts | 44 +++++++ .../services/memoryConsolidationService.ts | 9 +- src/node/services/memoryService.test.ts | 120 ++++++++++++++++++ src/node/services/memoryService.ts | 26 +++- .../services/refinement/refinementJournal.ts | 21 ++- .../services/refinement/refinementRollback.ts | 2 + .../refinement/sharedMemoryRowMigration.ts | 27 +++- src/node/services/turnRequestBuilder.ts | 7 + 9 files changed, 255 insertions(+), 9 deletions(-) diff --git a/src/common/types/message.ts b/src/common/types/message.ts index fc554e7257b..58ccdb92017 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -952,6 +952,14 @@ export interface ContextBudgetRejectedMessage { export interface MuxMetadata { /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; + /** + * The turn that produced this assistant row recorded its workspace-memory + * write policy before the row was appended (TurnRequestBuilder start()). + * Builds that do not maintain that policy (older ones, after a downgrade) + * leave it unset, so the post-compaction harvest cannot take their turns' + * user rows as accounted for (memoryConsolidationService.ts). + */ + workspaceMemoryPolicyRecorded?: true; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) /** Provider step boundaries in parts, persisted so continuous compaction can keep complete steps. */ stepStartPartIndices?: number[]; diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index d067eb7e8fc..09691d738ba 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -336,6 +336,7 @@ async function seedCompactionEpoch( workspaceId, createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyRecorded: true, }) ); await fixture.historyService.appendToHistory( @@ -1221,6 +1222,7 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyRecorded: true, }) ); await fixture.historyService.appendToHistory( @@ -1274,6 +1276,47 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: foreign.metadata?.historySequence, + workspaceMemoryPolicyRecorded: true, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(true); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + + it("takes no coverage from assistant rows of a build that did not record the policy", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // A downgraded build ran a (read-only) turn mid-epoch: its assistant row + // carries the request snapshot bound but no policy record, and it left + // the durable accumulator's stale grant untouched. The compaction then + // completes with that grant. + const prompt = createMuxMessage("pref-1", "user", "Read-only turn on the old build."); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, }) ); await fixture.historyService.appendToHistory( @@ -1339,6 +1382,7 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage(`${ids.summary}-reply`, "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyRecorded: true, }) ); await fixture.historyService.appendToHistory( diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 2d5d000f4d5..3dcb3fbb2ef 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -338,7 +338,10 @@ class HarvestRefusedError extends Error { * user message) and this turn's own row would be left without a turn of its * own — which is exactly what surfaces here as uncovered. Rows are matched by * exact id, never by adjacency, so no interleaved foreign row can ride along. - * Assistant rows without the bound cover nothing (fail closed). Token-budget + * Assistant rows without the bound, or without `workspaceMemoryPolicyRecorded` + * (a build that does not maintain the policy — e.g. turns run by a downgraded + * build mid-epoch, which also left the durable accumulator untouched), cover + * nothing (fail closed). Token-budget * control rows (rollover lead-in, budget warning) need no turn: backend * template text appended in the same durable batch as the turn they precede, * carrying neither agent nor repository content. @@ -352,7 +355,9 @@ function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { } const covered = new Set(); for (const message of messages) { - if (message.role !== "assistant") continue; + if (message.role !== "assistant" || message.metadata?.workspaceMemoryPolicyRecorded !== true) { + continue; + } const bound = message.metadata?.requestHistorySequence; if (typeof bound !== "number") continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 74ab1886869..92b025c61c1 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -27,6 +27,9 @@ import { import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; +import { reclaimExcessRefinementInverseBlobs } from "./refinement/refinementJournal"; +import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; +import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; import { sharedWorkspaceMemoryPeerSessionDirs } from "./memoryWorkspaceOwner"; import { createRefinementRollbackTool } from "./tools/refinement_rollback"; import type { MemoryScopeAccess } from "@/common/constants/memory"; @@ -1089,6 +1092,27 @@ describe("MemoryService", () => { expect(invalidated).toEqual([["ws-child"]]); }); + it("does not memoize the self fallback taken while config.json is unreadable but unchanged", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // The file stats the same (no stamp change) but cannot be read/parsed + // for a moment (EACCES interval, non-atomic writer): the lenient load + // yields the empty default while the strict one throws. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + // Readability returns without the stamp moving: the next resolution + // must see the real tree instead of a pinned fallback. + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + }); + it("advances the owner store's revision token on shared writes, visible to another backend", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1207,6 +1231,26 @@ describe("MemoryService", () => { ).toBe(false); }); + it("refuses a read whose workspace was tombstoned while the legacy adoption pass ran", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + // The adoption pass (owner-store lock) is the window: another backend's + // removal of ws-child publishes its tombstone after the readability + // check that opened the store, and the pass swallows its own refusal. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + spyOn( + fixture.service as unknown as { adoptLegacyPrivateStore: () => Promise }, + "adoptLegacyPrivateStore" + ).mockImplementationOnce(async () => { + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const refused = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("was removed"); + }); + it("refuses a pin toggle once the owner it was bound to is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1948,6 +1992,82 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(keep, "utf-8")).toBe("v2"); }); + it("migrates a row whose inverse payload was reclaimed as an audit-only record", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory (no post-state hash), then the child edits + // a file under the destination; the child's inverse payload is then + // reclaimed under its quota before the child is removed. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "v1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "v1", + "child-v2", + "agent" + ); + const childJournal = sharedDurableEventJournal(childSessionDir); + await reclaimExcessRefinementInverseBlobs(childJournal, [ + { ref: `sha256:${"e".repeat(64)}`, size: REFINEMENT_INVERSE_BLOB_QUOTA_BYTES }, + ]); + const childEdit = (await readRefinementEvents(childSessionDir)).find( + (row) => (row.data.action as { op: string }).op === "str_replace" + )!; + const childBlobRef = (childEdit.data.inverse as { files: Array<{ blobRef: string }> }) + .files[0].blobRef; + expect(await childJournal.blobs.has(childBlobRef as never)).toBe(false); + + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const copy = ownerRows.find((row) => row.data.migratedFrom === `ws-child:${childEdit.id}`)!; + // Paths and (dangling) payload reference preserved; nothing published. + expect( + (copy.data.inverse as { files: Array<{ path: string; blobRef: string }> }).files + ).toEqual([ + { path: path.join(ownerSessionDir, "memory", "moved", "a.md"), blobRef: childBlobRef }, + ]); + // Unrollbackable, like any evicted payload... + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(false); + if (!undo.success) expect(undo.error).toContain("no longer available"); + // ...but still evidence: rolling the owner's rename back would move the + // child's newer content, so it is reported as a conflict instead. + const renameRow = ownerRows.find( + (row) => (row.data.action as { op: string }).op === "rename" + )!; + const renameUndo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: renameRow.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(renameUndo.success).toBe(false); + if (!renameUndo.success) expect(renameUndo.error).toContain("diverges"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("child-v2"); + }); + it("concurrent migrations of the same child copy each row exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index a0cd41003be..34ea51766cd 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -721,7 +721,24 @@ export class MemoryService extends EventEmitter { } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; - const owner = resolveWorkspaceMemoryOwnerId(this.config.loadConfigOrDefault(), workspaceId); + // Only a successful load is memoized. A config.json that stats fine but + // cannot be read or parsed right now (EACCES interval, half-written by a + // non-atomic writer) yields the fresh-install default — the self + // fallback — and the stamp will not move when readability returns, so a + // memo taken now would pin the child to its private notebook until an + // unrelated config rewrite. The fallback is still returned (callers + // degrade to the private store), just re-resolved on the next call. + let cfg: ReturnType; + try { + cfg = this.config.loadConfigOrDefault({ throwOnError: true }); + } catch (error) { + log.debug("[MemoryService] config unreadable; workspace memory owner not memoized", { + workspaceId, + error, + }); + return resolveWorkspaceMemoryOwnerId(this.config.loadConfigOrDefault(), workspaceId); + } + const owner = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); this.workspaceMemoryOwnerById.set(workspaceId, owner); return owner; } @@ -973,6 +990,13 @@ export class MemoryService extends EventEmitter { private async openWorkspaceStore(ctx: MemoryScopeContext, store: MemoryStore): Promise { await this.assertWorkspaceStoreReadable(ctx, store); await this.adoptLegacyPrivateStore(ctx, store); + // The adoption pass waits for and holds the owner-store lock, a window in + // which another backend's removal can publish the acting workspace's (or + // the owner's) tombstone. The pass itself refuses on its commit guard + // and swallows that as a retryable adoption failure, so re-check here: + // the caller is about to read the owner's still-live notebook on behalf + // of a workspace that no longer exists. + await this.assertWorkspaceStoreReadable(ctx, store); } /** diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 9eddc5843cc..4ad1796664e 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -49,12 +49,28 @@ export interface RefinementFileCapture { content: string; } +/** + * A payload reference carried over as-is instead of content: shared-memory row + * migration uses it for a row whose inverse blob was already reclaimed in the + * source journal. The copied row keeps its paths and order for conflict + * detection (an audit record), and the rollback engine refuses it exactly as + * it refuses any evicted payload — the ref resolves to nothing. + */ +export interface RefinementFileReference { + path: string; + blobRef: string; +} + /** Inverse draft with captured contents inline; blob offload happens at append. */ export type RefinementInverseDraft = | { op: "delete-files"; paths: string[] } // deletePaths (r67): mixed force-apply pre-state — restore `files` AND // delete the paths the forced rollback created (see RefinementInverseSchema). - | { op: "restore-files"; files: RefinementFileCapture[]; deletePaths?: string[] } + | { + op: "restore-files"; + files: Array; + deletePaths?: string[]; + } | { op: "rename"; from: string; to: string }; export interface RefinementEmitArgs { @@ -137,6 +153,9 @@ export async function resolveRefinementInverse( const publishedBlobs: BlobQuotaEntry[] = []; const files = await Promise.all( draft.files.map(async (file) => { + // A bare reference (payload already gone at the source) is neither + // stored nor quota-charged: there is nothing to retain or reclaim. + if (!("content" in file)) return { path: file.path, blobRef: file.blobRef }; const { ref, size } = await blobs.put(file.content); publishedBlobs.push({ ref, size: inverseQuotaCharge(size) }); return { path: file.path, blobRef: ref }; diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 00a56b1e583..81d1b8ac28f 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -1234,6 +1234,8 @@ async function compensatePartialApply( ? preState.files.find((file) => file.path === p) : undefined; if (prior !== undefined) { + // Captured from disk moments ago; only migrated rows carry bare references. + assert("content" in prior, "pre-rollback capture carries file contents"); await fsPromises.mkdir(path.dirname(p), { recursive: true }); await writeFileAtomic(p, prior.content, { encoding: "utf-8" }); } else { diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index d910a90df61..bc505ed58dd 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -16,6 +16,8 @@ import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJour import { appendRefinementEventUnderBlobLock, reclaimRefinementInverseBlobsBestEffort, + type RefinementFileCapture, + type RefinementFileReference, type RefinementInverseDraft, } from "./refinementJournal"; import { listRefinements } from "./refinementRollback"; @@ -54,8 +56,10 @@ function isInside(root: string, filePath: string): boolean { * rollbackable. A row rolled back before its FIRST copy is simply dead and * stays behind with its whole lineage. * - * A row whose payload cannot be reconstructed (evicted blob, unparseable - * action) is skipped with a log line — nothing durable exists to preserve. + * A row whose inverse payload was reclaimed is copied as an audit-only record + * (RefinementFileReference) so conflict detection keeps seeing the edit; a + * row that cannot be parsed at all is skipped with a log line — nothing + * durable exists to preserve. * A row that CAN be reconstructed but cannot be persisted in the owner's * journal throws: the caller must not delete the source journal, or the * only inverse and rollback ID would be lost. Returns the number migrated. @@ -162,15 +166,28 @@ export async function migrateSharedMemoryRefinementRows(args: { let draft: RefinementInverseDraft; if (inverse.data.op === "restore-files") { - const files: Array<{ path: string; content: string }> = []; + const files: Array = []; for (const file of inverse.data.files) { // Contents are blob-offloaded at append (resolveRefinementInverse); older // rows may carry them inline. const content = file.text ?? (file.blobRef === undefined ? null : await childJournal.blobs.getText(file.blobRef)); - if (content === null) break; - files.push({ path: file.path, content }); + if (content !== null) { + files.push({ path: file.path, content }); + } else if (file.blobRef !== undefined) { + // Payload reclaimed under the child's inverse-blob quota. The row + // still travels as an audit record — its paths and source order + // are what conflict detection needs when the owner later rolls + // back an older edit over the same files (a directory rename has + // no post-state hash to notice the child's newer content by) — + // but it can no longer be rolled back: the reference resolves to + // nothing in the owner journal, which the rollback engine + // refuses exactly like an evicted payload of its own. + files.push({ path: file.path, blobRef: file.blobRef }); + } else { + break; // neither text nor blobRef: nothing durable to preserve + } } if (files.length !== inverse.data.files.length) { log.debug("[refinement] skipping shared-memory row migration: inverse payload missing", { diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index d298e6a69bb..621a6de7b31 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2834,6 +2834,13 @@ export class TurnRequestBuilder { } const assistantMessage = createMuxMessage(assistantMessageId, "assistant", "", { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), + // Proof for the harvest gate that this turn's policy was recorded + // (above, or the grant at onStreamStarted): a build without the sink + // — or an older build after a downgrade — leaves it unset and its + // turns' user rows stay unaccounted for. + ...(!isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink + ? { workspaceMemoryPolicyRecorded: true as const } + : {}), timestamp: Date.now(), model: canonicalModelString, routedThroughGateway, From 0cab99a8cdf0a431ff2848fce8a0645eb710bd1a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 08:51:39 +0000 Subject: [PATCH 46/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-fir?= =?UTF-8?q?st=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Epoch-bind the workspace-memory policy accumulator: config gains workspaceMemoryWritableEpoch and the deny marker an epoch (the opening boundary's history sequence, -1 before any), so readers ignore values of other epochs; a backend starting the new epoch cannot inherit the closing epoch's value before the compacting backend's reset lands. Resets are fenced by epoch; preserved-tail compactions re-bind the carried value/marker to the new epoch. - Completion-side policy observation loads config strictly and skips the harvest when unreadable. - Owner memo invalidation loads strictly and retains memoized owners (keeping the old stamp) when config.json cannot be read. - Legacy notebook adoption: stamp covers nested file size/mtime and the child-keyed sidecar fingerprint; manifest is a Map (__proto__-safe); removal throws when a note could not be represented and re-runs the adoption inside the removal locks before the tombstone. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/schemas/project.ts | 4 + .../utils/messages/compactionBoundary.ts | 20 +++ src/node/services/agentSession.ts | 106 +++++++++---- src/node/services/compactionHandler.ts | 16 +- src/node/services/memoryService.test.ts | 118 ++++++++++++++ src/node/services/memoryService.ts | 147 ++++++++++++++---- src/node/services/turnRequestBuilder.ts | 16 +- .../services/workspaceMemoryDenyMarker.ts | 102 ++++++++---- src/node/services/workspaceService.test.ts | 95 +++++++++-- src/node/services/workspaceService.ts | 79 ++++++++-- 10 files changed, 559 insertions(+), 144 deletions(-) diff --git a/src/common/schemas/project.ts b/src/common/schemas/project.ts index 0e40e3c2d1c..a4f28bfab87 100644 --- a/src/common/schemas/project.ts +++ b/src/common/schemas/project.ts @@ -109,6 +109,10 @@ export const WorkspaceConfigSchema = z.object({ description: "Whether this workspace's agent may write /memories/workspace, as resolved on its last normal turn. Persisted so a post-compaction memory harvest that resumes in a fresh session (restart, recovery) still knows the policy; harvest fails closed when unknown.", }), + workspaceMemoryWritableEpoch: z.number().optional().meta({ + description: + "Compaction epoch `workspaceMemoryWritable` accumulates over: the history sequence of the durable context boundary that opened it (-1 before any boundary). Readers ignore the value under any other epoch, so a backend starting the new epoch cannot inherit the closing epoch's value before the boundary reset lands.", + }), memoryOwnerWorkspaceId: z.string().optional().meta({ description: "Memory owner pinned when an intermediate ancestor was removed while this descendant stayed alive: the parentWorkspaceId chain no longer reaches the task-tree root, so this keeps /memories/workspace bound to the root's store (memoryWorkspaceOwner.ts). Set only by workspace removal.", diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 69aa23e0d3a..1d391bbad9a 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -71,6 +71,26 @@ export function isDurableContextBoundaryMarker(message: MuxMessage | undefined): return getContextBoundaryKind(message) !== null; } +/** + * History sequence of the latest durable context boundary among `messages` + * (any kind), or undefined when there is none. Identifies the compaction + * epoch the rows after it belong to: compaction completion metadata carries it + * as `previousBoundaryHistorySequence`, and the workspace-memory policy + * accumulator is bound to it (WorkspaceService.recordWorkspaceMemoryWritable). + */ +export function latestContextBoundaryHistorySequence( + messages: readonly MuxMessage[] +): number | undefined { + let latest: number | undefined; + for (const message of messages) { + if (!isDurableContextBoundaryMarker(message)) continue; + const sequence = message.metadata?.historySequence; + if (typeof sequence !== "number" || !Number.isInteger(sequence) || sequence < 0) continue; + if (latest === undefined || sequence > latest) latest = sequence; + } + return latest; +} + /** * Locate the latest durable context boundary in reverse chronological order. * diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 8bae162a270..d350009f90f 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -92,7 +92,10 @@ import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; -import { clearWorkspaceMemoryDenyMarker } from "@/node/services/workspaceMemoryDenyMarker"; +import { + carryWorkspaceMemoryDenyMarker, + clearWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; import { buildStreamErrorEventData, createStreamErrorMessage, @@ -1098,17 +1101,14 @@ export class AgentSession { * grant is never left behind by this path — grants are re-recorded per * turn). Nothing to do when the field is already absent. */ - private async resetWorkspaceMemoryWritable(options?: { - closing: boolean | undefined; - }): Promise { - const boundaryAt = Date.now(); + private async resetWorkspaceMemoryWritable(options?: { closingEpoch: number }): Promise { this.workspaceMemoryWritable = undefined; // The session-dir deny marker (fallback for an unwritable config.json) // belongs to the closing epoch too. Same fence idea as below: a deny - // recorded after this boundary is the new epoch's and survives. + // recorded for the new epoch survives. await clearWorkspaceMemoryDenyMarker( path.join(this.config.sessionsDir, this.workspaceId), - options !== undefined ? { notAfter: boundaryAt } : undefined + options ); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); if (entry?.workspace.workspaceMemoryWritable === undefined) return; @@ -1117,13 +1117,45 @@ export class AgentSession { if (current === null) return cfg; // Fenced to the epoch being closed: another backend may already have // recorded the first turn of the NEW epoch between the completion and - // this locked write; a value different from the one this session - // observed at the boundary is that newer epoch's and must survive - // (a same-valued newer deny survives through that backend's mirror). - if (options !== undefined && current.workspace.workspaceMemoryWritable !== options.closing) { + // this locked write; a value bound to a different epoch is that newer + // epoch's and must survive. (Readers ignore a stale epoch's value + // anyway — WorkspaceService.recordWorkspaceMemoryWritable — so this + // delete is hygiene, not the correctness boundary.) + if ( + options !== undefined && + current.workspace.workspaceMemoryWritableEpoch !== options.closingEpoch + ) { return cfg; } delete current.workspace.workspaceMemoryWritable; + delete current.workspace.workspaceMemoryWritableEpoch; + return cfg; + }); + } + + /** + * Preserved-tail compaction: the tail copies were produced under the + * closing epoch's policy, so its accumulator carries into the new epoch — + * durably, by re-binding the config value and the deny marker recorded for + * `closingEpoch` to `nextEpoch` (another backend's first turn of the new + * epoch reads by epoch and would otherwise see nothing). Fenced like the + * reset: a value already bound to another epoch is left alone. + */ + private async carryWorkspaceMemoryWritable( + closingEpoch: number, + nextEpoch: number + ): Promise { + await carryWorkspaceMemoryDenyMarker( + path.join(this.config.sessionsDir, this.workspaceId), + closingEpoch, + nextEpoch + ); + const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); + if (entry?.workspace.workspaceMemoryWritableEpoch !== closingEpoch) return; + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, this.workspaceId); + if (current?.workspace.workspaceMemoryWritableEpoch !== closingEpoch) return cfg; + current.workspace.workspaceMemoryWritableEpoch = nextEpoch; return cfg; }); } @@ -1299,30 +1331,36 @@ export class AgentSession { ); // New epoch. A preserved tail copies messages produced under this // epoch's policy into the next one, so the fail-closed accumulator - // carries over with them; otherwise the next normal turn restarts it. - // The mirror forgets the closing epoch right here, synchronously; the - // durable reset runs AFTER the completion observation settled (it - // reads the closing epoch's marker/config) and is awaited by the next - // turn's policy record (settleWorkspaceMemoryPolicyEpoch), so no turn - // of the new epoch can AND itself with the closing epoch's stale deny. - if ((metadata.preservedTailMessageCount ?? 0) === 0) { - this.workspaceMemoryWritable = undefined; - const reset = observed - .catch(() => undefined) - .then(() => this.resetWorkspaceMemoryWritable({ closing })) - .catch((error: unknown) => { - log.warn("Failed to reset the workspace memory policy epoch", { - workspaceId: this.workspaceId, - error, - }); - }) - .finally(() => { - if (this.workspaceMemoryEpochReset === reset) { - this.workspaceMemoryEpochReset = undefined; - } + // carries over with them (re-bound to the new epoch key, durably); + // otherwise the next normal turn restarts it. The mirror forgets the + // closing epoch right here, synchronously; the durable reset runs + // AFTER the completion observation settled (it reads the closing + // epoch's marker/config) and is awaited by this session's next turn + // (settleWorkspaceMemoryPolicyEpoch). Other backends need no such + // wait: every durable value is bound to its epoch, so the closing + // epoch's value is invisible to their new-epoch turns regardless. + const closingEpoch = metadata.previousBoundaryHistorySequence ?? -1; + const preservedTail = (metadata.preservedTailMessageCount ?? 0) > 0; + if (!preservedTail) this.workspaceMemoryWritable = undefined; + const reset = observed + .catch(() => undefined) + .then(() => + preservedTail + ? this.carryWorkspaceMemoryWritable(closingEpoch, metadata.summaryHistorySequence) + : this.resetWorkspaceMemoryWritable({ closingEpoch }) + ) + .catch((error: unknown) => { + log.warn("Failed to reset the workspace memory policy epoch", { + workspaceId: this.workspaceId, + error, }); - this.workspaceMemoryEpochReset = reset; - } + }) + .finally(() => { + if (this.workspaceMemoryEpochReset === reset) { + this.workspaceMemoryEpochReset = undefined; + } + }); + this.workspaceMemoryEpochReset = reset; }, onIdleCompactionOutcome, }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 012db8bf17e..8a0c7c70f86 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -39,6 +39,7 @@ import { import { isDurableCompactedMarker, isDurableContextBoundaryMarker, + latestContextBoundaryHistorySequence, sliceMessagesFromLatestCompactionBoundary, } from "@/common/utils/messages/compactionBoundary"; import { extractReadFilePaths, mergeReadFilePaths } from "@/common/utils/messages/extractReadFiles"; @@ -302,17 +303,6 @@ function isCompactedSummaryMessage(message: MuxMessage): boolean { return isDurableCompactedMarker(message.metadata?.compacted); } -function getLatestBoundaryHistorySequence(messages: readonly MuxMessage[]): number | undefined { - let latest: number | undefined; - for (const message of messages) { - if (!isDurableContextBoundaryMarker(message)) continue; - const sequence = message.metadata?.historySequence; - if (!isNonNegativeInteger(sequence)) continue; - if (latest === undefined || sequence > latest) latest = sequence; - } - return latest; -} - function getNextCompactionEpoch(messages: MuxMessage[]): number { let epochCursor = 0; @@ -1343,7 +1333,7 @@ export class CompactionHandler { summaryMessageId: boundary.id, summaryHistorySequence: sequence, compactionEpoch: epoch, - previousBoundaryHistorySequence: getLatestBoundaryHistorySequence(params.messages), + previousBoundaryHistorySequence: latestContextBoundaryHistorySequence(params.messages), compactionRequestMessageId: boundary.id, preservedTailMessageCount: copies.length, }); @@ -1406,7 +1396,7 @@ export class CompactionHandler { const nextCompactionEpoch = getNextCompactionEpoch(messages); assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); - const previousBoundaryHistorySequence = getLatestBoundaryHistorySequence(messages); + const previousBoundaryHistorySequence = latestContextBoundaryHistorySequence(messages); const maxExistingHistorySequence = this.getMaxExistingHistorySequence(messages); // For idle compaction, preserve the original recency timestamp so the workspace diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 92b025c61c1..5d2b9f39cbd 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -34,6 +34,7 @@ import { sharedWorkspaceMemoryPeerSessionDirs } from "./memoryWorkspaceOwner"; import { createRefinementRollbackTool } from "./tools/refinement_rollback"; import type { MemoryScopeAccess } from "@/common/constants/memory"; import { workspaceRemovalTombstonePath } from "./workspaceRemoval"; +import { memoryMutationLockKey, withTargetMutationLock } from "./refinement/targetMutationLocks"; import { TestTempDir, mockToolCallOptions } from "./tools/testHelpers"; function pathExists(target: string): Promise { @@ -1532,6 +1533,123 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); }); + it("re-adopts when a downgraded build edits a nested legacy note in place or only its pin", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v1"); + // In-place edit of an existing nested file on the old build: neither the + // legacy root's mtime nor the (unknown to it) store clock moves. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v2 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"), + "utf-8" + ) + ).toBe("v2 (downgrade)"); + // Sidecar-only change (a pin toggled on the old build under the child + // key): no file stat changes at all, yet the owner key must follow. + const childKey = memoryLogicalKey("workspace", "sub/note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The pin lands on the owner key of the note's current copy (the + // imported one, since the owner path holds the older bytes). + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "imported/ws-child/sub/note.md", { + projectPath: "", + workspaceId: "ws-owner", + }) + ) + ).toBe(true); + }); + + it("keeps adopting a legacy note named __proto__ exactly once", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "__proto__"), "proto notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "__proto__"), "utf-8")).toBe( + "proto notes" + ); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["__proto__"]); + // A fresh process (empty memo) finds the record and leaves the clock alone. + const revision = await fixture.service.workspaceMemoryRevision("ws-owner"); + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await restarted.listIndexEntries({ ...fixture.ctx }); + expect(await restarted.workspaceMemoryRevision("ws-owner")).toBe(revision); + }); + + it("removal adoption refuses to leave a note behind and runs under held locks", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner notebook at the cap: the child's note has no slot. + await Promise.all( + Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ) + ); + await fsPromises.writeFile(path.join(legacyRoot, "stranded.md"), "only copy"); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/could not be folded/); + // Space frees up; the in-lock delta pass (removal holds the owner-store + // lock already) folds the note in without re-acquiring the lock. + await fsPromises.rm(path.join(ownerRoot, "o0000.md")); + await withTargetMutationLock( + fixture.xumHome, + memoryMutationLockKey(fixture.xumHome, ownerRoot), + () => + fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner", { + locksHeld: true, + }) + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "stranded.md"), "utf-8")).toBe( + "only copy" + ); + }); + + it("keeps memoized owners when a changed config.json cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // The stamp moves (a rewrite) but the contents are unreadable for a + // moment: the memo must not be replaced by the empty default's self + // fallbacks, and the pass must be retried once readable. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + spyOn(fixture.config, "configFileStamp").mockReturnValue("rewritten"); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + }); + it("folds in a note written under a self-fallback once ownership resolves to the tree root again", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 34ea51766cd..c1096487558 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -419,28 +419,39 @@ function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { ); } -/** Self-healing read of the adoption manifest: anything malformed reads as not adopted. */ +/** + * Self-healing read of the adoption manifest: anything malformed reads as not + * adopted. A Map, not a plain object: a legacy note may legitimately be named + * `__proto__` (any store-valid relPath), and assigning that key on an + * ordinary object hits the prototype setter instead of creating an entry the + * serialization would carry — the note would then be re-adopted (and the + * owner clock advanced) on every access. JSON.parse and Object.fromEntries + * create own properties, so the round-trip below is exact. + */ async function readLegacyAdoptionManifest( manifestPath: string -): Promise> { +): Promise> { try { const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return {}; - return Object.fromEntries( + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); + return new Map( Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => isLegacyAdoptionRecord(entry[1]) ) ); } catch { - return {}; + return new Map(); } } /** * Change stamp of a sub-agent's legacy private store: its store clock (any * MemoryService write there advances it, including a foreign backend's - * self-fallback write) plus the root directory's mtime (top-level entry - * changes made outside MemoryService). Missing pieces read as fixed tokens. + * self-fallback write), the root directory's mtime, and every listed file's + * size + mtime — a DOWNGRADED build editing an existing nested note moves + * neither the clock (it does not know it) nor the root mtime. Bounded by the + * per-scope file cap and paid only while a legacy directory exists. Missing + * pieces read as fixed tokens. */ async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Promise { const revision = await readWorkspaceMemoryRevision(childSessionDir).catch(() => null); @@ -448,7 +459,17 @@ async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Pr .stat(legacyRoot) .then((stat) => String(stat.mtimeMs)) .catch(() => "missing"); - return `${revision ?? "none"}:${rootMtime}`; + const files = await new LocalMemoryStore(legacyRoot).listFiles().catch(() => []); + const fileStamps = await Promise.all( + files.map(async (relPath) => { + const stamp = await fsPromises + .lstat(path.join(legacyRoot, relPath), { bigint: true }) + .then((stat) => `${stat.size}:${stat.mtimeNs}`) + .catch(() => "missing"); + return `${relPath}=${stamp}`; + }) + ); + return `${revision ?? "none"}:${rootMtime}:${fileStamps.join("\u0001")}`; } /** Link-aware kind of a path: symlinks are reported as such, never followed. */ @@ -716,8 +737,10 @@ export class MemoryService extends EventEmitter { if (snapshot !== undefined) return resolveWorkspaceMemoryOwnerId(snapshot(), workspaceId); const stamp = this.config.configFileStamp(); if (stamp !== this.workspaceMemoryOwnerConfigStamp) { - this.workspaceMemoryOwnerConfigStamp = stamp; - this.invalidateWorkspaceMemoryOwnerMemo(); + // Adopt the new stamp only once its contents were actually read: a + // changed-but-unreadable file (see below) must be retried on the next + // call, not remembered as "seen". + if (this.invalidateWorkspaceMemoryOwnerMemo()) this.workspaceMemoryOwnerConfigStamp = stamp; } const cached = this.workspaceMemoryOwnerById.get(workspaceId); if (cached !== undefined) return cached; @@ -755,12 +778,26 @@ export class MemoryService extends EventEmitter { * Most config edits (titles, models, task status) leave the topology alone; * emitting for those would make every live child rebuild its index and hot * set from disk on ordinary churn, so only real owner changes are reported. - * One parse, only when something was memoized. + * One parse, only when something was memoized. Returns false when the + * config could not be read (readable stat, unreadable/unparseable content): + * the memoized mappings are RETAINED rather than replaced by the empty + * default's self fallbacks — those would be pinned until the stamp moved, + * which a restored permission bit never does — and the caller keeps the old + * stamp so the pass is retried on the next resolution. */ - private invalidateWorkspaceMemoryOwnerMemo(): void { - if (this.workspaceMemoryOwnerById.size === 0) return; + private invalidateWorkspaceMemoryOwnerMemo(): boolean { + if (this.workspaceMemoryOwnerById.size === 0) return true; // One parse and one ID index for the whole pass (O(n), not O(n²)). - const resolve = workspaceMemoryOwnerResolver(this.config.loadConfigOrDefault()); + let cfg: ReturnType; + try { + cfg = this.config.loadConfigOrDefault({ throwOnError: true }); + } catch (error) { + log.debug("[MemoryService] config unreadable; keeping memoized workspace memory owners", { + error, + }); + return false; + } + const resolve = workspaceMemoryOwnerResolver(cfg); const changed: string[] = []; for (const [workspaceId, previousOwner] of this.workspaceMemoryOwnerById) { const owner = resolve(workspaceId); @@ -768,6 +805,7 @@ export class MemoryService extends EventEmitter { if (owner !== previousOwner) changed.push(workspaceId); } if (changed.length > 0) this.emit("ownersInvalidated", changed); + return true; } /** @@ -1066,15 +1104,20 @@ export class MemoryService extends EventEmitter { * the deletion would discard its notes for good. Runs BEFORE any teardown * step (removal reuses the owner it verified with a strict config load), so * a failure aborts the removal with the workspace intact: this variant - * THROWS instead of deferring to a next access that will never come. Not - * run again under the removal locks — the owner-store lock this takes is - * the one removal's critical section holds, and the legacy directory is - * written only by downgraded builds or a self-fallback backend, neither of - * which the in-lock delta pass could fence anyway. + * THROWS instead of deferring to a next access that will never come — also + * when a listed note could not be represented in the owner store (shared + * notebook at its file cap, both destinations taken by different content, + * unreadable as text): the pass would count it as skipped and the deletion + * would take the only copy. Removal runs it twice: pre-teardown, and again + * inside the removal locks (`locksHeld`, the owner-store lock among them) + * right before the tombstone, catching a note a self-fallback backend + * committed into the legacy directory in between — the child's own store + * lock is held there too, so nothing can land after that pass. */ async adoptLegacyPrivateStoreForRemoval( childWorkspaceId: string, - ownerWorkspaceId: string + ownerWorkspaceId: string, + options?: { locksHeld: boolean } ): Promise { assert(childWorkspaceId.length > 0, "adoptLegacyPrivateStoreForRemoval requires a child id"); assert( @@ -1099,20 +1142,31 @@ export class MemoryService extends EventEmitter { `shared memory owner of ${childWorkspaceId} resolved to ${String(resolvedOwner)} while removal verified ${ownerWorkspaceId}` ); } - await this.adoptLegacyPrivateStoreOrThrow(ctx, store, ownerWorkspaceId, { force: true }); + const { skipped } = await this.adoptLegacyPrivateStoreOrThrow(ctx, store, ownerWorkspaceId, { + force: true, + locksHeld: options?.locksHeld === true, + }); + if (skipped > 0) { + throw new Error( + `${skipped} legacy workspace memory note(s) of ${childWorkspaceId} could not be folded into ${ownerWorkspaceId}'s shared notebook (full, conflicting, or not text); removing the session directory would discard them` + ); + } } /** * The adoption pass (see adoptLegacyPrivateStore). `force` skips the * per-process "already checked" memo: removal wants the pass to run against * the current legacy directory regardless of what an earlier access saw. + * `locksHeld`: the caller already holds the owner store's mutation lock + * (removal's in-lock delta pass), so it is not re-acquired. Returns how many + * listed legacy notes could NOT be represented in the owner store this pass. */ private async adoptLegacyPrivateStoreOrThrow( ctx: MemoryScopeContext, store: MemoryStore, owner: string, - options?: { force: boolean } - ): Promise { + options?: { force: boolean; locksHeld?: boolean } + ): Promise<{ skipped: number }> { const childId = ctx.workspaceId; // Checked once per (child, owner, legacy-store state) per process. The // owner is part of the key because ownership can move: a command served @@ -1121,15 +1175,32 @@ export class MemoryService extends EventEmitter { // it because ANOTHER backend can do the same while this process's // resolution never changes: its self-fallback write advances the child's // store clock (memory.revision in the child's session dir) and replaces a - // root entry, so either signal re-runs the pass. Two small stats per - // workspace-scope access; the pass itself is idempotent. + // root entry, so either signal re-runs the pass — as does any listed + // file's size/mtime. The child-keyed sidecar entries are the fourth + // input: a downgraded build can change only a pin or usage counter, which + // the manifest reconciles (sidecar fingerprint) but no file stat shows. + // The pass itself is idempotent. const childSessionDir = path.join(this.config.sessionsDir, childId); const legacyRoot = path.join(childSessionDir, "memory"); const legacyRootKind = await lstatKind(legacyRoot); + const childKeyPrefix = memoryLogicalKey("workspace", "", { + projectPath: ctx.projectPath, + workspaceId: childId, + }); + const childSidecarFingerprint = + legacyRootKind === "dir" + ? JSON.stringify( + [...(await this.metaService.getEntries())] + .filter(([key]) => key.startsWith(childKeyPrefix)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ) + : ""; const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" - }`; - if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) return; + }\u0000${childSidecarFingerprint}`; + if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) { + return { skipped: 0 }; + } if (legacyRootKind !== "dir") { if (legacyRootKind === "symlink") { log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { @@ -1138,12 +1209,13 @@ export class MemoryService extends EventEmitter { }); } this.legacyStoreCheckedAgainst.set(childId, checkKey); - return; + return { skipped: 0 }; } // Files adopted this pass (bytes written OR only their sidecar entries // folded in): either changes what the shared store's readers derive from it. let adoptedCount = 0; - await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), async () => { + let skipped = 0; + const pass = async (): Promise => { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock const legacy = new LocalMemoryStore(legacyRoot); @@ -1167,7 +1239,6 @@ export class MemoryService extends EventEmitter { let capacityExhausted = false; let manifestDirty = false; let imported = 0; - let skipped = 0; for (const relPath of files) { // Same read gates as a memory command: containment (no symlink // escape), size cap, and text-only (a lossy utf-8 decode cannot be @@ -1190,7 +1261,7 @@ export class MemoryService extends EventEmitter { sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), target: "", }; - const previous = adopted[relPath]; + const previous = adopted.get(relPath); if (previous?.content === record.content && previous.sidecar === record.sidecar) { continue; // folded in earlier, nothing changed since } @@ -1272,12 +1343,14 @@ export class MemoryService extends EventEmitter { continue; } } - adopted[relPath] = record; + adopted.set(relPath, record); manifestDirty = true; adoptedCount++; } if (manifestDirty) { - await writeFileAtomic(manifestPath, JSON.stringify(adopted), { encoding: "utf-8" }); + await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { + encoding: "utf-8", + }); } if (capacityExhausted) { log.warn( @@ -1295,11 +1368,17 @@ export class MemoryService extends EventEmitter { { childId, owner, imported, skipped } ); } - }); + }; + if (options?.locksHeld === true) { + await pass(); + } else { + await withTargetMutationLock(this.config.rootDir, this.storeLockKey(store), pass); + } // Recorded against the state observed BEFORE the pass: a foreign write // landing during it changes the stamp and re-runs the (idempotent) pass. this.legacyStoreCheckedAgainst.set(childId, checkKey); if (adoptedCount > 0) this.emitChange(ctx, "workspace", "", "agent"); + return { skipped }; } /** diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 621a6de7b31..4b006d216f7 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -32,6 +32,7 @@ import type { SendMessageError } from "@/common/types/errors"; import type { GoalRecordV1 } from "@/common/types/goal"; import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/types/message"; import { createMuxMessage } from "@/common/types/message"; +import { latestContextBoundaryHistorySequence } from "@/common/utils/messages/compactionBoundary"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; import { secretsToRecord } from "@/common/types/secrets"; import type { XumToolScope } from "@/common/types/toolScope"; @@ -557,7 +558,7 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean } + options: { epochHasPriorTurns: boolean; policyEpoch: number } ): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; @@ -1484,10 +1485,21 @@ export class TurnRequestBuilder { message.metadata?.muxMetadata?.type !== "compaction-request" ); })(); + // The compaction epoch this turn's policy accumulates over: the latest + // durable boundary's history sequence (any kind), -1 before any boundary + // — the same identity compaction completion reports as + // previousBoundaryHistorySequence, so the completion-side observation + // and every backend's turn records agree on which epoch a value belongs to. + const policyEpoch = latestContextBoundaryHistorySequence(messages) ?? -1; const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; - if (await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { epochHasPriorTurns })) { + if ( + await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { + epochHasPriorTurns, + policyEpoch, + }) + ) { return true; } if (!writable) return false; diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 55dfa44cee7..b92c9a5c18d 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -13,8 +13,14 @@ * it is consulted; it is cleared only at the epoch boundary that clears the * config bit. * - * Fail-closed by construction: a missing marker is "no deny"; a present, - * malformed, or unreadable marker is a deny. + * Like the config bit, the marker is bound to its compaction epoch (the + * opening boundary's history sequence, -1 before any boundary): a reader + * consulting it for another epoch ignores it, so a backend starting the new + * epoch before this one's boundary reset landed cannot inherit a stale deny. + * + * Fail-closed by construction: a missing marker (or one recorded for a + * different epoch) is "no deny"; a present marker for this epoch, or a + * malformed/unreadable one, is a deny. */ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -30,59 +36,93 @@ export function workspaceMemoryDenyMarkerPath(sessionDir: string): string { } /** Durable-or-throw: verified by reading the marker back. */ -export async function writeWorkspaceMemoryDenyMarker(sessionDir: string): Promise { +export async function writeWorkspaceMemoryDenyMarker( + sessionDir: string, + epoch: number +): Promise { + assert(Number.isInteger(epoch), "workspace memory deny marker epoch must be an integer"); const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); await fsPromises.mkdir(sessionDir, { recursive: true }); - await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now() }), { + await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now(), epoch }), { encoding: "utf-8", }); - if (!(await readWorkspaceMemoryDenyMarker(sessionDir))) { + if (!(await readWorkspaceMemoryDenyMarker(sessionDir, epoch))) { throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); } } -/** True when a deny is recorded (or the marker cannot be trusted); false only when absent. */ -export async function readWorkspaceMemoryDenyMarker(sessionDir: string): Promise { +/** Parsed marker, or null when it is unreadable/malformed (which readers treat as a deny). */ +async function readMarkerRecord( + markerPath: string +): Promise<{ deniedAt: number | null; epoch: number | null } | "absent" | null> { try { - await fsPromises.access(workspaceMemoryDenyMarkerPath(sessionDir)); - return true; + const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); + if (typeof parsed !== "object" || parsed === null) return null; + const { deniedAt, epoch } = parsed as { deniedAt?: unknown; epoch?: unknown }; + return { + deniedAt: typeof deniedAt === "number" && Number.isFinite(deniedAt) ? deniedAt : null, + epoch: typeof epoch === "number" && Number.isInteger(epoch) ? epoch : null, + }; } catch (error) { - return !hasErrorCode(error, "ENOENT"); + return hasErrorCode(error, "ENOENT") ? "absent" : null; } } +/** + * True when a deny is recorded for `epoch` (or the marker cannot be trusted: + * unreadable, malformed, or written by a build that did not record an epoch); + * false when absent or recorded for a different epoch. Without `epoch`, any + * present marker is a deny (epoch-agnostic existence check). + */ +export async function readWorkspaceMemoryDenyMarker( + sessionDir: string, + epoch?: number +): Promise { + const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); + if (record === "absent") return false; + if (record === null || epoch === undefined) return true; + return record.epoch === null || record.epoch === epoch; +} + /** * Epoch boundary: remove the marker, durable-or-throw (verified absent). - * `notAfter` fences the clear to denies recorded up to the boundary: a deny - * another backend recorded for the NEW epoch in the meantime must survive. - * Only a well-formed marker can claim to be newer; a truncated or malformed - * one is stale state from some earlier epoch (every reader already treated - * it as a deny for as long as it existed) and is healed here — otherwise one - * corrupt file would force every later epoch's accumulator to false until a - * destructive history clear. + * `closingEpoch` fences the clear to that epoch's deny: a deny another + * backend recorded for the NEW epoch in the meantime (different epoch) must + * survive. Only a well-formed marker can claim to be another epoch's; a + * truncated or malformed one is stale state from some earlier epoch (every + * reader already treated it as a deny for as long as it existed) and is + * healed here — otherwise one corrupt file would force every later epoch's + * accumulator to false until a destructive history clear. */ export async function clearWorkspaceMemoryDenyMarker( sessionDir: string, - options?: { notAfter: number } + options?: { closingEpoch: number } ): Promise { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); if (options !== undefined) { - let deniedAt: number | null; - try { - const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); - const candidate = - typeof parsed === "object" && parsed !== null - ? (parsed as { deniedAt?: unknown }).deniedAt - : undefined; - deniedAt = typeof candidate === "number" && Number.isFinite(candidate) ? candidate : null; - } catch (error) { - if (hasErrorCode(error, "ENOENT")) return; - deniedAt = null; // unreadable/malformed: cannot be a newer deny - } - if (deniedAt !== null && deniedAt > options.notAfter) return; + const record = await readMarkerRecord(markerPath); + if (record === "absent") return; + if (record !== null && record.epoch !== null && record.epoch !== options.closingEpoch) return; } await fsPromises.rm(markerPath, { force: true }); if (await readWorkspaceMemoryDenyMarker(sessionDir)) { throw new Error(`Workspace memory deny marker could not be removed at ${markerPath}`); } } + +/** + * A preserved-tail compaction carries the closing epoch's policy into the new + * one (the tail copies were produced under it): a deny marker recorded for + * `closingEpoch` is re-stamped with `nextEpoch` so readers of the new epoch + * keep seeing it. Markers of other epochs and absent markers are left alone; + * a malformed one stays a deny for every reader regardless. + */ +export async function carryWorkspaceMemoryDenyMarker( + sessionDir: string, + closingEpoch: number, + nextEpoch: number +): Promise { + const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); + if (record === "absent" || record?.epoch !== closingEpoch) return; + await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); +} diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7188c379b01..245fdba4888 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9316,16 +9316,26 @@ describe("WorkspaceService initialize", () => { const persisted = () => findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace .workspaceMemoryWritable; + const persistedEpoch = () => + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace + .workspaceMemoryWritableEpoch; + const EPOCH0 = { epochHasPriorTurns: false, policyEpoch: -1 }; try { // The durable bit is the epoch accumulator: the harvest reads every // message of the epoch, so a read-only turn denies the epoch even when // a writable turn follows — across restarts and backends, since the // conjunction lives in config.json rather than in one process. - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(true); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(false); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(false); // New epoch (field cleared at the boundary), then ANOTHER backend records @@ -9336,13 +9346,17 @@ describe("WorkspaceService initialize", () => { delete entry.workspace.workspaceMemoryWritable; return cfg; }); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(true); await realConfig.editConfig((cfg) => { findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable = false; return cfg; }); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(false); // New epoch again, and now config.json cannot take the deny: the turn's @@ -9360,17 +9374,21 @@ describe("WorkspaceService initialize", () => { spyOn(realConfig, "editConfig").mockImplementationOnce(() => Promise.reject(new Error("disk full")) ); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); expect(persisted()).toBeUndefined(); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(false); // Malformed marker still denies; an epoch boundary — even the fenced // compaction one, since a malformed file cannot claim to be a newer // deny — heals it and the next epoch can become writable again. await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() - 60_000 }); + await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); await clearWorkspaceMemoryDenyMarker(sessionDir); @@ -9379,15 +9397,50 @@ describe("WorkspaceService initialize", () => { delete entry.workspace.workspaceMemoryWritable; return cfg; }); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); expect(persisted()).toBe(true); - // The fenced clear (compaction boundary) keeps a deny recorded after it. - await writeWorkspaceMemoryDenyMarker(sessionDir); - await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() - 60_000 }); + // The fenced clear (compaction boundary) keeps a deny recorded for the + // NEW epoch; readers of any other epoch ignore that deny. + await writeWorkspaceMemoryDenyMarker(sessionDir, 7); + await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: -1 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(sessionDir, { notAfter: Date.now() + 60_000 }); + await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: 7 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + // The durable bit is bound to its epoch too: the closing epoch's + // `false` is invisible to the first turn of the next epoch on ANOTHER + // backend (which cannot await this backend's boundary reset), so it + // grants and re-binds the value to its own epoch. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritable = false; + entry.workspaceMemoryWritableEpoch = -1; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 12, + }) + ).toBe(true); + expect(persisted()).toBe(true); + expect(persistedEpoch()).toBe(12); + // ...while a turn of the closing epoch itself still sees its deny. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritable = false; + entry.workspaceMemoryWritableEpoch = -1; + return cfg; + }); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + true + ); + expect(persisted()).toBe(false); + // Unknown history fails closed: no accumulator, no marker, no mirror // (this service never recorded this epoch), yet the epoch already holds // turns — the record was lost (a deny that could not be made durable @@ -9401,6 +9454,7 @@ describe("WorkspaceService initialize", () => { expect( await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: true, + policyEpoch: -1, }) ).toBe(true); expect(persisted()).toBe(false); @@ -9412,6 +9466,7 @@ describe("WorkspaceService initialize", () => { expect( await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, + policyEpoch: -1, }) ).toBe(true); expect(persisted()).toBe(true); @@ -9436,7 +9491,7 @@ describe("WorkspaceService initialize", () => { }); let settled = false; const pendingRecord = service - .recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false }) + .recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0) .then((ok) => { settled = true; return ok; @@ -9466,11 +9521,15 @@ describe("WorkspaceService initialize", () => { throw new Error("config.json: unexpected token"); }; spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true)).toBe(false); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + false + ); expect(persisted()).toBeUndefined(); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); spyOn(realConfig, "loadConfigOrDefault").mockImplementationOnce(unreadable); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); expect(persisted()).toBeUndefined(); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); await clearWorkspaceMemoryDenyMarker(sessionDir); @@ -9490,7 +9549,9 @@ describe("WorkspaceService initialize", () => { spyOn(realConfig, "editConfig").mockImplementationOnce(() => Promise.reject(new Error("disk full")) ); - expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false)).toBe(true); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); expect( await fsPromises.stat(sessionDir).then( () => true, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index e5ed46f53cd..7f9637fbd86 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -33,7 +33,12 @@ import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import { STOP_UNRECORDED_MESSAGE } from "@/common/constants/workspace"; import { MULTI_PROJECT_CONFIG_KEY } from "@/common/constants/multiProject"; import type { CompactionCompletionMetadata } from "@/common/types/compaction"; -import { ProvidersConfigStore, SecretsStore, type Config } from "@/node/config"; +import { + ProvidersConfigStore, + SecretsStore, + type Config, + type Workspace as WorkspaceConfigEntry, +} from "@/node/config"; import type { ProjectsConfig, Workspace } from "@/common/types/project"; import type { Result } from "@/common/types/result"; import { Ok, Err } from "@/common/types/result"; @@ -4258,8 +4263,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options?: { epochHasPriorTurns: boolean } + options: { epochHasPriorTurns: boolean; policyEpoch: number } ): Promise { + // The accumulator (config bit and deny marker alike) is bound to the + // compaction epoch it accumulates over — the opening boundary's history + // sequence, -1 before any boundary — so a value recorded under another + // epoch reads as absent here. Without this, a backend starting the FIRST + // turn of a new epoch could read the closing epoch's `false` before the + // compacting backend's durable reset landed (the reset is awaited only by + // that backend's own session) and carry it, via its mirror, through an + // otherwise all-writable epoch. + const { policyEpoch } = options; + assert(Number.isInteger(policyEpoch), "policyEpoch must be an integer"); const session = this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); // A no-tail compaction's durable epoch reset may still be in flight: read @@ -4286,7 +4301,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // removed workspace has nothing left to harvest: recorded as done. const written = await withTargetMutationLock(this.config.rootDir, sessionDir, async () => { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)) return false; - await writeWorkspaceMemoryDenyMarker(sessionDir); + await writeWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); return true; }); if (!written) { @@ -4351,7 +4366,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // removed the durable field underneath it. // A fourth input: the session-dir deny marker, the durable fallback taken // when config.json could not record a deny (denyDurableFallback above). - const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir); + const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); // Unknown history fails closed, like the harvest's own unknown → closed // rule: with no durable accumulator, no marker and no mirror, an epoch // that already holds turns has a policy nobody recorded — the record was @@ -4360,9 +4375,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // until the next boundary rather than granted by whichever turn comes // first. The first turn of a fresh epoch has no prior turns and grants // normally. - const stored = before.workspace.workspaceMemoryWritable; + const storedFor = (entry: WorkspaceConfigEntry): boolean | undefined => + entry.workspaceMemoryWritableEpoch === policyEpoch + ? entry.workspaceMemoryWritable + : undefined; + const stored = storedFor(before.workspace); const unknownHistory = - stored === undefined && mirror === undefined && options?.epochHasPriorTurns === true; + stored === undefined && mirror === undefined && options.epochHasPriorTurns; const conjunction = (durable: boolean | undefined): boolean => !denyMarker && !unknownHistory && (durable ?? true) && (mirror ?? true) && writable; // Fast path (no write): the outcome cannot differ from the stored value — @@ -4376,8 +4395,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); if (current !== null) { - effective = conjunction(current.workspace.workspaceMemoryWritable); + effective = conjunction(storedFor(current.workspace)); current.workspace.workspaceMemoryWritable = effective; + current.workspace.workspaceMemoryWritableEpoch = policyEpoch; } return cfg; }); @@ -4390,8 +4410,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return denyDurableFallback(getErrorMessage(error)); } session?.recordWorkspaceMemoryWritable(effective); - const persisted = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId)?.workspace - .workspaceMemoryWritable; + const persistedEntry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); + const persisted = persistedEntry === null ? undefined : storedFor(persistedEntry.workspace); if (persisted !== effective) { log.error("Workspace memory write policy did not persist (config write swallowed?)", { workspaceId, @@ -4500,14 +4520,38 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // to the completion) and the durable accumulator (which other backends // sharing this chat.jsonl also write) — and either deny is // authoritative; both unknown (fresh recovery session, field absent) → - // the harvest fails closed. - const persistedWritable = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId) - ?.workspace.workspaceMemoryWritable; + // the harvest fails closed. The durable value counts only under the + // CLOSING epoch's key (see recordWorkspaceMemoryWritable). Strict + // load: an unreadable config.json would read as the empty default and + // silently drop another backend's persisted deny, letting this + // session's own writable mirror grant the harvest — skip it instead + // (fail closed; nothing is recorded, the epoch is simply not harvested). + const closingEpoch = metadata.previousBoundaryHistorySequence ?? -1; + let persistedWritable: boolean | undefined; + try { + const entry = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + workspaceId + )?.workspace; + persistedWritable = + entry?.workspaceMemoryWritableEpoch === closingEpoch + ? entry.workspaceMemoryWritable + : undefined; + } catch (error: unknown) { + log.warn("Skipping post-compaction memory harvest: config.json unreadable", { + workspaceId, + error: getErrorMessage(error), + }); + return; + } // Third observation: the session-dir deny marker (fallback taken when // config.json could not record a deny; see recordWorkspaceMemoryWritable). // Returned so the session orders its epoch reset (which clears the // marker) after this observation; the harvest runs in the background. - return readWorkspaceMemoryDenyMarker(path.join(this.config.sessionsDir, workspaceId)) + return readWorkspaceMemoryDenyMarker( + path.join(this.config.sessionsDir, workspaceId), + closingEpoch + ) .then((denyMarker) => { const observed = [ metadata.workspaceMemoryWritable, @@ -6650,6 +6694,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ownerSessionDir === undefined ? undefined : async () => { + // Legacy-notebook delta too (locks held: the owner store's + // AND this child's own, so a self-fallback backend's late + // note into /memory either landed before this pass + // or is refused). Throws → removal aborts, session intact. + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + memoryOwnerId, + { locksHeld: true } + ); await migrateSharedMemoryRefinementRows({ childSessionDir: sessionDir, childWorkspaceId: workspaceId, From 944339288ffb936b613240486dbe3654fc8f4007 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 09:27:56 +0000 Subject: [PATCH 47/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-sec?= =?UTF-8?q?ond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A refused harvest (policy unrecorded for a turn of the epoch) returns Err and no longer sweeps the owner's shared notebook; the terminal record carries `refused` so later triggers stay refused too. - Config.loadExistingConfigOrThrow: strict load that also rejects an absent config.json; used by the policy accumulator, its completion observation, and removal's owner resolution (abort, retryable). - Deny-marker clear/carry run under the session-dir target lock the writer holds, so the epoch fence cannot go stale before the rm. - Legacy adoption takes the child's pin only when its pin bit changed since the last adoption; usage-only changes keep the owner's pin. - MemoryMetaService caches the sidecar stamp only after a real read; a transiently unreadable sidecar is retried instead of served empty. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/orpc/schemas/memory.ts | 6 + src/node/config/index.ts | 18 +++ src/node/services/agentSession.ts | 2 + .../memoryConsolidationService.test.ts | 43 +++++-- .../services/memoryConsolidationService.ts | 21 ++- src/node/services/memoryMeta.test.ts | 21 ++- src/node/services/memoryMeta.ts | 9 +- src/node/services/memoryService.test.ts | 31 +++++ src/node/services/memoryService.ts | 24 +++- .../services/workspaceMemoryDenyMarker.ts | 39 ++++-- src/node/services/workspaceService.test.ts | 120 ++++++++++++++---- src/node/services/workspaceService.ts | 28 ++-- 12 files changed, 291 insertions(+), 71 deletions(-) diff --git a/src/common/orpc/schemas/memory.ts b/src/common/orpc/schemas/memory.ts index d7310437d50..d857e31b781 100644 --- a/src/common/orpc/schemas/memory.ts +++ b/src/common/orpc/schemas/memory.ts @@ -116,6 +116,12 @@ export const MemoryHarvestRecordSchema = z.object({ acceptedCandidates: z.number(), skippedCandidates: z.number(), error: z.string().optional(), + /** + * Terminal refusal (policy unknown/read-only, or a turn of the epoch never + * recorded its policy): unlike an exhausted failure, the epoch's owner + * notebook must not be swept on its behalf either. + */ + refused: z.boolean().optional(), usage: z.object({ inputTokens: z.number(), outputTokens: z.number() }).optional(), completionMetadata: CompactionCompletionMetadataSchema.optional(), }); diff --git a/src/node/config/index.ts b/src/node/config/index.ts index c46a983b0e9..13096936631 100644 --- a/src/node/config/index.ts +++ b/src/node/config/index.ts @@ -1325,6 +1325,24 @@ export class Config { return { ids, hasWorkspaceEntriesWithoutIds }; } + /** + * Strict load that additionally REQUIRES config.json to exist. The strict + * mode of loadConfigOrDefault still treats ENOENT as a fresh install (an + * empty, valid config), which fail-closed callers — workspace removal's + * shared-memory handover, the workspace-memory policy accumulator — must + * not mistake for "this workspace is not registered" while the file is + * merely mid-rewrite. Stat before and after the read: a file present at + * both is taken as present during it. + */ + loadExistingConfigOrThrow(): ProjectsConfig { + const before = this.configFileStamp(); + const config = this.loadConfigOrDefault({ throwOnError: true }); + if (before === "missing" || this.configFileStamp() === "missing") { + throw new Error(`config.json is absent at ${this.configFile}`); + } + return config; + } + loadConfigOrDefault(options?: { throwOnError?: boolean }): ProjectsConfig { // Read as a Buffer and hand the same snapshot to the failure handler: backing up via a // second read could preserve a concurrent writer's replacement instead of the bytes that diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index d350009f90f..ca105f62c30 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1107,6 +1107,7 @@ export class AgentSession { // belongs to the closing epoch too. Same fence idea as below: a deny // recorded for the new epoch survives. await clearWorkspaceMemoryDenyMarker( + this.config.rootDir, path.join(this.config.sessionsDir, this.workspaceId), options ); @@ -1146,6 +1147,7 @@ export class AgentSession { nextEpoch: number ): Promise { await carryWorkspaceMemoryDenyMarker( + this.config.rootDir, path.join(this.config.sessionsDir, this.workspaceId), closingEpoch, nextEpoch diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 09691d738ba..c8aa57b9bab 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1245,13 +1245,29 @@ describe("MemoryConsolidationService", () => { compactionEpoch: 1, compactionRequestMessageId: "compact-request", }); - // The sweep still runs (success) while the harvest record is terminal - // (never completed, never retried), so recovery cannot replay the grant. - expect(result.success).toBe(true); + // Refused end to end: no sweep either (the Dream pass writes the shared + // notebook too), and the harvest record is terminal (never completed, + // never retried), so recovery cannot replay the grant. + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("never recorded"); + expect(fixture.modelCalls).toHaveLength(0); const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; expect(record?.status).toBe("failed"); expect(record?.attemptCount).toBe(HARVEST_MAX_ATTEMPTS); expect(record?.error).toContain("never recorded"); + expect(record?.refused).toBe(true); + // A later trigger for the same boundary (recovery, duplicate completion) + // finds the terminal refusal and does not fall through to the sweep. + const again = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(again.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); }); it("covers only a turn's own batch: a foreign row inside the request snapshot leaves the turn's own row uncovered", async () => { @@ -1299,7 +1315,8 @@ describe("MemoryConsolidationService", () => { compactionEpoch: 1, compactionRequestMessageId: "compact-request", }); - expect(result.success).toBe(true); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; expect(record?.status).toBe("failed"); expect(record?.error).toContain("never recorded"); @@ -1339,7 +1356,8 @@ describe("MemoryConsolidationService", () => { compactionEpoch: 1, compactionRequestMessageId: "compact-request", }); - expect(result.success).toBe(true); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; expect(record?.status).toBe("failed"); expect(record?.error).toContain("never recorded"); @@ -1408,22 +1426,27 @@ describe("MemoryConsolidationService", () => { ? { previousBoundaryHistorySequence } : {}), }); - expect(result.success).toBe(true); previousBoundaryHistorySequence = summary.metadata?.historySequence; - return (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + return { + result, + record: (await fixture.service.getStatus("ws-dream")).latestHarvestRecord, + }; }; // Only one of the two adjacent snapshot rows is listed: the other is a // foreign row that merely sits next to the batch. const partial = await harvest({ listed: ["s1-snap-a"], summary: "s1" }); - expect(partial?.status).toBe("failed"); - expect(partial?.error).toContain("never recorded"); + expect(partial.result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + expect(partial.record?.status).toBe("failed"); + expect(partial.record?.error).toContain("never recorded"); // Both listed: the whole batch is the turn's own and harvests. const complete = await harvest({ listed: ["s2-snap-a", "s2-snap-b"], summary: "s2", leadIn: true, }); - expect(complete?.status).toBe("completed"); + expect(complete.result.success).toBe(true); + expect(complete.record?.status).toBe("completed"); }); it("finalizes a removed workspace's retryable harvest records so they are never retried", async () => { diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 3dcb3fbb2ef..a77fa5a306e 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -741,6 +741,7 @@ export class MemoryConsolidationService extends EventEmitter { acceptedCandidates: 0, skippedCandidates: 0, error: reason, + refused: true, }, projectPath ) @@ -1116,6 +1117,11 @@ export class MemoryConsolidationService extends EventEmitter { const boundaryKey = metadata.summaryMessageId; const sidecar = yield* self.loadEffect(); const existing = sidecar.harvestsByWorkspace[metadata.workspaceId]?.[boundaryKey]; + // An epoch refused earlier (terminal record) stays refused on every + // later trigger — recovery, a duplicate completion — sweep included. + if (existing?.refused === true) { + return Err(existing.error ?? "harvest of this epoch was refused (fail closed)"); + } const existingAttemptCount = existing?.attemptCount ?? 0; const stalePending = existing === undefined ? false : isStalePendingHarvestRecord(existing); @@ -1160,11 +1166,19 @@ export class MemoryConsolidationService extends EventEmitter { // completed-record save rejecting — journals a failed record and // still falls through to the sweep, so the fold handles the error // channel AND defects identically. + // A refusal (policy never recorded for some turn of the epoch) + // already journaled its terminal record; re-journaling would make it + // retryable again. It also ends the run here, WITHOUT the owner sweep: + // the reasons the harvest is refused (an unaccounted turn, a stale + // grant) are exactly the reasons a model-driven Dream pass over the + // owner's shared notebook must not run on this epoch's behalf either. + const refused = { reason: null as string | null }; const journalHarvestFailure = (error: unknown): Effect.Effect => Effect.gen(function* () { - // A refusal already journaled its terminal record; re-journaling - // would make it retryable again. - if (error instanceof HarvestRefusedError) return; + if (error instanceof HarvestRefusedError) { + refused.reason = error.message; + return; + } yield* self.saveHarvestRecordEffect( metadata.workspaceId, boundaryKey, @@ -1196,6 +1210,7 @@ export class MemoryConsolidationService extends EventEmitter { removalSignal, }) .pipe(Effect.catch(journalHarvestFailure), Effect.catchDefect(journalHarvestFailure)); + if (refused.reason !== null) return Err(refused.reason); } // A sub-agent's inbox lives in the owner's store: wait on and run the diff --git a/src/node/services/memoryMeta.test.ts b/src/node/services/memoryMeta.test.ts index ee2e03868bb..dc7e6453c2e 100644 --- a/src/node/services/memoryMeta.test.ts +++ b/src/node/services/memoryMeta.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "bun:test"; +import { describe, it, expect, spyOn } from "bun:test"; import { Effect } from "effect"; import * as fsPromises from "node:fs/promises"; @@ -45,6 +45,25 @@ describe("memoryLogicalKey", () => { }); describe("MemoryMetaService", () => { + it("does not cache an empty view taken while the sidecar was unreadable", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + const service = new MemoryMetaService(tempDir.path); + await service.setPinned("global:prefs.md", true); + // Transient read failure (EACCES interval) with the file's stamp unchanged: + // this read heals to empty, but the next one must retry the file — not + // serve the empty view and then write it back over the real pins. + const reader = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))) as never); + const reloaded = new MemoryMetaService(tempDir.path); + expect(await reloaded.getPinnedKeys()).toEqual(new Set()); + reader.mockRestore(); + expect(await reloaded.getPinnedKeys()).toEqual(new Set(["global:prefs.md"])); + await reloaded.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + it("persists pins across instances via the sidecar file", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index bee2454557f..a731eebec63 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -306,6 +306,7 @@ export class MemoryMetaService { return Effect.gen(function* () { const stamp = yield* Effect.promise(() => self.fileStamp()); if (self.cache !== null && stamp === self.cacheStamp) return self.cache; + let readFailed = false; const parsed = yield* Effect.tryPromise({ try: async (): Promise => JSON.parse(await fsPromises.readFile(self.metaPath, "utf-8")), @@ -315,12 +316,18 @@ export class MemoryMetaService { // Missing file is the normal first-run case; anything else is healed to empty. if ((error as NodeJS.ErrnoException).code !== "ENOENT") { log.debug("[MemoryMetaService] healing unreadable sidecar", { error }); + readFailed = true; } return Effect.succeed(null); }) ); self.cache = sanitizeMetaFile(parsed); - self.cacheStamp = stamp; + // The stamp is remembered only for a real read (or a genuinely absent + // file): a transiently unreadable sidecar (EACCES interval, a writer + // mid-swap) heals to empty for THIS call, but caching that empty view + // under the file's unchanged stamp would keep serving it once readable + // again — and the next mutation would write the pins and stats away. + self.cacheStamp = readFailed ? null : stamp; return self.cache; }); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 5d2b9f39cbd..f4d3edafc27 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1573,6 +1573,37 @@ describe("MemoryService", () => { ).toBe(true); }); + it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + // Child had viewed it once on the old build (unpinned); adopted. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner pins the shared copy... + await fixture.metaService.setPinned(ownerKey, true); + // ...then the downgraded build merely views the legacy note again: the + // child sidecar changes (usage), its pin bit does not. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // A pin the child actually toggles on the old build is the newer intent. + await fixture.metaService.setPinned(ownerKey, false); + await fixture.metaService.setPinned(childKey, true); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index c1096487558..3753b73cf7d 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -419,6 +419,19 @@ function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { ); } +/** Pin bit of a manifest record's child sidecar fingerprint (null when none was recorded). */ +function legacySidecarPinned(sidecar: string): boolean | null { + if (sidecar === "") return null; + try { + const parsed: unknown = JSON.parse(sidecar); + return typeof parsed === "object" && parsed !== null + ? ((parsed as { pinned?: unknown }).pinned ?? false) === true + : null; + } catch { + return null; + } +} + /** * Self-healing read of the adoption manifest: anything malformed reads as not * adopted. A Map, not a plain object: a legacy note may legitimately be named @@ -1323,9 +1336,14 @@ export class MemoryService extends EventEmitter { // succeeded, so an adoption interrupted after its writeFile (or a // failing sidecar write) retries this step on the next access. A // first adoption keeps the owner's own pin (a note the owner tracked - // independently); a sidecar the CHILD changed since its last - // adoption (downgrade-time pin/unpin) is the newer intent and wins. + // independently); a PIN the CHILD toggled since its last adoption + // (downgrade-time pin/unpin) is the newer intent and wins. Only the + // pin bit counts for that: a downgraded build merely viewing the note + // changes its usage counters, which must not drag the owner's pin + // back to the child's unchanged value. if (childEntry !== undefined) { + const childPinChanged = + previous !== undefined && legacySidecarPinned(previous.sidecar) !== childEntry.pinned; try { await this.metaService.mergeKeys( childKey, @@ -1333,7 +1351,7 @@ export class MemoryService extends EventEmitter { projectPath: ctx.projectPath, workspaceId: owner, }), - { pinned: previous === undefined ? "target" : "source" } + { pinned: childPinChanged ? "source" : "target" } ); } catch (error) { log.warn( diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index b92c9a5c18d..b7e2e024c26 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -27,6 +27,7 @@ import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; export const WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME = "memory-policy-deny.json"; @@ -95,19 +96,27 @@ export async function readWorkspaceMemoryDenyMarker( * accumulator to false until a destructive history clear. */ export async function clearWorkspaceMemoryDenyMarker( + rootDir: string, sessionDir: string, options?: { closingEpoch: number } ): Promise { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); - if (options !== undefined) { - const record = await readMarkerRecord(markerPath); - if (record === "absent") return; - if (record !== null && record.epoch !== null && record.epoch !== options.closingEpoch) return; - } - await fsPromises.rm(markerPath, { force: true }); - if (await readWorkspaceMemoryDenyMarker(sessionDir)) { - throw new Error(`Workspace memory deny marker could not be removed at ${markerPath}`); - } + // Read-check-delete under the session-dir target lock the writer holds + // (WorkspaceService.recordWorkspaceMemoryWritable's fallback), so the fence + // cannot go stale between the read and the rm: a new-epoch deny written in + // that gap — possibly the only durable record of it — would otherwise be + // deleted right after its writer verified it. + await withTargetMutationLock(rootDir, sessionDir, async () => { + if (options !== undefined) { + const record = await readMarkerRecord(markerPath); + if (record === "absent") return; + if (record !== null && record.epoch !== null && record.epoch !== options.closingEpoch) return; + } + await fsPromises.rm(markerPath, { force: true }); + if (await readWorkspaceMemoryDenyMarker(sessionDir)) { + throw new Error(`Workspace memory deny marker could not be removed at ${markerPath}`); + } + }); } /** @@ -115,14 +124,18 @@ export async function clearWorkspaceMemoryDenyMarker( * one (the tail copies were produced under it): a deny marker recorded for * `closingEpoch` is re-stamped with `nextEpoch` so readers of the new epoch * keep seeing it. Markers of other epochs and absent markers are left alone; - * a malformed one stays a deny for every reader regardless. + * a malformed one stays a deny for every reader regardless. Same lock as the + * clear, for the same read→write reason. */ export async function carryWorkspaceMemoryDenyMarker( + rootDir: string, sessionDir: string, closingEpoch: number, nextEpoch: number ): Promise { - const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); - if (record === "absent" || record?.epoch !== closingEpoch) return; - await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); + await withTargetMutationLock(rootDir, sessionDir, async () => { + const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); + if (record === "absent" || record?.epoch !== closingEpoch) return; + await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); + }); } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 245fdba4888..c26fd2f29ed 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -6,6 +6,7 @@ import { writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; import { workspaceRemovalTombstonePath } from "@/node/services/workspaceRemoval"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; import { MutexMap } from "@/node/utils/concurrency/mutexMap"; @@ -213,6 +214,23 @@ function createMockAIService(overrides: Partial = {}): AIService { } as unknown as AIService; } +/** + * Mock configs model no file on disk. Production requires an EXISTING + * config.json for removal's shared-memory handover and the memory policy + * accumulator (Config.loadExistingConfigOrThrow); for a mock that only + * provides loadConfigOrDefault, treat that snapshot as the existing file. + */ +function withExistingConfigLoader>(config: T): T { + if ( + typeof config.loadExistingConfigOrThrow !== "function" && + typeof config.loadConfigOrDefault === "function" + ) { + const load = config.loadConfigOrDefault.bind(config); + return { ...config, loadExistingConfigOrThrow: () => load({ throwOnError: true }) }; + } + return config; +} + function createWorkspaceServiceForTest(options: { config: | (Partial & { getEffectiveSecrets?: SecretsStore["getEffectiveSecrets"] }) @@ -234,7 +252,7 @@ function createWorkspaceServiceForTest(options: { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions const defaultHistoryService: HistoryService = {} as HistoryService; return new WorkspaceService( - options.config as Config, + withExistingConfigLoader(options.config as Partial) as Config, options.historyService ?? defaultHistoryService, options.aiService ?? createMockAIService(), options.initStateManager ?? (mockInitStateManager as InitStateManager), @@ -7376,7 +7394,7 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { getInitState: mock(() => null), } as unknown as InitStateManager; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, aiService, initStateManager, @@ -9388,10 +9406,10 @@ describe("WorkspaceService initialize", () => { // deny — heals it and the next epoch can become writable again. await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: -1 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); - await clearWorkspaceMemoryDenyMarker(sessionDir); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; delete entry.workspace.workspaceMemoryWritable; @@ -9404,11 +9422,11 @@ describe("WorkspaceService initialize", () => { // The fenced clear (compaction boundary) keeps a deny recorded for the // NEW epoch; readers of any other epoch ignore that deny. await writeWorkspaceMemoryDenyMarker(sessionDir, 7); - await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: -1 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(sessionDir, { closingEpoch: 7 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 7 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); // The durable bit is bound to its epoch too: the closing epoch's @@ -9532,7 +9550,25 @@ describe("WorkspaceService initialize", () => { ); expect(persisted()).toBeUndefined(); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(sessionDir); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + // A MISSING config.json (mid-rewrite by another backend) is not a fresh + // install either: strict mode alone would read it as one and take the + // "unregistered" shortcut. Same fallback as unreadable. + const configFile = path.join(realConfig.rootDir, "config.json"); + await fsPromises.rename(configFile, `${configFile}.parked`); + try { + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( + false + ); + expect(await service.recordWorkspaceMemoryWritable("policy-scratch", false, EPOCH0)).toBe( + true + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + } finally { + await fsPromises.rename(`${configFile}.parked`, configFile); + } + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); + expect(persisted()).toBeUndefined(); // A late deny reaching the marker fallback after the workspace was // removed (tombstoned, session dir deleted) must not recreate the @@ -9563,6 +9599,34 @@ describe("WorkspaceService initialize", () => { } }); + test("the fenced deny-marker clear cannot delete a new-epoch deny written under the lock", async () => { + const { config: realConfig, cleanup } = await createTestHistoryService(); + try { + const sessionDir = path.join(realConfig.sessionsDir, "policy-lock"); + await writeWorkspaceMemoryDenyMarker(sessionDir, -1); + // Another backend's deny writer holds the session-dir lock while the + // boundary reset starts its read-check-delete: the reset must queue + // behind it and then see (and keep) the new epoch's marker. + let cleared = false; + let clear: Promise | undefined; + await withTargetMutationLock(realConfig.rootDir, sessionDir, async () => { + clear = clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { + closingEpoch: -1, + }).then(() => { + cleared = true; + }); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(cleared).toBe(false); + await writeWorkspaceMemoryDenyMarker(sessionDir, 5); + }); + await clear; + expect(cleared).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); + } finally { + await cleanup(); + } + }); + test("removes stale orphaned scratch workdirs but keeps referenced and recent ones", async () => { const { config: realConfig, historyService, cleanup } = await createTestHistoryService(); const scratchDirFor = (id: string) => path.join(realConfig.rootDir, "scratch", id); @@ -11317,7 +11381,7 @@ describe("WorkspaceService pending auto-title", () => { }; workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, aiService, mockInitStateManager as InitStateManager, @@ -14256,7 +14320,7 @@ describe("WorkspaceService remove timing rollup", () => { }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -14683,7 +14747,7 @@ describe("WorkspaceService metadata listeners", () => { }; new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -14746,7 +14810,7 @@ describe("WorkspaceService metadata listeners", () => { }; new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, aiService, mockInitStateManager as InitStateManager, @@ -16599,7 +16663,7 @@ describe("WorkspaceService archive init cancellation", () => { } as unknown as AIService; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -18773,7 +18837,7 @@ describe("WorkspaceService init cancellation", () => { try { const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -18864,7 +18928,7 @@ describe("WorkspaceService init cancellation", () => { loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager, @@ -18940,7 +19004,7 @@ describe("WorkspaceService init cancellation", () => { findWorkspace: mock(() => null), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager, @@ -19012,7 +19076,7 @@ describe("WorkspaceService init cancellation", () => { loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19092,7 +19156,7 @@ describe("WorkspaceService init cancellation", () => { loadConfigOrDefault: mock(() => ({ projects: new Map() })), }; const workspaceService = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19459,7 +19523,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19589,7 +19653,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19708,7 +19772,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19821,7 +19885,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -19932,7 +19996,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20042,7 +20106,7 @@ describe("WorkspaceService fork", () => { }; const workspaceService = new WorkspaceService( - config, + withExistingConfigLoader(config), historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20388,7 +20452,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { const mockBackgroundProcessManager = {}; const { historyService } = await createTestHistoryService(); return new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20521,7 +20585,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { }; const { historyService } = await createTestHistoryService(); const service = new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -20755,7 +20819,7 @@ describe("WorkspaceService.getGoalContinuationRuntimeState", () => { const mockExtensionMetadataService = {}; const mockBackgroundProcessManager = {}; return new WorkspaceService( - mockConfig as Config, + withExistingConfigLoader(mockConfig) as Config, historyService, mockAIService, mockInitStateManager as InitStateManager, @@ -21837,6 +21901,10 @@ describe("WorkspaceService disposal ownership", () => { async (externalRemoval) => { const h = await createAgentSessionHarness({ workspaceId: "leased-removal" }); const workspaceId = "leased-removal"; + // Removal requires an EXISTING config.json (an absent one reads as + // mid-rewrite, not as a fresh install); this workspace is simply not + // registered in it, exercising the phantom (metadata-less) path. + await h.config.editConfig((cfg) => cfg); const service = createWorkspaceServiceForTest({ config: h.config, historyService: h.historyService, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7f9637fbd86..55f90de8465 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4242,7 +4242,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { */ private loadConfigForRemovalOrAbort(workspaceId: string): ProjectsConfig { try { - return this.config.loadConfigOrDefault({ throwOnError: true }); + // Existing file required: strict mode alone reads ENOENT as a fresh + // install, which would verify this child as its own owner mid-rewrite. + return this.config.loadExistingConfigOrThrow(); } catch (error: unknown) { throw new SharedMemoryRemovalAbortedError(workspaceId, { cause: error }); } @@ -4323,20 +4325,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { session?.recordWorkspaceMemoryWritable(false); return true; }; - // Strict load: a config.json that is missing or malformed right now reads - // as the fresh-install default, in which this still-registered workspace - // is absent — the "unregistered" shortcut below would then report a deny - // as durable without persisting anything, while another backend keeps a - // prior durable grant. Treat an unreadable config like a failed config - // write: the deny takes the session-dir fallback (tombstone-gated, so a - // genuinely deregistered workspace still skips), a grant leaves the - // harvest closed. + // Strict load of an EXISTING file: a config.json that is missing (strict + // mode alone reads ENOENT as a fresh install) or malformed right now + // reads as the fresh-install default, in which this still-registered + // workspace is absent — the "unregistered" shortcut below would then + // report a deny as durable without persisting anything, while another + // backend keeps a prior durable grant. Treat an unreadable config like a + // failed config write: the deny takes the session-dir fallback + // (tombstone-gated, so a genuinely deregistered workspace still skips), a + // grant leaves the harvest closed. let before: ReturnType; try { - before = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), - workspaceId - ); + before = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), workspaceId); } catch (error: unknown) { log.error("Workspace memory write policy: config unreadable", { workspaceId, @@ -4530,7 +4530,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { let persistedWritable: boolean | undefined; try { const entry = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), + this.config.loadExistingConfigOrThrow(), workspaceId )?.workspace; persistedWritable = From fb6f5939d4191f504546d73a7b770615814a9aa7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 11:31:23 +0000 Subject: [PATCH 48/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-thi?= =?UTF-8?q?rd=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Assistant rows now carry `workspaceMemoryPolicyEpoch` (the compaction epoch the turn's policy was recorded under) on both the placeholder and the stream's initialMetadata, so the finalized row keeps the proof. The harvest gate accepts a turn's coverage only for the closing epoch and refuses a turn stamped for another epoch (a pre-reset turn from a foreign backend appended after the new boundary); RLM preserved-tail copies are excluded from gate and harvest. - The durable policy accumulator is stored per epoch (`workspaceMemoryWritableByEpoch`, newest few retained) and the deny marker holds a per-epoch set, so a new epoch's first turn on another backend can no longer displace the closing epoch's deny before the compacting backend observes it. - Consolidation refuses a cancelled or tombstoned sub-agent before redirecting its run to the shared-memory owner. - MemoryMetaService mutations fail instead of overwriting the sidecar from a view healed after a failed read (corrupt content still heals). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/schemas/project.ts | 8 +- src/common/types/message.ts | 16 ++- src/node/services/agentSession.ts | 56 +++++--- .../memoryConsolidationService.test.ts | 113 +++++++++++++++- .../services/memoryConsolidationService.ts | 123 ++++++++++++------ src/node/services/memoryMeta.test.ts | 30 ++++- src/node/services/memoryMeta.ts | 46 ++++++- src/node/services/turnRequestBuilder.ts | 19 ++- .../services/workspaceMemoryDenyMarker.ts | 112 +++++++++++----- .../services/workspaceMemoryPolicyEpochs.ts | 63 +++++++++ src/node/services/workspaceService.test.ts | 72 ++++++---- src/node/services/workspaceService.ts | 18 +-- 12 files changed, 519 insertions(+), 157 deletions(-) create mode 100644 src/node/services/workspaceMemoryPolicyEpochs.ts diff --git a/src/common/schemas/project.ts b/src/common/schemas/project.ts index a4f28bfab87..0d1e12043eb 100644 --- a/src/common/schemas/project.ts +++ b/src/common/schemas/project.ts @@ -105,13 +105,9 @@ export const WorkspaceConfigSchema = z.object({ description: "If set, this workspace is a child workspace spawned from the parent workspaceId (enables nesting in UI and backend orchestration).", }), - workspaceMemoryWritable: z.boolean().optional().meta({ + workspaceMemoryWritableByEpoch: z.record(z.string(), z.boolean()).optional().meta({ description: - "Whether this workspace's agent may write /memories/workspace, as resolved on its last normal turn. Persisted so a post-compaction memory harvest that resumes in a fresh session (restart, recovery) still knows the policy; harvest fails closed when unknown.", - }), - workspaceMemoryWritableEpoch: z.number().optional().meta({ - description: - "Compaction epoch `workspaceMemoryWritable` accumulates over: the history sequence of the durable context boundary that opened it (-1 before any boundary). Readers ignore the value under any other epoch, so a backend starting the new epoch cannot inherit the closing epoch's value before the boundary reset lands.", + "Whether this workspace's agent may write /memories/workspace, accumulated (fail-closed AND over its normal turns) per compaction epoch — keyed by the history sequence of the durable context boundary that opened the epoch (-1 before any boundary); only the newest few epochs are kept. Persisted so a post-compaction memory harvest that resumes in a fresh session (restart, recovery) or observes a closing epoch while another backend already records the next one still knows the policy; harvest fails closed when unknown.", }), memoryOwnerWorkspaceId: z.string().optional().meta({ description: diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 58ccdb92017..87f862952ce 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -953,13 +953,17 @@ export interface MuxMetadata { /** Highest persisted history sequence included in the provider request that produced this assistant. */ requestHistorySequence?: number; /** - * The turn that produced this assistant row recorded its workspace-memory - * write policy before the row was appended (TurnRequestBuilder start()). - * Builds that do not maintain that policy (older ones, after a downgrade) - * leave it unset, so the post-compaction harvest cannot take their turns' - * user rows as accounted for (memoryConsolidationService.ts). + * The compaction epoch (opening boundary's history sequence, -1 before any + * boundary) under which the turn that produced this assistant row recorded + * its workspace-memory write policy, before the row was appended + * (TurnRequestBuilder start()). The post-compaction harvest accepts a turn + * only when this matches the epoch being harvested: a turn started before + * a destructive reset but appended after the new boundary carries the old + * epoch, whose deny the reset discarded. Builds that do not maintain the + * policy (older ones, after a downgrade) leave it unset, so their turns' + * user rows are never taken as accounted for (memoryConsolidationService.ts). */ - workspaceMemoryPolicyRecorded?: true; + workspaceMemoryPolicyEpoch?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) /** Provider step boundaries in parts, persisted so continuous compaction can keep complete steps. */ stepStartPartIndices?: number[]; diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index ca105f62c30..352151de55d 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -92,6 +92,11 @@ import { ToolPolicySchema } from "@/common/orpc/schemas/stream"; import { normalizeAgentId, resolvePersistedAgentIdCandidates } from "@/common/utils/agentIds"; import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + deleteWorkspaceMemoryWritableForEpoch, + setWorkspaceMemoryWritableForEpoch, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; import { carryWorkspaceMemoryDenyMarker, clearWorkspaceMemoryDenyMarker, @@ -1112,24 +1117,29 @@ export class AgentSession { options ); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); - if (entry?.workspace.workspaceMemoryWritable === undefined) return; + if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) return; + if ( + options !== undefined && + workspaceMemoryWritableForEpoch(entry.workspace, options.closingEpoch) === undefined + ) { + return; + } await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, this.workspaceId); if (current === null) return cfg; // Fenced to the epoch being closed: another backend may already have // recorded the first turn of the NEW epoch between the completion and - // this locked write; a value bound to a different epoch is that newer - // epoch's and must survive. (Readers ignore a stale epoch's value - // anyway — WorkspaceService.recordWorkspaceMemoryWritable — so this - // delete is hygiene, not the correctness boundary.) - if ( - options !== undefined && - current.workspace.workspaceMemoryWritableEpoch !== options.closingEpoch - ) { - return cfg; + // this locked write; its record (a different epoch's) must survive. + // Records are per epoch and readers key by epoch anyway + // (WorkspaceService.recordWorkspaceMemoryWritable), so this delete is + // hygiene, not the correctness boundary. A destructive boundary + // (no closing epoch: /clear, reset, history replace) discards every + // epoch's transcript and so every record. + if (options !== undefined) { + deleteWorkspaceMemoryWritableForEpoch(current.workspace, options.closingEpoch); + } else { + delete current.workspace.workspaceMemoryWritableByEpoch; } - delete current.workspace.workspaceMemoryWritable; - delete current.workspace.workspaceMemoryWritableEpoch; return cfg; }); } @@ -1153,11 +1163,27 @@ export class AgentSession { nextEpoch ); const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); - if (entry?.workspace.workspaceMemoryWritableEpoch !== closingEpoch) return; + if ( + entry === null || + workspaceMemoryWritableForEpoch(entry.workspace, closingEpoch) === undefined + ) { + return; + } await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, this.workspaceId); - if (current?.workspace.workspaceMemoryWritableEpoch !== closingEpoch) return cfg; - current.workspace.workspaceMemoryWritableEpoch = nextEpoch; + if (current === null) return cfg; + const closing = workspaceMemoryWritableForEpoch(current.workspace, closingEpoch); + if (closing === undefined) return cfg; + // ANDed into a record another backend's first turn of the new epoch + // may already have made (its own conjunction could not see the closing + // value under the new key), never overwriting it. + const next = workspaceMemoryWritableForEpoch(current.workspace, nextEpoch); + setWorkspaceMemoryWritableForEpoch( + current.workspace, + nextEpoch, + next === undefined ? closing : next && closing + ); + deleteWorkspaceMemoryWritableForEpoch(current.workspace, closingEpoch); return cfg; }); } diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index c8aa57b9bab..a8049594e4c 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -336,7 +336,7 @@ async function seedCompactionEpoch( workspaceId, createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, - workspaceMemoryPolicyRecorded: true, + workspaceMemoryPolicyEpoch: -1, }) ); await fixture.historyService.appendToHistory( @@ -1222,7 +1222,7 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, - workspaceMemoryPolicyRecorded: true, + workspaceMemoryPolicyEpoch: -1, }) ); await fixture.historyService.appendToHistory( @@ -1292,7 +1292,7 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage("reply-1", "assistant", "Noted.", { requestHistorySequence: foreign.metadata?.historySequence, - workspaceMemoryPolicyRecorded: true, + workspaceMemoryPolicyEpoch: -1, }) ); await fixture.historyService.appendToHistory( @@ -1322,6 +1322,92 @@ describe("MemoryConsolidationService", () => { expect(record?.error).toContain("never recorded"); }); + it("refuses a turn whose policy was recorded for another epoch, and ignores preserved-tail copies", async () => { + using fixture = await createFixture(); + await fixture.addWorkspace("ws-clean"); + const seed = async (workspaceId: string, foreignTurn: boolean) => { + const reset = createMuxMessage("reset-1", "assistant", "", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory(workspaceId, reset); + const closingEpoch = reset.metadata?.historySequence ?? -1; + // RLM keep-recent copies of the previous epoch's turns: no stamps of + // their own, no coverage needed (their originals were judged already), + // and not harvested again. + for (const [id, role] of [ + ["copy-user", "user"], + ["copy-reply", "assistant"], + ] as const) { + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage(id, role, "Copied tail row.", { + synthetic: true, + rlmPreservedTailCopy: true, + }) + ); + } + const prompt = createMuxMessage("pref-1", "user", "Remember I prefer concise tests."); + await fixture.historyService.appendToHistory(workspaceId, prompt); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: closingEpoch, + }) + ); + if (foreignTurn) { + // Backend B started a read-only turn under the previous epoch (-1) and + // recorded its deny there; backend A then reset the context (the deny + // went with the epoch) and B's assistant landed after the new boundary + // — without its user row, which the reset removed. Only the epoch + // stamp can surface it. + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("b-reply", "assistant", "Read-only output.", { + requestHistorySequence: closingEpoch - 1, + workspaceMemoryPolicyEpoch: -1, + }) + ); + } + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 2, + }); + await fixture.historyService.appendToHistory(workspaceId, summary); + return fixture.service.maybeHarvestThenSweep({ + workspaceId, + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 2, + compactionRequestMessageId: "compact-request", + previousBoundaryHistorySequence: closingEpoch, + }); + }; + const refused = await seed("ws-dream", true); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); + expect((await fixture.service.getStatus("ws-dream")).latestHarvestRecord?.refused).toBe(true); + + // Without the foreign turn the copies alone refuse nothing, and the + // harvest transcript leaves them out. + const granted = await seed("ws-clean", false); + expect(granted.success).toBe(true); + expect(fixture.modelPrompts.length).toBeGreaterThan(0); + expect(fixture.modelPrompts[0]).toContain("concise tests"); + expect(fixture.modelPrompts[0]).not.toContain("Copied tail row"); + }); + it("takes no coverage from assistant rows of a build that did not record the policy", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); // A downgraded build ran a (read-only) turn mid-epoch: its assistant row @@ -1400,7 +1486,7 @@ describe("MemoryConsolidationService", () => { "ws-dream", createMuxMessage(`${ids.summary}-reply`, "assistant", "Noted.", { requestHistorySequence: prompt.metadata?.historySequence, - workspaceMemoryPolicyRecorded: true, + workspaceMemoryPolicyEpoch: previousBoundaryHistorySequence ?? -1, }) ); await fixture.historyService.appendToHistory( @@ -1831,6 +1917,25 @@ describe("MemoryConsolidationService", () => { if (!archive.success) expect(archive.error).toContain("owner"); expect(fixture.modelCalls).toHaveLength(2); + // A child mid-teardown must not start an owner run: locally cancelled + // (the child's removal drain could never cancel an owner-keyed run) or + // tombstoned by another backend. + await fixture.addWorkspace("ws-sub-gone", { parentWorkspaceId: "ws-dream" }); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-sub-gone"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile( + tombstonePath, + JSON.stringify({ workspaceId: "ws-sub-gone", removedAt: Date.now() }) + ); + const tombstoned = await fixture.service.maybeRun("ws-sub-gone", "manual"); + expect(tombstoned.success).toBe(false); + if (!tombstoned.success) expect(tombstoned.error).toContain("being removed"); + await fixture.service.cancelInFlightConsolidation("ws-sub"); + const cancelled = await fixture.service.maybeRun("ws-sub", "manual"); + expect(cancelled.success).toBe(false); + if (!cancelled.success) expect(cancelled.error).toContain("being removed"); + expect(fixture.modelCalls).toHaveLength(2); + // A dangling parent chain resolves to a PRIVATE store (owner == self), so // that workspace consolidates itself rather than being orphaned forever. await fixture.addWorkspace("ws-orphan", { parentWorkspaceId: "ws-gone" }); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index a77fa5a306e..834062ae39e 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -96,6 +96,12 @@ interface MemoryConsolidationRunOptions { */ skipWorkspaceDebounce?: boolean; skipHarvestRecovery?: boolean; + /** + * Set by maybeRun when a sub-agent's trigger is redirected to its owner: the + * child's own durable removal tombstone then gates the owner run too, since + * a child mid-teardown must not mutate the shared notebook. + */ + actingWorkspaceId?: string; } interface ExperimentsCheck { @@ -324,29 +330,38 @@ class HarvestRefusedError extends Error { } /** - * Whether some user row of the epoch belongs to no turn that recorded its - * write policy. A turn records that policy in start(), before its assistant - * row is appended, and the assistant row carries `requestHistorySequence` — - * the last history sequence its request was built from. The turn's own batch - * is the LAST user row at or below that bound (the request's latest user - * message) plus the snapshot/payload rows that row lists in - * `requestPreludeMessageIds`. Only that batch is covered — not every user row - * below the bound: with several backends on one chat.jsonl, another - * backend's read-only batch can land between this turn's user row and its - * request snapshot while that backend's deny has not been recorded yet; the - * snapshot would then include the foreign row (making it this turn's latest - * user message) and this turn's own row would be left without a turn of its - * own — which is exactly what surfaces here as uncovered. Rows are matched by - * exact id, never by adjacency, so no interleaved foreign row can ride along. - * Assistant rows without the bound, or without `workspaceMemoryPolicyRecorded` - * (a build that does not maintain the policy — e.g. turns run by a downgraded - * build mid-epoch, which also left the durable accumulator untouched), cover - * nothing (fail closed). Token-budget - * control rows (rollover lead-in, budget warning) need no turn: backend - * template text appended in the same durable batch as the turn they precede, - * carrying neither agent nor repository content. + * Why the compacted epoch's transcript cannot be harvested under the policy + * observed at completion, or null when every turn of it is accounted for. + * + * A turn records its write policy in start(), before its assistant row is + * appended, and the assistant row carries `requestHistorySequence` — the + * last history sequence its request was built from — plus + * `workspaceMemoryPolicyEpoch`, the epoch that policy was recorded under. A + * turn accounts for its own batch only when that epoch is the one being + * closed: a turn started in another backend before a destructive reset and + * appended after the new boundary recorded its (possibly read-only) policy + * for the epoch the reset discarded, and the reset may have removed its user + * row too, so nothing else would surface it — such a turn is refused + * outright. The turn's own batch is the LAST user row at or below the bound + * (the request's latest user message) plus the snapshot/payload rows that + * row lists in `requestPreludeMessageIds`. Only that batch is covered — not + * every user row below the bound: with several backends on one chat.jsonl, + * another backend's read-only batch can land between this turn's user row + * and its request snapshot while that backend's deny has not been recorded + * yet; the snapshot would then include the foreign row (making it this + * turn's latest user message) and this turn's own row would be left without + * a turn of its own — which is exactly what surfaces here as uncovered. Rows + * are matched by exact id, never by adjacency, so no interleaved foreign row + * can ride along. Assistant rows without the bound or without the epoch + * stamp (a build that does not maintain the policy — e.g. turns run by a + * downgraded build mid-epoch, which also left the durable accumulator + * untouched; synthetic payload/summary rows that are no turn) cover nothing + * (fail closed). Token-budget control rows (rollover lead-in, budget + * warning) need no turn: backend template text appended in the same durable + * batch as the turn they precede, carrying neither agent nor repository + * content. */ -function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { +function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: number): string | null { const userRows: Array<{ message: MuxMessage; sequence: number }> = []; for (const message of messages) { const sequence = message.metadata?.historySequence; @@ -355,11 +370,14 @@ function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { } const covered = new Set(); for (const message of messages) { - if (message.role !== "assistant" || message.metadata?.workspaceMemoryPolicyRecorded !== true) { - continue; - } + if (message.role !== "assistant") continue; const bound = message.metadata?.requestHistorySequence; if (typeof bound !== "number") continue; + const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; + if (typeof policyEpoch === "number" && policyEpoch !== closingEpoch) { + return "the compacted epoch holds a turn whose memory policy was recorded for another epoch; harvest refused (fail closed)"; + } + if (policyEpoch === undefined) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); @@ -367,10 +385,13 @@ function epochHasUncoveredUserRows(messages: readonly MuxMessage[]): boolean { covered.add(id); } } - return messages.some( + const uncovered = messages.some( (message) => message.role === "user" && !covered.has(message.id) && !isTokenBudgetInternalMessage(message) ); + return uncovered + ? "the compacted epoch holds user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)" + : null; } export class MemoryConsolidationService extends EventEmitter { @@ -874,6 +895,14 @@ export class MemoryConsolidationService extends EventEmitter { // one-shot promotion pass; a child archive must not trigger it. Compared // by resolved owner, not parentWorkspaceId: a dangling/cyclic chain falls // back to a private store that must stay consolidatable. + // + // The acting child's own teardown gate comes BEFORE the redirect: + // cancelInFlightConsolidation(child) marks only the child id, and an + // owner-keyed run reserved after that would neither be refused by the + // owner's check below nor be cancellable by the child's removal drain. + if (this.removalCancelled.has(workspaceId)) { + return Err("workspace is being removed; consolidation refused"); + } const ownerWorkspaceId = this.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); if (ownerWorkspaceId !== workspaceId) { if (trigger === "archive") { @@ -881,11 +910,13 @@ export class MemoryConsolidationService extends EventEmitter { "sub-agent workspaces share their owner's workspace memory; the owner's archive pass promotes it" ); } - // The owner run's recovery covers this child's harvest bucket too. - return this.maybeRun(ownerWorkspaceId, trigger, options); - } - if (this.removalCancelled.has(workspaceId)) { - return Err("workspace is being removed; consolidation refused"); + // The owner run's recovery covers this child's harvest bucket too. The + // child's durable tombstone is probed behind the reservation, like the + // owner's (actingWorkspaceId). + return this.maybeRun(ownerWorkspaceId, trigger, { + ...options, + actingWorkspaceId: options.actingWorkspaceId ?? workspaceId, + }); } const active = this.inFlight.get(workspaceId); if (active !== undefined) { @@ -939,6 +970,12 @@ export class MemoryConsolidationService extends EventEmitter { if (yield* self.isRemovalTombstonedEffect(workspaceId)) { return Err("workspace is being removed; consolidation refused"); } + if ( + options.actingWorkspaceId !== undefined && + (yield* self.isRemovalTombstonedEffect(options.actingWorkspaceId)) + ) { + return Err("workspace is being removed; consolidation refused"); + } // Manual runs bypass debounce (an explicit /dream is explicit intent). // Archive too: it is the workspace's one-shot final pass — the only @@ -1244,15 +1281,21 @@ export class MemoryConsolidationService extends EventEmitter { catch: (error) => error, }); if (!epoch.success) return yield* Effect.fail(new Error(epoch.error)); + // RLM keep-recent copies (compactionHandler) duplicate the previous + // epoch's last turns after its boundary: that epoch's harvest already + // judged the originals, and the copies carry no turn stamps of their + // own, so they neither need covering nor get harvested twice. + const messages = epoch.data.messages.filter( + (message) => message.metadata?.rlmPreservedTailCopy !== true + ); // Every scanned user row must be covered by a turn whose write policy - // was recorded (see epochHasUncoveredUserRows). Uncovered rows have an - // unknown policy the grant evaluated at completion could not have - // accounted for. Terminal refusal: a retry would replay that grant. - if (epochHasUncoveredUserRows(epoch.data.messages)) { - const reason = - "the compacted epoch holds user rows of a turn whose memory policy was never recorded; harvest refused (fail closed)"; - yield* Effect.promise(() => self.recordRefusedHarvest(metadata, reason)); - return yield* Effect.fail(new HarvestRefusedError(reason)); + // was recorded for THIS epoch (see epochHarvestRefusal). Uncovered rows + // have an unknown policy the grant evaluated at completion could not + // have accounted for. Terminal refusal: a retry would replay that grant. + const refusal = epochHarvestRefusal(messages, metadata.previousBoundaryHistorySequence ?? -1); + if (refusal !== null) { + yield* Effect.promise(() => self.recordRefusedHarvest(metadata, refusal)); + return yield* Effect.fail(new HarvestRefusedError(refusal)); } const modelString = resolveDreamModelString(self.config, metadata.workspaceId); @@ -1279,7 +1322,7 @@ export class MemoryConsolidationService extends EventEmitter { memoryService: self.memoryService, ctx, completionMetadata: metadata, - messages: epoch.data.messages, + messages, summary: epoch.data.summary, // Timeout + removal (r60); see the runLockedEffect signal for rationale. abortSignal: AbortSignal.any([ diff --git a/src/node/services/memoryMeta.test.ts b/src/node/services/memoryMeta.test.ts index dc7e6453c2e..bf4164338ae 100644 --- a/src/node/services/memoryMeta.test.ts +++ b/src/node/services/memoryMeta.test.ts @@ -3,7 +3,7 @@ import { Effect } from "effect"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; -import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; +import { MemoryMetaService, MemoryMetaWriteError, memoryLogicalKey } from "./memoryMeta"; import { TestTempDir } from "./tools/testHelpers"; describe("memoryLogicalKey", () => { @@ -64,6 +64,34 @@ describe("MemoryMetaService", () => { ); }); + it("refuses a mutation whose read of the sidecar failed instead of overwriting it", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + await new MemoryMetaService(tempDir.path).setPinned("global:prefs.md", true); + // The mutating call itself hits the transient failure: its healed empty + // view must not become the file, or every existing pin is erased. + const reader = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" }))) as never); + const fresh = new MemoryMetaService(tempDir.path); + try { + const failure = await fresh.setPinned("workspace:ws-1:scratch.md", true).then( + () => null, + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(MemoryMetaWriteError); + expect((failure as MemoryMetaWriteError).reason).toContain("could not be read"); + } finally { + reader.mockRestore(); + } + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md"]) + ); + // Once readable again the same instance mutates normally. + await fresh.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + it("persists pins across instances via the sidecar file", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index a731eebec63..e0bd00b4d86 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -301,15 +301,25 @@ export class MemoryMetaService { * (logged for diagnosis) and only writes can fail. */ private load(): Effect.Effect { + return Effect.map(this.loadWithHealth(), (loaded) => loaded.meta); + } + + /** + * `load()` plus whether this view is a healed substitute for a sidecar that + * exists but could not be read. Reads may serve that substitute; a mutation + * must not (see mutate()). + */ + private loadWithHealth(): Effect.Effect<{ meta: MemoryMetaFile; readFailed: boolean }> { // eslint-disable-next-line @typescript-eslint/no-this-alias -- Effect.gen generator bodies do not inherit `this` const self = this; return Effect.gen(function* () { const stamp = yield* Effect.promise(() => self.fileStamp()); - if (self.cache !== null && stamp === self.cacheStamp) return self.cache; + if (self.cache !== null && stamp === self.cacheStamp) { + return { meta: self.cache, readFailed: false }; + } let readFailed = false; - const parsed = yield* Effect.tryPromise({ - try: async (): Promise => - JSON.parse(await fsPromises.readFile(self.metaPath, "utf-8")), + const raw = yield* Effect.tryPromise({ + try: (): Promise => fsPromises.readFile(self.metaPath, "utf-8"), catch: (error) => error, }).pipe( Effect.catch((error) => { @@ -318,9 +328,19 @@ export class MemoryMetaService { log.debug("[MemoryMetaService] healing unreadable sidecar", { error }); readFailed = true; } - return Effect.succeed(null); + return Effect.succeed(null); }) ); + let parsed: unknown = null; + if (raw !== null) { + try { + parsed = JSON.parse(raw); + } catch (error) { + // Corrupt content (unlike a failed read) IS the file's state: healing + // it to empty and letting the next mutation rewrite it is the fix. + log.debug("[MemoryMetaService] healing corrupt sidecar", { error }); + } + } self.cache = sanitizeMetaFile(parsed); // The stamp is remembered only for a real read (or a genuinely absent // file): a transiently unreadable sidecar (EACCES interval, a writer @@ -328,7 +348,7 @@ export class MemoryMetaService { // under the file's unchanged stamp would keep serving it once readable // again — and the next mutation would write the pins and stats away. self.cacheStamp = readFailed ? null : stamp; - return self.cache; + return { meta: self.cache, readFailed }; }); } @@ -367,7 +387,19 @@ export class MemoryMetaService { }); yield* Effect.addFinalizer(() => Effect.promise(() => fileLock[Symbol.asyncDispose]())); // Stamp-validated: sees a foreign backend's write that landed since. - const meta = yield* self.load(); + const { meta, readFailed } = yield* self.loadWithHealth(); + // A read that healed to empty is fine to serve, but rewriting the + // sidecar from it would erase every existing pin and usage stat the + // moment the file becomes readable again. Fail the mutation instead; + // the caller retries on a later call, which re-reads. + if (readFailed) { + return yield* Effect.fail( + new MemoryMetaWriteError({ + metaPath: self.metaPath, + reason: "sidecar exists but could not be read; refusing to overwrite it", + }) + ); + } const entries = { ...meta.entries }; update(entries); for (const [key, entry] of Object.entries(entries)) { diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 4b006d216f7..bc105d8b882 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -2844,15 +2844,19 @@ export class TurnRequestBuilder { result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), }; } + // Proof for the harvest gate that this turn's policy was recorded + // (above, or the grant at onStreamStarted) and for which epoch: a build + // without the sink — or an older build after a downgrade — leaves it + // unset and its turns' user rows stay unaccounted for. Carried by the + // placeholder AND the stream's initialMetadata below: StreamManager + // builds the final assistant row from the latter, not the placeholder. + const workspaceMemoryPolicyStamp = + !isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink + ? { workspaceMemoryPolicyEpoch: policyEpoch } + : {}; const assistantMessage = createMuxMessage(assistantMessageId, "assistant", "", { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), - // Proof for the harvest gate that this turn's policy was recorded - // (above, or the grant at onStreamStarted): a build without the sink - // — or an older build after a downgrade — leaves it unset and its - // turns' user rows stay unaccounted for. - ...(!isCompactionRequest && this.dependencies.bindings.workspaceMemoryPolicySink - ? { workspaceMemoryPolicyRecorded: true as const } - : {}), + ...workspaceMemoryPolicyStamp, timestamp: Date.now(), model: canonicalModelString, routedThroughGateway, @@ -3213,6 +3217,7 @@ export class TurnRequestBuilder { contextBudgetLimit: primaryRequest.contextBudgetLimit, initialMetadata: { ...(requestHistorySequence >= 0 ? { requestHistorySequence } : {}), + ...workspaceMemoryPolicyStamp, systemMessageTokens, timestamp: Date.now(), agentId: effectiveAgentId, diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index b7e2e024c26..ae2158ed481 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -13,14 +13,17 @@ * it is consulted; it is cleared only at the epoch boundary that clears the * config bit. * - * Like the config bit, the marker is bound to its compaction epoch (the - * opening boundary's history sequence, -1 before any boundary): a reader - * consulting it for another epoch ignores it, so a backend starting the new - * epoch before this one's boundary reset landed cannot inherit a stale deny. + * Like the config records, the marker is bound to its compaction epoch (the + * opening boundary's history sequence, -1 before any boundary) and holds one + * entry per epoch (the newest few): a reader consulting it for another epoch + * ignores it, so a backend starting the new epoch before this one's boundary + * reset landed cannot inherit a stale deny — and a deny that backend records + * for the new epoch cannot displace the closing epoch's deny before the + * compacting backend observes it (see workspaceMemoryPolicyEpochs.ts). * - * Fail-closed by construction: a missing marker (or one recorded for a - * different epoch) is "no deny"; a present marker for this epoch, or a - * malformed/unreadable one, is a deny. + * Fail-closed by construction: a missing marker (or one without an entry for + * this epoch) is "no deny"; a present entry for this epoch, or a + * malformed/unreadable marker, is a deny. */ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; @@ -28,6 +31,7 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; +import { WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED } from "@/node/services/workspaceMemoryPolicyEpochs"; export const WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME = "memory-policy-deny.json"; @@ -36,7 +40,7 @@ export function workspaceMemoryDenyMarkerPath(sessionDir: string): string { return path.join(sessionDir, WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME); } -/** Durable-or-throw: verified by reading the marker back. */ +/** Durable-or-throw: verified by reading the marker back. Adds `epoch` to the recorded set. */ export async function writeWorkspaceMemoryDenyMarker( sessionDir: string, epoch: number @@ -44,26 +48,43 @@ export async function writeWorkspaceMemoryDenyMarker( assert(Number.isInteger(epoch), "workspace memory deny marker epoch must be an integer"); const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); await fsPromises.mkdir(sessionDir, { recursive: true }); - await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now(), epoch }), { - encoding: "utf-8", - }); + const record = await readMarkerRecord(markerPath); + // A malformed marker was a deny for every reader while it existed; a + // well-formed write for this epoch supersedes it (the boundary clear would + // heal it the same way). + const epochs = record === "absent" || record === null ? [] : record.epochs; + await writeMarkerRecord(markerPath, [...epochs.filter((e) => e !== epoch), epoch]); if (!(await readWorkspaceMemoryDenyMarker(sessionDir, epoch))) { throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); } } +async function writeMarkerRecord(markerPath: string, epochs: readonly number[]): Promise { + const retained = [...epochs] + .sort((a, b) => b - a) + .slice(0, WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED); + await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now(), epochs: retained }), { + encoding: "utf-8", + }); +} + /** Parsed marker, or null when it is unreadable/malformed (which readers treat as a deny). */ async function readMarkerRecord( markerPath: string -): Promise<{ deniedAt: number | null; epoch: number | null } | "absent" | null> { +): Promise<{ epochs: number[] } | "absent" | null> { try { const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); if (typeof parsed !== "object" || parsed === null) return null; - const { deniedAt, epoch } = parsed as { deniedAt?: unknown; epoch?: unknown }; - return { - deniedAt: typeof deniedAt === "number" && Number.isFinite(deniedAt) ? deniedAt : null, - epoch: typeof epoch === "number" && Number.isInteger(epoch) ? epoch : null, - }; + const { epochs } = parsed as { epochs?: unknown }; + if ( + !Array.isArray(epochs) || + !epochs.every( + (epoch): epoch is number => typeof epoch === "number" && Number.isInteger(epoch) + ) + ) { + return null; + } + return { epochs }; } catch (error) { return hasErrorCode(error, "ENOENT") ? "absent" : null; } @@ -71,9 +92,9 @@ async function readMarkerRecord( /** * True when a deny is recorded for `epoch` (or the marker cannot be trusted: - * unreadable, malformed, or written by a build that did not record an epoch); - * false when absent or recorded for a different epoch. Without `epoch`, any - * present marker is a deny (epoch-agnostic existence check). + * unreadable or malformed); false when absent or recorded only for other + * epochs. Without `epoch`, any present marker is a deny (epoch-agnostic + * existence check). */ export async function readWorkspaceMemoryDenyMarker( sessionDir: string, @@ -82,18 +103,20 @@ export async function readWorkspaceMemoryDenyMarker( const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); if (record === "absent") return false; if (record === null || epoch === undefined) return true; - return record.epoch === null || record.epoch === epoch; + return record.epochs.includes(epoch); } /** - * Epoch boundary: remove the marker, durable-or-throw (verified absent). - * `closingEpoch` fences the clear to that epoch's deny: a deny another - * backend recorded for the NEW epoch in the meantime (different epoch) must - * survive. Only a well-formed marker can claim to be another epoch's; a - * truncated or malformed one is stale state from some earlier epoch (every - * reader already treated it as a deny for as long as it existed) and is - * healed here — otherwise one corrupt file would force every later epoch's - * accumulator to false until a destructive history clear. + * Epoch boundary: remove the closing epoch's deny, durable-or-throw (verified + * absent). `closingEpoch` fences the clear to that epoch's entry: a deny + * another backend recorded for the NEW epoch in the meantime must survive, + * so the file is removed only once no entry is left. Only a well-formed + * marker can hold other epochs' entries; a truncated or malformed one is + * stale state from some earlier epoch (every reader already treated it as a + * deny for as long as it existed) and is healed here — otherwise one corrupt + * file would force every later epoch's accumulator to false until a + * destructive history clear. Without `closingEpoch` (destructive boundary), + * every epoch's deny goes. */ export async function clearWorkspaceMemoryDenyMarker( rootDir: string, @@ -110,7 +133,17 @@ export async function clearWorkspaceMemoryDenyMarker( if (options !== undefined) { const record = await readMarkerRecord(markerPath); if (record === "absent") return; - if (record !== null && record.epoch !== null && record.epoch !== options.closingEpoch) return; + if (record !== null) { + const remaining = record.epochs.filter((epoch) => epoch !== options.closingEpoch); + if (remaining.length === record.epochs.length) return; + if (remaining.length > 0) { + await writeMarkerRecord(markerPath, remaining); + if (await readWorkspaceMemoryDenyMarker(sessionDir, options.closingEpoch)) { + throw new Error(`Workspace memory deny marker could not be cleared at ${markerPath}`); + } + return; + } + } } await fsPromises.rm(markerPath, { force: true }); if (await readWorkspaceMemoryDenyMarker(sessionDir)) { @@ -121,9 +154,9 @@ export async function clearWorkspaceMemoryDenyMarker( /** * A preserved-tail compaction carries the closing epoch's policy into the new - * one (the tail copies were produced under it): a deny marker recorded for - * `closingEpoch` is re-stamped with `nextEpoch` so readers of the new epoch - * keep seeing it. Markers of other epochs and absent markers are left alone; + * one (the tail copies were produced under it): a deny recorded for + * `closingEpoch` is re-stamped as `nextEpoch` so readers of the new epoch + * keep seeing it. Entries of other epochs and absent markers are left alone; * a malformed one stays a deny for every reader regardless. Same lock as the * clear, for the same read→write reason. */ @@ -134,8 +167,15 @@ export async function carryWorkspaceMemoryDenyMarker( nextEpoch: number ): Promise { await withTargetMutationLock(rootDir, sessionDir, async () => { - const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); - if (record === "absent" || record?.epoch !== closingEpoch) return; - await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + const record = await readMarkerRecord(markerPath); + if (record === "absent" || !record?.epochs.includes(closingEpoch)) return; + await writeMarkerRecord(markerPath, [ + ...record.epochs.filter((epoch) => epoch !== closingEpoch && epoch !== nextEpoch), + nextEpoch, + ]); + if (!(await readWorkspaceMemoryDenyMarker(sessionDir, nextEpoch))) { + throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); + } }); } diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts new file mode 100644 index 00000000000..cfdab815275 --- /dev/null +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -0,0 +1,63 @@ +/** + * Per-epoch storage of the workspace-memory write-policy accumulator on a + * workspace's config.json entry (`workspaceMemoryWritableByEpoch`, see + * WorkspaceService.recordWorkspaceMemoryWritable). + * + * One record per compaction epoch rather than a single slot: with several + * backends over one chat.jsonl (XUM_ALLOW_MULTIPLE_INSTANCES), a backend can + * start the NEW epoch's first turn between another backend's compaction + * boundary and that backend's completion-side read of the CLOSING epoch's + * value. A single slot would be overwritten by the new epoch's grant, dropping + * a deny recorded for the closing epoch — and the compacting backend's own + * mirror (writable) would then grant the harvest of a read-only turn. Epochs + * are history sequences (-1 before any boundary), so the newest few are kept + * and older ones garbage-collected: a closing epoch is observed at its + * boundary, before more than a couple of further boundaries can exist. + */ +import type { Workspace as WorkspaceConfigEntry } from "@/node/config"; +import assert from "@/common/utils/assert"; + +/** The closing epoch plus the next ones that can be opened before it is observed. */ +export const WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED = 3; + +function epochKey(epoch: number): string { + assert(Number.isInteger(epoch), "workspace memory policy epoch must be an integer"); + return String(epoch); +} + +export function workspaceMemoryWritableForEpoch( + entry: WorkspaceConfigEntry, + epoch: number +): boolean | undefined { + return entry.workspaceMemoryWritableByEpoch?.[epochKey(epoch)]; +} + +/** Record `writable` for `epoch`; drops the oldest records beyond the retained window. */ +export function setWorkspaceMemoryWritableForEpoch( + entry: WorkspaceConfigEntry, + epoch: number, + writable: boolean +): void { + const next: Record = { + ...entry.workspaceMemoryWritableByEpoch, + [epochKey(epoch)]: writable, + }; + const keys = Object.keys(next).sort((a, b) => Number(b) - Number(a)); + for (const key of keys.slice(WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED)) delete next[key]; + entry.workspaceMemoryWritableByEpoch = next; +} + +/** Forget `epoch`'s record; removes the field once no record is left. */ +export function deleteWorkspaceMemoryWritableForEpoch( + entry: WorkspaceConfigEntry, + epoch: number +): void { + const current = entry.workspaceMemoryWritableByEpoch; + if (current === undefined) return; + const key = epochKey(epoch); + if (!(key in current)) return; + const next = { ...current }; + delete next[key]; + if (Object.keys(next).length === 0) delete entry.workspaceMemoryWritableByEpoch; + else entry.workspaceMemoryWritableByEpoch = next; +} diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c26fd2f29ed..1cdf2e52f82 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9331,12 +9331,10 @@ describe("WorkspaceService initialize", () => { aiService, initStateManager: mockInitStateManager as InitStateManager, }); - const persisted = () => + const persistedFor = (epoch: number) => findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace - .workspaceMemoryWritable; - const persistedEpoch = () => - findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")?.workspace - .workspaceMemoryWritableEpoch; + .workspaceMemoryWritableByEpoch?.[String(epoch)]; + const persisted = () => persistedFor(-1); const EPOCH0 = { epochHasPriorTurns: false, policyEpoch: -1 }; try { // The durable bit is the epoch accumulator: the harvest reads every @@ -9361,7 +9359,7 @@ describe("WorkspaceService initialize", () => { // not publish false→true. await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( @@ -9369,7 +9367,9 @@ describe("WorkspaceService initialize", () => { ); expect(persisted()).toBe(true); await realConfig.editConfig((cfg) => { - findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable = false; + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch = { + "-1": false, + }; return cfg; }); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( @@ -9385,7 +9385,7 @@ describe("WorkspaceService initialize", () => { // observation as well. await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); const sessionDir = path.join(realConfig.sessionsDir, "policy-scratch"); @@ -9412,7 +9412,7 @@ describe("WorkspaceService initialize", () => { await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( @@ -9420,8 +9420,12 @@ describe("WorkspaceService initialize", () => { ); expect(persisted()).toBe(true); // The fenced clear (compaction boundary) keeps a deny recorded for the - // NEW epoch; readers of any other epoch ignore that deny. + // NEW epoch; readers of any other epoch ignore that deny. A new-epoch + // deny written before the clear does not displace the closing one. + await writeWorkspaceMemoryDenyMarker(sessionDir, -1); await writeWorkspaceMemoryDenyMarker(sessionDir, 7); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); @@ -9432,11 +9436,13 @@ describe("WorkspaceService initialize", () => { // The durable bit is bound to its epoch too: the closing epoch's // `false` is invisible to the first turn of the next epoch on ANOTHER // backend (which cannot await this backend's boundary reset), so it - // grants and re-binds the value to its own epoch. + // grants under its own epoch's record — WITHOUT displacing the closing + // epoch's deny, which the compacting backend's completion observation + // may not have read yet (a single slot would let its writable mirror + // grant the harvest of the read-only turn). await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; - entry.workspaceMemoryWritable = false; - entry.workspaceMemoryWritableEpoch = -1; + entry.workspaceMemoryWritableByEpoch = { "-1": false }; return cfg; }); expect( @@ -9445,19 +9451,29 @@ describe("WorkspaceService initialize", () => { policyEpoch: 12, }) ).toBe(true); - expect(persisted()).toBe(true); - expect(persistedEpoch()).toBe(12); + expect(persistedFor(12)).toBe(true); + expect(persisted()).toBe(false); // ...while a turn of the closing epoch itself still sees its deny. - await realConfig.editConfig((cfg) => { - const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; - entry.workspaceMemoryWritable = false; - entry.workspaceMemoryWritableEpoch = -1; - return cfg; - }); expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( true ); expect(persisted()).toBe(false); + expect(persistedFor(12)).toBe(true); + // Only the newest few epochs are retained. + for (const policyEpoch of [20, 30, 40]) { + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch, + }) + ).toBe(true); + } + expect( + Object.keys( + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")!.workspace + .workspaceMemoryWritableByEpoch! + ).sort() + ).toEqual(["20", "30", "40"]); // Unknown history fails closed: no accumulator, no marker, no mirror // (this service never recorded this epoch), yet the epoch already holds @@ -9466,7 +9482,7 @@ describe("WorkspaceService initialize", () => { // must not grant the whole epoch; the first turn of a fresh epoch does. await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); expect( @@ -9478,7 +9494,7 @@ describe("WorkspaceService initialize", () => { expect(persisted()).toBe(false); await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); expect( @@ -9504,7 +9520,9 @@ describe("WorkspaceService initialize", () => { fakeSession ); await realConfig.editConfig((cfg) => { - findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable = false; + findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch = { + "-1": false, + }; return cfg; }); let settled = false; @@ -9518,7 +9536,7 @@ describe("WorkspaceService initialize", () => { expect(settled).toBe(false); // The reset finishes (field cleared) and only then does the record read. await realConfig.editConfig((cfg) => { - delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable; + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch; return cfg; }); releaseReset(); @@ -9532,7 +9550,7 @@ describe("WorkspaceService initialize", () => { // without a record anywhere, and a grant is reported unpersisted (the // harvest stays closed) rather than "done". await realConfig.editConfig((cfg) => { - delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritable; + delete findWorkspaceEntry(cfg, "policy-scratch")!.workspace.workspaceMemoryWritableByEpoch; return cfg; }); const unreadable = () => { @@ -9575,7 +9593,7 @@ describe("WorkspaceService initialize", () => { // session dir as an orphan; nothing is left to harvest, so it is done. await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!; - delete entry.workspace.workspaceMemoryWritable; + delete entry.workspace.workspaceMemoryWritableByEpoch; return cfg; }); await fsPromises.rm(sessionDir, { recursive: true, force: true }); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 55f90de8465..25baddd767f 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -358,6 +358,10 @@ import { type WorkspaceLiveActivity, } from "@/node/services/taskWorkspaceSeam"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { + setWorkspaceMemoryWritableForEpoch, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; import type { WorktreeArchiveSnapshotService } from "@/node/services/worktreeArchiveSnapshotService"; import type { DevToolsService } from "@/node/services/devToolsService"; @@ -4376,9 +4380,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // first. The first turn of a fresh epoch has no prior turns and grants // normally. const storedFor = (entry: WorkspaceConfigEntry): boolean | undefined => - entry.workspaceMemoryWritableEpoch === policyEpoch - ? entry.workspaceMemoryWritable - : undefined; + workspaceMemoryWritableForEpoch(entry, policyEpoch); const stored = storedFor(before.workspace); const unknownHistory = stored === undefined && mirror === undefined && options.epochHasPriorTurns; @@ -4396,8 +4398,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const current = findWorkspaceEntry(cfg, workspaceId); if (current !== null) { effective = conjunction(storedFor(current.workspace)); - current.workspace.workspaceMemoryWritable = effective; - current.workspace.workspaceMemoryWritableEpoch = policyEpoch; + // Per-epoch record: never overwrites the closing epoch's value, + // which the compacting backend may not have observed yet + // (workspaceMemoryPolicyEpochs.ts). + setWorkspaceMemoryWritableForEpoch(current.workspace, policyEpoch, effective); } return cfg; }); @@ -4534,9 +4538,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceId )?.workspace; persistedWritable = - entry?.workspaceMemoryWritableEpoch === closingEpoch - ? entry.workspaceMemoryWritable - : undefined; + entry === undefined ? undefined : workspaceMemoryWritableForEpoch(entry, closingEpoch); } catch (error: unknown) { log.warn("Skipping post-compaction memory harvest: config.json unreadable", { workspaceId, From cf4c8848364156a58721b458c41d02da6fe4b5ce Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 11:50:38 +0000 Subject: [PATCH 49/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20keep=20removal=20of?= =?UTF-8?q?=20unknown=20workspaces=20idempotent=20on=20a=20fresh=20root?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removing an unregistered workspace id on a root whose config.json was never written (nothing registered, no session dir) is a no-op again instead of aborting on the strict config load: there is no topology to resolve and nothing a tombstone could protect. A present session dir or config file still takes the strict path. - Test mocks: the multi-project removal fixture gets the same `loadExistingConfigOrThrow` shim as workspaceService.test.ts, and the post-compaction attachments fixture's config snapshot carries an empty project map for the boundary's policy reset. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/config/index.ts | 7 +- ...tSession.postCompactionAttachments.test.ts | 4 +- .../workspaceService.multiProject.test.ts | 18 +++- src/node/services/workspaceService.ts | 97 ++++++++++++------- 4 files changed, 86 insertions(+), 40 deletions(-) diff --git a/src/node/config/index.ts b/src/node/config/index.ts index 13096936631..2303821b5d1 100644 --- a/src/node/config/index.ts +++ b/src/node/config/index.ts @@ -884,6 +884,11 @@ interface ConfigLoadFailureState { // re-log the same corrupt-config error once per instance. const configLoadFailureStates = new Map(); +/** Location of config.json under a Xum root (also used by callers that must stat it without a Config). */ +export function configFilePath(rootDir: string): string { + return path.join(rootDir, "config.json"); +} + function configLoadFailureState(configFile: string): ConfigLoadFailureState { let state = configLoadFailureStates.get(configFile); if (!state) { @@ -984,7 +989,7 @@ export class Config { this.rootDir = sessionLocator.rootDir; this.sessionsDir = sessionLocator.sessionsDir; this.srcDir = sessionLocator.srcDir; - this.configFile = path.join(this.rootDir, "config.json"); + this.configFile = configFilePath(this.rootDir); this.providersConfigStore = providersConfigStore ?? new ProvidersConfigStore(this.rootDir); } diff --git a/src/node/services/agentSession.postCompactionAttachments.test.ts b/src/node/services/agentSession.postCompactionAttachments.test.ts index d7a119a3d6d..a340add08cc 100644 --- a/src/node/services/agentSession.postCompactionAttachments.test.ts +++ b/src/node/services/agentSession.postCompactionAttachments.test.ts @@ -140,7 +140,9 @@ function createSessionForHistory(historyService: HistoryService, sessionDir: str rootDir: path.dirname(sessionDir), sessionsDir: path.dirname(sessionDir), srcDir: "/tmp", - loadConfigOrDefault: mock(() => ({})), + // A context boundary resets the durable memory-policy accumulator, which + // looks this workspace up in the config snapshot. + loadConfigOrDefault: mock(() => ({ projects: new Map() })), } as unknown as Config; return new AgentSession({ diff --git a/src/node/services/workspaceService.multiProject.test.ts b/src/node/services/workspaceService.multiProject.test.ts index 59b956df223..aca12866496 100644 --- a/src/node/services/workspaceService.multiProject.test.ts +++ b/src/node/services/workspaceService.multiProject.test.ts @@ -82,9 +82,25 @@ function createMockAIService(metadata?: WorkspaceMetadata): AIService { off: mock(() => undefined), } as unknown as AIService; } +/** + * Mock configs model no file on disk. Removal's shared-memory handover + * requires an EXISTING config.json (Config.loadExistingConfigOrThrow); for a + * mock that only provides loadConfigOrDefault, treat that snapshot as the + * existing file (same shim as workspaceService.test.ts). + */ +function withExistingConfigLoader(config: Partial): Partial { + if ( + typeof config.loadExistingConfigOrThrow !== "function" && + typeof config.loadConfigOrDefault === "function" + ) { + const load = config.loadConfigOrDefault.bind(config); + return { ...config, loadExistingConfigOrThrow: () => load({ throwOnError: true }) }; + } + return config; +} function createWorkspaceServiceForTest(options: WorkspaceServiceTestOptions): WorkspaceService { return new WorkspaceService( - options.config as Config, + withExistingConfigLoader(options.config) as Config, options.historyService, options.aiService ?? createMockAIService(), options.initStateManager ?? createMockInitStateManager(), diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 25baddd767f..8abb3c10ed8 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -36,6 +36,7 @@ import type { CompactionCompletionMetadata } from "@/common/types/compaction"; import { ProvidersConfigStore, SecretsStore, + configFilePath, type Config, type Workspace as WorkspaceConfigEntry, } from "@/node/config"; @@ -6679,44 +6680,66 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // admitted under the real owner lock recreate the deleted session dir. // Without an earlier pass, resolve strictly — an unreadable config // aborts the removal (retryable) instead of guessing the topology. - const memoryOwnerId = - verifiedSharedMemoryOwnerId ?? - resolveWorkspaceMemoryOwnerId(this.loadConfigForRemovalOrAbort(workspaceId), workspaceId); - const ownerSessionDir = - memoryOwnerId === workspaceId - ? undefined - : path.join(this.config.sessionsDir, memoryOwnerId); - await removeSessionDirUnderMemoryLocks({ - rootDir: this.config.rootDir, - sessionDir, - workspaceId, - attemptId: removalAttemptId, - sharedWorkspaceMemorySessionDir: ownerSessionDir, - beforeTombstone: - ownerSessionDir === undefined + // Idempotent no-op short of that: a root whose config.json was never + // written (fresh install, nothing registered) and no session dir + // means there is no topology to resolve and nothing a tombstone + // could protect — removing an unknown id must still succeed. + const exists = (target: string): Promise => + fsPromises.stat(target).then( + () => true, + () => false + ); + const nothingToTearDown = + verifiedSharedMemoryOwnerId === null && + !(await exists(configFilePath(this.config.rootDir))) && + !(await exists(sessionDir)); + if (nothingToTearDown) { + log.debug("Skipping session teardown: no config.json and no session dir", { + workspaceId, + }); + } else { + const memoryOwnerId = + verifiedSharedMemoryOwnerId ?? + resolveWorkspaceMemoryOwnerId( + this.loadConfigForRemovalOrAbort(workspaceId), + workspaceId + ); + const ownerSessionDir = + memoryOwnerId === workspaceId ? undefined - : async () => { - // Legacy-notebook delta too (locks held: the owner store's - // AND this child's own, so a self-fallback backend's late - // note into /memory either landed before this pass - // or is refused). Throws → removal aborts, session intact. - await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( - workspaceId, - memoryOwnerId, - { locksHeld: true } - ); - await migrateSharedMemoryRefinementRows({ - childSessionDir: sessionDir, - childWorkspaceId: workspaceId, - ownerSessionDir, - ownerWorkspaceId: memoryOwnerId, - }); - }, - }); - // Only once the session (and with it the transcript) is gone are the - // retryable harvest records truly unrecoverable; an aborted removal - // above must leave them retryable. - await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); + : path.join(this.config.sessionsDir, memoryOwnerId); + await removeSessionDirUnderMemoryLocks({ + rootDir: this.config.rootDir, + sessionDir, + workspaceId, + attemptId: removalAttemptId, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + beforeTombstone: + ownerSessionDir === undefined + ? undefined + : async () => { + // Legacy-notebook delta too (locks held: the owner store's + // AND this child's own, so a self-fallback backend's late + // note into /memory either landed before this pass + // or is refused). Throws → removal aborts, session intact. + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + memoryOwnerId, + { locksHeld: true } + ); + await migrateSharedMemoryRefinementRows({ + childSessionDir: sessionDir, + childWorkspaceId: workspaceId, + ownerSessionDir, + ownerWorkspaceId: memoryOwnerId, + }); + }, + }); + // Only once the session (and with it the transcript) is gone are the + // retryable harvest records truly unrecoverable; an aborted removal + // above must leave them retryable. + await this.memoryConsolidationService?.finalizeHarvestsForRemoval(workspaceId); + } } catch (error) { // r63: without a durable tombstone the retained orphan stays // writable by foreign backends forever — abort the removal (the From f86659a596487cfc5db8504e931ee493b53888d7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 11:55:06 +0000 Subject: [PATCH 50/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-fou?= =?UTF-8?q?rth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Legacy-notebook adoption: a manifest record without a child sidecar entry now reads as the default unpinned state, so a usage-only entry a downgraded build creates by merely viewing the note is no pin transition and the owner's pin stands; only an actual boolean pin change overrides it. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 12 ++++++++---- src/node/services/memoryService.ts | 15 +++++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index f4d3edafc27..209db9c14c1 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1587,13 +1587,17 @@ describe("MemoryService", () => { projectPath: "", workspaceId: "ws-owner", }); - // Child had viewed it once on the old build (unpinned); adopted. - await fixture.metaService.recordAccess(childKey, { write: false }); + // Adopted before the child ever had a sidecar entry (no view, no pin). await fixture.service.listIndexEntries({ ...fixture.ctx }); // The owner pins the shared copy... await fixture.metaService.setPinned(ownerKey, true); - // ...then the downgraded build merely views the legacy note again: the - // child sidecar changes (usage), its pin bit does not. + // ...then the downgraded build merely views the legacy note: the child + // sidecar gains a usage-only entry — the default unpinned state, not a + // pin transition — so the owner's pin stands. + await fixture.metaService.recordAccess(childKey, { write: false }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // Another view once an entry exists: usage changes, the pin bit does not. await fixture.metaService.recordAccess(childKey, { write: false }); await fixture.service.listIndexEntries({ ...fixture.ctx }); expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 3753b73cf7d..9b2a2e6d514 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -419,9 +419,14 @@ function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { ); } -/** Pin bit of a manifest record's child sidecar fingerprint (null when none was recorded). */ +/** + * Pin bit of a manifest record's child sidecar fingerprint. No child entry at + * that adoption is the default, unpinned state (a usage entry a downgraded + * build creates by merely viewing the note is not a pin transition); null + * only for an unparsable fingerprint. + */ function legacySidecarPinned(sidecar: string): boolean | null { - if (sidecar === "") return null; + if (sidecar === "") return false; try { const parsed: unknown = JSON.parse(sidecar); return typeof parsed === "object" && parsed !== null @@ -1342,8 +1347,10 @@ export class MemoryService extends EventEmitter { // changes its usage counters, which must not drag the owner's pin // back to the child's unchanged value. if (childEntry !== undefined) { - const childPinChanged = - previous !== undefined && legacySidecarPinned(previous.sidecar) !== childEntry.pinned; + const priorPinned = previous === undefined ? null : legacySidecarPinned(previous.sidecar); + // Only an actual boolean transition of the child's pin overrides + // the owner's; an unknown prior state never does. + const childPinChanged = priorPinned !== null && priorPinned !== childEntry.pinned; try { await this.metaService.mergeKeys( childKey, From e7c214acbe08d54e2abcf9cd84be43815d4f92f1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 13:23:10 +0000 Subject: [PATCH 51/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-fif?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A sub-agent's memory probe token now folds in its legacy private notebook's adoption fingerprint (legacy store stamp + child-keyed sidecar entries), so a downgraded backend's edit or pin change — which never moves the owner store's clock — misses the cached session context and the next access adopts it. - Workspace removal that aborts before its point of no return (no tombstone published, workspace intact) lifts the consolidation teardown gate again instead of refusing every later Dream run and harvest until restart. - Harvest gate: a policy-epoch stamp that is present but not the integer equal to the closing epoch (another epoch's, or a corrupted raw-JSON value such as null) refuses the epoch; only an absent stamp is a non-turn row. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../memoryConsolidationService.test.ts | 18 ++++-- .../services/memoryConsolidationService.ts | 30 ++++++--- src/node/services/memoryService.test.ts | 24 +++++++- src/node/services/memoryService.ts | 61 +++++++++++++------ src/node/services/workspaceService.test.ts | 32 ++++++++++ src/node/services/workspaceService.ts | 18 +++++- 6 files changed, 147 insertions(+), 36 deletions(-) diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index a8049594e4c..eabb425f441 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1325,7 +1325,8 @@ describe("MemoryConsolidationService", () => { it("refuses a turn whose policy was recorded for another epoch, and ignores preserved-tail copies", async () => { using fixture = await createFixture(); await fixture.addWorkspace("ws-clean"); - const seed = async (workspaceId: string, foreignTurn: boolean) => { + await fixture.addWorkspace("ws-corrupt"); + const seed = async (workspaceId: string, foreignTurn: number | null | undefined) => { const reset = createMuxMessage("reset-1", "assistant", "", { compactionBoundary: true, compacted: "user", @@ -1357,17 +1358,18 @@ describe("MemoryConsolidationService", () => { workspaceMemoryPolicyEpoch: closingEpoch, }) ); - if (foreignTurn) { + if (foreignTurn !== undefined) { // Backend B started a read-only turn under the previous epoch (-1) and // recorded its deny there; backend A then reset the context (the deny // went with the epoch) and B's assistant landed after the new boundary // — without its user row, which the reset removed. Only the epoch - // stamp can surface it. + // stamp can surface it. A corrupted stamp (raw JSON row) refuses too. await fixture.historyService.appendToHistory( workspaceId, createMuxMessage("b-reply", "assistant", "Read-only output.", { requestHistorySequence: closingEpoch - 1, - workspaceMemoryPolicyEpoch: -1, + // `null` models a corrupted raw-JSON row. + workspaceMemoryPolicyEpoch: foreignTurn as unknown as number, }) ); } @@ -1393,15 +1395,19 @@ describe("MemoryConsolidationService", () => { previousBoundaryHistorySequence: closingEpoch, }); }; - const refused = await seed("ws-dream", true); + const refused = await seed("ws-dream", -1); expect(refused.success).toBe(false); if (!refused.success) expect(refused.error).toContain("another epoch"); expect(fixture.modelCalls).toHaveLength(0); expect((await fixture.service.getStatus("ws-dream")).latestHarvestRecord?.refused).toBe(true); + // `null` is neither "no stamp" nor this epoch's: fail closed. + const corrupt = await seed("ws-corrupt", null); + expect(corrupt.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); // Without the foreign turn the copies alone refuse nothing, and the // harvest transcript leaves them out. - const granted = await seed("ws-clean", false); + const granted = await seed("ws-clean", undefined); expect(granted.success).toBe(true); expect(fixture.modelPrompts.length).toBeGreaterThan(0); expect(fixture.modelPrompts[0]).toContain("concise tests"); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 834062ae39e..6ec04cac01b 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -356,7 +356,8 @@ class HarvestRefusedError extends Error { * stamp (a build that does not maintain the policy — e.g. turns run by a * downgraded build mid-epoch, which also left the durable accumulator * untouched; synthetic payload/summary rows that are no turn) cover nothing - * (fail closed). Token-budget control rows (rollover lead-in, budget + * (fail closed); a stamp that is present but malformed refuses like a foreign + * epoch's. Token-budget control rows (rollover lead-in, budget * warning) need no turn: backend template text appended in the same durable * batch as the turn they precede, carrying neither agent nor repository * content. @@ -374,10 +375,14 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb const bound = message.metadata?.requestHistorySequence; if (typeof bound !== "number") continue; const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; - if (typeof policyEpoch === "number" && policyEpoch !== closingEpoch) { + // No stamp at all: a row that is no turn of this build (covers nothing). + if (policyEpoch === undefined) continue; + // History rows are raw JSON: a stamp that is present but not the integer + // equal to the closing epoch — another epoch's, or a corrupted value such + // as null — proves no policy for this epoch and refuses the harvest. + if (!Number.isInteger(policyEpoch) || policyEpoch !== closingEpoch) { return "the compacted epoch holds a turn whose memory policy was recorded for another epoch; harvest refused (fail closed)"; } - if (policyEpoch === undefined) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); @@ -445,10 +450,9 @@ export class MemoryConsolidationService extends EventEmitter { * post-harvest sweep, and a cancelled run still starts retryable-harvest * recovery, each with a fresh un-aborted signal. Entry points refuse and * new controllers start pre-aborted while a workspace is in this set. - * Entries are never cleared: removal is terminal, and if a force=false - * removal fails after the drain, losing background consolidation for the - * surviving workspace (until restart) matches the documented drained- - * producers tradeoff in WorkspaceService.removeWorkspace. Cross-PROCESS + * Entries are cleared only when removal aborts before its point of no + * return (releaseRemovalCancellation); once the tombstone is published, + * removal is terminal. Cross-PROCESS * teardown is covered by the durable removal tombstone instead (see * workspaceRemoval.ts), checked at memory mutation commit points. */ @@ -736,6 +740,18 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.runPromise(this.cancelInFlightConsolidationEffect(workspaceId)); } + /** + * Removal aborted BEFORE its point of no return (no tombstone published, the + * workspace stays registered and intact — e.g. a non-forced removal whose + * shared-memory handover found the owner notebook full): lift the teardown + * gate again, or the surviving workspace would refuse every Dream run and + * post-compaction harvest until restart. The drained in-flight runs are + * gone regardless (retryable harvests recover on the next trigger). + */ + releaseRemovalCancellation(workspaceId: string): void { + this.removalCancelled.delete(workspaceId); + } + /** Terminal failed record for a policy-refused harvest (see maybeHarvestThenSweep). */ private async recordRefusedHarvest( metadata: CompactionCompletionMetadata, diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index ae1c5aecb5d..7a21ccd760a 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1209,8 +1209,26 @@ describe("MemoryService", () => { await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "v1", "agent"); const afterCreate = await foreign.workspaceMemoryRevision("ws-owner"); expect(afterCreate).not.toBe("missing"); - // Child and owner read the same (owner-keyed) token. - expect(await foreign.workspaceMemoryRevision("ws-child")).toBe(afterCreate); + // The child's token tracks the owner clock, plus its own legacy notebook + // state (see below); the child has none yet. + const childToken = await foreign.workspaceMemoryRevision("ws-child"); + expect(childToken.startsWith(`${afterCreate}\u0000`)).toBe(true); + // A downgraded backend writing the child's LEGACY notebook (or toggling a + // child-keyed pin) moves no owner clock, yet the child's cached context + // must miss so its next access adopts the change. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "legacy"); + const afterLegacyWrite = await foreign.workspaceMemoryRevision("ws-child"); + expect(afterLegacyWrite).not.toBe(childToken); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "old.md", { projectPath: "", workspaceId: "ws-child" }), + true + ); + expect(await foreign.workspaceMemoryRevision("ws-child")).not.toBe(afterLegacyWrite); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + await fsPromises.rm(legacyRoot, { recursive: true, force: true }); // Other scopes leave the workspace store's token alone... await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); @@ -2405,7 +2423,7 @@ describe("MemoryService", () => { expect(clocks[1]!).toBeLessThan(clocks[2]!); // The published token never lags a row's clock (change events tick it once more). expect( - Number(await fixture.service.workspaceMemoryRevision("ws-child")) + Number(await fixture.service.workspaceMemoryRevision("ws-owner")) ).toBeGreaterThanOrEqual(Math.max(...(clocks as number[]))); // The owner's edit is not the newest for that path: refused without force. diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 6e62ec23700..3b3dc242167 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1190,6 +1190,43 @@ export class MemoryService extends EventEmitter { } } + /** + * Fingerprint of a sub-agent's legacy private notebook as seen by the + * adoption pass: owner, legacy root kind, the legacy store's stamp (child + * store clock, root entry, listed files' size/mtime) and the child-keyed + * sidecar entries. Also folded into the child's memory probe token + * (workspaceMemoryRevision): a downgraded backend's edit to the legacy note + * or its pin never moves the OWNER store's clock, so a cached session + * context keyed on that clock alone would keep serving the pre-edit index + * until some unrelated owner mutation. Cheap when no legacy root exists + * (one lstat). + */ + private async legacyAdoptionCheckKey( + childId: string, + owner: string + ): Promise<{ legacyRootKind: Awaited>; checkKey: string }> { + const childSessionDir = path.join(this.config.sessionsDir, childId); + const legacyRoot = path.join(childSessionDir, "memory"); + const legacyRootKind = await lstatKind(legacyRoot); + // Workspace-scope keys embed only the workspace id (see logicalKeyFor). + const childKeyPrefix = memoryLogicalKey("workspace", "", { + projectPath: "", + workspaceId: childId, + }); + const childSidecarFingerprint = + legacyRootKind === "dir" + ? JSON.stringify( + [...(await this.metaService.getEntries())] + .filter(([key]) => key.startsWith(childKeyPrefix)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ) + : ""; + const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ + legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" + }\u0000${childSidecarFingerprint}`; + return { legacyRootKind, checkKey }; + } + /** * The adoption pass (see adoptLegacyPrivateStore). `force` skips the * per-process "already checked" memo: removal wants the pass to run against @@ -1219,22 +1256,7 @@ export class MemoryService extends EventEmitter { // The pass itself is idempotent. const childSessionDir = path.join(this.config.sessionsDir, childId); const legacyRoot = path.join(childSessionDir, "memory"); - const legacyRootKind = await lstatKind(legacyRoot); - const childKeyPrefix = memoryLogicalKey("workspace", "", { - projectPath: ctx.projectPath, - workspaceId: childId, - }); - const childSidecarFingerprint = - legacyRootKind === "dir" - ? JSON.stringify( - [...(await this.metaService.getEntries())] - .filter(([key]) => key.startsWith(childKeyPrefix)) - .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - ) - : ""; - const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ - legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" - }\u0000${childSidecarFingerprint}`; + const { legacyRootKind, checkKey } = await this.legacyAdoptionCheckKey(childId, owner); if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) { return { skipped: 0 }; } @@ -1786,7 +1808,12 @@ export class MemoryService extends EventEmitter { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return "revoked"; } const revision = await readWorkspaceMemoryRevision(path.join(this.config.sessionsDir, owner)); - return revision === null ? "missing" : String(revision); + const token = revision === null ? "missing" : String(revision); + // A redirected sub-agent's token also tracks its legacy private notebook + // (see legacyAdoptionCheckKey): its next store access adopts the change, + // so the cached context must miss as soon as the legacy state moves. + if (owner === workspaceId) return token; + return `${token}\u0000${(await this.legacyAdoptionCheckKey(workspaceId, owner)).checkKey}`; } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 1cdf2e52f82..feafbf1adeb 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -14682,6 +14682,38 @@ describe("WorkspaceService remove desktop session cleanup", () => { expect(reopened).toEqual([workspaceId]); }); + test("remove() lifts the consolidation teardown gate only when it aborts before committing", async () => { + const calls: string[] = []; + workspaceService.setMemoryConsolidationService({ + triggerInBackground: () => undefined, + triggerHarvestThenSweepInBackground: () => undefined, + cancelInFlightConsolidation: () => { + calls.push("cancel"); + return Promise.resolve(); + }, + releaseRemovalCancellation: () => { + calls.push("release"); + }, + finalizeHarvestsForRemoval: () => Promise.resolve(), + }); + // Aborted before the point of no return (live descendant tasks): the + // workspace stays intact, so any teardown gate is lifted again. + let descendants = true; + workspaceService.setAgentTaskIntegration( + makeAgentTaskIntegrationFake({ hasDescendantAgentTasks: () => descendants }) + ); + const aborted = await workspaceService.remove(workspaceId); + expect(aborted.success).toBe(false); + expect(calls).toEqual(["release"]); + // Committed removal: cancelled (drained) and never released. + calls.length = 0; + descendants = false; + const removed = await workspaceService.remove(workspaceId); + expect(removed.success).toBe(true); + expect(calls.filter((call) => call === "cancel").length).toBeGreaterThan(0); + expect(calls).not.toContain("release"); + }); + test("remove() flushes the timeline before deleting the session directory", async () => { const sessionDir = path.join(tempRoot, "sessions", workspaceId); await fsPromises.mkdir(sessionDir, { recursive: true }); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 8abb3c10ed8..d7a0a78b612 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -2757,6 +2757,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + releaseRemovalCancellation(workspaceId: string): void; finalizeHarvestsForRemoval(workspaceId: string): Promise; }; /** Narrow MemoryService surface for removal's shared-memory handover; wired by coreServices. */ @@ -3103,6 +3104,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { triggerInBackground(workspaceId: string, trigger: "compaction" | "archive"): void; triggerHarvestThenSweepInBackground(metadata: CompactionCompletionMetadata): void; cancelInFlightConsolidation(workspaceId: string): Promise; + releaseRemovalCancellation(workspaceId: string): void; finalizeHarvestsForRemoval(workspaceId: string): Promise; }): void { this.memoryConsolidationService = service; @@ -6105,6 +6107,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { .filter((session): session is AgentSession => session != null) .map((session) => session.holdTurnAdmission()); + // Set once removal passes its point of no return (session teardown and + // tombstone follow unconditionally); an abort before that leaves the + // workspace registered and intact, so the finally lifts the consolidation + // teardown gate the drains below installed. + let removalCommitted = false; // Try to remove from runtime (filesystem) try { if (this.agentTaskIntegration?.hasDescendantAgentTasks(workspaceId) === true) { @@ -6199,9 +6206,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // with no second rollup). // Trade-off: a force=false deletion failure below keeps the // workspace but its producers were already drained. That loss is - // recoverable (rerun /refine, refork); a checkout write racing - // deletion is not. Both calls are idempotent; they run again later - // for the phantom-metadata path. + // recoverable (rerun /refine, refork; the consolidation teardown gate + // is lifted again in the finally); a checkout write racing deletion + // is not. Both calls are idempotent; they run again later for the + // phantom-metadata path. // Dream/harvest consolidation is a third producer (r60): its runs // ride only a hard timeout, so removal must abort them explicitly or // a detached run could mutate memory and journal into the deleted @@ -6583,6 +6591,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // // Intentionally deferred until we're committed to removal: if runtime deletion fails with // force=false we return early and keep init state intact so init-end can refresh metadata. + removalCommitted = true; this.initStateManager.clearInMemoryState(workspaceId); // Dispose the session before deleting its directory: disposal aborts the active stream, and @@ -6874,6 +6883,9 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const message = getErrorMessage(error); return Err(`Failed to remove workspace: ${message}`); } finally { + if (!removalCommitted) { + this.memoryConsolidationService?.releaseRemovalCancellation(workspaceId); + } for (const hold of admissionHolds) { hold[Symbol.dispose](); } From b157b20f68815738668f36f8a99e1cb9a9c954bb Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 13:57:12 +0000 Subject: [PATCH 52/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-six?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy-notebook adoption memoizes only a complete pass: one that left a note unrepresented (owner store full, both destinations taken, unreadable, sidecar fold failed) is retried on the next access, since its retry depends on owner-side state the legacy check key does not observe. - A removal aborted inside the locked handover or by a non-durable tombstone (workspace left registered with its session directory) also lifts the consolidation teardown gate, not only pre-drain aborts. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 7 ++-- src/node/services/memoryService.ts | 13 +++++- src/node/services/workspaceService.test.ts | 46 +++++++++++++++++++++- src/node/services/workspaceService.ts | 5 +++ 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 7a21ccd760a..191dc11b05a 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1584,11 +1584,12 @@ describe("MemoryService", () => { "agent" ); expect(full.success).toBe(false); - // Freed capacity lets a later pass (fresh process) fold in the rest. + // Freed capacity lets a later pass of the SAME process fold in the rest: + // an incomplete pass is not memoized, since its retry depends on owner + // state the legacy check key does not observe. await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0000.md", "agent"); await fixture.service.deletePath({ ...fixture.ctx }, "/memories/workspace/o0001.md", "agent"); - const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); - const relisted = (await restarted.listIndexEntries({ ...fixture.ctx })) + const relisted = (await fixture.service.listIndexEntries({ ...fixture.ctx })) .filter((e) => e.scope === "workspace") .map((e) => e.relPath); expect(relisted).toHaveLength(MEMORY_MAX_FILES_PER_SCOPE); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 3b3dc242167..065eaafef08 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1274,6 +1274,11 @@ export class MemoryService extends EventEmitter { // folded in): either changes what the shared store's readers derive from it. let adoptedCount = 0; let skipped = 0; + // Some listed note was not represented this pass (owner store full, both + // destinations taken, unreadable, sidecar fold failed): the retry it + // promises depends on OWNER-side state the check key does not observe, + // so such a pass is never memoized — the next access runs it again. + let incomplete = false; const pass = async (): Promise => { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock @@ -1406,6 +1411,7 @@ export class MemoryService extends EventEmitter { "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", { relPath, error } ); + incomplete = true; continue; } } @@ -1442,7 +1448,12 @@ export class MemoryService extends EventEmitter { } // Recorded against the state observed BEFORE the pass: a foreign write // landing during it changes the stamp and re-runs the (idempotent) pass. - this.legacyStoreCheckedAgainst.set(childId, checkKey); + // Only a complete pass is memoized (see `incomplete`). + if (skipped === 0 && !incomplete) { + this.legacyStoreCheckedAgainst.set(childId, checkKey); + } else { + this.legacyStoreCheckedAgainst.delete(childId); + } if (adoptedCount > 0) this.emitChange(ctx, "workspace", "", "agent"); return { skipped }; } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index feafbf1adeb..f8c1f6b1814 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -14705,9 +14705,53 @@ describe("WorkspaceService remove desktop session cleanup", () => { const aborted = await workspaceService.remove(workspaceId); expect(aborted.success).toBe(false); expect(calls).toEqual(["release"]); + // Aborted inside the locked handover (a late legacy note the owner store + // cannot take), i.e. after the drain but BEFORE the tombstone: the session + // directory survives, so the gate is lifted too. + descendants = false; + calls.length = 0; + const sessionDir = path.join(tempRoot, "sessions", workspaceId); + await fsPromises.mkdir(sessionDir, { recursive: true }); + const topology = { + projects: new Map([ + [ + "/tmp/src/project", + { + workspaces: [ + { path: "/tmp/src/project/owner", id: "ws-owner" }, + { path: "/tmp/src/project/child", id: workspaceId, parentWorkspaceId: "ws-owner" }, + ], + }, + ], + ]), + }; + // The service holds its own copy of the mock config (createWorkspaceServiceForTest). + const config = (workspaceService as unknown as { config: MockWorkspaceConfig }).config; + const previousLoad = config.loadConfigOrDefault; + const previousLoadExisting = config.loadExistingConfigOrThrow; + config.loadConfigOrDefault = (() => topology) as MockWorkspaceConfig["loadConfigOrDefault"]; + config.loadExistingConfigOrThrow = (() => + topology) as MockWorkspaceConfig["loadExistingConfigOrThrow"]; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => + Promise.reject(new Error("1 legacy note could not be folded into the shared notebook")), + }); + try { + const lockedAbort = await workspaceService.remove(workspaceId); + expect(lockedAbort.success).toBe(false); + if (!lockedAbort.success) expect(lockedAbort.error).toContain("tombstone could be published"); + expect(existsSync(sessionDir)).toBe(true); + expect(calls).toContain("cancel"); + expect(calls).toContain("release"); + } finally { + config.loadConfigOrDefault = previousLoad; + config.loadExistingConfigOrThrow = previousLoadExisting; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => Promise.resolve(), + }); + } // Committed removal: cancelled (drained) and never released. calls.length = 0; - descendants = false; const removed = await workspaceService.remove(workspaceId); expect(removed.success).toBe(true); expect(calls.filter((call) => call === "cancel").length).toBeGreaterThan(0); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index d7a0a78b612..91bfeae91fc 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6758,6 +6758,11 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { error instanceof TombstoneNotDurableError || error instanceof SharedMemoryRemovalAbortedError ) { + // No durable tombstone was published (the locked handover or the + // tombstone write itself failed): the workspace stays registered + // with its session directory intact, so the consolidation teardown + // gate is lifted again in the finally like any pre-commit abort. + removalCommitted = false; throw error; } log.error(`Failed to remove session directory for ${workspaceId}:`, error); From d2dd83b21ab6d8834d7a4b38d69e50aa365b1424 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 14:17:45 +0000 Subject: [PATCH 53/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-sev?= =?UTF-8?q?enth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local config-change notification adopts the config-file stamp for the workspace-memory owner memo only once the memo was actually rebuilt from a readable file; an unreadable file at notification time keeps the old stamp so the next resolution retries, like the stamp check on resolve. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 26 +++++++++++++++++++++++++ src/node/services/memoryService.ts | 10 +++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 191dc11b05a..9a33031a7fb 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1198,6 +1198,32 @@ describe("MemoryService", () => { expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); }); + it("keeps the owner memo retryable when a local config edit notifies while the file is unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + // A local edit removes the owner. The change notification fires while + // the file cannot be read (a swallowed late write failure): the memo + // must not be stamped as current, or the stale mapping survives until + // an unrelated rewrite once readability returns without a stamp change. + const real = fixture.config.loadConfigOrDefault.bind(fixture.config); + const unreadable = spyOn(fixture.config, "loadConfigOrDefault").mockImplementation( + (options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("EACCES: permission denied"); + return { ...real(), projects: new Map() }; + } + ); + await fixture.config.editConfig((cfg) => { + for (const project of cfg.projects.values()) { + project.workspaces = project.workspaces.filter((ws) => ws.id !== "ws-owner"); + } + return cfg; + }); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-owner"); + unreadable.mockRestore(); + expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); + }); + it("advances the owner store's revision token on shared writes, visible to another backend", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 065eaafef08..17cf34acdde 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -730,10 +730,14 @@ export class MemoryService extends EventEmitter { // Local edits notify here; edits by ANOTHER backend (multi-instance) are // caught by the config-file stamp check in resolveWorkspaceMemoryOwnerId. // The notification fires after the file write, so adopting the new stamp - // here keeps the next resolve from repeating the invalidation. + // here keeps the next resolve from repeating the invalidation — but only + // once the memo was actually rebuilt from the file: an unreadable file at + // notification time (a swallowed late write failure, an EACCES interval) + // keeps the old stamp so the next resolve retries, exactly like the + // stamp check in resolveWorkspaceMemoryOwnerId. this.config.onConfigChanged(() => { - this.workspaceMemoryOwnerConfigStamp = this.config.configFileStamp(); - this.invalidateWorkspaceMemoryOwnerMemo(); + const stamp = this.config.configFileStamp(); + if (this.invalidateWorkspaceMemoryOwnerMemo()) this.workspaceMemoryOwnerConfigStamp = stamp; }); } From 58cabb3b3dbcd3c9f86bb97634ae19c9809ef7ad Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 15:14:00 +0000 Subject: [PATCH 54/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-eig?= =?UTF-8?q?hth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Sub-agent removal seals the final shared-memory handover and the removal tombstone under the removal locks BEFORE the checkout is deleted (sealSubAgentForRemovalUnderMemoryLocks): a handover the owner store cannot take aborts with the checkout intact, a refused checkout deletion rolls the tombstone back, and `force` accepts the loss instead of being blocked by a persistent handover failure. - Legacy adoption: a failed sidecar fold counts as skipped (strict removal fails instead of stranding the metadata); copies the adoption CREATED follow their legacy source out of the shared notebook when it is deleted or renamed on the old build, while pre-existing or owner-edited notes stay. - Owner memory probe token carries the canonical store's file stamps, so a downgraded build's direct owner-dir write invalidates cached contexts. - Deny marker: a malformed marker replaced by another epoch's deny write is carried as a wildcard deny until the closing boundary clears it. - epochHasPriorTurns ignores RLM preserved-tail copies. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 112 +++++++++-- src/node/services/memoryService.ts | 93 +++++++-- src/node/services/turnRequestBuilder.ts | 9 +- .../services/workspaceMemoryDenyMarker.ts | 60 ++++-- src/node/services/workspaceRemoval.ts | 163 +++++++++++----- src/node/services/workspaceService.test.ts | 181 +++++++++++++++++- src/node/services/workspaceService.ts | 116 ++++++++--- 7 files changed, 613 insertions(+), 121 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 9a33031a7fb..ce3a552151e 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -97,6 +97,9 @@ function projectMemoryRoot(fixture: MemoryFixture): string { ); } +/** Store clock segment of a workspaceMemoryRevision token (the rest are file/legacy stamps). */ +const clockOf = (token: string): number => Number(MemoryService.revisionClockOf(token)); + describe("MemoryService", () => { describe("create + view round-trip", () => { it("creates and views a global memory file at /memory/global", async () => { @@ -1230,7 +1233,9 @@ describe("MemoryService", () => { // A second MemoryService over the same Xum root stands in for another // backend process: it receives none of this instance's change events. const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); - expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe("missing"); + expect(MemoryService.revisionClockOf(await foreign.workspaceMemoryRevision("ws-owner"))).toBe( + "missing" + ); await fixture.service.create(fixture.ctx, "/memories/workspace/shared.md", "v1", "agent"); const afterCreate = await foreign.workspaceMemoryRevision("ws-owner"); @@ -1259,10 +1264,20 @@ describe("MemoryService", () => { // Other scopes leave the workspace store's token alone... await fixture.service.create(fixture.ctx, "/memories/global/g.md", "g", "agent"); expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(afterCreate); + // ...a downgraded build writing straight into the owner's canonical + // notebook moves no clock, but the token still changes (file stamps)... + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "old-build.md"), + "written by a downgraded build" + ); + const afterOldBuildWrite = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterOldBuildWrite).not.toBe(afterCreate); + expect(clockOf(afterOldBuildWrite)).toBe(clockOf(afterCreate)); // ...a pin toggle (hot-set input, no store write) advances it... await fixture.service.setPinned(fixture.ctx, "/memories/workspace/shared.md", true); const afterPin = await foreign.workspaceMemoryRevision("ws-owner"); - expect(Number(afterPin)).toBeGreaterThan(Number(afterCreate)); + expect(clockOf(afterPin)).toBeGreaterThan(clockOf(afterCreate)); // ...while every shared-store mutation advances it. await fixture.service.strReplace( fixture.ctx, @@ -1272,7 +1287,7 @@ describe("MemoryService", () => { "agent" ); const afterEdit = await foreign.workspaceMemoryRevision("ws-owner"); - expect(Number(afterEdit)).toBeGreaterThan(Number(afterPin)); + expect(clockOf(afterEdit)).toBeGreaterThan(clockOf(afterPin)); // A read-side access (view / recall) re-ranks the shared hot set through // the owner-keyed usage stats: it advances the clock and announces the // owner's store like a pin does, so the rest of the tree (and other @@ -1283,7 +1298,7 @@ describe("MemoryService", () => { (await fixture.service.view(fixture.ctx, "/memories/workspace/shared.md")).success ).toBe(true); const afterView = await foreign.workspaceMemoryRevision("ws-owner"); - expect(Number(afterView)).toBeGreaterThan(Number(afterEdit)); + expect(clockOf(afterView)).toBeGreaterThan(clockOf(afterEdit)); expect(events).toHaveLength(1); expect(events[0]).toMatchObject({ scope: "workspace", @@ -1291,8 +1306,8 @@ describe("MemoryService", () => { workspaceId: "ws-owner", }); await fixture.service.recordRecall(fixture.ctx, "/memories/workspace/shared.md"); - expect(Number(await foreign.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( - Number(afterView) + expect(clockOf(await foreign.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + clockOf(afterView) ); expect(events).toHaveLength(2); // Global reads have no store clock and stay silent. @@ -1493,11 +1508,11 @@ describe("MemoryService", () => { const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); const restartedEvents: unknown[] = []; restarted.on("change", (event) => restartedEvents.push(event)); - const revisionBefore = Number(await fixture.service.workspaceMemoryRevision("ws-owner")); + const revisionBefore = clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")); const relisted = await restarted.listIndexEntries(fixture.ctx); // The pass wrote one file and copied one pin: both change what other // backends derive from the store, so the clock moved and the tabs heard. - expect(Number(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + expect(clockOf(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( revisionBefore ); expect(restartedEvents).toHaveLength(1); @@ -1508,9 +1523,9 @@ describe("MemoryService", () => { const metaOnly = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); const metaOnlyEvents: unknown[] = []; metaOnly.on("change", (event) => metaOnlyEvents.push(event)); - const revisionMid = Number(await fixture.service.workspaceMemoryRevision("ws-owner")); + const revisionMid = clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")); await metaOnly.listIndexEntries(fixture.ctx); - expect(Number(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( + expect(clockOf(await fixture.service.workspaceMemoryRevision("ws-owner"))).toBeGreaterThan( revisionMid ); expect(metaOnlyEvents).toHaveLength(1); @@ -1702,6 +1717,77 @@ describe("MemoryService", () => { ).toBe(true); }); + it("follows legacy deletions and renames for copies the adoption created, never owner notes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // `same.md` pre-exists identically on the owner side (reused, not created); + // `mine.md` and `moved.md` are created by the adoption; `edited.md` too, + // but the owner edits it afterwards. + await fsPromises.writeFile(path.join(ownerRoot, "same.md"), "identical"); + for (const [name, body] of [ + ["same.md", "identical"], + ["mine.md", "child note"], + ["moved.md", "to be renamed"], + ["edited.md", "child draft"], + ]) { + await fsPromises.writeFile(path.join(legacyRoot, name), body); + } + await fixture.service.listIndexEntries({ ...fixture.ctx }); + for (const name of ["same.md", "mine.md", "moved.md", "edited.md"]) { + expect( + await fsPromises.stat(path.join(ownerRoot, name)).then( + () => true, + () => false + ) + ).toBe(true); + } + await fixture.service.setPinned({ ...fixture.ctx }, "/memories/workspace/mine.md", true); + await fixture.service.strReplace( + { ...fixture.ctx }, + "/memories/workspace/edited.md", + "draft", + "final", + "agent" + ); + // The downgraded build deletes same.md and mine.md, renames moved.md, and + // deletes edited.md. + await new Promise((resolve) => setTimeout(resolve, 5)); + for (const name of ["same.md", "mine.md", "edited.md"]) { + await fsPromises.rm(path.join(legacyRoot, name)); + } + await fsPromises.rename( + path.join(legacyRoot, "moved.md"), + path.join(legacyRoot, "renamed.md") + ); + const relisted = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + // Created + unchanged copies are gone (mine.md, moved.md); the reused + // owner note and the owner-edited copy stay; the rename's new name is + // adopted. + expect(relisted).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "edited.md"), "utf-8")).toBe( + "child final" + ); + // The removed copy's owner-side pin went with it. + expect( + (await fixture.metaService.getPinnedKeys()).has( + memoryLogicalKey("workspace", "mine.md", { projectPath: "", workspaceId: "ws-owner" }) + ) + ).toBe(false); + // Idempotent: a further pass changes nothing. + const again = (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort(); + expect(again).toEqual(relisted); + }); + it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2450,7 +2536,7 @@ describe("MemoryService", () => { expect(clocks[1]!).toBeLessThan(clocks[2]!); // The published token never lags a row's clock (change events tick it once more). expect( - Number(await fixture.service.workspaceMemoryRevision("ws-owner")) + clockOf(await fixture.service.workspaceMemoryRevision("ws-owner")) ).toBeGreaterThanOrEqual(Math.max(...(clocks as number[]))); // The owner's edit is not the newest for that path: refused without force. @@ -2471,13 +2557,13 @@ describe("MemoryService", () => { }); expect(undone.success).toBe(true); const after = await fixture.service.workspaceMemoryRevision("ws-owner"); - expect(Number(after)).toBeGreaterThan(Number(before)); + expect(clockOf(after)).toBeGreaterThan(clockOf(before)); // ...and its row takes the next clock value, so the owner's edit is now // the newest and rolls back cleanly. const rollbackRow = (await readRefinementEvents(childSessionDir)).find( (row) => row.data.rollbackOf === childEdit.id )!; - expect(rollbackRow.data.sourceTs).toBe(Number(after)); + expect(rollbackRow.data.sourceTs).toBe(clockOf(after)); expect( ( await rollbackRefinement({ diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 17cf34acdde..8d6170c70e6 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -408,11 +408,17 @@ const LEGACY_IMPORT_DIR = "imported"; */ const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; -/** One adopted legacy file: content hash, child sidecar fingerprint, owner-store relPath. */ +/** + * One adopted legacy file: content hash, child sidecar fingerprint, owner-store + * relPath, and whether the adoption CREATED that owner file (provenance: only + * such a copy may be removed again when the legacy source disappears; a + * pre-existing identical owner note is the owner's own). + */ interface LegacyAdoptionRecord { content: string; sidecar: string; target: string; + created?: boolean; } function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { @@ -1278,11 +1284,6 @@ export class MemoryService extends EventEmitter { // folded in): either changes what the shared store's readers derive from it. let adoptedCount = 0; let skipped = 0; - // Some listed note was not represented this pass (owner store full, both - // destinations taken, unreadable, sidecar fold failed): the retry it - // promises depends on OWNER-side state the check key does not observe, - // so such a pass is never memoized — the next access runs it again. - let incomplete = false; const pass = async (): Promise => { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock @@ -1328,6 +1329,7 @@ export class MemoryService extends EventEmitter { content: sha256Hex(content), sidecar: childEntry === undefined ? "" : JSON.stringify(childEntry), target: "", + created: false, }; const previous = adopted.get(relPath); if (previous?.content === record.content && previous.sidecar === record.sidecar) { @@ -1349,7 +1351,10 @@ export class MemoryService extends EventEmitter { (await this.readBoundedTextFile(store, previous.target, previous.target).catch( () => null )) === content; - if (stillAdopted) target = { relPath: previous.target, write: false }; + if (stillAdopted) { + target = { relPath: previous.target, write: false }; + record.created = previous.created === true; + } } if (target === null) { target = await this.legacyImportTarget(store, childId, relPath, content); @@ -1382,6 +1387,7 @@ export class MemoryService extends EventEmitter { await store.writeFile(target.relPath, content); remainingCapacity--; imported++; + record.created = true; } } record.target = target.relPath; @@ -1415,7 +1421,10 @@ export class MemoryService extends EventEmitter { "[MemoryService] failed to fold legacy memory stats into the shared store; retrying on next access", { relPath, error } ); - incomplete = true; + // Counts as skipped: the note's pin/usage metadata is still + // stranded under the child key, and removal must not delete the + // child session (the only trigger for a retry) on that basis. + skipped++; continue; } } @@ -1423,6 +1432,48 @@ export class MemoryService extends EventEmitter { manifestDirty = true; adoptedCount++; } + // Legacy notes deleted or renamed on the downgraded build: a copy THIS + // adoption created, still holding the adopted bytes, follows the source + // out of the shared notebook (a rename's new name is adopted above like + // a fresh note). Provenance and unchanged content are both required — + // an owner note that merely happened to be identical, or an adopted + // copy the owner has since edited, is the owner's and stays. Unlisted + // sources are only ever judged against the listing that succeeded + // above; a failed listing never reaches this point. + const listed = new Set(files); + for (const [relPath, previous] of adopted) { + if (listed.has(relPath)) continue; + if (previous.created === true) { + const current = + (await store.assertContained(previous.target).then( + () => true, + () => false + )) && (await store.kind(previous.target)) === "file" + ? await this.readBoundedTextFile(store, previous.target, previous.target).catch( + () => null + ) + : null; + const unchanged = current !== null && sha256Hex(current) === previous.content; + if (unchanged) { + await store.remove(previous.target); + await this.metaService.removeKeys( + memoryLogicalKey("workspace", previous.target, { + projectPath: ctx.projectPath, + workspaceId: owner, + }) + ); + adoptedCount++; + log.info("[MemoryService] removed an adopted legacy note deleted on the old build", { + childId, + owner, + relPath, + target: previous.target, + }); + } + } + adopted.delete(relPath); + manifestDirty = true; + } if (manifestDirty) { await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { encoding: "utf-8", @@ -1452,8 +1503,11 @@ export class MemoryService extends EventEmitter { } // Recorded against the state observed BEFORE the pass: a foreign write // landing during it changes the stamp and re-runs the (idempotent) pass. - // Only a complete pass is memoized (see `incomplete`). - if (skipped === 0 && !incomplete) { + // Only a complete pass is memoized: a note left unrepresented (owner + // store full, both destinations taken, unreadable, sidecar fold failed) + // is retried on the next access, and that retry depends on OWNER-side + // state the check key does not observe. + if (skipped === 0) { this.legacyStoreCheckedAgainst.set(childId, checkKey); } else { this.legacyStoreCheckedAgainst.delete(childId); @@ -1822,8 +1876,18 @@ export class MemoryService extends EventEmitter { for (const guarded of new Set([workspaceId, owner])) { if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return "revoked"; } - const revision = await readWorkspaceMemoryRevision(path.join(this.config.sessionsDir, owner)); - const token = revision === null ? "missing" : String(revision); + const ownerSessionDir = path.join(this.config.sessionsDir, owner); + const revision = await readWorkspaceMemoryRevision(ownerSessionDir); + // The clock is what this build's writers advance. A downgraded build + // sharing the root writes straight into the owner's canonical notebook + // without touching it, so the token also carries the store's own file + // stamps (root mtime, per-file size + mtime; bounded by the per-scope + // file cap): stat fingerprints are cache hints — enough to make a cached + // context miss — never proof that authorizes any mutation. + const token = `${revision === null ? "missing" : String(revision)}\u0000${await legacyStoreStamp( + ownerSessionDir, + workspaceMemoryStorePath(this.config.sessionsDir, owner) + )}`; // A redirected sub-agent's token also tracks its legacy private notebook // (see legacyAdoptionCheckKey): its next store access adopts the change, // so the cached context must miss as soon as the legacy state moves. @@ -1831,6 +1895,11 @@ export class MemoryService extends EventEmitter { return `${token}\u0000${(await this.legacyAdoptionCheckKey(workspaceId, owner)).checkKey}`; } + /** The store clock segment of a workspaceMemoryRevision token (tests, diagnostics). */ + static revisionClockOf(token: string): string { + return token.split("\u0000", 1)[0] ?? token; + } + /** * Announces memory files mutated outside this service by a refinement * rollback (which applies inverses straight to disk). Physical paths are diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index a52d2a6e75b..400cb76b141 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1512,11 +1512,18 @@ export class TurnRequestBuilder { ? [] : [latestUserMessage.id, ...(latestUserMessage.metadata?.requestPreludeMessageIds ?? [])] ); + // RLM keep-recent copies are the previous epoch's turns re-appended + // after the boundary (compactionHandler), not turns of this epoch: the + // harvest gate skips them too, and counting them would make another + // backend's first turn of the new epoch — racing the compacting + // backend's asynchronous policy carry — record an unknown-history + // deny for an all-writable epoch. return activeContextMessages.some( (message) => message.role === "user" && !currentBatch.has(message.id) && - message.metadata?.muxMetadata?.type !== "compaction-request" + message.metadata?.muxMetadata?.type !== "compaction-request" && + message.metadata?.rlmPreservedTailCopy !== true ); })(); // The compaction epoch this turn's policy accumulates over: the latest diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index ae2158ed481..1d0f0feffef 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -22,7 +22,8 @@ * compacting backend observes it (see workspaceMemoryPolicyEpochs.ts). * * Fail-closed by construction: a missing marker (or one without an entry for - * this epoch) is "no deny"; a present entry for this epoch, or a + * this epoch) is "no deny"; a present entry for this epoch, a wildcard entry + * (inherited from a malformed marker some later writer replaced), or a * malformed/unreadable marker, is a deny. */ import * as fsPromises from "node:fs/promises"; @@ -49,33 +50,44 @@ export async function writeWorkspaceMemoryDenyMarker( const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); await fsPromises.mkdir(sessionDir, { recursive: true }); const record = await readMarkerRecord(markerPath); - // A malformed marker was a deny for every reader while it existed; a - // well-formed write for this epoch supersedes it (the boundary clear would - // heal it the same way). + // A malformed marker was a deny for EVERY epoch's reader while it existed, + // possibly the only evidence of some other backend's read-only turn in + // the closing epoch. This writer, recording a deny for its own epoch, + // cannot know which epoch that was, so the well-formed file it leaves + // behind carries the deny forward as a wildcard until a boundary + // observation clears it (clearWorkspaceMemoryDenyMarker). const epochs = record === "absent" || record === null ? [] : record.epochs; - await writeMarkerRecord(markerPath, [...epochs.filter((e) => e !== epoch), epoch]); + const wildcard = record === null || (record !== "absent" && record.wildcard); + await writeMarkerRecord(markerPath, [...epochs.filter((e) => e !== epoch), epoch], wildcard); if (!(await readWorkspaceMemoryDenyMarker(sessionDir, epoch))) { throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); } } -async function writeMarkerRecord(markerPath: string, epochs: readonly number[]): Promise { +/** `wildcard`: a deny of unknown epoch (inherited from a malformed marker), denying every reader. */ +async function writeMarkerRecord( + markerPath: string, + epochs: readonly number[], + wildcard: boolean +): Promise { const retained = [...epochs] .sort((a, b) => b - a) .slice(0, WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED); - await writeFileAtomic(markerPath, JSON.stringify({ deniedAt: Date.now(), epochs: retained }), { - encoding: "utf-8", - }); + await writeFileAtomic( + markerPath, + JSON.stringify({ deniedAt: Date.now(), epochs: retained, wildcard }), + { encoding: "utf-8" } + ); } /** Parsed marker, or null when it is unreadable/malformed (which readers treat as a deny). */ async function readMarkerRecord( markerPath: string -): Promise<{ epochs: number[] } | "absent" | null> { +): Promise<{ epochs: number[]; wildcard: boolean } | "absent" | null> { try { const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); if (typeof parsed !== "object" || parsed === null) return null; - const { epochs } = parsed as { epochs?: unknown }; + const { epochs, wildcard } = parsed as { epochs?: unknown; wildcard?: unknown }; if ( !Array.isArray(epochs) || !epochs.every( @@ -84,7 +96,7 @@ async function readMarkerRecord( ) { return null; } - return { epochs }; + return { epochs, wildcard: wildcard === true }; } catch (error) { return hasErrorCode(error, "ENOENT") ? "absent" : null; } @@ -103,7 +115,7 @@ export async function readWorkspaceMemoryDenyMarker( const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); if (record === "absent") return false; if (record === null || epoch === undefined) return true; - return record.epochs.includes(epoch); + return record.wildcard || record.epochs.includes(epoch); } /** @@ -134,10 +146,12 @@ export async function clearWorkspaceMemoryDenyMarker( const record = await readMarkerRecord(markerPath); if (record === "absent") return; if (record !== null) { + // This boundary observed the closing epoch (deny included): the + // epoch-less wildcard, if any, is consumed with it. const remaining = record.epochs.filter((epoch) => epoch !== options.closingEpoch); - if (remaining.length === record.epochs.length) return; + if (remaining.length === record.epochs.length && !record.wildcard) return; if (remaining.length > 0) { - await writeMarkerRecord(markerPath, remaining); + await writeMarkerRecord(markerPath, remaining, false); if (await readWorkspaceMemoryDenyMarker(sessionDir, options.closingEpoch)) { throw new Error(`Workspace memory deny marker could not be cleared at ${markerPath}`); } @@ -169,11 +183,17 @@ export async function carryWorkspaceMemoryDenyMarker( await withTargetMutationLock(rootDir, sessionDir, async () => { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); const record = await readMarkerRecord(markerPath); - if (record === "absent" || !record?.epochs.includes(closingEpoch)) return; - await writeMarkerRecord(markerPath, [ - ...record.epochs.filter((epoch) => epoch !== closingEpoch && epoch !== nextEpoch), - nextEpoch, - ]); + if (record === "absent" || record === null) return; + if (!record.wildcard && !record.epochs.includes(closingEpoch)) return; + // The wildcard denied the closing epoch; carried as the new epoch's deny. + await writeMarkerRecord( + markerPath, + [ + ...record.epochs.filter((epoch) => epoch !== closingEpoch && epoch !== nextEpoch), + nextEpoch, + ], + false + ); if (!(await readWorkspaceMemoryDenyMarker(sessionDir, nextEpoch))) { throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); } diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 29761d8af4c..758adfe0dcf 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -153,6 +153,14 @@ export async function removeSessionDirUnderMemoryLocks(args: { rootDir: string; sessionDir: string; workspaceId: string; + /** + * The tombstone was already published under these same locks by + * sealSubAgentForRemovalUnderMemoryLocks (sub-agents: before the checkout + * was deleted). Nothing fallible remains before the rm, and a lock failure + * here takes the orphan path instead of aborting a removal whose checkout + * is already gone. + */ + tombstoneSealed?: boolean; /** * Unique ID of THIS removal attempt, stamped into the tombstone (r66). * The caller's compensating rollback deletes the marker only while it @@ -186,6 +194,108 @@ export async function removeSessionDirUnderMemoryLocks(args: { typeof args.rootDir === "string" && args.rootDir.length > 0, "removeSessionDirUnderMemoryLocks requires a rootDir" ); + assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); + let tombstonePublishedUnderLocks = args.tombstoneSealed === true; + try { + await withRemovalLocks(args, async () => { + await args.beforeTombstone?.(); + // Tombstone BEFORE rm: once the locks release, any waiting writer + // re-checks it pre-commit (inside its own lock) and refuses, so the + // deleted directory cannot be recreated by a late mutation or + // journal append. + await publishRemovalTombstone(args); + tombstonePublishedUnderLocks = true; + await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); + }); + } catch (error) { + // The orphan path below assumes a wedged writer's target is THIS + // workspace's retained session dir. A sub-agent's admitted memory write + // targets its OWNER's live notebook instead, so unless the tombstone was + // already published UNDER the owner-store lock, publishing it outside + // (after the lock was released, or never taken) would let a holder that + // passed its commit check — or a new writer — finish after removal. + // Abort instead: the workspace stays registered and removal is retried. + if (args.sharedWorkspaceMemorySessionDir !== undefined && !tombstonePublishedUnderLocks) { + throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); + } + // Fail-closed orphan path (r62): a wedged writer blocks the deletion, + // but the caller proceeds to deregister the workspace regardless — so + // the terminal marker must still become durable or a foreign backend + // would keep mutating memory and journaling into the retained orphan + // forever. Publishing outside the locks is safe on THIS path precisely + // because the directory is not deleted: a writer mid-commit lands in + // the orphan, and every later mutation observes the tombstone. + try { + await publishRemovalTombstone(args); + } catch (publishError) { + // No durable marker could be written at all (r63, e.g. ENOSPC): + // deregistering now would leave the orphan writable again the moment + // the transient failure clears. Signal the caller to ABORT the + // removal so the workspace stays registered and retryable. + throw new TombstoneNotDurableError(args.workspaceId, { cause: publishError }); + } + throw error; + } +} + +/** + * Sub-agent removal, BEFORE the checkout is deleted: run the final, fallible + * shared-memory handover (legacy-notebook adoption + refinement-row delta) + * under the full removal lock set and publish the removal tombstone in the + * same critical section. From here on no backend — including a downgraded + * one, which honors the same locks and tombstone at its memory commit + * points — can add to the child's notebooks, so the later session-dir + * deletion has nothing fallible left in front of it. Runs before runtime + * deletion so a handover failure aborts with the checkout intact (the + * caller rolls the tombstone back if the checkout deletion is then refused). + * Any failure aborts the removal (SharedMemoryRemovalAbortedError). + */ +export async function sealSubAgentForRemovalUnderMemoryLocks(args: { + rootDir: string; + sessionDir: string; + workspaceId: string; + attemptId: string; + sharedWorkspaceMemorySessionDir: string; + beforeTombstone: () => Promise; +}): Promise { + try { + await withRemovalLocks(args, async () => { + await args.beforeTombstone(); + await publishRemovalTombstone(args); + }); + } catch (error) { + throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); + } +} + +async function publishRemovalTombstone(args: { + rootDir: string; + workspaceId: string; + attemptId: string; +}): Promise { + assert(args.attemptId.length > 0, "removal tombstone requires an attemptId"); + const tombstonePath = workspaceRemovalTombstonePath(args.rootDir, args.workspaceId); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await writeFileAtomic( + tombstonePath, + JSON.stringify({ + workspaceId: args.workspaceId, + removedAt: Date.now(), + attemptId: args.attemptId, + }) + ); +} + +/** The removal critical section's lock set (see removeSessionDirUnderMemoryLocks). */ +async function withRemovalLocks( + args: { + rootDir: string; + sessionDir: string; + workspaceId: string; + sharedWorkspaceMemorySessionDir?: string; + }, + body: () => Promise +): Promise { // Same key derivations as MemoryService.storeLockKey: the workspace store // root lives inside the session directory; global/project mutations hold // the coarse `/memory` key while journaling into this session dir. @@ -207,21 +317,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { // sidecar writers (headless usage) serialize their tombstone check + // commit against this same key, closing their check→write window. const sessionDirKey = path.resolve(args.sessionDir); - assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); - const publishTombstone = async (): Promise => { - const tombstonePath = workspaceRemovalTombstonePath(args.rootDir, args.workspaceId); - await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); - await writeFileAtomic( - tombstonePath, - JSON.stringify({ - workspaceId: args.workspaceId, - removedAt: Date.now(), - attemptId: args.attemptId, - }) - ); - }; - let tombstonePublishedUnderLocks = false; - try { + { // Refine serialization (r66) — acquired FIRST (r67): a /refine apply in // ANOTHER backend is untouched by the remover's process-local // cancellation and holds this same (session-dir-external) lock across @@ -257,44 +353,9 @@ export async function removeSessionDirUnderMemoryLocks(args: { timeoutMs: 10_000, label: "history write lock (removal)", }); - await args.beforeTombstone?.(); - // Tombstone BEFORE rm: once the locks release, any waiting writer - // re-checks it pre-commit (inside its own lock) and refuses, so the - // deleted directory cannot be recreated by a late mutation or - // journal append. - await publishTombstone(); - tombstonePublishedUnderLocks = true; - await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); + await body(); } ); - } catch (error) { - // The orphan path below assumes a wedged writer's target is THIS - // workspace's retained session dir. A sub-agent's admitted memory write - // targets its OWNER's live notebook instead, so unless the tombstone was - // already published UNDER the owner-store lock, publishing it outside - // (after the lock was released, or never taken) would let a holder that - // passed its commit check — or a new writer — finish after removal. - // Abort instead: the workspace stays registered and removal is retried. - if (ownerMemoryKeys.length > 0 && !tombstonePublishedUnderLocks) { - throw new SharedMemoryRemovalAbortedError(args.workspaceId, { cause: error }); - } - // Fail-closed orphan path (r62): a wedged writer blocks the deletion, - // but the caller proceeds to deregister the workspace regardless — so - // the terminal marker must still become durable or a foreign backend - // would keep mutating memory and journaling into the retained orphan - // forever. Publishing outside the locks is safe on THIS path precisely - // because the directory is not deleted: a writer mid-commit lands in - // the orphan, and every later mutation observes the tombstone. - try { - await publishTombstone(); - } catch (publishError) { - // No durable marker could be written at all (r63, e.g. ENOSPC): - // deregistering now would leave the orphan writable again the moment - // the transient failure clears. Signal the caller to ABORT the - // removal so the workspace stays registered and retryable. - throw new TombstoneNotDurableError(args.workspaceId, { cause: publishError }); - } - throw error; } } diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index f8c1f6b1814..4cd4989c794 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -5,7 +5,10 @@ import { workspaceMemoryDenyMarkerPath, writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; -import { workspaceRemovalTombstonePath } from "@/node/services/workspaceRemoval"; +import { + workspaceRemovalTombstonePath, + isWorkspaceRemovalTombstoned, +} from "@/node/services/workspaceRemoval"; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; @@ -9406,6 +9409,19 @@ describe("WorkspaceService initialize", () => { // deny — heals it and the next epoch can become writable again. await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); + // Another backend's new-epoch deny write must not heal the malformed + // marker away (it may be the only evidence of a read-only turn in the + // closing epoch): the deny is carried as a wildcard until the closing + // boundary observes and clears it. + await writeWorkspaceMemoryDenyMarker(sessionDir, 7); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 3)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 7 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); @@ -14366,6 +14382,169 @@ describe("WorkspaceService remove timing rollup", () => { }); }); +describe("WorkspaceService remove sub-agent handover ordering", () => { + // A sub-agent's final shared-memory handover + removal tombstone are sealed + // under the removal locks BEFORE the checkout is deleted: a handover the + // owner store cannot take aborts with the checkout intact, and a refused + // checkout deletion rolls the tombstone back. + const projectPath = "/tmp/proj-handover"; + const workspaceId = "child-handover"; + const ownerId = "owner-handover"; + const workspacePath = path.join(projectPath, "child-ws"); + const runtimeConfig = { type: "worktree" as const, srcBaseDir: "/tmp/src" }; + let rootDir: string; + + beforeEach(async () => { + rootDir = path.join(tmpdir(), "mux-handover-order", `root-${crypto.randomUUID()}`); + await fsPromises.mkdir(path.join(rootDir, "sessions", workspaceId), { recursive: true }); + }); + afterEach(async () => { + await fsPromises.rm(rootDir, { recursive: true, force: true }); + }); + + function buildConfig(): Partial { + const topology = { + projects: new Map([ + [ + projectPath, + { + trusted: true, + workspaces: [ + { + id: ownerId, + name: "owner", + path: path.join(projectPath, "owner-ws"), + runtimeConfig, + }, + { + id: workspaceId, + name: "child", + path: workspacePath, + runtimeConfig, + parentWorkspaceId: ownerId, + }, + ], + }, + ], + ]), + }; + return { + rootDir, + srcDir: "/tmp/src", + sessionsDir: path.join(rootDir, "sessions"), + removeWorkspace: mock(() => Promise.resolve()), + findWorkspace: mock(() => ({ workspacePath, projectPath })), + loadConfigOrDefault: mock(() => topology), + editConfig: mock((edit: (cfg: typeof topology) => typeof topology) => + Promise.resolve(edit(topology)) + ), + } as unknown as Partial; + } + + function buildAiService(): AIService { + return { + ...createStreamLifecycleMocks(), + isStreaming: mock(() => false), + stopStream: mock(() => Promise.resolve(Ok(undefined))), + getWorkspaceMetadata: mock(() => + Promise.resolve( + Ok({ + id: workspaceId, + name: "child", + projectPath, + projectName: "proj", + runtimeConfig, + parentWorkspaceId: ownerId, + }) + ) + ), + on: mock(() => undefined), + off: mock(() => undefined), + } as unknown as AIService; + } + + test("a handover the owner cannot take aborts before the checkout is deleted", async () => { + const deleteWorkspace = mock(() => + Promise.resolve({ success: true as const, deletedPath: workspacePath }) + ); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockReturnValue({ + deleteWorkspace, + } as unknown as ReturnType); + try { + const workspaceService = createWorkspaceServiceForTest({ + config: buildConfig(), + aiService: buildAiService(), + }); + let adoptions = 0; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: (_child, _owner, options) => { + adoptions++; + // The unlocked pre-pass succeeds; the late note appears for the + // locked pass, which cannot place it. + return options?.locksHeld + ? Promise.reject(new Error("1 legacy note could not be folded")) + : Promise.resolve(); + }, + }); + const result = await workspaceService.remove(workspaceId); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("could not be folded"); + expect(adoptions).toBe(2); + expect(deleteWorkspace).not.toHaveBeenCalled(); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); + // force accepts the loss and completes the removal. + expect((await workspaceService.remove(workspaceId, true)).success).toBe(true); + expect(deleteWorkspace).toHaveBeenCalledTimes(1); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(false); + } finally { + createRuntimeSpy.mockRestore(); + } + }); + + test("a refused checkout deletion rolls the sealed tombstone back", async () => { + let refuse = true; + const deleteWorkspace = mock(() => + Promise.resolve( + refuse + ? { success: false as const, error: "Workspace has uncommitted changes" } + : { success: true as const, deletedPath: workspacePath } + ) + ); + const createRuntimeSpy = spyOn(runtimeFactory, "createRuntime").mockReturnValue({ + deleteWorkspace, + } as unknown as ReturnType); + try { + const workspaceService = createWorkspaceServiceForTest({ + config: buildConfig(), + aiService: buildAiService(), + }); + let sealedTombstone = false; + workspaceService.setSharedWorkspaceMemoryStore({ + adoptLegacyPrivateStoreForRemoval: () => Promise.resolve(), + }); + deleteWorkspace.mockImplementation(async () => { + // Runtime deletion runs with the tombstone already sealed. + sealedTombstone = await isWorkspaceRemovalTombstoned(rootDir, workspaceId); + return refuse + ? { success: false as const, error: "Workspace has uncommitted changes" } + : { success: true as const, deletedPath: workspacePath }; + }); + const refused = await workspaceService.remove(workspaceId); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("uncommitted changes"); + expect(sealedTombstone).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); + expect(existsSync(path.join(rootDir, "sessions", workspaceId))).toBe(true); + refuse = false; + expect((await workspaceService.remove(workspaceId)).success).toBe(true); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(true); + } finally { + createRuntimeSpy.mockRestore(); + } + }); +}); + describe("WorkspaceService remove shared-workspace guard", () => { const projectPath = "/tmp/proj-shared"; const workspaceId = "child-shared"; diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 91bfeae91fc..a2f438bc55b 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -138,6 +138,7 @@ import { healRemovalTombstonesForRegisteredWorkspaces, isWorkspaceRemovalTombstoned, removeSessionDirUnderMemoryLocks, + sealSubAgentForRemovalUnderMemoryLocks, SharedMemoryRemovalAbortedError, refineApplyLockPath, rollbackRemovalTombstoneIfOwned, @@ -4430,6 +4431,41 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return true; } + /** + * Removal's in-lock shared-memory handover for a sub-agent (owner store's + * lock AND the child's own held by the caller): legacy-notebook adoption — + * a self-fallback backend's late note into /memory either landed + * before this pass or is refused — then the refinement-row delta. Throws + * so removal aborts with the session intact; `force` logs and proceeds + * instead, accepting the loss (the user asked for the deletion regardless). + */ + private async lockedSharedMemoryHandover( + workspaceId: string, + ownerWorkspaceId: string, + force: boolean + ): Promise { + try { + await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( + workspaceId, + ownerWorkspaceId, + { locksHeld: true } + ); + await migrateSharedMemoryRefinementRows({ + childSessionDir: path.join(this.config.sessionsDir, workspaceId), + childWorkspaceId: workspaceId, + ownerSessionDir: path.join(this.config.sessionsDir, ownerWorkspaceId), + ownerWorkspaceId, + }); + } catch (error) { + if (!force) throw error; + log.warn("Forced removal: locked shared-memory handover to the owner failed", { + workspaceId, + ownerWorkspaceId, + error: getErrorMessage(error), + }); + } + } + /** Transfer destructive cleanup out of a callback that still owns a session lease. */ deferWorkspaceCleanup(run: () => Promise): void { this.trackWorkspaceCleanup(run).catch((error: unknown) => @@ -6112,11 +6148,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // workspace registered and intact, so the finally lifts the consolidation // teardown gate the drains below installed. let removalCommitted = false; + // r66: identifies THIS removal attempt in the durable tombstone so the + // compensating rollback below cannot delete a concurrent backend + // attempt's marker. + const removalAttemptId = crypto.randomUUID(); + // Sub-agents: the tombstone was published (with the final shared-memory + // handover) BEFORE the checkout deletion; an abort between the two rolls + // it back so the intact workspace stays usable. + let sealedForRemoval = false; // Try to remove from runtime (filesystem) try { if (this.agentTaskIntegration?.hasDescendantAgentTasks(workspaceId) === true) { return Err(DESCENDANT_WORKSPACE_REMOVE_ERROR); } + const sessionDir = path.join(this.config.sessionsDir, workspaceId); // The captured engine stop joins partial finalization and raw terminal delivery. try { @@ -6294,6 +6339,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { error: getErrorMessage(error), }); } + // Final handover + tombstone under the removal locks, BEFORE the + // checkout is deleted (sealSubAgentForRemovalUnderMemoryLocks): a + // late legacy note the owner store cannot take must abort while the + // checkout still exists, and once sealed no backend can add + // another (they honor the tombstone at their commit points), so the + // session-dir deletion after runtime deletion has nothing fallible + // left. `force` accepts the loss of notes the handover cannot place. + await sealSubAgentForRemovalUnderMemoryLocks({ + rootDir: this.config.rootDir, + sessionDir, + workspaceId, + attemptId: removalAttemptId, + sharedWorkspaceMemorySessionDir: path.join( + this.config.sessionsDir, + sharedMemoryOwnerId + ), + beforeTombstone: () => + this.lockedSharedMemoryHandover(workspaceId, sharedMemoryOwnerId, force), + }); + sealedForRemoval = true; } if (isMultiProject(metadata)) { @@ -6643,11 +6708,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { ); // Remove session data - const sessionDir = path.join(this.config.sessionsDir, workspaceId); - // r66: identifies THIS removal attempt in the durable tombstone so the - // compensating rollback below cannot delete a concurrent backend - // attempt's marker. - const removalAttemptId = crypto.randomUUID(); try { if (parentWorkspaceId) { try { @@ -6723,26 +6783,16 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceId, attemptId: removalAttemptId, sharedWorkspaceMemorySessionDir: ownerSessionDir, + tombstoneSealed: sealedForRemoval, + // Sealed above (metadata path): handover done and tombstone + // published under these locks already. Otherwise (phantom, + // metadata-less path) the handover runs here, inside the locks + // and right before the tombstone. Throws → removal aborts, + // session intact. beforeTombstone: - ownerSessionDir === undefined + ownerSessionDir === undefined || sealedForRemoval ? undefined - : async () => { - // Legacy-notebook delta too (locks held: the owner store's - // AND this child's own, so a self-fallback backend's late - // note into /memory either landed before this pass - // or is refused). Throws → removal aborts, session intact. - await this.sharedWorkspaceMemoryStore?.adoptLegacyPrivateStoreForRemoval( - workspaceId, - memoryOwnerId, - { locksHeld: true } - ); - await migrateSharedMemoryRefinementRows({ - childSessionDir: sessionDir, - childWorkspaceId: workspaceId, - ownerSessionDir, - ownerWorkspaceId: memoryOwnerId, - }); - }, + : () => this.lockedSharedMemoryHandover(workspaceId, memoryOwnerId, force), }); // Only once the session (and with it the transcript) is gone are the // retryable harvest records truly unrecoverable; an aborted removal @@ -6889,6 +6939,26 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return Err(`Failed to remove workspace: ${message}`); } finally { if (!removalCommitted) { + // A sealed sub-agent whose checkout deletion was then refused + // (force=false) keeps its session dir and config entry: lift the + // tombstone again (ownership-checked, r66) so it stays usable. + if (sealedForRemoval) { + try { + await rollbackRemovalTombstoneIfOwned({ + rootDir: this.config.rootDir, + sessionDir: path.join(this.config.sessionsDir, workspaceId), + workspaceId, + attemptId: removalAttemptId, + workspaceStillRegistered: () => this.config.findWorkspace(workspaceId) != null, + }); + } catch (rollbackError) { + log.error( + "Failed to roll back the removal tombstone after an aborted removal; " + + "the startup self-heal will reclaim it", + { workspaceId, rollbackError } + ); + } + } this.memoryConsolidationService?.releaseRemovalCancellation(workspaceId); } for (const hold of admissionHolds) { From 09c8068c900a07b0c547c9cb2cd755c475fe4192 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 15:46:25 +0000 Subject: [PATCH 55/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20forty-nin?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption reconciles a deleted source only on a proven ENOENT of the source itself (a tolerant, possibly partial listing is no proof), and removes the owner-key sidecar entries BEFORE the adopted copy so a sidecar failure retries both. - A sealed removal tombstone is not rewritten by the later session-dir teardown (a failing redundant write cannot abort a removal whose checkout is already gone). - Policy epoch records (config and deny marker) are never pruned by count: only the boundary observation that consumes them removes them. A persisted non-boolean record reads as a deny. - Deny marker: an unreadable file is distinguished from malformed content; clears, carries and writes refuse instead of replacing or deleting epoch state they could not read. - MemoryMetaService: a failed sidecar stat is not the cacheable "missing" first-run stamp; mutations on it fail instead of writing a stale cache. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryMeta.test.ts | 29 +++++++++++ src/node/services/memoryMeta.ts | 18 ++++--- src/node/services/memoryService.test.ts | 16 +++++++ src/node/services/memoryService.ts | 16 ++++++- .../services/workspaceMemoryDenyMarker.ts | 44 +++++++++++++---- .../services/workspaceMemoryPolicyEpochs.ts | 35 +++++++++----- src/node/services/workspaceRemoval.ts | 16 ++++++- src/node/services/workspaceService.test.ts | 48 ++++++++++++++++++- 8 files changed, 188 insertions(+), 34 deletions(-) diff --git a/src/node/services/memoryMeta.test.ts b/src/node/services/memoryMeta.test.ts index bf4164338ae..daff2743e0d 100644 --- a/src/node/services/memoryMeta.test.ts +++ b/src/node/services/memoryMeta.test.ts @@ -92,6 +92,35 @@ describe("MemoryMetaService", () => { ); }); + it("does not serve a cached first-run view when the sidecar stat itself fails", async () => { + using tempDir = new TestTempDir("test-memory-meta"); + const service = new MemoryMetaService(tempDir.path); + // Cached "missing" (normal first run in this process)... + expect(await service.getPinnedKeys()).toEqual(new Set()); + // ...then another backend creates the sidecar. + await new MemoryMetaService(tempDir.path).setPinned("global:prefs.md", true); + // A transient stat failure must not read as "still missing": the stale + // empty cache would otherwise be written over the foreign pins. + const stat = spyOn(fsPromises, "stat").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + const failure = await service.setPinned("workspace:ws-1:scratch.md", true).then( + () => null, + (error: unknown) => error + ); + expect(failure).toBeInstanceOf(MemoryMetaWriteError); + } finally { + stat.mockRestore(); + } + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md"]) + ); + await service.setPinned("workspace:ws-1:scratch.md", true); + expect(await new MemoryMetaService(tempDir.path).getPinnedKeys()).toEqual( + new Set(["global:prefs.md", "workspace:ws-1:scratch.md"]) + ); + }); + it("persists pins across instances via the sidecar file", async () => { using tempDir = new TestTempDir("test-memory-meta"); const service = new MemoryMetaService(tempDir.path); diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index e0bd00b4d86..270d47caf96 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -314,7 +314,9 @@ export class MemoryMetaService { const self = this; return Effect.gen(function* () { const stamp = yield* Effect.promise(() => self.fileStamp()); - if (self.cache !== null && stamp === self.cacheStamp) { + // A failed stat (null) never matches: another backend may have written + // the file since the cached "missing" was taken. + if (stamp !== null && self.cache !== null && stamp === self.cacheStamp) { return { meta: self.cache, readFailed: false }; } let readFailed = false; @@ -348,17 +350,21 @@ export class MemoryMetaService { // under the file's unchanged stamp would keep serving it once readable // again — and the next mutation would write the pins and stats away. self.cacheStamp = readFailed ? null : stamp; - return { meta: self.cache, readFailed }; + return { meta: self.cache, readFailed: readFailed || stamp === null }; }); } - /** Cheap change signal for the sidecar (same scheme as Config.configFileStamp). */ - private async fileStamp(): Promise { + /** + * Cheap change signal for the sidecar (same scheme as Config.configFileStamp). + * "missing" only on a proven ENOENT (the normal first run); null when the + * stat itself failed — not cacheable, and a mutation must not proceed on it. + */ + private async fileStamp(): Promise { try { const st = await fsPromises.stat(this.metaPath, { bigint: true }); return `${st.dev}:${st.ino}:${st.size}:${st.mtimeNs}`; - } catch { - return "missing"; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? "missing" : null; } } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index ce3a552151e..0291c570e05 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1786,6 +1786,22 @@ describe("MemoryService", () => { .map((entry) => entry.relPath) .sort(); expect(again).toEqual(relisted); + // A lossy legacy listing (readdir failure tolerated by listFiles) is not + // proof of deletion: the copies stay while the sources provably exist. + await fsPromises.writeFile(path.join(legacyRoot, "renamed.md"), "to be renamed (v2)"); + const lossy = spyOn(fsPromises, "readdir").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + lossy.mockRestore(); + } + expect( + (await fixture.service.listIndexEntries({ ...fixture.ctx })) + .filter((entry) => entry.scope === "workspace") + .map((entry) => entry.relPath) + .sort() + ).toEqual(["edited.md", "imported/ws-child/renamed.md", "renamed.md", "same.md"]); }); it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8d6170c70e6..3b0b0fd7f34 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -24,6 +24,7 @@ import writeFileAtomic from "write-file-atomic"; import YAML from "yaml"; import assert from "@/common/utils/assert"; import { CONTEXT_NOTES_MEMORY_PATH } from "@/common/constants/contextBudget"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; import { MEMORY_HOT_SET_MAX_ITEM_BYTES, MEMORY_INDEX_DESCRIPTION_MAX_CHARS, @@ -1443,6 +1444,15 @@ export class MemoryService extends EventEmitter { const listed = new Set(files); for (const [relPath, previous] of adopted) { if (listed.has(relPath)) continue; + // Absence from the listing is not proof: LocalMemoryStore.listFiles + // tolerates readdir failures (a partial list). Only a provable ENOENT + // on the source itself counts; any other outcome keeps the entry + // (and the copy) for a later pass. + const sourceGone = await fsPromises.lstat(path.join(legacyRoot, relPath)).then( + () => false, + (error: unknown) => hasErrorCode(error, "ENOENT") + ); + if (!sourceGone) continue; if (previous.created === true) { const current = (await store.assertContained(previous.target).then( @@ -1455,13 +1465,17 @@ export class MemoryService extends EventEmitter { : null; const unchanged = current !== null && sha256Hex(current) === previous.content; if (unchanged) { - await store.remove(previous.target); + // Metadata first: a sidecar failure then aborts the pass with the + // file and manifest entry intact, so the retry repeats both; + // the reverse order would strand the owner-key pin/usage once + // the file was gone and the entry dropped. await this.metaService.removeKeys( memoryLogicalKey("workspace", previous.target, { projectPath: ctx.projectPath, workspaceId: owner, }) ); + await store.remove(previous.target); adoptedCount++; log.info("[MemoryService] removed an adopted legacy note deleted on the old build", { childId, diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 1d0f0feffef..323db0bddc8 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -32,7 +32,6 @@ import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; -import { WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED } from "@/node/services/workspaceMemoryPolicyEpochs"; export const WORKSPACE_MEMORY_DENY_MARKER_FILE_NAME = "memory-policy-deny.json"; @@ -50,6 +49,9 @@ export async function writeWorkspaceMemoryDenyMarker( const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); await fsPromises.mkdir(sessionDir, { recursive: true }); const record = await readMarkerRecord(markerPath); + if (record === "unreadable") { + throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); + } // A malformed marker was a deny for EVERY epoch's reader while it existed, // possibly the only evidence of some other backend's read-only turn in // the closing epoch. This writer, recording a deny for its own epoch, @@ -70,9 +72,9 @@ async function writeMarkerRecord( epochs: readonly number[], wildcard: boolean ): Promise { - const retained = [...epochs] - .sort((a, b) => b - a) - .slice(0, WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED); + // Entries are removed only by the boundary observation that consumes them + // (clear/carry), never by count — see workspaceMemoryPolicyEpochs.ts. + const retained = [...new Set(epochs)].sort((a, b) => b - a); await writeFileAtomic( markerPath, JSON.stringify({ deniedAt: Date.now(), epochs: retained, wildcard }), @@ -80,12 +82,23 @@ async function writeMarkerRecord( ); } -/** Parsed marker, or null when it is unreadable/malformed (which readers treat as a deny). */ +/** + * Parsed marker; "absent" on a proven ENOENT; null when the content is + * malformed (readers deny; a boundary clear heals it); "unreadable" when the + * file could not be read at all (EACCES, I/O) — readers deny, and mutators + * throw rather than replace or delete epoch state they could not see. + */ async function readMarkerRecord( markerPath: string -): Promise<{ epochs: number[]; wildcard: boolean } | "absent" | null> { +): Promise<{ epochs: number[]; wildcard: boolean } | "absent" | "unreadable" | null> { + let raw: string; try { - const parsed: unknown = JSON.parse(await fsPromises.readFile(markerPath, "utf-8")); + raw = await fsPromises.readFile(markerPath, "utf-8"); + } catch (error) { + return hasErrorCode(error, "ENOENT") ? "absent" : "unreadable"; + } + try { + const parsed: unknown = JSON.parse(raw); if (typeof parsed !== "object" || parsed === null) return null; const { epochs, wildcard } = parsed as { epochs?: unknown; wildcard?: unknown }; if ( @@ -97,8 +110,8 @@ async function readMarkerRecord( return null; } return { epochs, wildcard: wildcard === true }; - } catch (error) { - return hasErrorCode(error, "ENOENT") ? "absent" : null; + } catch { + return null; } } @@ -114,7 +127,7 @@ export async function readWorkspaceMemoryDenyMarker( ): Promise { const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); if (record === "absent") return false; - if (record === null || epoch === undefined) return true; + if (record === null || record === "unreadable" || epoch === undefined) return true; return record.wildcard || record.epochs.includes(epoch); } @@ -145,6 +158,12 @@ export async function clearWorkspaceMemoryDenyMarker( if (options !== undefined) { const record = await readMarkerRecord(markerPath); if (record === "absent") return; + // Unreadable is not malformed: the file may hold a newer epoch's deny + // (possibly the only durable record of a read-only turn). Refuse; the + // caller's reset fails and is retried rather than deleting unknown state. + if (record === "unreadable") { + throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); + } if (record !== null) { // This boundary observed the closing epoch (deny included): the // epoch-less wildcard, if any, is consumed with it. @@ -183,6 +202,11 @@ export async function carryWorkspaceMemoryDenyMarker( await withTargetMutationLock(rootDir, sessionDir, async () => { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); const record = await readMarkerRecord(markerPath); + if (record === "unreadable") { + throw new Error( + `Workspace memory deny marker is unreadable at ${workspaceMemoryDenyMarkerPath(sessionDir)}` + ); + } if (record === "absent" || record === null) return; if (!record.wildcard && !record.epochs.includes(closingEpoch)) return; // The wildcard denied the closing epoch; carried as the new epoch's deny. diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts index cfdab815275..24e956e2261 100644 --- a/src/node/services/workspaceMemoryPolicyEpochs.ts +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -9,42 +9,51 @@ * boundary and that backend's completion-side read of the CLOSING epoch's * value. A single slot would be overwritten by the new epoch's grant, dropping * a deny recorded for the closing epoch — and the compacting backend's own - * mirror (writable) would then grant the harvest of a read-only turn. Epochs - * are history sequences (-1 before any boundary), so the newest few are kept - * and older ones garbage-collected: a closing epoch is observed at its - * boundary, before more than a couple of further boundaries can exist. + * mirror (writable) would then grant the harvest of a read-only turn. A + * record is removed only by the observation that consumes it — the + * compacting session's boundary reset/carry (AgentSession) or a destructive + * boundary — never by count: a backend suspended between persisting its + * boundary and observing the closing policy must still find the record + * however many epochs other backends opened meanwhile. Records of a boundary + * whose observer never ran (crash in between) linger until the next + * destructive boundary; that residue is bounded by such crashes. */ import type { Workspace as WorkspaceConfigEntry } from "@/node/config"; import assert from "@/common/utils/assert"; -/** The closing epoch plus the next ones that can be opened before it is observed. */ -export const WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED = 3; - function epochKey(epoch: number): string { assert(Number.isInteger(epoch), "workspace memory policy epoch must be an integer"); return String(epoch); } +/** + * The recorded policy for `epoch`: `undefined` when none was recorded. Config + * entries are loaded from raw JSON without schema validation, so anything + * present that is not an actual boolean (a corrupted `"false"` or `null`) + * reads as a DENY — a truthy string or a null-coalesced default would + * otherwise turn corrupted deny state into a grant. + */ export function workspaceMemoryWritableForEpoch( entry: WorkspaceConfigEntry, epoch: number ): boolean | undefined { - return entry.workspaceMemoryWritableByEpoch?.[epochKey(epoch)]; + const records: Record | undefined = entry.workspaceMemoryWritableByEpoch; + const key = epochKey(epoch); + if (records === undefined || !Object.hasOwn(records, key)) return undefined; + const value = records[key]; + return typeof value === "boolean" ? value : false; } -/** Record `writable` for `epoch`; drops the oldest records beyond the retained window. */ +/** Record `writable` for `epoch`. */ export function setWorkspaceMemoryWritableForEpoch( entry: WorkspaceConfigEntry, epoch: number, writable: boolean ): void { - const next: Record = { + entry.workspaceMemoryWritableByEpoch = { ...entry.workspaceMemoryWritableByEpoch, [epochKey(epoch)]: writable, }; - const keys = Object.keys(next).sort((a, b) => Number(b) - Number(a)); - for (const key of keys.slice(WORKSPACE_MEMORY_POLICY_EPOCHS_RETAINED)) delete next[key]; - entry.workspaceMemoryWritableByEpoch = next; } /** Forget `epoch`'s record; removes the field once no record is left. */ diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index 758adfe0dcf..b8fe408fbb7 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -196,6 +196,18 @@ export async function removeSessionDirUnderMemoryLocks(args: { ); assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); let tombstonePublishedUnderLocks = args.tombstoneSealed === true; + // A sealed tombstone is not rewritten: the redundant write could fail + // (storage read-only/full after the checkout deletion) and would then abort + // a removal whose durable marker is already in place. + const publish = async (): Promise => { + if ( + args.tombstoneSealed === true && + (await isWorkspaceRemovalTombstoned(args.rootDir, args.workspaceId)) + ) { + return; + } + await publishRemovalTombstone(args); + }; try { await withRemovalLocks(args, async () => { await args.beforeTombstone?.(); @@ -203,7 +215,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { // re-checks it pre-commit (inside its own lock) and refuses, so the // deleted directory cannot be recreated by a late mutation or // journal append. - await publishRemovalTombstone(args); + await publish(); tombstonePublishedUnderLocks = true; await fsPromises.rm(args.sessionDir, { recursive: true, force: true }); }); @@ -226,7 +238,7 @@ export async function removeSessionDirUnderMemoryLocks(args: { // because the directory is not deleted: a writer mid-commit lands in // the orphan, and every later mutation observes the tombstone. try { - await publishRemovalTombstone(args); + await publish(); } catch (publishError) { // No durable marker could be written at all (r63, e.g. ENOSPC): // deregistering now would leave the orphan writable again the moment diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 4cd4989c794..c8764c35695 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9,6 +9,7 @@ import { workspaceRemovalTombstonePath, isWorkspaceRemovalTombstoned, } from "@/node/services/workspaceRemoval"; +import { workspaceMemoryWritableForEpoch } from "@/node/services/workspaceMemoryPolicyEpochs"; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; @@ -9424,6 +9425,22 @@ describe("WorkspaceService initialize", () => { await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + // Unreadable is not malformed: a marker that cannot be read may hold a + // newer epoch's deny, so the fenced clear refuses instead of deleting it. + await writeWorkspaceMemoryDenyMarker(sessionDir, 9); + const unreadableMarker = spyOn(fsPromises, "readFile").mockImplementationOnce((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + const refusedClear = await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { + closingEpoch: -1, + }).then( + () => null, + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ); + expect(refusedClear).toContain("unreadable"); + unreadableMarker.mockRestore(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(true); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 9 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); await realConfig.editConfig((cfg) => { @@ -9475,7 +9492,9 @@ describe("WorkspaceService initialize", () => { ); expect(persisted()).toBe(false); expect(persistedFor(12)).toBe(true); - // Only the newest few epochs are retained. + // Records are never pruned by count: a suspended compacting backend + // must still find its closing epoch's record however many epochs + // others opened meanwhile. for (const policyEpoch of [20, 30, 40]) { expect( await service.recordWorkspaceMemoryWritable("policy-scratch", true, { @@ -9489,7 +9508,32 @@ describe("WorkspaceService initialize", () => { findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")!.workspace .workspaceMemoryWritableByEpoch! ).sort() - ).toEqual(["20", "30", "40"]); + ).toEqual(["-1", "12", "20", "30", "40"]); + // Raw config is not schema-validated: a corrupted non-boolean record + // reads as a deny, never as a grant. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + (entry.workspaceMemoryWritableByEpoch as Record)["50"] = "false"; + (entry.workspaceMemoryWritableByEpoch as Record)["51"] = null; + return cfg; + }); + for (const policyEpoch of [50, 51]) { + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch, + }) + ).toBe(true); + // The deny stands (no grant written over the corrupted value)... + expect(persistedFor(policyEpoch)).not.toBe(true); + // ...and every reader sees it as false. + expect( + workspaceMemoryWritableForEpoch( + findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")!.workspace, + policyEpoch + ) + ).toBe(false); + } // Unknown history fails closed: no accumulator, no marker, no mirror // (this service never recorded this epoch), yet the epoch already holds From 2ca7f1aac7649df2e9b1c74ff50c6ff51b89aaf5 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 16:36:48 +0000 Subject: [PATCH 56/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fiftieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy-notebook adoption lists the legacy directory strictly: a traversal failure fails the pass (retried on the next access; removal aborts with the session intact) instead of a partial listing counting as "nothing to adopt" and removal deleting a never-seen note's only copy. - The destructive-boundary policy reset loads config strictly (an unreadable config.json is a retryable partial failure of the boundary, not a "successful" reset that leaves the stale `-1` deny behind) and forgets the in-memory mirror only after the durable clear completed. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- ...tSession.postCompactionAttachments.test.ts | 23 +++++++++++++++++-- src/node/services/agentSession.ts | 23 ++++++++++++++++--- src/node/services/memoryService.test.ts | 14 +++++++++++ src/node/services/memoryService.ts | 22 ++++++++++++++---- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/src/node/services/agentSession.postCompactionAttachments.test.ts b/src/node/services/agentSession.postCompactionAttachments.test.ts index a340add08cc..be4ef966429 100644 --- a/src/node/services/agentSession.postCompactionAttachments.test.ts +++ b/src/node/services/agentSession.postCompactionAttachments.test.ts @@ -250,8 +250,27 @@ describe("AgentSession post-compaction attachments", () => { expect(injected).not.toBeNull(); expect(getReadFilePaths(injected ?? [])).toEqual(["/tmp/pre-boundary-read.ts"]); - // A new context segment starts (context reset / full history clear): - // the reset was meant to discard that context, so... + // A new context segment starts (context reset / full history clear). + // Its durable policy-epoch reset reads config strictly: an unreadable + // config.json must surface as a retryable failure rather than a + // "successful" reset that leaves stale per-epoch records behind. + const sessionConfig = (session as unknown as { config: Config }).config; + const readable = sessionConfig.loadConfigOrDefault.bind(sessionConfig); + sessionConfig.loadConfigOrDefault = ((options?: { throwOnError?: boolean }) => { + if (options?.throwOnError) throw new Error("config.json: unexpected token"); + return readable(options); + }) as Config["loadConfigOrDefault"]; + try { + expect( + await session.clearPostCompactionState().then( + () => null, + (error: unknown) => (error instanceof Error ? error.message : String(error)) + ) + ).toContain("unexpected token"); + } finally { + sessionConfig.loadConfigOrDefault = readable; + } + // The reset was meant to discard that context, so... await session.clearPostCompactionState(); // ...no later turn may re-inject pre-boundary paths — neither diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 89bdefebd27..1b7d2ef13c6 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1126,7 +1126,12 @@ export class AgentSession { * turn). Nothing to do when the field is already absent. */ private async resetWorkspaceMemoryWritable(options?: { closingEpoch: number }): Promise { - this.workspaceMemoryWritable = undefined; + // Compaction (closing epoch given): the completion callback already + // forgot the mirror synchronously. Destructive boundary: the mirror is + // forgotten only once the durable clear below completed — a reset that + // reports success while the persisted `-1` deny survives would pin the + // whole new segment to the stored-false fast path. + if (options !== undefined) this.workspaceMemoryWritable = undefined; // The session-dir deny marker (fallback for an unwritable config.json) // belongs to the closing epoch too. Same fence idea as below: a deny // recorded for the new epoch survives. @@ -1135,8 +1140,19 @@ export class AgentSession { path.join(this.config.sessionsDir, this.workspaceId), options ); - const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); - if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) return; + // Strict load: an unreadable config.json would read as the empty default, + // in which this workspace has no records to clear — the reset would + // report success and leave the stale records behind. Throwing makes the + // boundary a retryable partial failure instead. (A genuinely absent file + // holds no records and is the empty default for real.) + const entry = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + this.workspaceId + ); + if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) { + this.workspaceMemoryWritable = undefined; + return; + } if ( options !== undefined && workspaceMemoryWritableForEpoch(entry.workspace, options.closingEpoch) === undefined @@ -1161,6 +1177,7 @@ export class AgentSession { } return cfg; }); + this.workspaceMemoryWritable = undefined; } /** diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 0291c570e05..79d647a3b8b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1880,6 +1880,20 @@ describe("MemoryService", () => { .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") .then(() => null, getErrorMessage) ).toMatch(/could not be folded/); + // A legacy listing that cannot be completed (readdir failure) is no + // "nothing to adopt": removal must abort rather than delete a note it + // never saw. + const lossy = spyOn(fsPromises, "readdir").mockImplementation((() => + Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EIO/); + } finally { + lossy.mockRestore(); + } // Space frees up; the in-lock delta pass (removal holds the owner-store // lock already) folds the note in without re-acquiring the lock. await fsPromises.rm(path.join(ownerRoot, "o0000.md")); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 3b0b0fd7f34..90a5ce2bb79 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -374,7 +374,12 @@ interface MemoryStore { /** assertRootSafe + create the root if missing (write paths only). */ ensureRoot(): Promise; /** Relative paths of all non-dotfile files under the root, sorted. */ - listFiles(): Promise; + /** + * Files under the root. Tolerant by default (self-healing: an unreadable + * directory lists as empty); `strict` throws on any traversal failure, for + * callers whose decision must not rest on a possibly partial listing. + */ + listFiles(options?: { strict?: boolean }): Promise; kind(relPath: string): Promise; /** * Read at most `maxBytes` from the head of the file. Index/hot-set builds @@ -555,7 +560,7 @@ class LocalMemoryStore implements MemoryStore { await fsPromises.mkdir(this.physicalRoot, { recursive: true }); } - async listFiles(): Promise { + async listFiles(options?: { strict?: boolean }): Promise { const results: string[] = []; const walk = async (dirRel: string): Promise => { // Bounded walk: files may have been edited outside MemoryService. +1 lets @@ -564,7 +569,12 @@ class LocalMemoryStore implements MemoryStore { let entries; try { entries = await fsPromises.readdir(this.abs(dirRel), { withFileTypes: true }); - } catch { + } catch (error) { + // Strict callers (removal's legacy handover) must not take a partial + // listing for the whole; a missing ROOT is the genuine empty case. + if (options?.strict === true && !(dirRel === "" && hasErrorCode(error, "ENOENT"))) { + throw error; + } return; // Self-healing: missing/unreadable dirs list as empty. } // Iterate in path-string order — directories key as "name/" so the DFS @@ -1289,7 +1299,11 @@ export class MemoryService extends EventEmitter { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock const legacy = new LocalMemoryStore(legacyRoot); - const files = await legacy.listFiles(); + // Strict: a note omitted by a partial listing would count as "nothing + // to adopt" (skipped stays 0) and removal would then delete its only + // copy. A traversal failure fails the pass instead (access-time: + // retried on the next access; removal: aborted, session intact). + const files = await legacy.listFiles({ strict: true }); // What was already folded in, kept beside the legacy files (a dotfile, // so neither build lists it): per relPath the content hash, the // fingerprint of the child-keyed sidecar entry, and where the copy From 6b32c8d7b2d4e393c7fe53ebed5c5bf72b68cf72 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 19:28:49 +0000 Subject: [PATCH 57/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-fir?= =?UTF-8?q?st=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Strict legacy listings (adoption/removal) walk the whole notebook; the per-scope traversal cap applies to tolerant callers only. - Adoption manifest records provenance (`pending: true`, `created`) before the copy lands, so a retry after a crash or failed sidecar fold keeps `created: true` and legacy deletions still follow the copy out. - Owner-keyed sidecar entries are part of the workspaceMemoryRevision token: a pin whose best-effort clock write failed still changes the probe token. - Memory subscriptions announce a root refresh after each per-event token read, so a foreign write absorbed by the async read is refetched. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/orpc/routerSubscriptions.test.ts | 10 ++++ src/node/orpc/routerSubscriptions.ts | 8 ++- src/node/services/memoryService.test.ts | 48 +++++++++++++++++ src/node/services/memoryService.ts | 66 ++++++++++++++++++----- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/src/node/orpc/routerSubscriptions.test.ts b/src/node/orpc/routerSubscriptions.test.ts index 7cb2d69c02a..dd4011b1e4b 100644 --- a/src/node/orpc/routerSubscriptions.test.ts +++ b/src/node/orpc/routerSubscriptions.test.ts @@ -91,6 +91,16 @@ test("memory subscriptions match workspace-scope events on the shared memory own memoryService.emit("change", marker); expect((await first).value).toEqual({ ...base, workspaceId: "ws-owner" }); expect((await stream.next()).value).toEqual(marker); + // A workspace-scope event re-reads the store token asynchronously; the + // token is announced once read so a foreign write it may have absorbed is + // refetched too (see refreshStoreRevision in subscribeMemoryChanges). + expect((await stream.next()).value).toEqual({ + scope: "workspace", + path: "/memories/workspace", + actor: "agent", + workspaceId: "ws-owner", + projectPath: "", + }); // Ownership change for THIS workspace (owner removed): synthesized // root-addressed refresh + status refresh, now addressed to the new owner. diff --git a/src/node/orpc/routerSubscriptions.ts b/src/node/orpc/routerSubscriptions.ts index 93cebf8d494..c212bb3720f 100644 --- a/src/node/orpc/routerSubscriptions.ts +++ b/src/node/orpc/routerSubscriptions.ts @@ -260,7 +260,13 @@ export function subscribeMemoryChanges( ) return; if (event.scope === "project" && event.projectPath !== projectPath) return; - if (event.scope === "workspace") refreshStoreRevision(); + // The token read below is asynchronous: it may absorb a foreign + // backend's write that landed after the client already refetched for + // this event, and the interval probe would then never announce it. + // Announcing the token once read (as the baseline handshake does) + // orders one more client refresh behind whatever the token saw; the + // cost is a redundant listing refetch per local mutation. + if (event.scope === "workspace") refreshStoreRevision({ announce: true }); emit.push(event); }; const onStatusChange = (event: MemoryConsolidationStatusChangeEventPayload) => diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 79d647a3b8b..2f8cd0d8f3b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1278,6 +1278,16 @@ describe("MemoryService", () => { await fixture.service.setPinned(fixture.ctx, "/memories/workspace/shared.md", true); const afterPin = await foreign.workspaceMemoryRevision("ws-owner"); expect(clockOf(afterPin)).toBeGreaterThan(clockOf(afterCreate)); + // ...and an owner-keyed sidecar change whose clock write was lost (the + // revision write is best-effort; a downgraded build toggling the pin + // moves no clock either) still changes the token. + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "shared.md", { projectPath: "", workspaceId: "ws-owner" }), + false + ); + const afterSidecarOnly = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterSidecarOnly).not.toBe(afterPin); + expect(clockOf(afterSidecarOnly)).toBe(clockOf(afterPin)); // ...while every shared-store mutation advances it. await fixture.service.strReplace( fixture.ctx, @@ -1839,6 +1849,44 @@ describe("MemoryService", () => { expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); }); + it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "note.md", { projectPath: "", workspaceId: "ws-child" }), + true + ); + // The copy lands, then the sidecar fold fails before the manifest + // records the adoption as complete. + const failing = spyOn(fixture.metaService, "mergeKeys").mockImplementationOnce(() => + Promise.reject(new Error("sidecar unavailable")) + ); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + failing.mockRestore(); + } + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "child notes" + ); + // The retry finds identical bytes at the target (no-write path) and must + // still know this adoption created the copy... + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(manifest["note.md"]).toMatchObject({ created: true }); + expect(manifest["note.md"].pending).toBeUndefined(); + // ...so a deletion on the downgraded build still follows it out. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 90a5ce2bb79..b2baae85133 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -375,8 +375,9 @@ interface MemoryStore { ensureRoot(): Promise; /** Relative paths of all non-dotfile files under the root, sorted. */ /** - * Files under the root. Tolerant by default (self-healing: an unreadable - * directory lists as empty); `strict` throws on any traversal failure, for + * Files under the root. Tolerant and bounded by default (self-healing: an + * unreadable directory lists as empty; the walk stops past the per-scope + * cap); `strict` throws on any traversal failure and is unbounded, for * callers whose decision must not rest on a possibly partial listing. */ listFiles(options?: { strict?: boolean }): Promise; @@ -418,13 +419,17 @@ const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; * One adopted legacy file: content hash, child sidecar fingerprint, owner-store * relPath, and whether the adoption CREATED that owner file (provenance: only * such a copy may be removed again when the legacy source disappears; a - * pre-existing identical owner note is the owner's own). + * pre-existing identical owner note is the owner's own). `pending`: written + * BEFORE the copy lands (provenance must not depend on the copy's existence: a + * retry finding the bytes already at the target could not tell an interrupted + * adoption from an owner note); cleared once the sidecar fold completed. */ interface LegacyAdoptionRecord { content: string; sidecar: string; target: string; created?: boolean; + pending?: boolean; } function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { @@ -564,8 +569,11 @@ class LocalMemoryStore implements MemoryStore { const results: string[] = []; const walk = async (dirRel: string): Promise => { // Bounded walk: files may have been edited outside MemoryService. +1 lets - // callers detect overflow (e.g. the index logs its truncation). - if (results.length > MEMORY_MAX_FILES_PER_SCOPE) return; + // callers detect overflow (e.g. the index logs its truncation). Strict + // callers need the COMPLETE set (an omitted file would silently count + // as "nothing to adopt" and could lose its only copy), so the bound + // does not apply to them. + if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; let entries; try { entries = await fsPromises.readdir(this.abs(dirRel), { withFileTypes: true }); @@ -1323,6 +1331,10 @@ export class MemoryService extends EventEmitter { let capacityExhausted = false; let manifestDirty = false; let imported = 0; + const writeManifest = () => + writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { + encoding: "utf-8", + }); for (const relPath of files) { // Same read gates as a memory command: containment (no symlink // escape), size cap, and text-only (a lossy utf-8 decode cannot be @@ -1347,7 +1359,11 @@ export class MemoryService extends EventEmitter { created: false, }; const previous = adopted.get(relPath); - if (previous?.content === record.content && previous.sidecar === record.sidecar) { + if ( + previous?.content === record.content && + previous.sidecar === record.sidecar && + previous.pending !== true + ) { continue; // folded in earlier, nothing changed since } let target: { relPath: string; write: boolean } | null = null; @@ -1399,6 +1415,18 @@ export class MemoryService extends EventEmitter { skipped++; continue; } + // Provenance BEFORE the copy: interrupted here (crash, or the + // sidecar fold below failing), the retry finds the owner file + // already identical and takes the no-write path — without this + // record it would read as the owner's own note, and a legacy + // deletion could then never follow it out of the shared store. + adopted.set(relPath, { + ...record, + target: target.relPath, + created: true, + pending: true, + }); + await writeManifest(); await store.writeFile(target.relPath, content); remainingCapacity--; imported++; @@ -1418,7 +1446,11 @@ export class MemoryService extends EventEmitter { // changes its usage counters, which must not drag the owner's pin // back to the child's unchanged value. if (childEntry !== undefined) { - const priorPinned = previous === undefined ? null : legacySidecarPinned(previous.sidecar); + // A pending record's fold never ran: still a first adoption. + const priorPinned = + previous === undefined || previous.pending === true + ? null + : legacySidecarPinned(previous.sidecar); // Only an actual boolean transition of the child's pin overrides // the owner's; an unknown prior state never does. const childPinChanged = priorPinned !== null && priorPinned !== childEntry.pinned; @@ -1502,11 +1534,7 @@ export class MemoryService extends EventEmitter { adopted.delete(relPath); manifestDirty = true; } - if (manifestDirty) { - await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { - encoding: "utf-8", - }); - } + if (manifestDirty) await writeManifest(); if (capacityExhausted) { log.warn( "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", @@ -1912,10 +1940,22 @@ export class MemoryService extends EventEmitter { // stamps (root mtime, per-file size + mtime; bounded by the per-scope // file cap): stat fingerprints are cache hints — enough to make a cached // context miss — never proof that authorizes any mutation. + // The owner-keyed sidecar entries (pins, usage) rank the shared hot set + // and are written by every backend; a pin whose revision write then + // failed (advanceStoreRevision is best-effort) is still visible here. + const ownerKeyPrefix = memoryLogicalKey("workspace", "", { + projectPath: "", + workspaceId: owner, + }); + const ownerSidecar = JSON.stringify( + [...(await this.metaService.getEntries())] + .filter(([key]) => key.startsWith(ownerKeyPrefix)) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + ); const token = `${revision === null ? "missing" : String(revision)}\u0000${await legacyStoreStamp( ownerSessionDir, workspaceMemoryStorePath(this.config.sessionsDir, owner) - )}`; + )}\u0000${ownerSidecar}`; // A redirected sub-agent's token also tracks its legacy private notebook // (see legacyAdoptionCheckKey): its next store access adopts the change, // so the cached context must miss as soon as the legacy state moves. From ae8d9d5ff6507b8446467c05b9221c6c10c990c4 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 20:10:07 +0000 Subject: [PATCH 58/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-sec?= =?UTF-8?q?ond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption: an unreadable legacy root (EACCES/EIO) is not "missing"; the pass throws so removal aborts with the session intact. Destination probes use a strict `kind` that only treats a proven ENOENT as free. - Policy epoch records: a malformed container (null/array/string) reads as a deny for every epoch instead of throwing out of every turn start, and the next write or forget replaces it. - AgentSession boundary: the durable reset is read back before the mirror is cleared (throws when the clear did not persist); the preserved-tail carry loads strictly, verifies the new-epoch value, and falls back to the session-dir deny marker when a deny cannot be proven carried. - Removal: the tombstone lease is held from before the early seal so a slow runtime deletion cannot age the fresh tombstone into healable residue. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 115 ++++++++--- ...Session.workspaceMemoryPolicyEpoch.test.ts | 187 ++++++++++++++++++ src/node/services/memoryService.test.ts | 43 ++++ src/node/services/memoryService.ts | 49 ++++- .../services/workspaceMemoryPolicyEpochs.ts | 36 +++- src/node/services/workspaceService.test.ts | 32 ++- src/node/services/workspaceService.ts | 22 ++- 7 files changed, 430 insertions(+), 54 deletions(-) create mode 100644 src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 1b7d2ef13c6..d004d4a78b3 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -105,7 +105,10 @@ import { import { carryWorkspaceMemoryDenyMarker, clearWorkspaceMemoryDenyMarker, + writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; +import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import { buildStreamErrorEventData, createStreamErrorMessage, @@ -1177,6 +1180,25 @@ export class AgentSession { } return cfg; }); + // Verified read-back: Config.saveConfig swallows write failures, so the + // awaited edit alone does not prove the clear landed. A destructive + // boundary reuses epoch -1, and a surviving `-1: false` would pin the + // new segment to the stored-false fast path once the mirror is cleared + // below — so the mirror is cleared only after the durable state agrees. + const after = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + this.workspaceId + ); + const stale = + after !== null && + (options === undefined + ? after.workspace.workspaceMemoryWritableByEpoch !== undefined + : workspaceMemoryWritableForEpoch(after.workspace, options.closingEpoch) !== undefined); + if (stale) { + throw new Error( + `Workspace memory policy reset did not persist for ${this.workspaceId} (config write swallowed?)` + ); + } this.workspaceMemoryWritable = undefined; } @@ -1192,35 +1214,72 @@ export class AgentSession { closingEpoch: number, nextEpoch: number ): Promise { - await carryWorkspaceMemoryDenyMarker( - this.config.rootDir, - path.join(this.config.sessionsDir, this.workspaceId), - closingEpoch, - nextEpoch - ); - const entry = findWorkspaceEntry(this.config.loadConfigOrDefault(), this.workspaceId); - if ( - entry === null || - workspaceMemoryWritableForEpoch(entry.workspace, closingEpoch) === undefined - ) { - return; - } - await this.config.editConfig((cfg) => { - const current = findWorkspaceEntry(cfg, this.workspaceId); - if (current === null) return cfg; - const closing = workspaceMemoryWritableForEpoch(current.workspace, closingEpoch); - if (closing === undefined) return cfg; - // ANDed into a record another backend's first turn of the new epoch - // may already have made (its own conjunction could not see the closing - // value under the new key), never overwriting it. - const next = workspaceMemoryWritableForEpoch(current.workspace, nextEpoch); - setWorkspaceMemoryWritableForEpoch( - current.workspace, - nextEpoch, - next === undefined ? closing : next && closing + const sessionDir = path.join(this.config.sessionsDir, this.workspaceId); + await carryWorkspaceMemoryDenyMarker(this.config.rootDir, sessionDir, closingEpoch, nextEpoch); + // Fail closed: the carry must be PROVEN, not assumed. A tolerant load + // would read a transiently unreadable config.json as "no closing record" + // and skip the carry; Config.saveConfig swallows write failures, so the + // awaited edit does not prove the new-epoch record landed either. Tail + // copies are excluded from the prior-turn check by design, so a fresh or + // foreign backend's first turn of the new epoch would then grant, and + // output conditioned on the read-only tail would become harvestable. + // When a DENY cannot be shown to have reached the new epoch, the + // session-dir marker (the same fallback recordWorkspaceMemoryWritable + // takes for an unwritable config) denies the new epoch instead. + let carried: boolean | undefined; + let failure: string | undefined; + try { + const entry = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + this.workspaceId ); - deleteWorkspaceMemoryWritableForEpoch(current.workspace, closingEpoch); - return cfg; + const closingBefore = + entry === null ? undefined : workspaceMemoryWritableForEpoch(entry.workspace, closingEpoch); + if (closingBefore === undefined) return; + await this.config.editConfig((cfg) => { + const current = findWorkspaceEntry(cfg, this.workspaceId); + if (current === null) return cfg; + const closing = workspaceMemoryWritableForEpoch(current.workspace, closingEpoch); + if (closing === undefined) return cfg; + // ANDed into a record another backend's first turn of the new epoch + // may already have made (its own conjunction could not see the closing + // value under the new key), never overwriting it. + const next = workspaceMemoryWritableForEpoch(current.workspace, nextEpoch); + carried = next === undefined ? closing : next && closing; + setWorkspaceMemoryWritableForEpoch(current.workspace, nextEpoch, carried); + deleteWorkspaceMemoryWritableForEpoch(current.workspace, closingEpoch); + return cfg; + }); + // Consumed by another backend's boundary meanwhile: nothing to carry. + if (carried === undefined) return; + const after = findWorkspaceEntry( + this.config.loadConfigOrDefault({ throwOnError: true }), + this.workspaceId + ); + if ( + after === null || + workspaceMemoryWritableForEpoch(after.workspace, nextEpoch) !== carried + ) { + failure = "config write swallowed"; + } + } catch (error: unknown) { + failure = getErrorMessage(error); + } + // A grant that failed to persist leaves the new epoch without a record, + // which every reader treats as "grants normally" — nothing to fail + // closed on. A deny (or an unknown value) must be made durable somewhere. + if (failure === undefined || carried === true) return; + log.warn("Workspace memory policy carry not durable in config; recording a deny marker", { + workspaceId: this.workspaceId, + closingEpoch, + nextEpoch, + failure, + }); + // Same gate as WorkspaceService.denyDurableFallback: the marker's mkdir + // must not recreate a session dir a concurrent remover already deleted. + await withTargetMutationLock(this.config.rootDir, sessionDir, async () => { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, this.workspaceId)) return; + await writeWorkspaceMemoryDenyMarker(sessionDir, nextEpoch); }); } diff --git a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts new file mode 100644 index 00000000000..3e21d1fa877 --- /dev/null +++ b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts @@ -0,0 +1,187 @@ +import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; +import * as fsPromises from "fs/promises"; +import * as path from "path"; +import type { Config } from "@/node/config"; +import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; +import { getErrorMessage } from "@/common/utils/errors"; +import { findWorkspaceEntry } from "@/node/services/taskUtils"; +import { readWorkspaceMemoryDenyMarker } from "@/node/services/workspaceMemoryDenyMarker"; +import { AgentSession } from "./agentSession"; +import { createStreamLifecycleMocks } from "./agentSession.testHarness"; +import type { AIService } from "./aiService"; +import type { BackgroundProcessManager } from "./backgroundProcessManager"; +import type { InitStateManager } from "./initStateManager"; +import { createTestHistoryService } from "./testHistoryService"; + +/** + * Durable epoch boundary of the workspace-memory write policy + * (AgentSession.resetWorkspaceMemoryWritable / carryWorkspaceMemoryWritable): + * Config.saveConfig swallows write failures, so both must prove their effect + * by reading back, and a deny the carry cannot prove reached the new epoch + * falls back to the session-dir marker. + */ +interface SessionInternals { + resetWorkspaceMemoryWritable(options?: { closingEpoch: number }): Promise; + carryWorkspaceMemoryWritable(closingEpoch: number, nextEpoch: number): Promise; +} + +const WORKSPACE_ID = "policy-epoch-ws"; + +describe("AgentSession workspace memory policy epoch boundary", () => { + let cleanup: (() => Promise) | undefined; + const sessions: AgentSession[] = []; + + afterEach(async () => { + for (const session of sessions.splice(0)) await session.dispose(); + await cleanup?.(); + cleanup = undefined; + mock.restore(); + }); + + const createSession = async () => { + const harness = await createTestHistoryService(); + cleanup = harness.cleanup; + const config: Config = harness.config; + await fsPromises.mkdir(path.join(config.sessionsDir, WORKSPACE_ID), { recursive: true }); + await config.editConfig((cfg) => { + cfg.projects.set(SCRATCH_PROJECT_CONFIG_KEY, { + workspaces: [ + { + kind: "scratch", + path: path.join(config.rootDir, "scratch", WORKSPACE_ID), + id: WORKSPACE_ID, + name: WORKSPACE_ID, + runtimeConfig: { type: "local" }, + }, + ], + projectKind: "system", + trusted: true, + }); + return cfg; + }); + const session = new AgentSession({ + workspaceId: WORKSPACE_ID, + config, + historyService: harness.historyService, + aiService: { + on() { + return this; + }, + off() { + return this; + }, + ...createStreamLifecycleMocks(), + isStreaming: () => false, + } as unknown as AIService, + initStateManager: { + on() { + return this; + }, + off() { + return this; + }, + } as unknown as InitStateManager, + backgroundProcessManager: { + cleanup: mock(() => Promise.resolve()), + setMessageQueued: mock(() => undefined), + } as unknown as BackgroundProcessManager, + }); + sessions.push(session); + const records = () => + findWorkspaceEntry(config.loadConfigOrDefault(), WORKSPACE_ID)?.workspace + .workspaceMemoryWritableByEpoch; + const setRecords = (value: Record) => + config.editConfig((cfg) => { + findWorkspaceEntry(cfg, WORKSPACE_ID)!.workspace.workspaceMemoryWritableByEpoch = value; + return cfg; + }); + // A config write the Config layer swallowed: the edit runs on a loaded + // copy and resolves, nothing lands on disk. + const swallowNextWrite = () => + spyOn(config, "editConfig").mockImplementationOnce((edit) => { + edit(config.loadConfigOrDefault()); + return Promise.resolve(); + }); + return { + session, + config, + internals: session as unknown as SessionInternals, + sessionDir: path.join(config.sessionsDir, WORKSPACE_ID), + records, + setRecords, + swallowNextWrite, + }; + }; + + test("carry re-binds the closing epoch's value and proves it, falling back to the deny marker", async () => { + const { internals, sessionDir, records, setRecords, swallowNextWrite, config } = + await createSession(); + // Proven carry: the closing deny moves to the new epoch key. + await setRecords({ "-1": false }); + await internals.carryWorkspaceMemoryWritable(-1, 7); + expect(records()).toEqual({ "7": false }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(false); + + // Swallowed write: the deny never reached epoch 9 in config, so the + // session-dir marker denies epoch 9 instead of nothing. + await setRecords({ "-1": false }); + swallowNextWrite(); + await internals.carryWorkspaceMemoryWritable(-1, 9); + expect(records()).toEqual({ "-1": false }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(false); + + // Unreadable config: the closing value is unknown, which also fails closed. + const real = config.loadConfigOrDefault.bind(config); + const unreadable = spyOn(config, "loadConfigOrDefault").mockImplementation((options) => { + if (options?.throwOnError) throw new Error("EIO"); + return { ...real(), projects: new Map() }; + }); + try { + await internals.carryWorkspaceMemoryWritable(-1, 11); + } finally { + unreadable.mockRestore(); + } + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 11)).toBe(true); + + // A GRANT that failed to persist leaves the new epoch without a record, + // which readers treat as "grants normally": no marker. + await setRecords({ "-1": true }); + swallowNextWrite(); + await internals.carryWorkspaceMemoryWritable(-1, 13); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 13)).toBe(false); + }); + + test("reset clears the mirror only once the durable clear is proven", async () => { + const { session, internals, records, setRecords, swallowNextWrite } = await createSession(); + // Destructive boundary: every record goes, then the mirror. + session.recordWorkspaceMemoryWritable(false); + await setRecords({ "-1": false, "5": true }); + await internals.resetWorkspaceMemoryWritable(); + expect(records()).toBeUndefined(); + expect(session.workspaceMemoryWritableMirror()).toBeUndefined(); + + // Swallowed write: a surviving `-1: false` would pin the new segment to + // the stored-false fast path — the reset must fail (retryable) and keep + // the mirror rather than report success. + session.recordWorkspaceMemoryWritable(false); + await setRecords({ "-1": false }); + swallowNextWrite(); + expect( + await internals.resetWorkspaceMemoryWritable().then(() => null, getErrorMessage) + ).toMatch(/did not persist/); + expect(records()).toEqual({ "-1": false }); + expect(session.workspaceMemoryWritableMirror()).toBe(false); + + // Compaction boundary, fenced to the closing epoch: the same proof. + await setRecords({ "3": false, "8": true }); + swallowNextWrite(); + expect( + await internals + .resetWorkspaceMemoryWritable({ closingEpoch: 3 }) + .then(() => null, getErrorMessage) + ).toMatch(/did not persist/); + await internals.resetWorkspaceMemoryWritable({ closingEpoch: 3 }); + expect(records()).toEqual({ "8": true }); + }); +}); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 2f8cd0d8f3b..f61f4c1ac60 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1942,9 +1942,52 @@ describe("MemoryService", () => { } finally { lossy.mockRestore(); } + // A legacy root that cannot be inspected (EACCES) is not "nothing to + // adopt" either: removal must abort rather than delete it unseen. + const realLstat = fsPromises.lstat.bind(fsPromises); + const unreadableRoot = spyOn(fsPromises, "lstat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === legacyRoot + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realLstat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/could not be inspected/); + } finally { + unreadableRoot.mockRestore(); + } // Space frees up; the in-lock delta pass (removal holds the owner-store // lock already) folds the note in without re-acquiring the lock. await fsPromises.rm(path.join(ownerRoot, "o0000.md")); + // A destination whose stat fails (not a proven absence) is not free: the + // pass aborts instead of overwriting whatever the owner keeps there. + await fsPromises.writeFile(path.join(ownerRoot, "stranded.md"), "owner's own"); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadableTarget = spyOn(fsPromises, "stat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === path.join(ownerRoot, "stranded.md") + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realStat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EIO/); + } finally { + unreadableTarget.mockRestore(); + } + expect(await fsPromises.readFile(path.join(ownerRoot, "stranded.md"), "utf-8")).toBe( + "owner's own" + ); + await fsPromises.rm(path.join(ownerRoot, "stranded.md")); await withTargetMutationLock( fixture.xumHome, memoryMutationLockKey(fixture.xumHome, ownerRoot), diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index b2baae85133..756c3e47626 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -381,7 +381,12 @@ interface MemoryStore { * callers whose decision must not rest on a possibly partial listing. */ listFiles(options?: { strict?: boolean }): Promise; - kind(relPath: string): Promise; + /** + * Kind of an entry, null when absent. Tolerant by default (any stat failure + * reads as absent); `strict` throws unless the absence is proven (ENOENT / + * ENOTDIR), for callers about to overwrite whatever is there. + */ + kind(relPath: string, options?: { strict?: boolean }): Promise; /** * Read at most `maxBytes` from the head of the file. Index/hot-set builds * use this so files edited outside MemoryService cannot force unbounded reads @@ -513,13 +518,26 @@ async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Pr return `${revision ?? "none"}:${rootMtime}:${fileStamps.join("\u0001")}`; } -/** Link-aware kind of a path: symlinks are reported as such, never followed. */ -async function lstatKind(absPath: string): Promise<"dir" | "symlink" | "other" | "missing"> { +/** A stat failure that proves the path is absent (vs. one that says nothing about it). */ +function isMissingPathError(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException | null)?.code; + return code === "ENOENT" || code === "ENOTDIR"; +} + +/** + * Link-aware kind of a path: symlinks are reported as such, never followed. + * "missing" only when proven (ENOENT/ENOTDIR); any other failure (EACCES, + * EIO) is "unreadable" — a legacy notebook whose root cannot be inspected + * must not read as "nothing to adopt" to a removal about to delete it. + */ +async function lstatKind( + absPath: string +): Promise<"dir" | "symlink" | "other" | "missing" | "unreadable"> { try { const stat = await fsPromises.lstat(absPath); return stat.isSymbolicLink() ? "symlink" : stat.isDirectory() ? "dir" : "other"; - } catch { - return "missing"; + } catch (error) { + return isMissingPathError(error) ? "missing" : "unreadable"; } } @@ -611,11 +629,12 @@ class LocalMemoryStore implements MemoryStore { return results.sort(); } - async kind(relPath: string): Promise { + async kind(relPath: string, options?: { strict?: boolean }): Promise { try { const stat = await fsPromises.stat(this.abs(relPath)); return stat.isDirectory() ? "dir" : "file"; - } catch { + } catch (error) { + if (options?.strict === true && !isMissingPathError(error)) throw error; return null; } } @@ -1289,6 +1308,12 @@ export class MemoryService extends EventEmitter { if (options?.force !== true && this.legacyStoreCheckedAgainst.get(childId) === checkKey) { return { skipped: 0 }; } + // Not "nothing to adopt": a root that could not be inspected may hold the + // only copy of downgrade-era notes. Access-time callers log and retry; + // removal aborts with the session intact. + const unreadableRoot = (): Error => + new Error(`the legacy workspace memory root of ${childId} could not be inspected`); + if (legacyRootKind === "unreadable") throw unreadableRoot(); if (legacyRootKind !== "dir") { if (legacyRootKind === "symlink") { log.warn("[MemoryService] ignoring a symlinked legacy workspace memory root", { @@ -1305,7 +1330,9 @@ export class MemoryService extends EventEmitter { let skipped = 0; const pass = async (): Promise => { await this.assertMutationCommittable(ctx, store, undefined, toVirtualPath("workspace", "")); - if ((await lstatKind(legacyRoot)) !== "dir") return; // swapped while waiting for the lock + const rootKindUnderLock = await lstatKind(legacyRoot); + if (rootKindUnderLock === "unreadable") throw unreadableRoot(); + if (rootKindUnderLock !== "dir") return; // swapped while waiting for the lock const legacy = new LocalMemoryStore(legacyRoot); // Strict: a note omitted by a partial listing would count as "nothing // to adopt" (skipped stays 0) and removal would then delete its only @@ -1591,7 +1618,11 @@ export class MemoryService extends EventEmitter { () => false ); if (!contained) continue; - const kind = await store.kind(candidate); + // Strict: a destination that merely could not be stat'ed (EACCES, EIO) + // is not free — declaring it so would overwrite whatever the owner + // keeps there once the copy runs. The failure aborts the pass instead + // (access-time: retried; removal: session intact). + const kind = await store.kind(candidate, { strict: true }); if (kind === null) return { relPath: candidate, write: true }; if (kind === "file") { const existing = await this.readBoundedTextFile(store, candidate, candidate).catch( diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts index 24e956e2261..7fabd3f9139 100644 --- a/src/node/services/workspaceMemoryPolicyEpochs.ts +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -37,21 +37,40 @@ export function workspaceMemoryWritableForEpoch( entry: WorkspaceConfigEntry, epoch: number ): boolean | undefined { - const records: Record | undefined = entry.workspaceMemoryWritableByEpoch; + const records = policyRecords(entry); + if (records === undefined) return undefined; + // A container of the wrong shape (null, array, string) is corruption too: + // fail closed for every epoch rather than throw out of every turn start + // (Object.hasOwn(null) would). The next write or forget heals it. + if (records === null) return false; const key = epochKey(epoch); - if (records === undefined || !Object.hasOwn(records, key)) return undefined; + if (!Object.hasOwn(records, key)) return undefined; const value = records[key]; return typeof value === "boolean" ? value : false; } +/** + * The persisted container: `undefined` when absent, `null` when present but + * not a plain object (raw JSON, no schema validation upstream). + */ +function policyRecords(entry: WorkspaceConfigEntry): Record | null | undefined { + const records: unknown = entry.workspaceMemoryWritableByEpoch; + if (records === undefined) return undefined; + return typeof records === "object" && records !== null && !Array.isArray(records) + ? (records as Record) + : null; +} + /** Record `writable` for `epoch`. */ export function setWorkspaceMemoryWritableForEpoch( entry: WorkspaceConfigEntry, epoch: number, writable: boolean ): void { + // A malformed container is replaced, not spread (spreading a string would + // persist its characters as epoch keys). entry.workspaceMemoryWritableByEpoch = { - ...entry.workspaceMemoryWritableByEpoch, + ...(policyRecords(entry) === null ? {} : entry.workspaceMemoryWritableByEpoch), [epochKey(epoch)]: writable, }; } @@ -61,11 +80,16 @@ export function deleteWorkspaceMemoryWritableForEpoch( entry: WorkspaceConfigEntry, epoch: number ): void { - const current = entry.workspaceMemoryWritableByEpoch; + const current = policyRecords(entry); if (current === undefined) return; + if (current === null) { + // Malformed container: nothing recoverable in it, heal by dropping it. + delete entry.workspaceMemoryWritableByEpoch; + return; + } const key = epochKey(epoch); - if (!(key in current)) return; - const next = { ...current }; + if (!Object.hasOwn(current, key)) return; + const next = { ...entry.workspaceMemoryWritableByEpoch }; delete next[key]; if (Object.keys(next).length === 0) delete entry.workspaceMemoryWritableByEpoch; else entry.workspaceMemoryWritableByEpoch = next; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index c8764c35695..91403d5f6f6 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9,7 +9,10 @@ import { workspaceRemovalTombstonePath, isWorkspaceRemovalTombstoned, } from "@/node/services/workspaceRemoval"; -import { workspaceMemoryWritableForEpoch } from "@/node/services/workspaceMemoryPolicyEpochs"; +import { + setWorkspaceMemoryWritableForEpoch, + workspaceMemoryWritableForEpoch, +} from "@/node/services/workspaceMemoryPolicyEpochs"; import { withTargetMutationLock } from "@/node/services/refinement/targetMutationLocks"; import type { TurnCompletion } from "./streamManager"; import type { TurnCoordinator } from "./turnCoordinator"; @@ -9534,6 +9537,33 @@ describe("WorkspaceService initialize", () => { ) ).toBe(false); } + // A corrupted CONTAINER (null, array, string) must not throw out of + // every turn start; it reads as a deny for every epoch and is healed + // (replaced, not spread) by the next write. + for (const container of [null, "false", ["-1"]]) { + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace as Record< + string, + unknown + >; + entry.workspaceMemoryWritableByEpoch = container; + return cfg; + }); + const corrupted = findWorkspaceEntry( + realConfig.loadConfigOrDefault(), + "policy-scratch" + )!.workspace; + expect(workspaceMemoryWritableForEpoch(corrupted, 60)).toBe(false); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 60, + }) + ).toBe(true); + expect(persistedFor(60)).not.toBe(true); + setWorkspaceMemoryWritableForEpoch(corrupted, 61, true); + expect(corrupted.workspaceMemoryWritableByEpoch).toEqual({ "61": true }); + } // Unknown history fails closed: no accumulator, no marker, no mirror // (this service never recorded this epoch), yet the epoch already holds diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index a2f438bc55b..464e6f7eabf 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -6162,6 +6162,18 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { return Err(DESCENDANT_WORKSPACE_REMOVE_ERROR); } const sessionDir = path.join(this.config.sessionsDir, workspaceId); + // r65: keep renewing the removal tombstone's mtime until this removal + // settles so a foreign backend's startup self-heal cannot mistake a + // merely SLOW removal (a hung runtime deletion or MCP server close) for + // crash residue and delete the marker while removal is live — a healed + // marker would readmit child writes after the final shared-memory + // handover (sealSubAgentForRemovalUnderMemoryLocks), which the later + // republish (tombstoneSealed) does not migrate. Held from before the + // earliest publish point: ticks against a not-yet-published marker are + // swallowed ENOENTs, as are ticks after a rollback deleted it, and + // disposal at scope exit (after deregistration or its rollback) is safe + // since a late renewal of a retained terminal marker is meaningless. + using _tombstoneLease = startRemovalTombstoneLease(this.config.rootDir, workspaceId); // The captured engine stop joins partial finalization and raw terminal delivery. try { @@ -6817,16 +6829,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { } log.error(`Failed to remove session directory for ${workspaceId}:`, error); } - // r65: the tombstone is durable here (both the locked path and the - // orphan fallback published it). Keep renewing its mtime until this - // removal settles so a foreign backend's startup self-heal cannot - // mistake a merely SLOW removal (e.g. a hung MCP server close below) - // for crash residue and delete the marker while removal is live. - // Disposal at scope exit (after deregistration or its rollback) is - // safe: a late renewal of a retained terminal marker is meaningless, - // and utimes on a rolled-back (deleted) marker is a swallowed ENOENT. - using _tombstoneLease = startRemovalTombstoneLease(this.config.rootDir, workspaceId); - // The on-disk devtools.jsonl died with the session directory above; also drop any // in-memory DevTools state so stale runs cannot outlive the workspace. try { From 3fb79ba51d1b08510c60c199e1e25942eea50812 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 20:47:42 +0000 Subject: [PATCH 59/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-thi?= =?UTF-8?q?rd=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Strict legacy listings ignore the per-entry cap inside a directory too. - Preserved-tail epochs name the epoch their tail was copied out of (`carriedPolicyEpoch`); the policy sink ANDs that epoch's record and deny marker directly, so another backend's first turn of the new epoch cannot grant before the compacting session's asynchronous carry lands. - Refinement rollbacks of a sub-agent's pre-sharing rows are retargeted through the adoption manifest to the owner copy the shared notebook serves; unadopted legacy paths refuse instead of mutating hidden files. Manifest primitives move to memoryLegacyAdoption.ts. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.ts | 6 +- src/node/services/memoryLegacyAdoption.ts | 134 ++++++++++++++++++ src/node/services/memoryService.test.ts | 26 ++++ src/node/services/memoryService.ts | 66 +-------- .../refinement/refinementRollback.test.ts | 55 +++++++ .../services/refinement/refinementRollback.ts | 109 +++++++++++--- src/node/services/turnRequestBuilder.ts | 16 ++- src/node/services/workspaceService.test.ts | 53 +++++++ src/node/services/workspaceService.ts | 41 +++++- 9 files changed, 414 insertions(+), 92 deletions(-) create mode 100644 src/node/services/memoryLegacyAdoption.ts diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 1d391bbad9a..a5997cbf491 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -79,13 +79,17 @@ export function isDurableContextBoundaryMarker(message: MuxMessage | undefined): * accumulator is bound to it (WorkspaceService.recordWorkspaceMemoryWritable). */ export function latestContextBoundaryHistorySequence( - messages: readonly MuxMessage[] + messages: readonly MuxMessage[], + options?: { before: number } ): number | undefined { let latest: number | undefined; for (const message of messages) { if (!isDurableContextBoundaryMarker(message)) continue; const sequence = message.metadata?.historySequence; if (typeof sequence !== "number" || !Number.isInteger(sequence) || sequence < 0) continue; + // `before`: the boundary preceding a given one (the epoch a preserved + // tail was copied out of). + if (options !== undefined && sequence >= options.before) continue; if (latest === undefined || sequence > latest) latest = sequence; } return latest; diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts new file mode 100644 index 00000000000..a59f1444a9a --- /dev/null +++ b/src/node/services/memoryLegacyAdoption.ts @@ -0,0 +1,134 @@ +/** + * Legacy-notebook adoption manifest: the durable record of which files of a + * sub-agent's PRE-SHARING private notebook (`/memory`, written by + * builds that kept `/memories/workspace` per workspace) were folded into the + * task-tree owner's shared store, and where each landed + * (MemoryService.adoptLegacyPrivateStore). Shared with the refinement + * rollback engine: refinement rows journaled before the upgrade address the + * legacy files, while the note the user sees since is the owner copy. + */ +import * as fsPromises from "node:fs/promises"; +import * as path from "node:path"; +import type { RefinementInverse } from "@/common/types/refinement"; + +/** + * Dotfile inside a sub-agent's legacy `memory` dir recording, per relPath, the + * sha256 of the content already copied into the shared store + * (adoptLegacyPrivateStore). Dotfiles are invisible to every build's listing. + */ +export const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; + +/** + * One adopted legacy file: content hash, child sidecar fingerprint, owner-store + * relPath, and whether the adoption CREATED that owner file (provenance: only + * such a copy may be removed again when the legacy source disappears; a + * pre-existing identical owner note is the owner's own). `pending`: written + * BEFORE the copy lands (provenance must not depend on the copy's existence: a + * retry finding the bytes already at the target could not tell an interrupted + * adoption from an owner note); cleared once the sidecar fold completed. + */ +export interface LegacyAdoptionRecord { + content: string; + sidecar: string; + target: string; + created?: boolean; + pending?: boolean; +} + +function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { + if (typeof value !== "object" || value === null) return false; + const record = value as Record; + return ( + typeof record.content === "string" && + typeof record.sidecar === "string" && + typeof record.target === "string" + ); +} + +/** + * Self-healing read of the adoption manifest: anything malformed reads as not + * adopted. A Map, not a plain object: a legacy note may legitimately be named + * `__proto__` (any store-valid relPath), and assigning that key on an + * ordinary object hits the prototype setter instead of creating an entry the + * serialization would carry — the note would then be re-adopted (and the + * owner clock advanced) on every access. JSON.parse and Object.fromEntries + * create own properties, so the round-trip below is exact. + */ +export async function readLegacyAdoptionManifest( + manifestPath: string +): Promise> { + try { + const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); + return new Map( + Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => + isLegacyAdoptionRecord(entry[1]) + ) + ); + } catch { + return new Map(); + } +} + +/** Thrown for a legacy path the shared store does not represent (see below). */ +export class LegacyPathNotAdoptedError extends Error { + constructor(legacyPath: string) { + super( + `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` + ); + this.name = "LegacyPathNotAdoptedError"; + } +} + +/** + * Retargets refinement inverses (and any other recorded path) of a sub-agent + * whose legacy private notebook was adopted into the owner's store: a path + * under `/memory` becomes the owner-store path its note was + * folded into, so a rollback reverts the copy the shared notebook actually + * serves rather than the hidden legacy file (which the next adoption pass + * would re-import as a conflicting duplicate). Paths outside the legacy root + * pass through unchanged. A legacy path the manifest does not know throws + * LegacyPathNotAdoptedError — fail closed rather than mutate an invisible + * file. Only the manifest's `target` is trusted for the destination's + * relPath; callers re-run their confinement checks on the mapped result. + */ +export async function createLegacyPathRemapper(args: { + childSessionDir: string; + ownerSessionDir: string; +}): Promise<{ + path(filePath: string): string; + inverse(inverse: RefinementInverse): RefinementInverse; +}> { + const legacyRoot = path.join(path.resolve(args.childSessionDir), "memory"); + const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const adopted = await readLegacyAdoptionManifest( + path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME) + ); + const remapPath = (filePath: string): string => { + const relative = path.relative(legacyRoot, path.resolve(filePath)); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return filePath; + const record = adopted.get(relative.split(path.sep).join("/")); + if (record === undefined || record.pending === true) + throw new LegacyPathNotAdoptedError(filePath); + return path.join(ownerRoot, ...record.target.split("/")); + }; + return { + path: remapPath, + inverse: (inverse) => { + switch (inverse.op) { + case "delete-files": + return { ...inverse, paths: inverse.paths.map(remapPath) }; + case "rename": + return { ...inverse, from: remapPath(inverse.from), to: remapPath(inverse.to) }; + case "restore-files": + return { + ...inverse, + files: inverse.files.map((file) => ({ ...file, path: remapPath(file.path) })), + ...(inverse.deletePaths === undefined + ? {} + : { deletePaths: inverse.deletePaths.map(remapPath) }), + }; + } + }, + }; +} diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index f61f4c1ac60..ae08ecb3d91 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1687,6 +1687,32 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); }); + it("removal lists an oversized legacy notebook completely, counting every unplaceable note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner store with one free slot; legacy notebook two notes past the + // per-scope cap in one flat directory. A capped walk would list cap+1 + // notes and report cap skipped; every note beyond the cap must be + // listed and reported so removal cannot delete an unlisted one. + await Promise.all([ + ...Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE - 1 }, (_, i) => + fsPromises.writeFile(path.join(ownerRoot, `o${String(i).padStart(4, "0")}.md`), "o") + ), + ...Array.from({ length: MEMORY_MAX_FILES_PER_SCOPE + 2 }, (_, i) => + fsPromises.writeFile(path.join(legacyRoot, `n${String(i).padStart(4, "0")}.md`), "n") + ), + ]); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(new RegExp(`^${MEMORY_MAX_FILES_PER_SCOPE + 1} legacy workspace memory note`)); + }); + it("re-adopts when a downgraded build edits a nested legacy note in place or only its pin", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 756c3e47626..22c1d79f557 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -48,6 +48,11 @@ import { withTargetMutationLock, } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; +import { + LEGACY_ADOPTION_MANIFEST_FILE_NAME, + readLegacyAdoptionManifest, + type LegacyAdoptionRecord, +} from "@/node/services/memoryLegacyAdoption"; import { resolveWorkspaceMemoryOwnerId, workspaceMemoryOwnerResolver, @@ -413,40 +418,6 @@ interface MemoryStore { */ const LEGACY_IMPORT_DIR = "imported"; -/** - * Dotfile inside a sub-agent's legacy `memory` dir recording, per relPath, the - * sha256 of the content already copied into the shared store - * (adoptLegacyPrivateStore). Dotfiles are invisible to every build's listing. - */ -const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; - -/** - * One adopted legacy file: content hash, child sidecar fingerprint, owner-store - * relPath, and whether the adoption CREATED that owner file (provenance: only - * such a copy may be removed again when the legacy source disappears; a - * pre-existing identical owner note is the owner's own). `pending`: written - * BEFORE the copy lands (provenance must not depend on the copy's existence: a - * retry finding the bytes already at the target could not tell an interrupted - * adoption from an owner note); cleared once the sidecar fold completed. - */ -interface LegacyAdoptionRecord { - content: string; - sidecar: string; - target: string; - created?: boolean; - pending?: boolean; -} - -function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { - if (typeof value !== "object" || value === null) return false; - const record = value as Record; - return ( - typeof record.content === "string" && - typeof record.sidecar === "string" && - typeof record.target === "string" - ); -} - /** * Pin bit of a manifest record's child sidecar fingerprint. No child entry at * that adoption is the default, unpinned state (a usage entry a downgraded @@ -465,31 +436,6 @@ function legacySidecarPinned(sidecar: string): boolean | null { } } -/** - * Self-healing read of the adoption manifest: anything malformed reads as not - * adopted. A Map, not a plain object: a legacy note may legitimately be named - * `__proto__` (any store-valid relPath), and assigning that key on an - * ordinary object hits the prototype setter instead of creating an entry the - * serialization would carry — the note would then be re-adopted (and the - * owner clock advanced) on every access. JSON.parse and Object.fromEntries - * create own properties, so the round-trip below is exact. - */ -async function readLegacyAdoptionManifest( - manifestPath: string -): Promise> { - try { - const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); - return new Map( - Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => - isLegacyAdoptionRecord(entry[1]) - ) - ); - } catch { - return new Map(); - } -} - /** * Change stamp of a sub-agent's legacy private store: its store clock (any * MemoryService write there advances it, including a foreign backend's @@ -615,7 +561,7 @@ class LocalMemoryStore implements MemoryStore { }); for (const entry of entries) { // Per-entry cap: a single flat directory can exceed the cap on its own. - if (results.length > MEMORY_MAX_FILES_PER_SCOPE) return; + if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; if (entry.name.startsWith(".")) continue; const childRel = dirRel === "" ? entry.name : `${dirRel}/${entry.name}`; if (entry.isDirectory()) { diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 2c0cfa06d35..b34da1d4309 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -873,6 +873,61 @@ describe("refinementRollback", () => { expect(await pathExists(path.join(fixture.muxHome, "memory", "global", "new.md"))).toBe(false); }); + it("retargets pre-sharing workspace rows to the adopted owner copy, refusing unadopted ones", async () => { + using fixture = await createFixture(); + // Rows journaled while the workspace owned its store address + // /memory (the legacy private notebook after an upgrade). + await fixture.service.create(fixture.ctx, "/memories/workspace/note.md", "v1\n", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/note.md", + "v1", + "v2", + "agent" + ); + const editRow = await lastRow(fixture.sessionDir); + await fixture.service.create(fixture.ctx, "/memories/workspace/orphan.md", "o1\n", "agent"); + const orphanRow = await lastRow(fixture.sessionDir); + // The upgrade folded note.md into the task-tree owner's store (adoption + // manifest beside the legacy files); orphan.md could not be placed. + const ownerSessionDir = path.join(path.dirname(fixture.sessionDir), "ws-owner"); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "sub", "note.md"), "v2\n"); + await fsPromises.writeFile( + path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + }) + ); + const result = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(result.success).toBe(true); + // The note the shared notebook serves is reverted; the hidden legacy file + // is left alone (it must keep matching the manifest, or the next adoption + // pass would re-import it as a conflicting duplicate). + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "sub", "note.md"), "utf-8") + ).toBe("v1\n"); + expect( + await fsPromises.readFile(path.join(fixture.sessionDir, "memory", "note.md"), "utf-8") + ).toBe("v2\n"); + const refused = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: orphanRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "not folded into the shared workspace store" + ); + expect(await pathExists(path.join(fixture.sessionDir, "memory", "orphan.md"))).toBe(true); + }); + it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { using fixture = await createFixture(); const virtualPath = "/memories/global/order.md"; diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 81d1b8ac28f..c86813684ce 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -55,6 +55,10 @@ import { import { withTargetMutationLocks } from "./targetMutationLocks"; import { advanceWorkspaceMemoryRevision } from "./workspaceMemoryRevision"; import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, +} from "@/node/services/memoryLegacyAdoption"; export type RefinementEvent = Extract; @@ -579,11 +583,41 @@ async function readSharedMemoryPeerRows( return peerRows; } +/** + * Path retargeting applied to every recorded path of the acting session's + * journal before it is compared or applied (see createLegacyPathRemapper): + * identity for a session that owns its store. Rows whose paths cannot be + * mapped (a legacy note the shared store never took) contribute nothing to + * divergence — they address an invisible file the target cannot overlap. + */ +interface RecordedPathRemapper { + path(filePath: string): string; + inverse(inverse: RefinementInverse): RefinementInverse; +} +const identityRemapper: RecordedPathRemapper = { + path: (filePath) => filePath, + inverse: (inverse) => inverse, +}; +function parseRemappedInverse( + row: RefinementEvent, + remap: RecordedPathRemapper +): RefinementInverse | null { + const parsed = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsed.success) return null; + try { + return remap.inverse(parsed.data); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } +} + async function collectDivergence( rows: RefinementEvent[], target: RefinementEvent, inverse: RefinementInverse, - readContent: InverseContentReader + readContent: InverseContentReader, + remap: RecordedPathRemapper ): Promise { const complaints: string[] = []; const targetPaths = inversePaths(inverse); @@ -601,11 +635,9 @@ async function collectDivergence( if (!isAfter(row, target)) continue; if (rolledBackIds.has(row.id)) continue; // Effect undone by a later rollback row. if (!liveRowConflictsWithTarget(rows, row, target)) continue; - const parsed = RefinementInverseSchema.safeParse(row.data.inverse); - if (!parsed.success) continue; - const overlap = inversePaths(parsed.data).some((p) => - targetPaths.some((t) => pathsOverlap(p, t)) - ); + const parsed = parseRemappedInverse(row, remap); + if (parsed === null) continue; + const overlap = inversePaths(parsed).some((p) => targetPaths.some((t) => pathsOverlap(p, t))); if (overlap) { complaints.push(`later refinement row ${row.id} (seq ${row.seq}) touched the same paths`); } @@ -637,7 +669,7 @@ async function collectDivergence( // so the current state must still match that applied inverse — // content-exact where the original restored files. complaints.push( - ...(await collectRollbackTargetDivergence(rows, rollbackAction.data, readContent)) + ...(await collectRollbackTargetDivergence(rows, rollbackAction.data, readContent, remap)) ); break; } @@ -666,7 +698,7 @@ async function collectDivergence( // Content-exact check via the row's recorded post-action hashes: a manual // or cross-workspace edit after the target row never appears in this // session's journal, so the seq-based scan above cannot see it. - complaints.push(...(await collectPostStateDivergence(target))); + complaints.push(...(await collectPostStateDivergence(target, remap))); return complaints; } @@ -678,22 +710,27 @@ async function collectDivergence( * complaints — their expected post-edit contents cannot be reconstructed from * the journal, so the presence-only checks above are the best we can do. */ -async function collectPostStateDivergence(target: RefinementEvent): Promise { +async function collectPostStateDivergence( + target: RefinementEvent, + remap: RecordedPathRemapper +): Promise { const postState = RefinementPostStateSchema.safeParse(target.data.postState); if (!postState.success) { return []; } const complaints: string[] = []; for (const file of postState.data.files) { + let filePath: string; let current: string; try { - current = await fsPromises.readFile(file.path, "utf-8"); + filePath = remap.path(file.path); + current = await fsPromises.readFile(filePath, "utf-8"); } catch { - continue; // Missing files are already reported by the presence checks. + continue; // Missing (or unmappable) files are already reported by the presence checks. } if (sha256Hex(current) !== file.sha256) { complaints.push( - `'${file.path}' was modified after the target refinement (current content no longer matches the state it left behind)` + `'${filePath}' was modified after the target refinement (current content no longer matches the state it left behind)` ); } } @@ -755,20 +792,21 @@ async function dirExists(target: string): Promise { async function collectRollbackTargetDivergence( rows: RefinementEvent[], action: RollbackRefinementAction, - readContent: InverseContentReader + readContent: InverseContentReader, + remap: RecordedPathRemapper ): Promise { const original = rows.find((row) => row.id === action.of); if (original === undefined) { return [`the original row '${action.of}' this rollback applied is missing from the journal`]; } - const applied = RefinementInverseSchema.safeParse(original.data.inverse); - if (!applied.success) { - return [`the original row '${action.of}' has an unparseable inverse`]; + const applied = parseRemappedInverse(original, remap); + if (applied === null) { + return [`the original row '${action.of}' has an unparseable or unmappable inverse`]; } const complaints: string[] = []; - switch (applied.data.op) { + switch (applied.op) { case "delete-files": - for (const p of applied.data.paths) { + for (const p of applied.paths) { if (await fileExists(p)) { complaints.push( `expected '${p}' to be absent (the rollback deleted it), but it was recreated since` @@ -777,7 +815,7 @@ async function collectRollbackTargetDivergence( } break; case "restore-files": - for (const file of applied.data.files) { + for (const file of applied.files) { if (!(await fileExists(file.path))) { complaints.push( `expected '${file.path}' to exist (the rollback restored it), but it was deleted since` @@ -792,7 +830,7 @@ async function collectRollbackTargetDivergence( } // Mixed force-apply inverse (r67): the rollback also deleted these // paths, so their recreation since is divergence too. - for (const p of applied.data.deletePaths ?? []) { + for (const p of applied.deletePaths ?? []) { if (await fileExists(p)) { complaints.push( `expected '${p}' to be absent (the rollback deleted it), but it was recreated since` @@ -868,7 +906,30 @@ export async function rollbackRefinement( `Row '${opts.id}' has an unparseable inverse payload: ${parsedInverse.error.message}` ); } - const inverse = parsedInverse.data; + // Sub-agent whose `/memories/workspace` is the owner's store: rows + // journaled before sharing address its own /memory, whose + // notes were since folded into the owner's store (memoryLegacyAdoption). + // Every recorded path of this journal — the target's inverse, later rows + // for overlap, post-state hashes, a rollback chain's root — is retargeted + // to the adopted copy, so the rollback reverts the note the shared + // notebook serves. Mapped BEFORE confinement: the manifest's targets are + // checked like any other recorded path. + const remap: RecordedPathRemapper = + kind === "memory" && + opts.sharedWorkspaceMemorySessionDir !== undefined && + path.resolve(opts.sharedWorkspaceMemorySessionDir) !== path.resolve(opts.sessionDir) + ? await createLegacyPathRemapper({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + }) + : identityRemapper; + let inverse: RefinementInverse; + try { + inverse = remap.inverse(parsedInverse.data); + } catch (error) { + if (!(error instanceof LegacyPathNotAdoptedError)) throw error; + throw new RollbackError(`Refusing rollback of '${opts.id}': ${error.message}`); + } // Confinement first — never overridable. A corrupted inverse must never // write outside the memory/skill roots (repo AGENTS.md, built-in skills, @@ -919,7 +980,8 @@ export async function rollbackRefinement( kind === "memory" ? [...rows, ...(await readSharedMemoryPeerRows(opts))] : rows, target, inverse, - readContent + readContent, + remap ); if (divergence.length > 0 && opts.force !== true) { throw new RollbackError( @@ -990,7 +1052,8 @@ export async function rollbackRefinement( : lockedRows, target, inverse, - readContent + readContent, + remap ); if (raced.length > 0) { throw new RollbackError( diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 400cb76b141..b20ea1972e5 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -567,7 +567,7 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number } + options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpoch?: number } ): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; @@ -1532,6 +1532,19 @@ export class TurnRequestBuilder { // previousBoundaryHistorySequence, so the completion-side observation // and every backend's turn records agree on which epoch a value belongs to. const policyEpoch = latestContextBoundaryHistorySequence(messages) ?? -1; + // A preserved-tail boundary (RLM keep-recent copies follow it) re-appends + // rows produced under the PREVIOUS epoch's policy: that epoch's + // accumulator is part of this one. The compacting session re-binds it to + // this epoch durably (AgentSession.carryWorkspaceMemoryWritable), but + // asynchronously — another backend's first turn here can precede that + // carry. Naming the carried epoch lets the sink AND its value directly + // (under whichever key it currently sits), so no window exists in which + // a read-only tail reads as writable. + const carriedPolicyEpoch = activeContextMessages.some( + (message) => message.metadata?.rlmPreservedTailCopy === true + ) + ? (latestContextBoundaryHistorySequence(messages, { before: policyEpoch }) ?? -1) + : undefined; const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; @@ -1539,6 +1552,7 @@ export class TurnRequestBuilder { await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { epochHasPriorTurns, policyEpoch, + ...(carriedPolicyEpoch === undefined ? {} : { carriedPolicyEpoch }), }) ) { return true; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 91403d5f6f6..7f0e24bc14d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9489,6 +9489,59 @@ describe("WorkspaceService initialize", () => { ).toBe(true); expect(persistedFor(12)).toBe(true); expect(persisted()).toBe(false); + // A PRESERVED-TAIL epoch names the epoch its tail was copied out of: + // that epoch's deny is ANDed in directly, under whichever key it sits — + // the closing key (compacting backend's carry not landed yet), the new + // key (carry landed), or the session-dir marker — so no window exists + // in which the read-only tail reads as writable to another backend. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": false }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 14, + carriedPolicyEpoch: -1, + }) + ).toBe(true); + expect(persistedFor(14)).toBe(false); + expect(persisted()).toBe(false); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "16": true }; + return cfg; + }); + await writeWorkspaceMemoryDenyMarker(sessionDir, 12); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 16, + carriedPolicyEpoch: 12, + }) + ).toBe(true); + expect(persistedFor(16)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 12 }); + // A carried grant (or no carried record at all) changes nothing. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": true }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 18, + carriedPolicyEpoch: -1, + }) + ).toBe(true); + expect(persistedFor(18)).toBe(true); + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": false, "12": true }; + return cfg; + }); // ...while a turn of the closing epoch itself still sees its deny. expect(await service.recordWorkspaceMemoryWritable("policy-scratch", true, EPOCH0)).toBe( true diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 464e6f7eabf..7445db8f80a 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4273,7 +4273,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number } + options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpoch?: number } ): Promise { // The accumulator (config bit and deny marker alike) is bound to the // compaction epoch it accumulates over — the opening boundary's history @@ -4283,8 +4283,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // compacting backend's durable reset landed (the reset is awaited only by // that backend's own session) and carry it, via its mirror, through an // otherwise all-writable epoch. - const { policyEpoch } = options; + const { policyEpoch, carriedPolicyEpoch } = options; assert(Number.isInteger(policyEpoch), "policyEpoch must be an integer"); + assert( + carriedPolicyEpoch === undefined || + (Number.isInteger(carriedPolicyEpoch) && carriedPolicyEpoch < policyEpoch), + "carriedPolicyEpoch must be an earlier epoch" + ); const session = this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); // A no-tail compaction's durable epoch reset may still be in flight: read @@ -4375,6 +4380,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // A fourth input: the session-dir deny marker, the durable fallback taken // when config.json could not record a deny (denyDurableFallback above). const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); + // Preserved-tail epoch: the previous epoch's accumulator is part of this + // one (its rows were copied in). The compacting session's carry moves the + // record/marker from the carried key to this one asynchronously; reading + // BOTH keys (record inside the transaction below, marker here) makes the + // conjunction independent of that carry's timing — a deny is visible + // under one key or the other at every instant, never under neither. + const carriedDenyMarker = + carriedPolicyEpoch !== undefined && + (await readWorkspaceMemoryDenyMarker(sessionDir, carriedPolicyEpoch)); + const carriedFor = (entry: WorkspaceConfigEntry): boolean | undefined => + carriedPolicyEpoch === undefined + ? undefined + : workspaceMemoryWritableForEpoch(entry, carriedPolicyEpoch); // Unknown history fails closed, like the harvest's own unknown → closed // rule: with no durable accumulator, no marker and no mirror, an epoch // that already holds turns has a policy nobody recorded — the record was @@ -4388,20 +4406,29 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const stored = storedFor(before.workspace); const unknownHistory = stored === undefined && mirror === undefined && options.epochHasPriorTurns; - const conjunction = (durable: boolean | undefined): boolean => - !denyMarker && !unknownHistory && (durable ?? true) && (mirror ?? true) && writable; + const conjunction = (durable: boolean | undefined, carried: boolean | undefined): boolean => + !denyMarker && + !carriedDenyMarker && + !unknownHistory && + (durable ?? true) && + (carried ?? true) && + (mirror ?? true) && + writable; // Fast path (no write): the outcome cannot differ from the stored value — // it is already false, or already true and this turn grants. - if (stored === false || (stored === true && conjunction(stored))) { + if ( + stored === false || + (stored === true && conjunction(stored, carriedFor(before.workspace))) + ) { session?.recordWorkspaceMemoryWritable(stored); return true; } - effective = conjunction(stored); + effective = conjunction(stored, carriedFor(before.workspace)); try { await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, workspaceId); if (current !== null) { - effective = conjunction(storedFor(current.workspace)); + effective = conjunction(storedFor(current.workspace), carriedFor(current.workspace)); // Per-epoch record: never overwrites the closing epoch's value, // which the compacting backend may not have observed yet // (workspaceMemoryPolicyEpochs.ts). From 2f2bfd4731b79551b15df1084d3f8f1702a3a52c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 21:32:47 +0000 Subject: [PATCH 60/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-fou?= =?UTF-8?q?rth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tail copies persist the policy epoch they were originally produced under (`rlmPreservedTailSourcePolicyEpoch`, kept through re-copies); the turn builder passes every carried epoch to the policy sink, which ANDs each one's record and deny marker, replacing the history-derived lookup. - The refinement_rollback tool's policy preflight and peer-row conflict detection both retarget pre-sharing legacy paths through the adoption manifest, matching the rollback engine. - A shared-store mutation whose clock write failed is journaled as `orderUnknown` and conflicts with every overlapping row in rollback instead of being ordered by an incomparable journal-local timestamp. - Reconciling a deleted legacy note probes its adopted copy strictly and keeps the manifest entry while the copy cannot be inspected. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/durableEvent.ts | 8 + src/common/types/message.ts | 12 ++ .../utils/messages/compactionBoundary.ts | 6 +- src/node/services/compactionHandler.test.ts | 20 +++ src/node/services/compactionHandler.ts | 30 +++- src/node/services/memoryService.test.ts | 144 ++++++++++++++++++ src/node/services/memoryService.ts | 38 ++++- .../services/refinement/refinementJournal.ts | 3 + .../services/refinement/refinementRollback.ts | 44 +++++- .../refinement/sharedMemoryRowMigration.ts | 1 + .../services/tools/refinement_rollback.ts | 50 ++++-- src/node/services/turnRequestBuilder.ts | 36 +++-- src/node/services/workspaceService.test.ts | 6 +- src/node/services/workspaceService.ts | 44 +++--- 14 files changed, 381 insertions(+), 61 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 4fcaaffc5be..0b3577c0679 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -101,6 +101,14 @@ export const RefinementDataSchema = z.object({ * comparable — still order totally; migrated rows keep their source value. */ sourceTs: z.number().optional(), + /** + * The mutation landed but the shared store's clock write failed, so this + * row has NO defensible position relative to other rows (its `ts`/`seq` + * are journal-local). Rollback conflict detection treats such a row as + * conflicting with every overlapping row in either direction (force + * overrides), instead of ordering it by an incomparable timestamp. + */ + orderUnknown: z.literal(true).optional(), /** Expected post-action file hashes (RefinementPostStateSchema in refinement.ts). */ postState: JsonValueSchema.optional(), /** diff --git a/src/common/types/message.ts b/src/common/types/message.ts index ad8796ff1af..f43673b7151 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -1085,6 +1085,18 @@ export interface MuxMetadata { */ rlmPreservedTailCopy?: boolean; + /** + * Compaction epoch (opening boundary's history sequence, -1 before any) the + * copied row was ORIGINALLY produced under — carried unchanged through + * repeated copies, so a copy of a copy still names the first epoch. The + * workspace-memory write policy of that epoch applies to the copy + * (TurnRequestBuilder → WorkspaceService.recordWorkspaceMemoryWritable): + * derived from history it would be wrong whenever the active-epoch read + * holds only the newest boundary. Absent on copies written before the + * field existed, whose source epochs never had a policy record either. + */ + rlmPreservedTailSourcePolicyEpoch?: number; + /** * @file mention snapshot token(s) this message provides content for. * Marks send-time materialized snapshot rows (the only @mention expansion diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index a5997cbf491..1d391bbad9a 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -79,17 +79,13 @@ export function isDurableContextBoundaryMarker(message: MuxMessage | undefined): * accumulator is bound to it (WorkspaceService.recordWorkspaceMemoryWritable). */ export function latestContextBoundaryHistorySequence( - messages: readonly MuxMessage[], - options?: { before: number } + messages: readonly MuxMessage[] ): number | undefined { let latest: number | undefined; for (const message of messages) { if (!isDurableContextBoundaryMarker(message)) continue; const sequence = message.metadata?.historySequence; if (typeof sequence !== "number" || !Number.isInteger(sequence) || sequence < 0) continue; - // `before`: the boundary preceding a given one (the epoch a preserved - // tail was copied out of). - if (options !== undefined && sequence >= options.before) continue; if (latest === undefined || sequence > latest) latest = sequence; } return latest; diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 2f23490aaf3..2afc08514ad 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1880,12 +1880,32 @@ describe("CompactionHandler", () => { expect(copy.metadata?.contextUsage).toBeUndefined(); // Copies must never masquerade as boundaries. expect(copy.metadata?.compactionBoundary).toBeUndefined(); + // The epoch the row was produced under (none before this boundary): + // its workspace-memory write policy governs the copy. + expect(copy.metadata?.rlmPreservedTailSourcePolicyEpoch).toBe(-1); } // Informational metadata survives. expect(epoch[2].metadata?.model).toBe("claude-x"); const metadata = onCompactionComplete.mock.calls[0]?.[0]; expect(metadata?.preservedTailMessageCount).toBe(2); + + // A second tail compaction re-copies the copies: they keep their + // ORIGINAL epoch (-1) while the new epoch's own rows carry this + // boundary's epoch — the chain stays visible to the policy conjunction. + const boundarySequence = epoch[0].metadata?.historySequence; + if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); + await seedHistory( + createMuxMessage("u2", "user", "second question"), + createMuxMessage("a2", "assistant", "second answer"), + createStampedCompactionRequest("compact-req-2", boundarySequence + 1) + ); + expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); + const secondEpoch = await historyService.getHistoryFromLatestBoundary(workspaceId); + if (!secondEpoch.success) throw new Error(secondEpoch.error); + expect( + secondEpoch.data.slice(1).map((copy) => copy.metadata?.rlmPreservedTailSourcePolicyEpoch) + ).toEqual([-1, -1, boundarySequence, boundarySequence]); }); it("rewrites MCP snapshot invoking IDs to the copy IDs of LATER tail rows", async () => { diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 8a0c7c70f86..3f133c70510 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1267,8 +1267,10 @@ export class CompactionHandler { } ); const idMap = new Map(params.tail.map((row) => [row.id, createPreservedTailCopyMessageId()])); + // Same closing epoch the completion metadata reports below. + const closingPolicyEpoch = latestContextBoundaryHistorySequence(params.messages) ?? -1; const copies = params.tail.map((row) => { - const copy = this.buildPreservedTailCopy(row, idMap); + const copy = this.buildPreservedTailCopy(row, idMap, closingPolicyEpoch); // Continuous compaction prunes the just-finished answer too. Keep recent pages // visible below the boundary while retaining RLM's usage/snapshot sanitizer. copy.metadata = { ...copy.metadata, uiVisible: true }; @@ -1503,7 +1505,8 @@ export class CompactionHandler { const preservedTailCopies = this.buildPreservedTailCopies( messages, compactionRequestMessageId, - summaryMessage.id + summaryMessage.id, + previousBoundaryHistorySequence ?? -1 ); const persistenceResult = @@ -1589,7 +1592,8 @@ export class CompactionHandler { private buildPreservedTailCopies( messages: MuxMessage[], compactionRequestMessageId: string, - summaryMessageId: string + summaryMessageId: string, + closingPolicyEpoch: number ): MuxMessage[] { const requestIndex = messages.findIndex((message) => message.id === compactionRequestMessageId); if (requestIndex === -1) { @@ -1629,7 +1633,7 @@ export class CompactionHandler { for (const row of tailRows) { idMap.set(row.id, createPreservedTailCopyMessageId()); } - return tailRows.map((row) => this.buildPreservedTailCopy(row, idMap)); + return tailRows.map((row) => this.buildPreservedTailCopy(row, idMap, closingPolicyEpoch)); } /** @@ -1642,7 +1646,11 @@ export class CompactionHandler { * original rows remain visible above the boundary; fresh IDs keep UI * aggregation from collapsing a hidden copy over its visible original. */ - private buildPreservedTailCopy(row: MuxMessage, idMap: Map): MuxMessage { + private buildPreservedTailCopy( + row: MuxMessage, + idMap: Map, + closingPolicyEpoch: number + ): MuxMessage { // IDs are preassigned for the whole tail (see caller) so forward-pointing // references (snapshot row → later invoking user row) rewrite correctly. const copyId = idMap.get(row.id); @@ -1661,12 +1669,24 @@ export class CompactionHandler { } : source?.mcpPromptSnapshot; + // The epoch whose workspace-memory write policy governs this row: a copy + // of a copy keeps its ORIGINAL epoch (the chain must stay visible to the + // policy conjunction); a first-time copy was produced under the epoch + // this compaction closes. Copies from before the field existed carry + // nothing forward — no policy record ever existed for their epochs. + const sourcePolicyEpoch = + source?.rlmPreservedTailCopy === true + ? source.rlmPreservedTailSourcePolicyEpoch + : closingPolicyEpoch; return { ...row, id: copyId, metadata: { synthetic: true, rlmPreservedTailCopy: true, + ...(sourcePolicyEpoch !== undefined + ? { rlmPreservedTailSourcePolicyEpoch: sourcePolicyEpoch } + : {}), ...(source?.timestamp !== undefined ? { timestamp: source.timestamp } : {}), ...(source?.model !== undefined ? { model: source.model } : {}), ...(source?.thinkingLevel !== undefined ? { thinkingLevel: source.thinkingLevel } : {}), diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index ae08ecb3d91..469b2a7b5fd 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1913,6 +1913,42 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); }); + it("retains adoption provenance while the copy of a deleted legacy note cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + expect(await pathExists(target)).toBe(true); + // The downgraded build deletes the source while the copy's stat fails + // transiently: neither the copy nor its provenance may go. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "stat").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realStat as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + // Recovered: the retained provenance lets the copy follow its source out. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2722,6 +2758,63 @@ describe("MemoryService", () => { ).toBe(true); }); + it("a shared-store row whose clock write failed conflicts with every overlapping row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/dir/s.md", "o1", "agent"); + // The child's edit lands, but the owner store's clock cannot be written: + // the row must not fall back to its journal-local `ts` (incomparable + // with the owner journal's rows) — it is journaled as order-unknown. + const revisionPath = path.join(ownerSessionDir, "memory.revision"); + await fsPromises.rm(revisionPath); + await fsPromises.mkdir(revisionPath); // a directory: the clock write fails + try { + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dir/s.md", + "o1", + "c1", + "agent" + ); + } finally { + await fsPromises.rmdir(revisionPath); + } + const [childEdit] = await readRefinementEvents(childSessionDir); + expect(childEdit.data.sourceTs).toBeUndefined(); + expect(childEdit.data.orderUnknown).toBe(true); + // The owner renames the directory afterwards (ordered by the clock). + await fixture.service.rename( + ownerCtx, + "/memories/workspace/dir", + "/memories/workspace/moved", + "agent" + ); + const [, ownerRename] = await readRefinementEvents(ownerSessionDir); + // Rolling the rename back would move the child's edit without seeing + // it if the row were ordered by `ts`; unknown order fails closed. + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "order relative to this row is unknown" + ); + const forced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + force: true, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(forced.success).toBe(true); + }); + it("the refinement_rollback tool refuses memory rollbacks into a read-only scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2779,6 +2872,57 @@ describe("MemoryService", () => { }); expect(allowed.success).toBe(true); expect(await pathExists(physical)).toBe(false); + + // A pre-sharing row (journaled while the child owned its store, so its + // inverse addresses /memory) whose note was since adopted: the + // policy gate classifies the ADOPTED owner path — the one the engine + // will touch — instead of refusing the legacy path as unclassifiable. + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "legacy.md"), "v2"); + await fsPromises.writeFile( + path.join(legacyRoot, ".adopted-into-shared-store.json"), + JSON.stringify({ + "legacy.md": { content: "x", sidecar: "", target: "legacy.md", created: true }, + }) + ); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "legacy.md"), "v2"); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/legacy.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "legacy.md"), text: "v1" }], + }, + }, + }); + const legacyRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + const legacyRefused = (await makeTool({ + global: "read", + project: "read", + workspace: "read", + }).execute!({ id: legacyRow.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + expect(legacyRefused.success).toBe(false); + expect(legacyRefused.error).toContain("read-only"); + const legacyAllowed = (await makeTool({ + global: "readwrite", + project: "readwrite", + workspace: "readwrite", + }).execute!({ id: legacyRow.id, reason: "test" }, mockToolCallOptions)) as { + success: boolean; + error?: string; + }; + expect(legacyAllowed.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "legacy.md"), "utf-8") + ).toBe("v1"); + expect(await fsPromises.readFile(path.join(legacyRoot, "legacy.md"), "utf-8")).toBe("v2"); }); it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 22c1d79f557..68387e2aac2 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1473,11 +1473,35 @@ export class MemoryService extends EventEmitter { ); if (!sourceGone) continue; if (previous.created === true) { - const current = - (await store.assertContained(previous.target).then( + // Strict probe: a target that merely could not be stat'ed is not + // "changed" — dropping the entry on that basis would lose the + // provenance for good and leave the obsolete copy visible forever + // once the filesystem recovers. Keep the entry (and the pass + // incomplete) so the next access reconciles it. + let targetKind: MemoryEntryKind; + try { + targetKind = (await store.assertContained(previous.target).then( () => true, () => false - )) && (await store.kind(previous.target)) === "file" + )) + ? await store.kind(previous.target, { strict: true }) + : null; + } catch (error) { + log.warn( + "[MemoryService] cannot inspect an adopted legacy note's copy; retrying later", + { + childId, + owner, + relPath, + target: previous.target, + error, + } + ); + skipped++; + continue; + } + const current = + targetKind === "file" ? await this.readBoundedTextFile(store, previous.target, previous.target).catch( () => null ) @@ -1658,7 +1682,14 @@ export class MemoryService extends EventEmitter { } // Workspace-scope rows carry the owner store's clock so rows from every // tree member's journal order consistently (see workspaceMemoryRevision.ts). + // The mutation is already on disk, so a failed clock write cannot fail the + // command; the row is journaled as `orderUnknown` instead — the rollback + // engine then treats it as conflicting with every overlapping row rather + // than ordering it by its journal-local `ts`, which another journal's rows + // cannot be compared against (a child's later edit beneath a directory + // the owner renamed would otherwise read as older and be moved silently). const sourceTs = await this.advanceStoreRevision(store); + const orderUnknown = sourceTs === undefined && this.storeOwnerWorkspaceId(store) !== null; await appendRefinementEvent({ sessionDir: path.join(this.config.sessionsDir, ctx.workspaceId), workspaceId: ctx.workspaceId, @@ -1672,6 +1703,7 @@ export class MemoryService extends EventEmitter { }, ...(postFiles !== undefined ? { postFiles } : {}), ...(sourceTs !== undefined ? { sourceTs } : {}), + ...(orderUnknown ? { orderUnknown: true } : {}), }); } diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 4ad1796664e..bfdaf493920 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -108,6 +108,8 @@ export interface RefinementEmitArgs { * source row's value/`ts` for a migrated row. */ sourceTs?: number; + /** See the durable event schema: the store clock write failed for this row. */ + orderUnknown?: true; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -369,6 +371,7 @@ export async function appendRefinementEventUnderBlobLock( ...(args.migratedFrom !== undefined ? { migratedFrom: args.migratedFrom } : {}), ...(args.rollbackOf !== undefined ? { rollbackOf: args.rollbackOf } : {}), ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), + ...(args.orderUnknown === true ? { orderUnknown: true } : {}), ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), }, }); diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index c86813684ce..820fe7e68e2 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -540,6 +540,16 @@ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { return rowTs > otherTs || (rowTs === otherTs && row.seq > other.seq); } +/** + * Whether the order of two rows cannot be established: a row journaled while + * the shared store's clock write failed (`orderUnknown`) has only + * journal-local `ts`/`seq`, incomparable with other journals' rows. Callers + * fail closed — such a pair conflicts in either direction (force overrides). + */ +function orderUnknown(row: RefinementEvent, other: RefinementEvent): boolean { + return row.data.orderUnknown === true || other.data.orderUnknown === true; +} + /** * Memory rows from the other task-tree members' journals that touched the * shared workspace store (owner's /memory). Read without their @@ -567,17 +577,32 @@ async function readSharedMemoryPeerRows( path.resolve(opts.sharedWorkspaceMemorySessionDir ?? opts.sessionDir), "memory" ); + const ownerSessionDir = path.dirname(sharedRoot); const peerRows: RefinementEvent[] = []; for (const peerDir of peerDirs) { assert( path.resolve(peerDir) !== path.resolve(opts.sessionDir), "peer session dirs exclude the acting session" ); + // A peer sub-agent's pre-sharing rows address ITS legacy private + // notebook; the notes live in the shared store now (adoption manifest + // beside the legacy files). Retarget them through that peer's manifest + // before the overlap test — the peer's later edit of an adopted note must + // surface against the owner's rollback like any shared-store row. The + // remapped inverse replaces the recorded one on the returned row, so the + // acting session's later checks (its own remapper is a no-op for owner + // paths) compare the paths the note actually lives at. Legacy paths the + // shared store never took address invisible files and are dropped. + const remap = + path.resolve(peerDir) === path.resolve(ownerSessionDir) + ? identityRemapper + : await createLegacyPathRemapper({ childSessionDir: peerDir, ownerSessionDir }); for (const row of await listRefinements(peerDir)) { if (row.data.kind !== "memory") continue; - const parsed = RefinementInverseSchema.safeParse(row.data.inverse); - if (!parsed.success) continue; - if (inversePaths(parsed.data).some((p) => pathsOverlap(p, sharedRoot))) peerRows.push(row); + const parsed = parseRemappedInverse(row, remap); + if (parsed === null) continue; + if (!inversePaths(parsed).some((p) => pathsOverlap(p, sharedRoot))) continue; + peerRows.push({ ...row, data: { ...row.data, inverse: parsed } }); } } return peerRows; @@ -632,14 +657,19 @@ async function collectDivergence( rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) ); for (const row of rows) { - if (!isAfter(row, target)) continue; + if (row.id === target.id) continue; + if (!isAfter(row, target) && !orderUnknown(row, target)) continue; if (rolledBackIds.has(row.id)) continue; // Effect undone by a later rollback row. if (!liveRowConflictsWithTarget(rows, row, target)) continue; const parsed = parseRemappedInverse(row, remap); if (parsed === null) continue; const overlap = inversePaths(parsed).some((p) => targetPaths.some((t) => pathsOverlap(p, t))); if (overlap) { - complaints.push(`later refinement row ${row.id} (seq ${row.seq}) touched the same paths`); + complaints.push( + orderUnknown(row, target) + ? `refinement row ${row.id} (seq ${row.seq}) touched the same paths and its order relative to this row is unknown (its store clock write failed)` + : `later refinement row ${row.id} (seq ${row.seq}) touched the same paths` + ); } } @@ -771,7 +801,9 @@ function liveRowConflictsWithTarget( if (rollbackCount % 2 === 0) { return true; // Even chain: the root row's edit was re-applied. } - return !isAfter(current, target); // Odd chain: rewound to just before root. + // Odd chain: rewound to just before root — a conflict unless the root is + // provably after the target. + return !isAfter(current, target) || orderUnknown(current, target); } async function dirExists(target: string): Promise { diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index bc505ed58dd..42d4dc40ba1 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -227,6 +227,7 @@ export async function migrateSharedMemoryRefinementRows(args: { migratedFrom, ...(rollbackOf !== undefined ? { rollbackOf } : {}), sourceTs: row.data.sourceTs ?? row.ts, + ...(row.data.orderUnknown === true ? { orderUnknown: true as const } : {}), ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); publishedBlobs.push(...appended.publishedBlobs); diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 751adb8dd55..509b05fb9b0 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -1,11 +1,16 @@ +import * as path from "node:path"; import { tool, type Tool } from "ai"; import type { RefinementRollbackToolResult } from "@/common/types/tools"; import { TOOL_DEFINITIONS } from "@/common/utils/tools/toolDefinitions"; import { listRefinements, rollbackRefinement } from "@/node/services/refinement/refinementRollback"; -import { RefinementInverseSchema } from "@/common/types/refinement"; +import { RefinementInverseSchema, type RefinementInverse } from "@/common/types/refinement"; import type { MemoryScopeAccess } from "@/common/constants/memory"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, +} from "@/node/services/memoryLegacyAdoption"; interface RefinementRollbackToolArgs { id: string; @@ -27,19 +32,41 @@ interface RefinementRollbackToolArgs { */ async function refuseReadOnlyMemoryRollback( sessionDir: string, + sharedWorkspaceMemorySessionDir: string | undefined, id: string, memory: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess } ): Promise { const row = (await listRefinements(sessionDir)).find((candidate) => candidate.id === id); if (row?.data.kind !== "memory") return null; - const inverse = RefinementInverseSchema.safeParse(row.data.inverse); - if (!inverse.success) return null; + const parsed = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsed.success) return null; + // The paths the engine will actually touch: a sub-agent's pre-sharing rows + // address its legacy private notebook, which the engine retargets to the + // adopted owner copy (refinementRollback.ts) — classify THOSE paths, or the + // legacy ones would read as unclassifiable and refuse every such row here. + // A legacy path the engine cannot map falls through to its refusal. + let inverse: RefinementInverse = parsed.data; + if ( + sharedWorkspaceMemorySessionDir !== undefined && + path.resolve(sharedWorkspaceMemorySessionDir) !== path.resolve(sessionDir) + ) { + const remap = await createLegacyPathRemapper({ + childSessionDir: sessionDir, + ownerSessionDir: sharedWorkspaceMemorySessionDir, + }); + try { + inverse = remap.inverse(parsed.data); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } + } const paths = - inverse.data.op === "delete-files" - ? inverse.data.paths - : inverse.data.op === "rename" - ? [inverse.data.from, inverse.data.to] - : [...inverse.data.files.map((file) => file.path), ...(inverse.data.deletePaths ?? [])]; + inverse.op === "delete-files" + ? inverse.paths + : inverse.op === "rename" + ? [inverse.from, inverse.to] + : [...inverse.files.map((file) => file.path), ...(inverse.deletePaths ?? [])]; for (const physicalPath of paths) { const scope = memory.service.scopeOfPhysicalPath(memory.ctx, physicalPath); // Fail closed: a memory row's paths always lie in some scope root, so @@ -81,7 +108,12 @@ export function createRefinementRollbackTool(ctx: { { toolCallId } ): Promise => { if (ctx.memory !== undefined) { - const refusal = await refuseReadOnlyMemoryRollback(ctx.sessionDir, id, ctx.memory); + const refusal = await refuseReadOnlyMemoryRollback( + ctx.sessionDir, + ctx.sharedWorkspaceMemorySessionDir, + id, + ctx.memory + ); if (refusal !== null) return { success: false, error: refusal }; } const result = await rollbackRefinement({ diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index b20ea1972e5..8959ec020fa 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -567,7 +567,7 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpoch?: number } + options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpochs?: number[] } ): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; @@ -1533,18 +1533,30 @@ export class TurnRequestBuilder { // and every backend's turn records agree on which epoch a value belongs to. const policyEpoch = latestContextBoundaryHistorySequence(messages) ?? -1; // A preserved-tail boundary (RLM keep-recent copies follow it) re-appends - // rows produced under the PREVIOUS epoch's policy: that epoch's - // accumulator is part of this one. The compacting session re-binds it to + // rows produced under EARLIER epochs' policies: those accumulators are + // part of this epoch. The compacting session re-binds the closing one to // this epoch durably (AgentSession.carryWorkspaceMemoryWritable), but // asynchronously — another backend's first turn here can precede that - // carry. Naming the carried epoch lets the sink AND its value directly - // (under whichever key it currently sits), so no window exists in which - // a read-only tail reads as writable. - const carriedPolicyEpoch = activeContextMessages.some( - (message) => message.metadata?.rlmPreservedTailCopy === true - ) - ? (latestContextBoundaryHistorySequence(messages, { before: policyEpoch }) ?? -1) - : undefined; + // carry. Naming the carried epochs lets the sink AND their values + // directly (under whichever key each currently sits), so no window exists + // in which a read-only tail reads as writable. Each copy records the + // epoch it was originally produced under (compactionHandler stamps it; a + // copy of a copy keeps the first), so a chain of tail compactions names + // every epoch involved — `messages` here holds only the active epoch, so + // nothing about earlier boundaries can be derived from it. + const carriedPolicyEpochs = [ + ...new Set( + activeContextMessages.flatMap((message) => { + const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; + return message.metadata?.rlmPreservedTailCopy === true && + typeof epoch === "number" && + Number.isInteger(epoch) && + epoch < policyEpoch + ? [epoch] + : []; + }) + ), + ].sort((a, b) => a - b); const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; @@ -1552,7 +1564,7 @@ export class TurnRequestBuilder { await sink.recordWorkspaceMemoryWritable(workspaceId, writable, { epochHasPriorTurns, policyEpoch, - ...(carriedPolicyEpoch === undefined ? {} : { carriedPolicyEpoch }), + ...(carriedPolicyEpochs.length === 0 ? {} : { carriedPolicyEpochs }), }) ) { return true; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7f0e24bc14d..0cc6c65a112 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9503,7 +9503,7 @@ describe("WorkspaceService initialize", () => { await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, policyEpoch: 14, - carriedPolicyEpoch: -1, + carriedPolicyEpochs: [-1], }) ).toBe(true); expect(persistedFor(14)).toBe(false); @@ -9518,7 +9518,7 @@ describe("WorkspaceService initialize", () => { await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, policyEpoch: 16, - carriedPolicyEpoch: 12, + carriedPolicyEpochs: [-1, 12], }) ).toBe(true); expect(persistedFor(16)).toBe(false); @@ -9533,7 +9533,7 @@ describe("WorkspaceService initialize", () => { await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, policyEpoch: 18, - carriedPolicyEpoch: -1, + carriedPolicyEpochs: [-1], }) ).toBe(true); expect(persistedFor(18)).toBe(true); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 7445db8f80a..6d8f0c82c35 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4273,7 +4273,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpoch?: number } + options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpochs?: number[] } ): Promise { // The accumulator (config bit and deny marker alike) is bound to the // compaction epoch it accumulates over — the opening boundary's history @@ -4283,12 +4283,12 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // compacting backend's durable reset landed (the reset is awaited only by // that backend's own session) and carry it, via its mirror, through an // otherwise all-writable epoch. - const { policyEpoch, carriedPolicyEpoch } = options; + const { policyEpoch } = options; + const carriedPolicyEpochs = options.carriedPolicyEpochs ?? []; assert(Number.isInteger(policyEpoch), "policyEpoch must be an integer"); assert( - carriedPolicyEpoch === undefined || - (Number.isInteger(carriedPolicyEpoch) && carriedPolicyEpoch < policyEpoch), - "carriedPolicyEpoch must be an earlier epoch" + carriedPolicyEpochs.every((epoch) => Number.isInteger(epoch) && epoch < policyEpoch), + "carriedPolicyEpochs must be earlier epochs" ); const session = this.sessions.get(workspaceId) ?? this.transientStartupRecoverySessions.get(workspaceId); @@ -4380,19 +4380,27 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // A fourth input: the session-dir deny marker, the durable fallback taken // when config.json could not record a deny (denyDurableFallback above). const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); - // Preserved-tail epoch: the previous epoch's accumulator is part of this - // one (its rows were copied in). The compacting session's carry moves the - // record/marker from the carried key to this one asynchronously; reading - // BOTH keys (record inside the transaction below, marker here) makes the - // conjunction independent of that carry's timing — a deny is visible - // under one key or the other at every instant, never under neither. - const carriedDenyMarker = - carriedPolicyEpoch !== undefined && - (await readWorkspaceMemoryDenyMarker(sessionDir, carriedPolicyEpoch)); - const carriedFor = (entry: WorkspaceConfigEntry): boolean | undefined => - carriedPolicyEpoch === undefined - ? undefined - : workspaceMemoryWritableForEpoch(entry, carriedPolicyEpoch); + // Preserved-tail epoch: the accumulators of the epochs its tail copies + // were produced under are part of this one. The compacting session's + // carry moves a record/marker from the carried key to this one + // asynchronously; reading EVERY key (records inside the transaction + // below, markers here) makes the conjunction independent of that carry's + // timing — a deny is visible under one key or another at every instant, + // never under none. + let carriedDenyMarker = false; + for (const epoch of carriedPolicyEpochs) { + if (await readWorkspaceMemoryDenyMarker(sessionDir, epoch)) carriedDenyMarker = true; + } + // undefined: no carried epoch recorded anything; false: some carried deny. + const carriedFor = (entry: WorkspaceConfigEntry): boolean | undefined => { + let carried: boolean | undefined; + for (const epoch of carriedPolicyEpochs) { + const value = workspaceMemoryWritableForEpoch(entry, epoch); + if (value === undefined) continue; + carried = (carried ?? true) && value; + } + return carried; + }; // Unknown history fails closed, like the harvest's own unknown → closed // rule: with no durable accumulator, no marker and no mirror, an epoch // that already holds turns has a policy nobody recorded — the record was From 736791e0afdecd7baaa8733e91ddc618167f9bdb Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 22:12:16 +0000 Subject: [PATCH 61/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-fif?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tail copies without a recorded source epoch deny the epoch's harvest (carriedPolicyUnknown) until no such copy remains in the active context. - The legacy adoption pass reads the manifest, sidecar and owner listing strictly: an unreadable one fails the pass (removal aborts) instead of standing in as empty. Reconciling a deleted source transfers provenance to a listed note that now references the same target. - Removal migrates pre-sharing refinement rows through the adoption manifest so adopted copies stay rollbackable from the owner journal. - The phantom removal path skips teardown only on proven absence (ENOENT/ENOTDIR); any other probe failure aborts the removal. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 32 ++-- src/node/services/memoryMeta.ts | 15 ++ src/node/services/memoryService.test.ts | 149 ++++++++++++++++-- src/node/services/memoryService.ts | 31 +++- .../refinement/sharedMemoryRowMigration.ts | 43 ++++- src/node/services/turnRequestBuilder.ts | 27 +++- src/node/services/workspaceService.test.ts | 52 ++++++ src/node/services/workspaceService.ts | 29 +++- 8 files changed, 339 insertions(+), 39 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index a59f1444a9a..9b9bbad528b 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -46,19 +46,33 @@ function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { } /** - * Self-healing read of the adoption manifest: anything malformed reads as not - * adopted. A Map, not a plain object: a legacy note may legitimately be named - * `__proto__` (any store-valid relPath), and assigning that key on an - * ordinary object hits the prototype setter instead of creating an entry the - * serialization would carry — the note would then be re-adopted (and the - * owner clock advanced) on every access. JSON.parse and Object.fromEntries - * create own properties, so the round-trip below is exact. + * Self-healing read of the adoption manifest: a missing or malformed file + * reads as "nothing adopted" (malformed content IS the file's state; the next + * pass rewrites it). An UNREADABLE file (EACCES, EIO) says nothing about that + * state: tolerant callers read it as empty too, `strict` callers throw — the + * removal handover decides what may be deleted from the manifest, and an + * empty substitute would delete the child session with the only provenance + * for a stale owner copy. A Map, not a plain object: a legacy note may + * legitimately be named `__proto__` (any store-valid relPath), and assigning + * that key on an ordinary object hits the prototype setter instead of + * creating an entry the serialization would carry — the note would then be + * re-adopted (and the owner clock advanced) on every access. JSON.parse and + * Object.fromEntries create own properties, so the round-trip is exact. */ export async function readLegacyAdoptionManifest( - manifestPath: string + manifestPath: string, + options?: { strict?: boolean } ): Promise> { + let raw: string; try { - const parsed: unknown = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")); + raw = await fsPromises.readFile(manifestPath, "utf-8"); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (options?.strict === true && code !== "ENOENT" && code !== "ENOTDIR") throw error; + return new Map(); + } + try { + const parsed: unknown = JSON.parse(raw); if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); return new Map( Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => diff --git a/src/node/services/memoryMeta.ts b/src/node/services/memoryMeta.ts index 270d47caf96..26050d60e4f 100644 --- a/src/node/services/memoryMeta.ts +++ b/src/node/services/memoryMeta.ts @@ -452,6 +452,21 @@ export class MemoryMetaService { return Effect.runPromise(this.effects.getEntries()); } + /** + * `getEntries()` that refuses a healed substitute: throws when the sidecar + * exists but could not be read. For decisions that consume the entries + * destructively — the legacy-notebook handover before a sub-agent's session + * is deleted folds child-keyed pins/usage into the owner key; an empty + * substitute would fold nothing, report success, and strand the entries. + */ + async getEntriesOrThrow(): Promise> { + const { meta, readFailed } = await Effect.runPromise(this.loadWithHealth()); + if (readFailed) { + throw new Error(`memory metadata sidecar could not be read at ${this.metaPath}`); + } + return new Map(Object.entries(meta.entries).map(([key, entry]) => [key, { ...entry }])); + } + async setPinned(logicalKey: string, pinned: boolean): Promise { await Effect.runPromise(this.effects.setPinned(logicalKey, pinned)); } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 469b2a7b5fd..8af963bde81 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -28,7 +28,7 @@ import { import { applyRefinementInverse, readRefinementEvents } from "./refinement/refinementTestHelpers"; import { rollbackRefinement } from "./refinement/refinementRollback"; import { migrateSharedMemoryRefinementRows } from "./refinement/sharedMemoryRowMigration"; -import { reclaimExcessRefinementInverseBlobs } from "./refinement/refinementJournal"; +import { reclaimExcessRefinementInverseBlobs, sha256Hex } from "./refinement/refinementJournal"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; import { sharedDurableEventJournal } from "@/node/utils/journal/durableEventJournal"; import { sharedWorkspaceMemoryPeerSessionDirs } from "./memoryWorkspaceOwner"; @@ -1675,16 +1675,82 @@ describe("MemoryService", () => { .then(() => null, getErrorMessage) ).toMatch(/resolved to ws-owner/); // Failures surface (the access-time pass only logs and retries later). - spyOn(fixture.metaService, "getEntries").mockImplementationOnce(() => - Promise.reject(new Error("sidecar unreadable")) - ); + // A sidecar that exists but cannot be read is no "no metadata": the + // handover would copy the note without its pin and report success. await fsPromises.writeFile(path.join(legacyRoot, "late.md"), "written later"); - expect( - await fixture.service - .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") - .then(() => null, getErrorMessage) - ).toMatch(/sidecar unreadable/); + const metaPath = path.join(fixture.xumHome, "memory-meta.json"); + const savedMeta = await fsPromises.readFile(metaPath).catch(() => null); + await fsPromises.rm(metaPath, { force: true }); + await fsPromises.mkdir(metaPath); // EISDIR on read + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/sidecar could not be read/); + } finally { + await fsPromises.rmdir(metaPath); + if (savedMeta !== null) await fsPromises.writeFile(metaPath, savedMeta); + } + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + // Same for the adoption manifest: unreadable (not missing) aborts. + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const savedManifest = await fsPromises.readFile(manifestPath); + await fsPromises.rm(manifestPath); + await fsPromises.mkdir(manifestPath); + try { + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/EISDIR/); + } finally { + await fsPromises.rmdir(manifestPath); + await fsPromises.writeFile(manifestPath, savedManifest); + } expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(true); + }); + + it("transfers provenance when a renamed legacy note lands on its own conflict copy", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Conflict: the owner has a different a.md, so the child's is adopted + // under imported//a.md. + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + // The downgraded build renames the source to exactly that imported + // path: the new record reuses the identical target; the old record's + // reconciliation must hand the copy over, not delete it. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["imported/ws-child/a.md"]); + expect(manifest["imported/ws-child/a.md"]).toMatchObject({ + target: "imported/ws-child/a.md", + created: true, + }); + // With provenance transferred, deleting the renamed source removes the copy. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); }); it("removal lists an oversized legacy notebook completely, counting every unplaceable note", async () => { @@ -2480,6 +2546,71 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(keep, "utf-8")).toBe("v1"); }); + it("migrates a pre-sharing row through the adoption manifest so the adopted copy stays rollbackable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Journaled before sharing: the inverse addresses the legacy notebook. + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + }, + }); + // Also a legacy row for a note the shared store never took (unplaceable). + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/never.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "never.md")] }, + }, + }); + // Adoption folds old.md into the owner store. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + await fsPromises.rm(childSessionDir, { recursive: true, force: true }); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom?.startsWith("ws-child:") === true + )!; + expect((copy.data.inverse as { files: Array<{ path: string }> }).files[0].path).toBe( + ownerCopy + ); + expect((copy.data.postState as { files: Array<{ path: string }> }).files[0].path).toBe( + ownerCopy + ); + const rolledBack = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + it("follows a row rolled back between the two handover passes with its rollback row", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 68387e2aac2..9a8663a726c 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1292,15 +1292,22 @@ export class MemoryService extends EventEmitter { // store would be re-imported as a stale duplicate on every backend // start. Sidecar: a downgraded build can change only a pin or usage // stats, which must reach the owner key without the bytes changing. + // Strict reads throughout: this pass decides what the handover may + // consider done (and removal then deletes the child session on that + // basis), so a transiently unreadable manifest, sidecar or owner + // listing must fail the pass rather than stand in as "empty". const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); - const adopted = await readLegacyAdoptionManifest(manifestPath); - const sidecarEntries = await this.metaService.getEntries(); + const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); + const sidecarEntries = await this.metaService.getEntriesOrThrow(); // The per-scope file cap is a store invariant (create/rename enforce // it): the copy stops at the owner store's remaining capacity so a // combined notebook cannot exceed it — an over-full scope is silently // truncated by the index and refuses every later create. Files left - // behind stay unrecorded and are retried once space frees up. - let remainingCapacity = MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles()).length; + // behind stay unrecorded and are retried once space frees up. Complete + // owner listing: an undercount would let the copy push the store past + // the cap and hide an adopted note's only copy once readable again. + let remainingCapacity = + MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles({ strict: true })).length; let capacityExhausted = false; let manifestDirty = false; let imported = 0; @@ -1507,7 +1514,21 @@ export class MemoryService extends EventEmitter { ) : null; const unchanged = current !== null && sha256Hex(current) === previous.content; - if (unchanged) { + // A listed note may now point at this very target (the downgraded + // build renamed `a.md` to the path its conflict copy was adopted + // under, and the new record reused the identical file): the target + // is that note's copy now. Provenance transfers to the successor + // record instead of the file being deleted from under it. + const successor = [...adopted].find( + ([rel, record]) => + rel !== relPath && listed.has(rel) && record.target === previous.target + ); + if (successor !== undefined) { + if (successor[1].created !== true) { + successor[1].created = true; + manifestDirty = true; + } + } else if (unchanged) { // Metadata first: a sidecar failure then aborts the pass with the // file and manifest entry intact, so the retry repeats both; // the reverse order would strand the owner-key pin/usage once diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 42d4dc40ba1..7b0d42f1c3d 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -21,6 +21,10 @@ import { type RefinementInverseDraft, } from "./refinementJournal"; import { listRefinements } from "./refinementRollback"; +import { + createLegacyPathRemapper, + LegacyPathNotAdoptedError, +} from "@/node/services/memoryLegacyAdoption"; function inversePaths(inverse: RefinementInverse): string[] { switch (inverse.op) { @@ -88,6 +92,23 @@ export async function migrateSharedMemoryRefinementRows(args: { ); const ownerMemoryRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); const rows = await listRefinements(args.childSessionDir); + // Pre-sharing rows address the child's legacy private notebook; their notes + // live in the owner store now (adoption manifest, read while the child + // session still exists). Retargeted like the rollback engine does, so the + // adopted copy stays rollbackable once the child journal is gone; legacy + // paths the shared store never took are skipped below like other roots. + const remap = await createLegacyPathRemapper({ + childSessionDir: args.childSessionDir, + ownerSessionDir: args.ownerSessionDir, + }); + const remapInverse = (inverse: RefinementInverse): RefinementInverse | null => { + try { + return remap.inverse(inverse); + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return null; + throw error; + } + }; // Liveness follows the whole rollback chain (rollback → rollback of the // rollback re-applies): an original row is live when it has been rolled // back an even number of times. Rollback rows themselves are never copied. @@ -160,8 +181,11 @@ export async function migrateSharedMemoryRefinementRows(args: { if (!parsed.success) continue; action = { ...parsed.data, of: rollbackOf }; } - const inverse = RefinementInverseSchema.safeParse(row.data.inverse); - if (!inverse.success) continue; + const parsedInverse = RefinementInverseSchema.safeParse(row.data.inverse); + if (!parsedInverse.success) continue; + const remapped = remapInverse(parsedInverse.data); + if (remapped === null) continue; + const inverse = { success: true as const, data: remapped }; if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; let draft: RefinementInverseDraft; @@ -223,7 +247,20 @@ export async function migrateSharedMemoryRefinementRows(args: { ? { actor: evidence.data.actor } : {}), }, - ...(postState.success ? { postState: postState.data } : {}), + ...(postState.success + ? { + postState: { + files: postState.data.files.flatMap((file) => { + try { + return [{ ...file, path: remap.path(file.path) }]; + } catch (error) { + if (error instanceof LegacyPathNotAdoptedError) return []; + throw error; + } + }), + }, + } + : {}), migratedFrom, ...(rollbackOf !== undefined ? { rollbackOf } : {}), sourceTs: row.data.sourceTs ?? row.ts, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 8959ec020fa..c125bf12203 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -567,7 +567,12 @@ export interface TurnRequestBuilderBindings extends OauthServiceBindings { recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpochs?: number[] } + options: { + epochHasPriorTurns: boolean; + policyEpoch: number; + carriedPolicyEpochs?: number[]; + carriedPolicyUnknown?: boolean; + } ): Promise; }; analyticsService?: { executeRawQuery(sql: string): Promise }; @@ -1544,19 +1549,28 @@ export class TurnRequestBuilder { // copy of a copy keeps the first), so a chain of tail compactions names // every epoch involved — `messages` here holds only the active epoch, so // nothing about earlier boundaries can be derived from it. + const tailCopies = activeContextMessages.filter( + (message) => message.metadata?.rlmPreservedTailCopy === true + ); const carriedPolicyEpochs = [ ...new Set( - activeContextMessages.flatMap((message) => { + tailCopies.flatMap((message) => { const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; - return message.metadata?.rlmPreservedTailCopy === true && - typeof epoch === "number" && - Number.isInteger(epoch) && - epoch < policyEpoch + return typeof epoch === "number" && Number.isInteger(epoch) && epoch < policyEpoch ? [epoch] : []; }) ), ].sort((a, b) => a - b); + // A copy without a source epoch (persisted by a build before the field, + // or a re-copy of one) carries a policy nobody can look up: it is + // excluded from the prior-turn check like every copy, so without this + // the epoch would grant on the strength of the turns it can see. Unknown + // fails closed — the epoch is denied until a no-tail boundary (or the + // tail turning over) leaves no such copy in the active context. + const carriedPolicyUnknown = tailCopies.some( + (message) => typeof message.metadata?.rlmPreservedTailSourcePolicyEpoch !== "number" + ); const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; if (isCompactionRequest || !sink) return true; @@ -1565,6 +1579,7 @@ export class TurnRequestBuilder { epochHasPriorTurns, policyEpoch, ...(carriedPolicyEpochs.length === 0 ? {} : { carriedPolicyEpochs }), + ...(carriedPolicyUnknown ? { carriedPolicyUnknown } : {}), }) ) { return true; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 0cc6c65a112..74a1fe6f7cc 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9537,6 +9537,17 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persistedFor(18)).toBe(true); + // A tail copy whose source epoch is unknown (persisted before the field + // existed) carries a policy nobody can look up: denied, like unknown + // history, even for an otherwise writable first turn. + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 22, + carriedPolicyUnknown: true, + }) + ).toBe(true); + expect(persistedFor(22)).toBe(false); await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; entry.workspaceMemoryWritableByEpoch = { "-1": false, "12": true }; @@ -22295,6 +22306,47 @@ describe("WorkspaceService.fork branch-summary rollback ordering", () => { }); }); +describe("WorkspaceService phantom removal probes", () => { + test("skips teardown only on PROVEN absence; an unreadable probe aborts the removal", async () => { + const { config, historyService, cleanup } = await createTestHistoryService(); + const service = createWorkspaceServiceForTest({ + config, + historyService, + aiService: createMockAIService({ + getWorkspaceMetadata: mock(() => Promise.resolve(Err("not found"))), + }), + }); + const workspaceId = "phantom-probe"; + const sessionDir = path.join(config.sessionsDir, workspaceId); + try { + // No config.json and no session dir: nothing to tear down (idempotent). + expect((await service.remove(workspaceId, true)).success).toBe(true); + // (Deregistration wrote config.json; take it away again so the probe + // pair is exercised.) The session dir probe now fails for a reason + // other than absence: the removal must not deregister on a guess — + // abort, retryable. + await fsPromises.rm(path.join(config.rootDir, "config.json"), { force: true }); + const realStat = fsPromises.stat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "stat").mockImplementation((( + target: Parameters[0], + ...rest: unknown[] + ) => + String(target) === sessionDir + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realStat as (...args: unknown[]) => unknown)(target, ...rest)) as never); + try { + const aborted = await service.remove(workspaceId, true); + expect(aborted.success).toBe(false); + expect(aborted.success ? "" : aborted.error).toContain("removal aborted"); + } finally { + unreadable.mockRestore(); + } + } finally { + await cleanup(); + } + }); +}); + describe("WorkspaceService disposal ownership", () => { test.each([false, true])( "leased cleanup removes real session files without a task-tree self-join (external=%s)", diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6d8f0c82c35..c272b455418 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -402,6 +402,7 @@ import { upsertSubagentTranscriptArtifactIndexEntry, } from "@/node/services/subagentTranscriptArtifacts"; import { getErrorMessage } from "@/common/utils/errors"; +import { hasErrorCode } from "@/node/services/tools/skillFileUtils"; /** Maximum number of retry attempts when workspace name collides */ const MAX_WORKSPACE_NAME_COLLISION_RETRIES = 3; @@ -4273,7 +4274,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { async recordWorkspaceMemoryWritable( workspaceId: string, writable: boolean, - options: { epochHasPriorTurns: boolean; policyEpoch: number; carriedPolicyEpochs?: number[] } + options: { + epochHasPriorTurns: boolean; + policyEpoch: number; + carriedPolicyEpochs?: number[]; + /** The epoch holds tail copies whose source epoch is unknown: denied (see TurnRequestBuilder). */ + carriedPolicyUnknown?: boolean; + } ): Promise { // The accumulator (config bit and deny marker alike) is bound to the // compaction epoch it accumulates over — the opening boundary's history @@ -4413,7 +4420,8 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { workspaceMemoryWritableForEpoch(entry, policyEpoch); const stored = storedFor(before.workspace); const unknownHistory = - stored === undefined && mirror === undefined && options.epochHasPriorTurns; + (stored === undefined && mirror === undefined && options.epochHasPriorTurns) || + options.carriedPolicyUnknown === true; const conjunction = (durable: boolean | undefined, carried: boolean | undefined): boolean => !denyMarker && !carriedDenyMarker && @@ -6800,15 +6808,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // written (fresh install, nothing registered) and no session dir // means there is no topology to resolve and nothing a tombstone // could protect — removing an unknown id must still succeed. - const exists = (target: string): Promise => + // Proven absence only: a probe that fails for any other reason + // (EACCES, EIO) says nothing, and declaring "nothing to tear down" + // on it would deregister the workspace without deleting its session + // or publishing a tombstone — an orphan foreign writers keep mutating. + const provenAbsent = (target: string): Promise => fsPromises.stat(target).then( - () => true, - () => false + () => false, + (error: unknown) => { + if (hasErrorCode(error, "ENOENT") || hasErrorCode(error, "ENOTDIR")) return true; + throw new SharedMemoryRemovalAbortedError(workspaceId, { cause: error }); + } ); const nothingToTearDown = verifiedSharedMemoryOwnerId === null && - !(await exists(configFilePath(this.config.rootDir))) && - !(await exists(sessionDir)); + (await provenAbsent(configFilePath(this.config.rootDir))) && + (await provenAbsent(sessionDir)); if (nothingToTearDown) { log.debug("Skipping session teardown: no config.json and no session dir", { workspaceId, From b7bee8582d35c90a0b0eeffaf2c2e33305daef90 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 22:52:03 +0000 Subject: [PATCH 62/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-six?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The policy sink reads the deny marker once for the current and every carried epoch (one snapshot cannot straddle the carry's re-stamp), and any tail copy whose source epoch is unusable marks the epoch unknown. - Legacy adoption: ENOTDIR proves a nested source's deletion; a target whose content cannot be read keeps its manifest entry; the remapper refuses records the adoption did not create (owner-owned notes) and offers a strict manifest read, which removal's row migration uses. - Rollback rows journaled after a failed clock advance carry orderUnknown; peer conflict detection ignores migrated copies of the acting journal's own rows; the debug CLI resolves the memory owner from a strict config read. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/cli/debug/refinements.ts | 5 +- src/node/services/memoryLegacyAdoption.ts | 26 +++- src/node/services/memoryService.test.ts | 140 ++++++++++++++++++ src/node/services/memoryService.ts | 34 ++++- .../refinement/refinementRollback.test.ts | 21 +++ .../services/refinement/refinementRollback.ts | 14 +- .../refinement/sharedMemoryRowMigration.ts | 4 + src/node/services/turnRequestBuilder.ts | 28 ++-- .../services/workspaceMemoryDenyMarker.ts | 23 ++- src/node/services/workspaceService.test.ts | 5 + src/node/services/workspaceService.ts | 19 +-- 11 files changed, 280 insertions(+), 39 deletions(-) diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index e58e3d04609..72ef3b1e1c3 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -56,7 +56,10 @@ export async function refinementsCommand( if (opts.rollback !== undefined) { // Sub-agents journal workspace-scope rows that point into the owner's // session dir; admit that root the same way the in-app tool does. - const cfg = defaultConfig.loadConfigOrDefault(); + // Strict: a tolerant read of an unreadable config.json would resolve a + // sub-agent to ITSELF, and the rollback would then mutate its hidden + // legacy notebook (no owner root, no adoption remap) and report success. + const cfg = defaultConfig.loadConfigOrDefault({ throwOnError: true }); const memoryOwnerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); const result = await rollbackRefinement({ sessionDir, diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 9b9bbad528b..2e9ffe852c6 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -86,9 +86,11 @@ export async function readLegacyAdoptionManifest( /** Thrown for a legacy path the shared store does not represent (see below). */ export class LegacyPathNotAdoptedError extends Error { - constructor(legacyPath: string) { + constructor(legacyPath: string, reason: "not-adopted" | "owner-owned") { super( - `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` + reason === "owner-owned" + ? `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook; the shared workspace store holds an identical note the owner already had (adoption created nothing), so a rollback there would alter the owner's own note` + : `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` ); this.name = "LegacyPathNotAdoptedError"; } @@ -103,12 +105,19 @@ export class LegacyPathNotAdoptedError extends Error { * would re-import as a conflicting duplicate). Paths outside the legacy root * pass through unchanged. A legacy path the manifest does not know throws * LegacyPathNotAdoptedError — fail closed rather than mutate an invisible - * file. Only the manifest's `target` is trusted for the destination's - * relPath; callers re-run their confinement checks on the mapped result. + * file. So does a record the adoption did NOT create (`created` unset: the + * owner already had an identical note of its own): the child's rows never + * touched that file, and applying their inverses there — a create row's + * delete-files in particular — would alter or remove the owner's own note. + * Only the manifest's `target` is trusted for the destination's relPath; + * callers re-run their confinement checks on the mapped result. `strict` + * (removal's row migration) throws on an unreadable manifest instead of + * treating every legacy path as unadopted. */ export async function createLegacyPathRemapper(args: { childSessionDir: string; ownerSessionDir: string; + strict?: boolean; }): Promise<{ path(filePath: string): string; inverse(inverse: RefinementInverse): RefinementInverse; @@ -116,14 +125,17 @@ export async function createLegacyPathRemapper(args: { const legacyRoot = path.join(path.resolve(args.childSessionDir), "memory"); const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); const adopted = await readLegacyAdoptionManifest( - path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME) + path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME), + { strict: args.strict === true } ); const remapPath = (filePath: string): string => { const relative = path.relative(legacyRoot, path.resolve(filePath)); if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return filePath; const record = adopted.get(relative.split(path.sep).join("/")); - if (record === undefined || record.pending === true) - throw new LegacyPathNotAdoptedError(filePath); + if (record === undefined || record.pending === true) { + throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); + } + if (record.created !== true) throw new LegacyPathNotAdoptedError(filePath, "owner-owned"); return path.join(ownerRoot, ...record.target.split("/")); }; return { diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 8af963bde81..38404010a44 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2015,6 +2015,68 @@ describe("MemoryService", () => { expect(await pathExists(target)).toBe(false); }); + it("treats a legacy directory replaced by a note as deleting its adopted descendants", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir", "note.md"), "nested"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(true); + // The downgraded build replaces dir/ with a regular note: the old + // descendant's probe fails ENOTDIR — proof of deletion, like ENOENT. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir"), "now a note"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "dir", "note.md"))).toBe(false); + // The new note itself lands under imported/ (the owner still has a + // directory at that path). + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "dir"), "utf-8") + ).toBe("now a note"); + }); + + it("retains adoption provenance while the copy of a deleted legacy note cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + // stat succeeds, the content read fails: not "changed" — keep the entry. + const realOpen = fsPromises.open.bind(fsPromises); + const unreadable = spyOn(fsPromises, "open").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realOpen as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(target)).toBe(true); + expect( + Object.keys( + JSON.parse( + await fsPromises.readFile( + path.join(legacyRoot, ".adopted-into-shared-store.json"), + "utf-8" + ) + ) as Record + ) + ).toEqual(["note.md"]); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(target)).toBe(false); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2944,6 +3006,84 @@ describe("MemoryService", () => { evidence: { toolName: "test", actor: "user" }, }); expect(forced.success).toBe(true); + // A rollback whose own clock write fails is journaled order-unknown too. + await fsPromises.rm(revisionPath, { force: true }); + await fsPromises.mkdir(revisionPath); + try { + expect( + ( + await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + force: true, + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); + } finally { + await fsPromises.rmdir(revisionPath); + } + const rollbackRow = (await readRefinementEvents(childSessionDir)).find( + (row) => row.data.rollbackOf === childEdit.id + )!; + expect(rollbackRow.data.sourceTs).toBeUndefined(); + expect(rollbackRow.data.orderUnknown).toBe(true); + }); + + it("ignores migrated copies of its own rows when a child rolls back after an aborted removal", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/dup.md", "v1", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dup.md", + "v1", + "v2", + "agent" + ); + // Pre-teardown migration ran, then the removal aborted: the owner + // journal holds copies of the child's rows while the child lives on. + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(2); + const [, childEdit] = await readRefinementEvents(childSessionDir); + const undone = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undone.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "dup.md"), "utf-8") + ).toBe("v1"); + }); + + it("aborts removal-time row migration when the adoption manifest is unreadable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/m.md", "v1", "agent"); + const manifestPath = path.join(childSessionDir, "memory", ".adopted-into-shared-store.json"); + await fsPromises.mkdir(manifestPath, { recursive: true }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }).then(() => null, getErrorMessage) + ).toMatch(/EISDIR/); }); it("the refinement_rollback tool refuses memory rollbacks into a read-only scope", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 9a8663a726c..1587dbf7fb0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1474,9 +1474,11 @@ export class MemoryService extends EventEmitter { // tolerates readdir failures (a partial list). Only a provable ENOENT // on the source itself counts; any other outcome keeps the entry // (and the copy) for a later pass. + // ENOTDIR is proof too: the downgraded build replaced `dir/` with a + // regular note, deleting every descendant. const sourceGone = await fsPromises.lstat(path.join(legacyRoot, relPath)).then( () => false, - (error: unknown) => hasErrorCode(error, "ENOENT") + (error: unknown) => isMissingPathError(error) ); if (!sourceGone) continue; if (previous.created === true) { @@ -1507,12 +1509,30 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - const current = - targetKind === "file" - ? await this.readBoundedTextFile(store, previous.target, previous.target).catch( - () => null - ) - : null; + // Same for the content read: a failure other than the cap check + // (MemoryCommandError: the owner grew the copy past the cap, which + // IS a change) says nothing about the content. + let current: string | null = null; + if (targetKind === "file") { + try { + current = await this.readBoundedTextFile(store, previous.target, previous.target); + } catch (error) { + if (!(error instanceof MemoryCommandError)) { + log.warn( + "[MemoryService] cannot read an adopted legacy note's copy; retrying later", + { + childId, + owner, + relPath, + target: previous.target, + error, + } + ); + skipped++; + continue; + } + } + } const unchanged = current !== null && sha256Hex(current) === previous.content; // A listed note may now point at this very target (the downgraded // build renamed `a.md` to the path its conflict copy was adopted diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index b34da1d4309..cdf3587634d 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -926,6 +926,27 @@ describe("refinementRollback", () => { "not folded into the shared workspace store" ); expect(await pathExists(path.join(fixture.sessionDir, "memory", "orphan.md"))).toBe(true); + // A note the owner already had (adoption created nothing, `created` + // unset): the child's create row must not delete the owner's own file. + await fixture.service.create(fixture.ctx, "/memories/workspace/same.md", "same\n", "agent"); + const sameRow = await lastRow(fixture.sessionDir); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "same.md"), "same\n"); + await fsPromises.writeFile( + path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + }) + ); + const ownerOwned = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: sameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(ownerOwned.success).toBe(false); + expect(ownerOwned.success ? "" : ownerOwned.error).toContain("owner's own note"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "same.md"))).toBe(true); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 820fe7e68e2..9d8e16d427d 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -578,6 +578,7 @@ async function readSharedMemoryPeerRows( "memory" ); const ownerSessionDir = path.dirname(sharedRoot); + const actingWorkspaceId = path.basename(path.resolve(opts.sessionDir)); const peerRows: RefinementEvent[] = []; for (const peerDir of peerDirs) { assert( @@ -599,6 +600,11 @@ async function readSharedMemoryPeerRows( : await createLegacyPathRemapper({ childSessionDir: peerDir, ownerSessionDir }); for (const row of await listRefinements(peerDir)) { if (row.data.kind !== "memory") continue; + // A removal that aborted after its pre-teardown pass leaves the owner + // journal holding COPIES of this session's rows (migratedFrom = + // ":") while this session lives on. They are + // this journal's rows seen twice, not later peer edits. + if (row.data.migratedFrom?.startsWith(`${actingWorkspaceId}:`) === true) continue; const parsed = parseRemappedInverse(row, remap); if (parsed === null) continue; if (!inversePaths(parsed).some((p) => pathsOverlap(p, sharedRoot))) continue; @@ -1217,13 +1223,16 @@ export async function rollbackRefinement( isWorkspaceMemoryRoot(opts.sessionDir, root, opts.sharedWorkspaceMemorySessionDir) ); let sourceTs: number | undefined; + let orderUnknownRow = false; if (kind === "memory" && workspaceMemoryRoot !== undefined) { try { sourceTs = await advanceWorkspaceMemoryRevision(path.dirname(workspaceMemoryRoot)); } catch (error) { - // Best-effort like the rest of this block: the row must still be - // journaled (falling back to its own `ts` for ordering). + // The inverse is already applied, so the row must still be + // journaled — as order-unknown (see MemoryService.journalRefinement): + // its journal-local `ts` is incomparable with other journals' rows. log.debug("[refinement] failed to advance workspace memory revision", { error }); + orderUnknownRow = true; } } let publishedBlobs: BlobQuotaEntry[] = []; @@ -1247,6 +1256,7 @@ export async function rollbackRefinement( }, rollbackOf: opts.id, ...(sourceTs !== undefined ? { sourceTs } : {}), + ...(orderUnknownRow ? { orderUnknown: true as const } : {}), }, }); }); diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 7b0d42f1c3d..ccd38426dcb 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -97,9 +97,13 @@ export async function migrateSharedMemoryRefinementRows(args: { // session still exists). Retargeted like the rollback engine does, so the // adopted copy stays rollbackable once the child journal is gone; legacy // paths the shared store never took are skipped below like other roots. + // Strict: an unreadable manifest must abort the removal (throws), not read + // as "nothing adopted" and let the child journal be deleted with the only + // rollback IDs and inverse payloads of adopted notes. const remap = await createLegacyPathRemapper({ childSessionDir: args.childSessionDir, ownerSessionDir: args.ownerSessionDir, + strict: true, }); const remapInverse = (inverse: RefinementInverse): RefinementInverse | null => { try { diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index c125bf12203..b8546aa5cce 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1552,24 +1552,30 @@ export class TurnRequestBuilder { const tailCopies = activeContextMessages.filter( (message) => message.metadata?.rlmPreservedTailCopy === true ); + // A usable source epoch is an integer earlier than this epoch (persisted + // history is unvalidated). A copy without one — persisted by a build + // before the field, a re-copy of such a copy, or a malformed value — + // carries a policy nobody can look up: it is excluded from the prior-turn + // check like every copy, so without this the epoch would grant on the + // strength of the turns it can see. Unknown fails closed — the epoch is + // denied until a no-tail boundary (or the tail turning over) leaves no + // such copy in the active context. + const usableSourceEpoch = (message: MuxMessage): number | undefined => { + const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; + return typeof epoch === "number" && Number.isInteger(epoch) && epoch < policyEpoch + ? epoch + : undefined; + }; const carriedPolicyEpochs = [ ...new Set( tailCopies.flatMap((message) => { - const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; - return typeof epoch === "number" && Number.isInteger(epoch) && epoch < policyEpoch - ? [epoch] - : []; + const epoch = usableSourceEpoch(message); + return epoch === undefined ? [] : [epoch]; }) ), ].sort((a, b) => a - b); - // A copy without a source epoch (persisted by a build before the field, - // or a re-copy of one) carries a policy nobody can look up: it is - // excluded from the prior-turn check like every copy, so without this - // the epoch would grant on the strength of the turns it can see. Unknown - // fails closed — the epoch is denied until a no-tail boundary (or the - // tail turning over) leaves no such copy in the active context. const carriedPolicyUnknown = tailCopies.some( - (message) => typeof message.metadata?.rlmPreservedTailSourcePolicyEpoch !== "number" + (message) => usableSourceEpoch(message) === undefined ); const persistWorkspaceMemoryWritable = async (writable: boolean): Promise => { const sink = this.dependencies.bindings.workspaceMemoryPolicySink; diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 323db0bddc8..e58547c4457 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -124,11 +124,30 @@ async function readMarkerRecord( export async function readWorkspaceMemoryDenyMarker( sessionDir: string, epoch?: number +): Promise { + return readWorkspaceMemoryDenyMarkerForEpochs( + sessionDir, + epoch === undefined ? undefined : [epoch] + ); +} + +/** + * `readWorkspaceMemoryDenyMarker` for several epochs from ONE read of the + * marker: a deny recorded for any of them. A preserved-tail turn consults its + * own epoch and the epochs its tail copies were produced under; the carry + * (carryWorkspaceMemoryDenyMarker) re-stamps an entry from the closing epoch + * to the new one atomically, so two separate reads could each miss it (old + * gone, new not yet seen) while a single snapshot always holds it under one + * key or the other. Without `epochs`, any present marker is a deny. + */ +export async function readWorkspaceMemoryDenyMarkerForEpochs( + sessionDir: string, + epochs?: readonly number[] ): Promise { const record = await readMarkerRecord(workspaceMemoryDenyMarkerPath(sessionDir)); if (record === "absent") return false; - if (record === null || record === "unreadable" || epoch === undefined) return true; - return record.wildcard || record.epochs.includes(epoch); + if (record === null || record === "unreadable" || epochs === undefined) return true; + return record.wildcard || epochs.some((epoch) => record.epochs.includes(epoch)); } /** diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 74a1fe6f7cc..20550ce7d4f 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -2,6 +2,7 @@ import { findWorkspaceEntry } from "@/node/services/taskUtils"; import { clearWorkspaceMemoryDenyMarker, readWorkspaceMemoryDenyMarker, + readWorkspaceMemoryDenyMarkerForEpochs, workspaceMemoryDenyMarkerPath, writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; @@ -9514,6 +9515,10 @@ describe("WorkspaceService initialize", () => { return cfg; }); await writeWorkspaceMemoryDenyMarker(sessionDir, 12); + // One snapshot answers for every epoch consulted (a carry re-stamping + // 12 → 16 between two separate reads could hide the entry from both). + expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1, 12])).toBe(true); + expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1])).toBe(false); expect( await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c272b455418..6fe9f3a4e09 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -147,6 +147,7 @@ import { } from "@/node/services/workspaceRemoval"; import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; import { + readWorkspaceMemoryDenyMarkerForEpochs, readWorkspaceMemoryDenyMarker, writeWorkspaceMemoryDenyMarker, } from "@/node/services/workspaceMemoryDenyMarker"; @@ -4386,18 +4387,19 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // removed the durable field underneath it. // A fourth input: the session-dir deny marker, the durable fallback taken // when config.json could not record a deny (denyDurableFallback above). - const denyMarker = await readWorkspaceMemoryDenyMarker(sessionDir, policyEpoch); // Preserved-tail epoch: the accumulators of the epochs its tail copies // were produced under are part of this one. The compacting session's // carry moves a record/marker from the carried key to this one // asynchronously; reading EVERY key (records inside the transaction - // below, markers here) makes the conjunction independent of that carry's - // timing — a deny is visible under one key or another at every instant, - // never under none. - let carriedDenyMarker = false; - for (const epoch of carriedPolicyEpochs) { - if (await readWorkspaceMemoryDenyMarker(sessionDir, epoch)) carriedDenyMarker = true; - } + // below, marker entries from ONE snapshot here) makes the conjunction + // independent of that carry's timing — a deny is visible under one key + // or another at every instant, never under none. One read for all the + // epochs: separate reads could straddle the carry's atomic re-stamp and + // each miss the entry. + const denyMarker = await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [ + policyEpoch, + ...carriedPolicyEpochs, + ]); // undefined: no carried epoch recorded anything; false: some carried deny. const carriedFor = (entry: WorkspaceConfigEntry): boolean | undefined => { let carried: boolean | undefined; @@ -4424,7 +4426,6 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { options.carriedPolicyUnknown === true; const conjunction = (durable: boolean | undefined, carried: boolean | undefined): boolean => !denyMarker && - !carriedDenyMarker && !unknownHistory && (durable ?? true) && (carried ?? true) && From e57ca7ac88be51f338f989fdf967f2d32aef6f63 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 9 Sep 2026 23:19:58 +0000 Subject: [PATCH 63/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-sev?= =?UTF-8?q?enth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adoption inspects a note's prior owner copy strictly before reusing it; an uninspectable copy is retried later instead of being duplicated. - Preserved-tail source epochs must lie in [-1, policyEpoch). - Rollback conflict detection reads peer adoption manifests strictly and refuses while one cannot be read. - The shared-store clock is advanced only from a readable, well-formed prior value; an unreadable existing clock throws so the row is journaled as order-unknown rather than ordered by a regressed value. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 87 +++++++++++++++++++ src/node/services/memoryService.ts | 39 +++++++-- .../services/refinement/refinementRollback.ts | 22 ++++- .../refinement/workspaceMemoryRevision.ts | 36 ++++++-- src/node/services/turnRequestBuilder.ts | 10 ++- 5 files changed, 174 insertions(+), 20 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 38404010a44..f302b38d272 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2077,6 +2077,51 @@ describe("MemoryService", () => { expect(await pathExists(target)).toBe(false); }); + it("retries instead of duplicating when an adopted note's prior copy cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const target = path.join(ownerRoot, "note.md"); + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + // Sidecar-only change (downgraded build pinned the note) while the + // prior copy cannot be read: no imported/ duplicate, record untouched. + await fixture.metaService.setPinned(childKey, true); + const realOpen = fsPromises.open.bind(fsPromises); + const unreadable = spyOn(fsPromises, "open").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" })) + : (realOpen as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + await fixture.service.listIndexEntries({ ...fixture.ctx }); + } finally { + unreadable.mockRestore(); + } + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); + // Readable again: the pin folds into the same copy. + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -3029,6 +3074,48 @@ describe("MemoryService", () => { )!; expect(rollbackRow.data.sourceTs).toBeUndefined(); expect(rollbackRow.data.orderUnknown).toBe(true); + // A clock that EXISTS but is unreadable/malformed must not be advanced + // from zero (a lower value would order this mutation before rows it + // followed): the row is order-unknown instead. + await fsPromises.writeFile(revisionPath, "garbage"); + await fixture.service.create(ownerCtx, "/memories/workspace/after.md", "x", "agent"); + const afterRow = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + expect(afterRow.data.sourceTs).toBeUndefined(); + expect(afterRow.data.orderUnknown).toBe(true); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe("garbage"); + }); + + it("refuses a rollback while a peer's adoption manifest cannot be read", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/o.md", "v1", "agent"); + const [ownerCreate] = await readRefinementEvents(ownerSessionDir); + // The child's (pre-sharing) manifest is unreadable: its adopted rows + // cannot be consulted, so the owner's rollback must not proceed blind. + const manifestPath = path.join(childSessionDir, "memory", ".adopted-into-shared-store.json"); + await fsPromises.mkdir(manifestPath, { recursive: true }); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerCreate.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain("adoption manifest could not be read"); + await fsPromises.rmdir(manifestPath); + expect( + ( + await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerCreate.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }) + ).success + ).toBe(true); }); it("ignores migrated copies of its own rows when a child rolls back after an aborted removal", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 1587dbf7fb0..910ead424c9 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1353,16 +1353,39 @@ export class MemoryService extends EventEmitter { // the owner may have edited, replaced or deleted it since, and the // child's pin must not land on unrelated content or a missing // file. Otherwise the note is placed anew like a fresh adoption. - const stillAdopted = - (await store.assertContained(previous.target).then( + // Inspected strictly: a prior target that merely cannot be stat'ed + // or read right now is not "replaced" — placing the note anew would + // leave the original copy visible without provenance for good. + // Retry on the next access instead (the pass stays incomplete). + let priorContent: string | null; + try { + const contained = await store.assertContained(previous.target).then( () => true, () => false - )) && - (await store.kind(previous.target)) === "file" && - (await this.readBoundedTextFile(store, previous.target, previous.target).catch( - () => null - )) === content; - if (stillAdopted) { + ); + priorContent = + contained && (await store.kind(previous.target, { strict: true })) === "file" + ? await this.readBoundedTextFile(store, previous.target, previous.target) + : null; + } catch (error) { + if (error instanceof MemoryCommandError) { + priorContent = null; // over the cap: the owner changed it + } else { + log.warn( + "[MemoryService] cannot inspect an adopted note's prior copy; retrying later", + { + childId, + owner, + relPath, + target: previous.target, + error, + } + ); + skipped++; + continue; + } + } + if (priorContent === content) { target = { relPath: previous.target, write: false }; record.created = previous.created === true; } diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 9d8e16d427d..b3c457f6ebb 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -594,10 +594,24 @@ async function readSharedMemoryPeerRows( // acting session's later checks (its own remapper is a no-op for owner // paths) compare the paths the note actually lives at. Legacy paths the // shared store never took address invisible files and are dropped. - const remap = - path.resolve(peerDir) === path.resolve(ownerSessionDir) - ? identityRemapper - : await createLegacyPathRemapper({ childSessionDir: peerDir, ownerSessionDir }); + // Strict: an unreadable peer manifest must refuse the rollback, not read + // as "nothing adopted" and silently drop that peer's later mutations. + let remap: RecordedPathRemapper; + if (path.resolve(peerDir) === path.resolve(ownerSessionDir)) { + remap = identityRemapper; + } else { + try { + remap = await createLegacyPathRemapper({ + childSessionDir: peerDir, + ownerSessionDir, + strict: true, + }); + } catch (error) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': a tree member's adoption manifest could not be read (${getErrorMessage(error)})` + ); + } + } for (const row of await listRefinements(peerDir)) { if (row.data.kind !== "memory") continue; // A removal that aborted after its pre-teardown pass leaves the owner diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts index 3ac23a79c82..5b0df102f90 100644 --- a/src/node/services/refinement/workspaceMemoryRevision.ts +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -29,22 +29,48 @@ export function workspaceMemoryRevisionPath(ownerSessionDir: string): string { /** Current clock value, or null when the store was never written (or the file is unreadable). */ export async function readWorkspaceMemoryRevision(ownerSessionDir: string): Promise { try { - const raw = await fsPromises.readFile(workspaceMemoryRevisionPath(ownerSessionDir), "utf-8"); - const value = Number.parseInt(raw, 10); - return Number.isSafeInteger(value) && value > 0 ? value : null; + return await readWorkspaceMemoryRevisionStrict(ownerSessionDir); } catch { return null; } } +/** + * `readWorkspaceMemoryRevision` that distinguishes a never-written clock + * (null: proven ENOENT) from one that exists but cannot be trusted — an + * unreadable file, or content that is not a positive safe integer — which + * throws. + */ +async function readWorkspaceMemoryRevisionStrict(ownerSessionDir: string): Promise { + const revisionPath = workspaceMemoryRevisionPath(ownerSessionDir); + let raw: string; + try { + raw = await fsPromises.readFile(revisionPath, "utf-8"); + } catch (error) { + if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return null; + throw error; + } + const value = Number.parseInt(raw, 10); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error( + `workspace memory revision at ${revisionPath} is malformed: ${raw.slice(0, 32)}` + ); + } + return value; +} + /** * Advance and persist the clock; returns the new value. Callers MUST hold the * store's target mutation lock (cross-process) — the read→write here is what * that lock makes atomic. Throws when the owner session dir is missing: the - * file is never allowed to recreate a removed owner's directory. + * file is never allowed to recreate a removed owner's directory. Throws too + * when an EXISTING clock cannot be read or parsed: advancing from 0 instead + * could persist a value below the prior counter (which may run ahead of wall + * time) and hand callers a `sourceTs` that orders the mutation before rows + * it followed — callers treat the throw as "order unknown". */ export async function advanceWorkspaceMemoryRevision(ownerSessionDir: string): Promise { - const previous = (await readWorkspaceMemoryRevision(ownerSessionDir)) ?? 0; + const previous = (await readWorkspaceMemoryRevisionStrict(ownerSessionDir)) ?? 0; const next = Math.max(Date.now(), previous + 1); await fsPromises.writeFile(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); return next; diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index b8546aa5cce..e472eed2b56 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1552,8 +1552,9 @@ export class TurnRequestBuilder { const tailCopies = activeContextMessages.filter( (message) => message.metadata?.rlmPreservedTailCopy === true ); - // A usable source epoch is an integer earlier than this epoch (persisted - // history is unvalidated). A copy without one — persisted by a build + // A usable source epoch is an integer in [-1, policyEpoch) — -1 before + // any boundary, else a boundary's history sequence (persisted history is + // unvalidated). A copy without one — persisted by a build // before the field, a re-copy of such a copy, or a malformed value — // carries a policy nobody can look up: it is excluded from the prior-turn // check like every copy, so without this the epoch would grant on the @@ -1562,7 +1563,10 @@ export class TurnRequestBuilder { // such copy in the active context. const usableSourceEpoch = (message: MuxMessage): number | undefined => { const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; - return typeof epoch === "number" && Number.isInteger(epoch) && epoch < policyEpoch + return typeof epoch === "number" && + Number.isInteger(epoch) && + epoch >= -1 && + epoch < policyEpoch ? epoch : undefined; }; From 6778f658ea490a17267a81d816d7bd870b4a82f3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 00:04:16 +0000 Subject: [PATCH 64/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-eig?= =?UTF-8?q?hth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption replaces an untouched copy it created in place when the legacy bytes change, keeps reconciled deletions as tombstoned manifest records (rollbacks of pre-sharing rows still map), and never represents a note through a symlinked owner path. - The shared-store clock accepts only the exact decimal format, and deny markers accept only epochs in the valid domain; anything else reads as malformed (order unknown / deny). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 9 + src/node/services/memoryService.test.ts | 115 ++++++++++-- src/node/services/memoryService.ts | 176 ++++++++++-------- .../refinement/refinementRollback.test.ts | 24 +++ .../refinement/workspaceMemoryRevision.ts | 5 +- .../services/workspaceMemoryDenyMarker.ts | 5 +- src/node/services/workspaceService.test.ts | 10 + 7 files changed, 258 insertions(+), 86 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 2e9ffe852c6..858c96fc9bf 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -33,6 +33,15 @@ export interface LegacyAdoptionRecord { target: string; created?: boolean; pending?: boolean; + /** + * The legacy source was deleted (or renamed away) on a downgraded build and + * the copy reconciled. Kept rather than dropped: the child's pre-sharing + * refinement rows for this note (a delete's restore inverse, a rename's + * mirrored rename) still address the legacy path and need the mapping to + * be rolled back into the shared store; a reappearing source is adopted + * afresh (the record's other fields are stale then). + */ + deleted?: boolean; } function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index f302b38d272..be58e6c9e7e 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1741,7 +1741,10 @@ describe("MemoryService", () => { const manifest = JSON.parse( await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") ) as Record; - expect(Object.keys(manifest)).toEqual(["imported/ws-child/a.md"]); + // The old record stays as a tombstone (rollbacks of the child's + // pre-sharing rows for a.md still need its mapping). + expect(Object.keys(manifest).sort()).toEqual(["a.md", "imported/ws-child/a.md"]); + expect(manifest["a.md"]).toMatchObject({ deleted: true }); expect(manifest["imported/ws-child/a.md"]).toMatchObject({ target: "imported/ws-child/a.md", created: true, @@ -1793,12 +1796,15 @@ describe("MemoryService", () => { await new Promise((resolve) => setTimeout(resolve, 5)); await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v2 (downgrade)"); await fixture.service.listIndexEntries({ ...fixture.ctx }); - expect( - await fsPromises.readFile( - path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"), - "utf-8" - ) - ).toBe("v2 (downgrade)"); + // The copy this adoption created was untouched by the owner: the new + // bytes replace it in place (an imported/ duplicate would strand the + // old copy, provenance lost, in the shared notebook). + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "v2 (downgrade)" + ); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"))).toBe( + false + ); // Sidecar-only change (a pin toggled on the old build under the child // key): no file stat changes at all, yet the owner key must follow. const childKey = memoryLogicalKey("workspace", "sub/note.md", { @@ -1807,16 +1813,35 @@ describe("MemoryService", () => { }); await fixture.metaService.setPinned(childKey, true); await fixture.service.listIndexEntries({ ...fixture.ctx }); - // The pin lands on the owner key of the note's current copy (the - // imported one, since the owner path holds the older bytes). expect( (await fixture.metaService.getPinnedKeys()).has( - memoryLogicalKey("workspace", "imported/ws-child/sub/note.md", { + memoryLogicalKey("workspace", "sub/note.md", { projectPath: "", workspaceId: "ws-owner", }) ) ).toBe(true); + // Once the OWNER edited the copy it is the owner's: a further legacy + // edit is placed anew under imported/. + await fixture.service.strReplace( + { ...fixture.ctx, workspaceId: "ws-owner" }, + "/memories/workspace/sub/note.md", + "v2", + "owner's v3", + "agent" + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "sub", "note.md"), "v4 (downgrade)"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile( + path.join(ownerRoot, "imported", "ws-child", "sub", "note.md"), + "utf-8" + ) + ).toBe("v4 (downgrade)"); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe( + "owner's v3 (downgrade)" + ); }); it("follows legacy deletions and renames for copies the adoption created, never owner notes", async () => { @@ -1898,12 +1923,16 @@ describe("MemoryService", () => { } finally { lossy.mockRestore(); } + // ...and the edited source replaces its untouched adopted copy in place. expect( (await fixture.service.listIndexEntries({ ...fixture.ctx })) .filter((entry) => entry.scope === "workspace") .map((entry) => entry.relPath) .sort() - ).toEqual(["edited.md", "imported/ws-child/renamed.md", "renamed.md", "same.md"]); + ).toEqual(["edited.md", "renamed.md", "same.md"]); + expect(await fsPromises.readFile(path.join(ownerRoot, "renamed.md"), "utf-8")).toBe( + "to be renamed (v2)" + ); }); it("keeps the owner's pin when a downgraded build only viewed the adopted note", async () => { @@ -2122,6 +2151,64 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); }); + it("never represents a legacy note through a symlinked owner path", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Owner path is a link to an unlisted dotfile with identical bytes: + // following it would call the note "already represented" while the + // shared notebook never lists it. + await fsPromises.writeFile(path.join(ownerRoot, ".hidden"), "child notes"); + await fsPromises.symlink(".hidden", path.join(ownerRoot, "note.md")); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "child notes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "note.md"), "utf-8") + ).toBe("child notes"); + expect((await fsPromises.lstat(path.join(ownerRoot, "note.md"))).isSymbolicLink()).toBe(true); + }); + + it("keeps a tombstoned record for a deleted legacy source and re-adopts a reappearing one", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const tombstoned = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; target: string } + >; + expect(tombstoned["note.md"]).toMatchObject({ target: "note.md", deleted: true }); + // A copy restored into the shared store (a rollback of the deletion) + // is not reconciled away again: the tombstone is final. + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v1"); + await fixture.service.create(fixture.ctx, "/memories/workspace/other.md", "o", "agent"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(true); + // The source reappears on the old build: adopted as a fresh note. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + const readopted = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + >; + expect(readopted["note.md"]).toMatchObject({ created: true }); + expect(readopted["note.md"].deleted).toBeUndefined(); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -3083,6 +3170,12 @@ describe("MemoryService", () => { expect(afterRow.data.sourceTs).toBeUndefined(); expect(afterRow.data.orderUnknown).toBe(true); expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe("garbage"); + // A numeric PREFIX is malformed too (parseInt would accept it). + await fsPromises.writeFile(revisionPath, "2000000000000000e1"); + await fixture.service.create(ownerCtx, "/memories/workspace/after2.md", "x", "agent"); + const after2 = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + expect(after2.data.orderUnknown).toBe(true); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe("2000000000000000e1"); }); it("refuses a rollback while a peer's adoption manifest cannot be read", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 910ead424c9..23ccba93b17 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1338,7 +1338,10 @@ export class MemoryService extends EventEmitter { target: "", created: false, }; - const previous = adopted.get(relPath); + // A record whose source was reconciled as deleted is kept only for + // the path mapping (rollbacks); a reappearing source is a fresh note. + const priorRecord = adopted.get(relPath); + const previous = priorRecord?.deleted === true ? undefined : priorRecord; if ( previous?.content === record.content && previous.sidecar === record.sidecar && @@ -1346,48 +1349,40 @@ export class MemoryService extends EventEmitter { ) { continue; // folded in earlier, nothing changed since } - let target: { relPath: string; write: boolean } | null = null; - if (previous?.content === record.content) { - // Bytes already adopted and only the sidecar changed: the recorded - // target is reused only while it still holds the adopted bytes — - // the owner may have edited, replaced or deleted it since, and the - // child's pin must not land on unrelated content or a missing - // file. Otherwise the note is placed anew like a fresh adoption. + let target: { relPath: string; write: boolean; replaces?: boolean } | null = null; + if (previous !== undefined) { + // The recorded target is reused only while it still holds bytes + // this adoption put there — the owner may have edited, replaced or + // deleted it since, and the child's note must not land on unrelated + // content or a missing file. Unchanged legacy bytes (only the + // sidecar moved): reuse without a write. Legacy bytes edited on the + // downgraded build while the copy THIS adoption created is still + // untouched: the copy is replaced in place — placing the new bytes + // elsewhere would strand the old copy, provenance lost, in the + // model-visible notebook. Otherwise the note is placed anew. // Inspected strictly: a prior target that merely cannot be stat'ed - // or read right now is not "replaced" — placing the note anew would - // leave the original copy visible without provenance for good. - // Retry on the next access instead (the pass stays incomplete). + // or read right now is not "replaced" — retry on the next access + // instead (the pass stays incomplete). let priorContent: string | null; try { - const contained = await store.assertContained(previous.target).then( - () => true, - () => false - ); - priorContent = - contained && (await store.kind(previous.target, { strict: true })) === "file" - ? await this.readBoundedTextFile(store, previous.target, previous.target) - : null; + priorContent = await this.inspectAdoptedCopy(store, previous.target); } catch (error) { - if (error instanceof MemoryCommandError) { - priorContent = null; // over the cap: the owner changed it - } else { - log.warn( - "[MemoryService] cannot inspect an adopted note's prior copy; retrying later", - { - childId, - owner, - relPath, - target: previous.target, - error, - } - ); - skipped++; - continue; - } + log.warn( + "[MemoryService] cannot inspect an adopted note's prior copy; retrying later", + { childId, owner, relPath, target: previous.target, error } + ); + skipped++; + continue; } if (priorContent === content) { target = { relPath: previous.target, write: false }; record.created = previous.created === true; + } else if ( + previous.created === true && + priorContent !== null && + sha256Hex(priorContent) === previous.content + ) { + target = { relPath: previous.target, write: true, replaces: true }; } } if (target === null) { @@ -1396,45 +1391,45 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - if (target.write) { - if (remainingCapacity <= 0) { - capacityExhausted = true; - skipped++; - continue; - } - // Destination containment immediately before the write (the - // same check a memory create runs): a symlinked component under - // the owner root — e.g. imported/ pointing elsewhere — - // must never let the copy land outside the store. - try { - await store.assertContained(target.relPath); - } catch (error) { - log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { - childId, - relPath, - target: target.relPath, - error, - }); - skipped++; - continue; - } - // Provenance BEFORE the copy: interrupted here (crash, or the - // sidecar fold below failing), the retry finds the owner file - // already identical and takes the no-write path — without this - // record it would read as the owner's own note, and a legacy - // deletion could then never follow it out of the shared store. - adopted.set(relPath, { - ...record, + } + if (target.write) { + if (target.replaces !== true && remainingCapacity <= 0) { + capacityExhausted = true; + skipped++; + continue; + } + // Destination containment immediately before the write (the + // same check a memory create runs): a symlinked component under + // the owner root — e.g. imported/ pointing elsewhere — + // must never let the copy land outside the store. + try { + await store.assertContained(target.relPath); + } catch (error) { + log.warn("[MemoryService] refusing to adopt a legacy note into an escaping path", { + childId, + relPath, target: target.relPath, - created: true, - pending: true, + error, }); - await writeManifest(); - await store.writeFile(target.relPath, content); - remainingCapacity--; - imported++; - record.created = true; + skipped++; + continue; } + // Provenance BEFORE the copy: interrupted here (crash, or the + // sidecar fold below failing), the retry finds the owner file + // already identical and takes the no-write path — without this + // record it would read as the owner's own note, and a legacy + // deletion could then never follow it out of the shared store. + adopted.set(relPath, { + ...record, + target: target.relPath, + created: true, + pending: true, + }); + await writeManifest(); + await store.writeFile(target.relPath, content); + if (target.replaces !== true) remainingCapacity--; + imported++; + record.created = true; } record.target = target.relPath; // Pins/stats were keyed by the child: fold them into the owner key. @@ -1492,7 +1487,7 @@ export class MemoryService extends EventEmitter { // above; a failed listing never reaches this point. const listed = new Set(files); for (const [relPath, previous] of adopted) { - if (listed.has(relPath)) continue; + if (listed.has(relPath) || previous.deleted === true) continue; // Absence from the listing is not proof: LocalMemoryStore.listFiles // tolerates readdir failures (a partial list). Only a provable ENOENT // on the source itself counts; any other outcome keeps the entry @@ -1592,7 +1587,11 @@ export class MemoryService extends EventEmitter { }); } } - adopted.delete(relPath); + // Kept as a tombstone, not dropped: the child's pre-sharing rows for + // this note still need relPath → target to be rolled back into the + // shared store (a delete's restore lands at the reconciled target; + // the reconciliation above never runs again for it). + adopted.set(relPath, { ...previous, deleted: true }); manifestDirty = true; } if (manifestDirty) await writeManifest(); @@ -1652,6 +1651,14 @@ export class MemoryService extends EventEmitter { () => false ); if (!contained) continue; + // A symlink is never a destination: the store's listing excludes links + // (and dotfiles), so a note "represented" through one would be + // invisible to the shared notebook, and a write would land through it. + const linkKind = await lstatKind(store.physicalPath(candidate)); + if (linkKind === "unreadable") { + throw new Error(`cannot inspect adoption destination ${candidate}`); + } + if (linkKind === "symlink") continue; // Strict: a destination that merely could not be stat'ed (EACCES, EIO) // is not free — declaring it so would overwrite whatever the owner // keeps there once the copy runs. The failure aborts the pass instead @@ -1668,6 +1675,29 @@ export class MemoryService extends EventEmitter { return null; } + /** + * Content of an adopted note's copy in the owner store, or null when no + * regular listed file is there (absent, a directory, a symlink, or grown + * past the cap — each a change the owner made). Throws when the copy + * cannot be inspected at all (EACCES, EIO): callers retry later. + */ + private async inspectAdoptedCopy(store: MemoryStore, relPath: string): Promise { + const contained = await store.assertContained(relPath).then( + () => true, + () => false + ); + if (!contained) return null; + const linkKind = await lstatKind(store.physicalPath(relPath)); + if (linkKind === "unreadable") throw new Error(`cannot inspect adopted copy ${relPath}`); + if (linkKind !== "other") return null; // missing, dir, or a symlink + try { + return await this.readBoundedTextFile(store, relPath, relPath); + } catch (error) { + if (error instanceof MemoryCommandError) return null; // over the cap + throw error; + } + } + /** * The workspace whose session dir physically holds `store` * (//memory → owner), or null for global/project roots, diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index cdf3587634d..20957c4138d 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -947,6 +947,30 @@ describe("refinementRollback", () => { expect(ownerOwned.success).toBe(false); expect(ownerOwned.success ? "" : ownerOwned.error).toContain("owner's own note"); expect(await pathExists(path.join(ownerSessionDir, "memory", "same.md"))).toBe(true); + // A note deleted on the downgraded build and reconciled out of the shared + // store keeps a tombstoned record: the child's pre-sharing delete row + // still maps, and rolling it back restores the note in the shared store. + await fixture.service.create(fixture.ctx, "/memories/workspace/gone.md", "g1\n", "agent"); + await fixture.service.deletePath(fixture.ctx, "/memories/workspace/gone.md", "agent"); + const deleteRow = await lastRow(fixture.sessionDir); + await fsPromises.writeFile( + path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + }) + ); + const restoredDelete = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: deleteRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(restoredDelete.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "gone.md"), "utf-8") + ).toBe("g1\n"); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts index 5b0df102f90..10f2bd69733 100644 --- a/src/node/services/refinement/workspaceMemoryRevision.ts +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -50,7 +50,10 @@ async function readWorkspaceMemoryRevisionStrict(ownerSessionDir: string): Promi if ((error as NodeJS.ErrnoException | null)?.code === "ENOENT") return null; throw error; } - const value = Number.parseInt(raw, 10); + // The persisted format is exactly the decimal digits of a positive safe + // integer (parseInt would accept a numeric prefix of anything). + const trimmed = raw.trim(); + const value = /^[1-9][0-9]{0,15}$/.test(trimmed) ? Number(trimmed) : Number.NaN; if (!Number.isSafeInteger(value) || value <= 0) { throw new Error( `workspace memory revision at ${revisionPath} is malformed: ${raw.slice(0, 32)}` diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index e58547c4457..06626db06c1 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -101,10 +101,13 @@ async function readMarkerRecord( const parsed: unknown = JSON.parse(raw); if (typeof parsed !== "object" || parsed === null) return null; const { epochs, wildcard } = parsed as { epochs?: unknown; wildcard?: unknown }; + // Epochs are -1 (before any boundary) or a boundary's history sequence: + // anything outside that domain is corruption, read as malformed (deny). if ( !Array.isArray(epochs) || !epochs.every( - (epoch): epoch is number => typeof epoch === "number" && Number.isInteger(epoch) + (epoch): epoch is number => + typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch >= -1 ) ) { return null; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 20550ce7d4f..a9da5393038 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9519,6 +9519,16 @@ describe("WorkspaceService initialize", () => { // 12 → 16 between two separate reads could hide the entry from both). expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1, 12])).toBe(true); expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1])).toBe(false); + // Epochs outside the domain (-1 or a history sequence) are corruption: + // the marker reads as malformed, a deny for every epoch. + const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); + const savedMarker = await fsPromises.readFile(markerPath, "utf-8"); + await fsPromises.writeFile( + markerPath, + JSON.stringify({ deniedAt: Date.now(), epochs: [-2], wildcard: false }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 16)).toBe(true); + await fsPromises.writeFile(markerPath, savedMarker); expect( await service.recordWorkspaceMemoryWritable("policy-scratch", true, { epochHasPriorTurns: false, From 8fe7e9f49fac6c497ff97cb72d8ee385cda904b7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 00:32:36 +0000 Subject: [PATCH 65/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20fifty-nin?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - An in-place replacement of an adopted copy records the prior manifest record as its pending state, so a crash before the write is recovered by replacing the surviving old bytes rather than importing a duplicate. - A sealed removal republishes its own tombstone when another attempt's marker occupies the slot, instead of relying on a marker that attempt's rollback may delete. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 41 ++++++++++++++++++++++ src/node/services/memoryService.ts | 16 +++++---- src/node/services/workspaceRemoval.test.ts | 39 ++++++++++++++++++++ src/node/services/workspaceRemoval.ts | 27 +++++++++++--- 4 files changed, 113 insertions(+), 10 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index be58e6c9e7e..81824145c45 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2209,6 +2209,47 @@ describe("MemoryService", () => { expect(readopted["note.md"].deleted).toBeUndefined(); }); + it("recovers an interrupted in-place replacement without duplicating the note", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The downgraded build edits the note; the replacement pass crashed + // after recording its pending state but before writing the bytes — + // the on-disk state that leaves: the PRIOR record marked pending, the + // owner copy still holding the old bytes. + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + expect(prior).toMatchObject({ target: "note.md", created: true }); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: true } }) + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // The retry recognizes the surviving old bytes as this adoption's copy + // and replaces them in place — no imported/ duplicate, provenance kept. + const restarted = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + await restarted.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); + expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual(["note.md"]); + expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); + expect(manifest["note.md"].pending).toBeUndefined(); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 23ccba93b17..5ed4cca5cf8 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1419,12 +1419,16 @@ export class MemoryService extends EventEmitter { // already identical and takes the no-write path — without this // record it would read as the owner's own note, and a legacy // deletion could then never follow it out of the shared store. - adopted.set(relPath, { - ...record, - target: target.relPath, - created: true, - pending: true, - }); + // A replacement keeps the PRIOR record (old hash, same target) + // while pending: interrupted before the write, the retry must still + // recognize the surviving old bytes as this adoption's copy and + // replace them, not import the new bytes elsewhere. + adopted.set( + relPath, + target.replaces === true && previous !== undefined + ? { ...previous, pending: true } + : { ...record, target: target.relPath, created: true, pending: true } + ); await writeManifest(); await store.writeFile(target.relPath, content); if (target.replaces !== true) remainingCapacity--; diff --git a/src/node/services/workspaceRemoval.test.ts b/src/node/services/workspaceRemoval.test.ts index 7b1c7a8a63e..f08fa07f997 100644 --- a/src/node/services/workspaceRemoval.test.ts +++ b/src/node/services/workspaceRemoval.test.ts @@ -452,6 +452,45 @@ describe("workspaceRemoval", () => { expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(false); }); + test("a sealed removal republishes its own tombstone over a foreign attempt's marker", async () => { + using tmp = new DisposableTempDir("workspace-removal-test"); + const rootDir = path.join(tmp.path, "xum-home"); + const sessionDir = path.join(tmp.path, "sessions", "ws-sealed"); + const workspaceId = "ws-sealed"; + await fsPromises.mkdir(sessionDir, { recursive: true }); + const tombstonePath = workspaceRemovalTombstonePath(rootDir, workspaceId); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + // Two backends sealed the same sub-agent; B's marker overwrote A's. + await fsPromises.writeFile( + tombstonePath, + JSON.stringify({ workspaceId, removedAt: Date.now(), attemptId: "attempt-B" }) + ); + // A's deletion must not rely on B's marker (B's rollback may delete it): + // it republishes its own before deleting the session. + await removeSessionDirUnderMemoryLocks({ + rootDir, + sessionDir, + workspaceId, + attemptId: "attempt-A", + tombstoneSealed: true, + }); + expect( + (JSON.parse(await fsPromises.readFile(tombstonePath, "utf-8")) as { attemptId: string }) + .attemptId + ).toBe("attempt-A"); + // B's compensating rollback now leaves A's marker alone. + expect( + await rollbackRemovalTombstoneIfOwned({ + rootDir, + sessionDir, + workspaceId, + attemptId: "attempt-B", + workspaceStillRegistered: () => true, + }) + ).toBe(false); + expect(await isWorkspaceRemovalTombstoned(rootDir, workspaceId)).toBe(true); + }); + test("startup heal reclaims old tombstones only for still-registered workspaces (r63)", async () => { using tmp = new DisposableTempDir("workspace-removal-test"); const rootDir = path.join(tmp.path, "xum-home"); diff --git a/src/node/services/workspaceRemoval.ts b/src/node/services/workspaceRemoval.ts index b8fe408fbb7..6a9495d960b 100644 --- a/src/node/services/workspaceRemoval.ts +++ b/src/node/services/workspaceRemoval.ts @@ -196,13 +196,17 @@ export async function removeSessionDirUnderMemoryLocks(args: { ); assert(args.attemptId.length > 0, "removeSessionDirUnderMemoryLocks requires an attemptId"); let tombstonePublishedUnderLocks = args.tombstoneSealed === true; - // A sealed tombstone is not rewritten: the redundant write could fail - // (storage read-only/full after the checkout deletion) and would then abort - // a removal whose durable marker is already in place. + // A sealed tombstone is not rewritten while it is still THIS attempt's: the + // redundant write could fail (storage read-only/full after the checkout + // deletion) and would then abort a removal whose durable marker is already + // in place. It IS republished when another attempt's marker sits there + // (two backends sealing the same sub-agent): relying on a foreign marker + // would let that attempt's compensating rollback delete the only tombstone + // while this one proceeds to delete the session and deregister. const publish = async (): Promise => { if ( args.tombstoneSealed === true && - (await isWorkspaceRemovalTombstoned(args.rootDir, args.workspaceId)) + (await readRemovalTombstoneAttemptId(args.rootDir, args.workspaceId)) === args.attemptId ) { return; } @@ -280,6 +284,21 @@ export async function sealSubAgentForRemovalUnderMemoryLocks(args: { } } +/** The attempt ID stamped in the tombstone, or null when missing, unreadable or malformed. */ +async function readRemovalTombstoneAttemptId( + rootDir: string, + workspaceId: string +): Promise { + try { + const parsed = JSON.parse( + await fsPromises.readFile(workspaceRemovalTombstonePath(rootDir, workspaceId), "utf-8") + ) as { attemptId?: unknown }; + return typeof parsed.attemptId === "string" ? parsed.attemptId : null; + } catch { + return null; + } +} + async function publishRemovalTombstone(args: { rootDir: string; workspaceId: string; From 4460d8a10fd93f991bc909aa722a16f4881c1ca1 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 01:03:15 +0000 Subject: [PATCH 66/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixtieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption transfers provenance only while the old copy still holds the adopted bytes, and a pending replacement keeps the prior sidecar state so a pin toggled with the edit survives an interrupted pass. - The shared-store clock is written atomically. - Tail copies of assistant rows keep the policy epoch the row recorded. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/compactionHandler.test.ts | 12 ++- src/node/services/compactionHandler.ts | 12 ++- src/node/services/memoryService.test.ts | 76 +++++++++++++++++++ src/node/services/memoryService.ts | 16 ++-- .../refinement/workspaceMemoryRevision.ts | 5 +- 5 files changed, 108 insertions(+), 13 deletions(-) diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 2afc08514ad..9369c56e621 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1895,9 +1895,17 @@ describe("CompactionHandler", () => { // boundary's epoch — the chain stays visible to the policy conjunction. const boundarySequence = epoch[0].metadata?.historySequence; if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); + // An assistant row that recorded its policy under an OLDER epoch (a + // turn that straddled a boundary) keeps that epoch on its copy. await seedHistory( createMuxMessage("u2", "user", "second question"), - createMuxMessage("a2", "assistant", "second answer"), + createMuxMessage("a2", "assistant", "second answer", { + workspaceMemoryPolicyEpoch: -1, + }), + createMuxMessage("u3", "user", "third question"), + createMuxMessage("a3", "assistant", "third answer", { + workspaceMemoryPolicyEpoch: boundarySequence, + }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1905,7 +1913,7 @@ describe("CompactionHandler", () => { if (!secondEpoch.success) throw new Error(secondEpoch.error); expect( secondEpoch.data.slice(1).map((copy) => copy.metadata?.rlmPreservedTailSourcePolicyEpoch) - ).toEqual([-1, -1, boundarySequence, boundarySequence]); + ).toEqual([-1, -1, boundarySequence, -1, boundarySequence, boundarySequence]); }); it("rewrites MCP snapshot invoking IDs to the copy IDs of LATER tail rows", async () => { diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 3f133c70510..dba5a669525 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1671,13 +1671,17 @@ export class CompactionHandler { // The epoch whose workspace-memory write policy governs this row: a copy // of a copy keeps its ORIGINAL epoch (the chain must stay visible to the - // policy conjunction); a first-time copy was produced under the epoch - // this compaction closes. Copies from before the field existed carry - // nothing forward — no policy record ever existed for their epochs. + // policy conjunction); an assistant row keeps the epoch its policy was + // recorded under (a turn that started before a destructive reset and + // landed after the new boundary belongs to the OLD epoch, whose policy + // the new one cannot vouch for); any other first-time copy was produced + // under the epoch this compaction closes. Copies from before the field + // existed carry nothing forward — no policy record ever existed for + // their epochs. const sourcePolicyEpoch = source?.rlmPreservedTailCopy === true ? source.rlmPreservedTailSourcePolicyEpoch - : closingPolicyEpoch; + : (source?.workspaceMemoryPolicyEpoch ?? closingPolicyEpoch); return { ...row, id: copyId, diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 81824145c45..c123f8bba76 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1756,6 +1756,49 @@ describe("MemoryService", () => { expect(await pathExists(importedCopy)).toBe(false); }); + it("keeps an owner-edited conflict copy the owner's when a renamed legacy note lands on it", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The owner edits the conflict copy: it is the owner's now. + await fixture.service.strReplace( + ownerCtx, + "/memories/workspace/imported/ws-child/a.md", + "child's a", + "owner's edit", + "agent" + ); + // The downgraded build renames the source onto that path with the + // owner's bytes: the new record reuses the file, but no provenance + // transfers — the old copy no longer holds the adopted bytes. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rm(path.join(legacyRoot, "a.md")); + await fsPromises.writeFile( + path.join(legacyRoot, "imported", "ws-child", "a.md"), + "owner's edit" + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifest = JSON.parse( + await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + ) as Record; + expect(manifest["imported/ws-child/a.md"].created).not.toBe(true); + // Deleting the renamed source leaves the owner's note in place. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "a.md"), "utf-8") + ).toBe("owner's edit"); + }); + it("removal lists an oversized legacy notebook completely, counting every unplaceable note", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2248,6 +2291,39 @@ describe("MemoryService", () => { expect(Object.keys(manifest)).toEqual(["note.md"]); expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); expect(manifest["note.md"].pending).toBeUndefined(); + // A pin the child toggled together with an edit survives an interrupted + // replacement: the pending record keeps the PRIOR sidecar state, so the + // retry still sees the transition and applies it over the owner's pin. + const childKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-child", + }); + const ownerKey = memoryLogicalKey("workspace", "note.md", { + projectPath: "", + workspaceId: "ws-owner", + }); + await fixture.metaService.setPinned(ownerKey, false); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3"); + await fixture.metaService.setPinned(childKey, true); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...settled, pending: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v3"); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); }); it("keeps adopting a legacy note named __proto__ exactly once", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 5ed4cca5cf8..84a9ffc0a59 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1448,11 +1448,12 @@ export class MemoryService extends EventEmitter { // changes its usage counters, which must not drag the owner's pin // back to the child's unchanged value. if (childEntry !== undefined) { - // A pending record's fold never ran: still a first adoption. - const priorPinned = - previous === undefined || previous.pending === true - ? null - : legacySidecarPinned(previous.sidecar); + // A pending record still carries the sidecar state it was recorded + // with: a fresh adoption's is the child's current state (no + // transition → first-adoption semantics), a pending replacement's + // is the prior record's — the child's toggle since must not be + // lost to the interrupted pass. + const priorPinned = previous === undefined ? null : legacySidecarPinned(previous.sidecar); // Only an actual boolean transition of the child's pin overrides // the owner's; an unknown prior state never does. const childPinChanged = priorPinned !== null && priorPinned !== childEntry.pinned; @@ -1566,7 +1567,10 @@ export class MemoryService extends EventEmitter { rel !== relPath && listed.has(rel) && record.target === previous.target ); if (successor !== undefined) { - if (successor[1].created !== true) { + // Only a copy still holding the adopted bytes is ours to hand + // over; one the owner edited since is the owner's, and the + // successor keeps its own (non-created) provenance. + if (unchanged && successor[1].created !== true) { successor[1].created = true; manifestDirty = true; } diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts index 10f2bd69733..a2933db9c18 100644 --- a/src/node/services/refinement/workspaceMemoryRevision.ts +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -1,4 +1,5 @@ import * as fsPromises from "node:fs/promises"; +import writeFileAtomic from "write-file-atomic"; import * as path from "node:path"; import assert from "@/common/utils/assert"; import { WORKSPACE_MEMORY_REVISION_FILE_NAME } from "@/common/constants/memory"; @@ -75,6 +76,8 @@ async function readWorkspaceMemoryRevisionStrict(ownerSessionDir: string): Promi export async function advanceWorkspaceMemoryRevision(ownerSessionDir: string): Promise { const previous = (await readWorkspaceMemoryRevisionStrict(ownerSessionDir)) ?? 0; const next = Math.max(Date.now(), previous + 1); - await fsPromises.writeFile(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); + // Atomic: a crash mid-write must not leave a truncated value the strict + // reader would reject forever (every later row order-unknown). + await writeFileAtomic(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); return next; } From b2e8c6951160481bd262c84d24bd78a4a65bd799 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 01:36:21 +0000 Subject: [PATCH 67/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-fir?= =?UTF-8?q?st=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Turn preparation sanitizes persisted prelude IDs before spreading them. - Legacy adoption: a tombstone keeps destructive provenance only for a copy that was still this adoption's; a pending replacement records both the pre-write and replacement hashes so either crash side recovers. - Removal migrates rows without a store-clock value as order-unknown. - Tail copies naming epochs whose policy was never recorded anywhere deny the epoch like unknown history. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 7 ++++ src/node/services/memoryService.test.ts | 34 +++++++++++++++++++ src/node/services/memoryService.ts | 21 +++++++++--- .../refinement/sharedMemoryRowMigration.ts | 10 ++++-- src/node/services/turnRequestBuilder.ts | 6 +++- src/node/services/workspaceService.test.ts | 16 +++++++++ src/node/services/workspaceService.ts | 13 ++++++- 7 files changed, 99 insertions(+), 8 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 858c96fc9bf..9ea2c60462b 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -33,6 +33,13 @@ export interface LegacyAdoptionRecord { target: string; created?: boolean; pending?: boolean; + /** + * Hash of the bytes an in-place replacement is about to write (set on the + * pending prior record, cleared once the pass completes). With `content` + * (the pre-write bytes) this lets a retry recognize the copy as this + * adoption's on either side of an interrupted write. + */ + replacementContent?: string; /** * The legacy source was deleted (or renamed away) on a downgraded build and * the copy reconciled. Kept rather than dropped: the child's pre-sharing diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index c123f8bba76..b9f20fe63ff 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1790,6 +1790,10 @@ describe("MemoryService", () => { await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") ) as Record; expect(manifest["imported/ws-child/a.md"].created).not.toBe(true); + // ...and the obsolete record's tombstone drops its destructive + // provenance: the child's old rows may not map onto the owner's note. + expect(manifest["a.md"]).toMatchObject({ deleted: true }); + expect(manifest["a.md"].created).not.toBe(true); // Deleting the renamed source leaves the owner's note in place. await new Promise((resolve) => setTimeout(resolve, 5)); await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); @@ -2324,6 +2328,32 @@ describe("MemoryService", () => { }); expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v3"); expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // The opposite crash window: the replacement bytes landed but the final + // manifest write did not, and the downgraded build deletes the source + // before the retry. The pending record names both hashes, so the copy + // is still recognized as this adoption's and follows the source out. + const settled3 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v4"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { ...settled3, pending: true, replacementContent: sha256Hex("v4") }, + }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); }); it("keeps adopting a legacy note named __proto__ exactly once", async () => { @@ -2913,6 +2943,10 @@ describe("MemoryService", () => { expect((copy.data.postState as { files: Array<{ path: string }> }).files[0].path).toBe( ownerCopy ); + // No store-clock value existed for the pre-sharing row: its order is + // unknown, not its journal-local timestamp dressed up as a clock value. + expect(copy.data.sourceTs).toBeUndefined(); + expect(copy.data.orderUnknown).toBe(true); const rolledBack = await rollbackRefinement({ sessionDir: ownerSessionDir, id: copy.id, diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 84a9ffc0a59..82df49e2f49 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1380,7 +1380,7 @@ export class MemoryService extends EventEmitter { } else if ( previous.created === true && priorContent !== null && - sha256Hex(priorContent) === previous.content + [previous.content, previous.replacementContent].includes(sha256Hex(priorContent)) ) { target = { relPath: previous.target, write: true, replaces: true }; } @@ -1426,7 +1426,7 @@ export class MemoryService extends EventEmitter { adopted.set( relPath, target.replaces === true && previous !== undefined - ? { ...previous, pending: true } + ? { ...previous, pending: true, replacementContent: record.content } : { ...record, target: target.relPath, created: true, pending: true } ); await writeManifest(); @@ -1504,6 +1504,7 @@ export class MemoryService extends EventEmitter { (error: unknown) => isMissingPathError(error) ); if (!sourceGone) continue; + let unchangedForTombstone = false; if (previous.created === true) { // Strict probe: a target that merely could not be stat'ed is not // "changed" — dropping the entry on that basis would lose the @@ -1556,7 +1557,10 @@ export class MemoryService extends EventEmitter { } } } - const unchanged = current !== null && sha256Hex(current) === previous.content; + // Either side of an interrupted in-place replacement counts as ours. + const unchanged = + current !== null && + [previous.content, previous.replacementContent].includes(sha256Hex(current)); // A listed note may now point at this very target (the downgraded // build renamed `a.md` to the path its conflict copy was adopted // under, and the new record reused the identical file): the target @@ -1566,6 +1570,7 @@ export class MemoryService extends EventEmitter { ([rel, record]) => rel !== relPath && listed.has(rel) && record.target === previous.target ); + unchangedForTombstone = unchanged && successor === undefined; if (successor !== undefined) { // Only a copy still holding the adopted bytes is ours to hand // over; one the owner edited since is the owner's, and the @@ -1599,7 +1604,15 @@ export class MemoryService extends EventEmitter { // this note still need relPath → target to be rolled back into the // shared store (a delete's restore lands at the reconciled target; // the reconciliation above never runs again for it). - adopted.set(relPath, { ...previous, deleted: true }); + // Destructive provenance survives only while the target was still + // this adoption's copy and nobody took it over: a copy the owner + // edited (or one handed to a successor record) is not the old path's + // to delete or restore any more. + adopted.set(relPath, { + ...previous, + deleted: true, + created: previous.created === true && unchangedForTombstone, + }); manifestDirty = true; } if (manifestDirty) await writeManifest(); diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index ccd38426dcb..fe3bb900907 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -267,8 +267,14 @@ export async function migrateSharedMemoryRefinementRows(args: { : {}), migratedFrom, ...(rollbackOf !== undefined ? { rollbackOf } : {}), - sourceTs: row.data.sourceTs ?? row.ts, - ...(row.data.orderUnknown === true ? { orderUnknown: true as const } : {}), + // A row without a store-clock value (pre-sharing, or its clock write + // failed) has only a journal-local `ts`, incomparable with the + // owner's clock-stamped rows: carried as order-unknown rather than + // dressed up as a clock value. + ...(row.data.sourceTs !== undefined ? { sourceTs: row.data.sourceTs } : {}), + ...(row.data.orderUnknown === true || row.data.sourceTs === undefined + ? { orderUnknown: true as const } + : {}), ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); publishedBlobs.push(...appended.publishedBlobs); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index e472eed2b56..be63ebcb117 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -38,6 +38,7 @@ import type { GoalRecordV1 } from "@/common/types/goal"; import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/types/message"; import { createMuxMessage } from "@/common/types/message"; import { latestContextBoundaryHistorySequence } from "@/common/utils/messages/compactionBoundary"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; import { secretsToRecord } from "@/common/types/secrets"; import type { XumToolScope } from "@/common/types/toolScope"; @@ -1515,7 +1516,10 @@ export class TurnRequestBuilder { const currentBatch = new Set( latestUserMessage === undefined ? [] - : [latestUserMessage.id, ...(latestUserMessage.metadata?.requestPreludeMessageIds ?? [])] + : [ + latestUserMessage.id, + ...getRequestPreludeMessageIds(latestUserMessage.metadata?.requestPreludeMessageIds), + ] ); // RLM keep-recent copies are the previous epoch's turns re-appended // after the boundary (compactionHandler), not turns of this epoch: the diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index a9da5393038..ccea9dcefb5 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9552,6 +9552,22 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persistedFor(18)).toBe(true); + // A carried epoch with no record anywhere (no carried key, nothing under + // this epoch, no marker — e.g. the first tail compaction after an + // upgrade) is unknown history: denied. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + delete entry.workspaceMemoryWritableByEpoch; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 24, + carriedPolicyEpochs: [-1], + }) + ).toBe(true); + expect(persistedFor(24)).toBe(false); // A tail copy whose source epoch is unknown (persisted before the field // existed) carries a policy nobody can look up: denied, like unknown // history, even for an otherwise writable first turn. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 6fe9f3a4e09..581f86f99ad 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4421,9 +4421,20 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const storedFor = (entry: WorkspaceConfigEntry): boolean | undefined => workspaceMemoryWritableForEpoch(entry, policyEpoch); const stored = storedFor(before.workspace); + // Carried epochs whose policy was never recorded anywhere — no record + // under any carried key, none under this epoch's (where a completed + // carry would have moved it), no marker — are unknown history too: the + // tail copies ARE turns of those epochs (excluded from epochHasPriorTurns + // by design), e.g. the first tail compaction after upgrading a chat. + const carriedUnrecorded = + carriedPolicyEpochs.length > 0 && + stored === undefined && + !denyMarker && + carriedFor(before.workspace) === undefined; const unknownHistory = (stored === undefined && mirror === undefined && options.epochHasPriorTurns) || - options.carriedPolicyUnknown === true; + options.carriedPolicyUnknown === true || + carriedUnrecorded; const conjunction = (durable: boolean | undefined, carried: boolean | undefined): boolean => !denyMarker && !unknownHistory && From caf2c6ac0d19734f1debb8ef8a472c907ffef56c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 02:14:46 +0000 Subject: [PATCH 68/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-sec?= =?UTF-8?q?ond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption records a pending deletion before removing a copy, so a crash before the tombstone is recovered as "removed by us". - Rows whose paths the adoption remapper retargeted are order-unknown: at removal migration (no private-clock sourceTs carried over) and in live rollback conflict detection (retargeted peers and targets vs. other journals). - The harvest gate validates a turn's policy stamp before its request bound, so a foreign-epoch turn with a missing bound still refuses. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../memoryConsolidationService.test.ts | 15 ++++- .../services/memoryConsolidationService.ts | 7 ++- src/node/services/memoryLegacyAdoption.ts | 7 +++ src/node/services/memoryService.test.ts | 49 +++++++++++++++- src/node/services/memoryService.ts | 13 ++++- .../refinement/refinementRollback.test.ts | 27 +++++++++ .../services/refinement/refinementRollback.ts | 57 +++++++++++++++---- .../refinement/sharedMemoryRowMigration.ts | 10 +++- 8 files changed, 163 insertions(+), 22 deletions(-) diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index eabb425f441..f9766bc29ab 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1326,7 +1326,12 @@ describe("MemoryConsolidationService", () => { using fixture = await createFixture(); await fixture.addWorkspace("ws-clean"); await fixture.addWorkspace("ws-corrupt"); - const seed = async (workspaceId: string, foreignTurn: number | null | undefined) => { + await fixture.addWorkspace("ws-unbounded"); + const seed = async ( + workspaceId: string, + foreignTurn: number | null | undefined, + options?: { withoutBound: boolean } + ) => { const reset = createMuxMessage("reset-1", "assistant", "", { compactionBoundary: true, compacted: "user", @@ -1367,7 +1372,7 @@ describe("MemoryConsolidationService", () => { await fixture.historyService.appendToHistory( workspaceId, createMuxMessage("b-reply", "assistant", "Read-only output.", { - requestHistorySequence: closingEpoch - 1, + ...(options?.withoutBound === true ? {} : { requestHistorySequence: closingEpoch - 1 }), // `null` models a corrupted raw-JSON row. workspaceMemoryPolicyEpoch: foreignTurn as unknown as number, }) @@ -1404,6 +1409,12 @@ describe("MemoryConsolidationService", () => { const corrupt = await seed("ws-corrupt", null); expect(corrupt.success).toBe(false); expect(fixture.modelCalls).toHaveLength(0); + // A foreign stamp refuses even when the row's request bound is missing + // or corrupt: its user row may be gone, so nothing else would surface it. + const unbounded = await seed("ws-unbounded", -1, { withoutBound: true }); + expect(unbounded.success).toBe(false); + if (!unbounded.success) expect(unbounded.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); // Without the foreign turn the copies alone refuse nothing, and the // harvest transcript leaves them out. diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 6ec04cac01b..dac4daaad93 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -372,17 +372,20 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb const covered = new Set(); for (const message of messages) { if (message.role !== "assistant") continue; - const bound = message.metadata?.requestHistorySequence; - if (typeof bound !== "number") continue; const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; // No stamp at all: a row that is no turn of this build (covers nothing). if (policyEpoch === undefined) continue; // History rows are raw JSON: a stamp that is present but not the integer // equal to the closing epoch — another epoch's, or a corrupted value such // as null — proves no policy for this epoch and refuses the harvest. + // Checked BEFORE the bound: a foreign-epoch turn whose bound is missing + // or corrupt must still refuse (its user row may be gone with the reset + // that made it foreign, so nothing else would surface it). if (!Number.isInteger(policyEpoch) || policyEpoch !== closingEpoch) { return "the compacted epoch holds a turn whose memory policy was recorded for another epoch; harvest refused (fail closed)"; } + const bound = message.metadata?.requestHistorySequence; + if (typeof bound !== "number") continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 9ea2c60462b..c3d2e645d96 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -40,6 +40,13 @@ export interface LegacyAdoptionRecord { * adoption's on either side of an interrupted write. */ replacementContent?: string; + /** + * Reconciliation of a deleted source is under way: the copy is about to be + * (or was just) removed. Set before the removal so a crash between the + * removal and the tombstone write is recovered as "removed by us" rather + * than "changed by the owner". + */ + pendingDeletion?: boolean; /** * The legacy source was deleted (or renamed away) on a downgraded build and * the copy reconciled. Kept rather than dropped: the child's pre-sharing diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index b9f20fe63ff..743b802c7e1 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2356,6 +2356,48 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); }); + it("recovers a deletion interrupted between the copy's removal and the tombstone", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The crash state: source deleted, deletion recorded as pending, copy + // already removed, tombstone never written. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + // Removed by us, not changed by the owner: destructive provenance kept, + // so the child's delete row still maps onto the shared store. + expect(tombstone).toMatchObject({ deleted: true, created: true }); + expect(tombstone.pendingDeletion).toBeUndefined(); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2909,6 +2951,9 @@ describe("MemoryService", () => { postState: { files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], }, + // A self-fallback write's clock value: the child's PRIVATE store's, + // not the owner's — meaningless once the row is retargeted. + sourceTs: 42, }, }); // Also a legacy row for a note the shared store never took (unplaceable). @@ -2943,8 +2988,8 @@ describe("MemoryService", () => { expect((copy.data.postState as { files: Array<{ path: string }> }).files[0].path).toBe( ownerCopy ); - // No store-clock value existed for the pre-sharing row: its order is - // unknown, not its journal-local timestamp dressed up as a clock value. + // A retargeted row's clock value belonged to the private store: its + // order among the owner's rows is unknown, not that value. expect(copy.data.sourceTs).toBeUndefined(); expect(copy.data.orderUnknown).toBe(true); const rolledBack = await rollbackRefinement({ diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 82df49e2f49..f84195ee4b3 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1570,7 +1570,10 @@ export class MemoryService extends EventEmitter { ([rel, record]) => rel !== relPath && listed.has(rel) && record.target === previous.target ); - unchangedForTombstone = unchanged && successor === undefined; + // A target already gone while a deletion was pending was removed by + // the interrupted pass, not changed by the owner. + const removedByUs = previous.pendingDeletion === true && current === null; + unchangedForTombstone = (unchanged || removedByUs) && successor === undefined; if (successor !== undefined) { // Only a copy still holding the adopted bytes is ours to hand // over; one the owner edited since is the owner's, and the @@ -1580,7 +1583,12 @@ export class MemoryService extends EventEmitter { manifestDirty = true; } } else if (unchanged) { - // Metadata first: a sidecar failure then aborts the pass with the + // Deletion provenance first: a crash after the removal but before + // the tombstone write must not make the retry read the missing + // copy as owner-changed (and drop the child's rollback mapping). + adopted.set(relPath, { ...previous, pendingDeletion: true }); + await writeManifest(); + // Metadata next: a sidecar failure then aborts the pass with the // file and manifest entry intact, so the retry repeats both; // the reverse order would strand the owner-key pin/usage once // the file was gone and the entry dropped. @@ -1610,6 +1618,7 @@ export class MemoryService extends EventEmitter { // to delete or restore any more. adopted.set(relPath, { ...previous, + pendingDeletion: undefined, deleted: true, created: previous.created === true && unchangedForTombstone, }); diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 20957c4138d..beb650d365d 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -899,6 +899,33 @@ describe("refinementRollback", () => { "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, }) ); + // The retargeted row was journaled against this session's private clock: + // against an overlapping row of the OWNER's journal its order is unknown, + // so the rollback is refused unless forced. + await sharedDurableEventJournal(ownerSessionDir).append({ + workspaceId: "ws-owner", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/sub/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "sub", "note.md"), text: "v2\n" }], + }, + sourceTs: 1, + }, + }); + const unordered = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + }); + expect(unordered.success).toBe(false); + expect(unordered.success ? "" : unordered.error).toContain( + "order relative to this row is unknown" + ); const result = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: editRow.id, diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index b3c457f6ebb..ed69e521b64 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -546,8 +546,14 @@ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { * journal-local `ts`/`seq`, incomparable with other journals' rows. Callers * fail closed — such a pair conflicts in either direction (force overrides). */ -function orderUnknown(row: RefinementEvent, other: RefinementEvent): boolean { - return row.data.orderUnknown === true || other.data.orderUnknown === true; +function orderUnknown( + row: RefinementEvent, + target: RefinementEvent, + targetRetargeted: boolean +): boolean { + if (row.data.orderUnknown === true || target.data.orderUnknown === true) return true; + // A retargeted target (see wasRetargeted) vs. a row of another journal. + return targetRetargeted && row.workspaceId !== target.workspaceId; } /** @@ -619,10 +625,19 @@ async function readSharedMemoryPeerRows( // ":") while this session lives on. They are // this journal's rows seen twice, not later peer edits. if (row.data.migratedFrom?.startsWith(`${actingWorkspaceId}:`) === true) continue; + const original = RefinementInverseSchema.safeParse(row.data.inverse); const parsed = parseRemappedInverse(row, remap); - if (parsed === null) continue; + if (parsed === null || !original.success) continue; if (!inversePaths(parsed).some((p) => pathsOverlap(p, sharedRoot))) continue; - peerRows.push({ ...row, data: { ...row.data, inverse: parsed } }); + // A retargeted peer row carries its private clock: order unknown. + peerRows.push({ + ...row, + data: { + ...row.data, + inverse: parsed, + ...(wasRetargeted(original.data, parsed) ? { orderUnknown: true as const } : {}), + }, + }); } } return peerRows; @@ -657,12 +672,22 @@ function parseRemappedInverse( } } +/** + * Whether retargeting changed the inverse's paths: such a row was journaled + * against a PRIVATE store (pre-sharing), so its `ts`/`sourceTs` belong to + * that store's clock and are incomparable with the shared store's rows. + */ +function wasRetargeted(original: RefinementInverse, remapped: RefinementInverse): boolean { + return JSON.stringify(inversePaths(original)) !== JSON.stringify(inversePaths(remapped)); +} + async function collectDivergence( rows: RefinementEvent[], target: RefinementEvent, inverse: RefinementInverse, readContent: InverseContentReader, - remap: RecordedPathRemapper + remap: RecordedPathRemapper, + targetRetargeted: boolean ): Promise { const complaints: string[] = []; const targetPaths = inversePaths(inverse); @@ -678,15 +703,15 @@ async function collectDivergence( ); for (const row of rows) { if (row.id === target.id) continue; - if (!isAfter(row, target) && !orderUnknown(row, target)) continue; + if (!isAfter(row, target) && !orderUnknown(row, target, targetRetargeted)) continue; if (rolledBackIds.has(row.id)) continue; // Effect undone by a later rollback row. - if (!liveRowConflictsWithTarget(rows, row, target)) continue; + if (!liveRowConflictsWithTarget(rows, row, target, targetRetargeted)) continue; const parsed = parseRemappedInverse(row, remap); if (parsed === null) continue; const overlap = inversePaths(parsed).some((p) => targetPaths.some((t) => pathsOverlap(p, t))); if (overlap) { complaints.push( - orderUnknown(row, target) + orderUnknown(row, target, targetRetargeted) ? `refinement row ${row.id} (seq ${row.seq}) touched the same paths and its order relative to this row is unknown (its store clock write failed)` : `later refinement row ${row.id} (seq ${row.seq}) touched the same paths` ); @@ -801,7 +826,8 @@ async function collectPostStateDivergence( function liveRowConflictsWithTarget( rows: RefinementEvent[], row: RefinementEvent, - target: RefinementEvent + target: RefinementEvent, + targetRetargeted: boolean ): boolean { if (row.data.rollbackOf === undefined) { return true; // Plain later row: its edit is live on disk. @@ -823,7 +849,7 @@ function liveRowConflictsWithTarget( } // Odd chain: rewound to just before root — a conflict unless the root is // provably after the target. - return !isAfter(current, target) || orderUnknown(current, target); + return !isAfter(current, target) || orderUnknown(current, target, targetRetargeted); } async function dirExists(target: string): Promise { @@ -982,6 +1008,11 @@ export async function rollbackRefinement( if (!(error instanceof LegacyPathNotAdoptedError)) throw error; throw new RollbackError(`Refusing rollback of '${opts.id}': ${error.message}`); } + // A retargeted target was journaled against this session's private + // clock: its order relative to OTHER journals' rows is unknown, so those + // are compared as order-unknown (conflict unless forced). Same-journal + // rows still order by their shared sequence. + const targetRetargeted = wasRetargeted(parsedInverse.data, inverse); // Confinement first — never overridable. A corrupted inverse must never // write outside the memory/skill roots (repo AGENTS.md, built-in skills, @@ -1033,7 +1064,8 @@ export async function rollbackRefinement( target, inverse, readContent, - remap + remap, + targetRetargeted ); if (divergence.length > 0 && opts.force !== true) { throw new RollbackError( @@ -1105,7 +1137,8 @@ export async function rollbackRefinement( target, inverse, readContent, - remap + remap, + targetRetargeted ); if (raced.length > 0) { throw new RollbackError( diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index fe3bb900907..bf3393f340e 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -189,6 +189,12 @@ export async function migrateSharedMemoryRefinementRows(args: { if (!parsedInverse.success) continue; const remapped = remapInverse(parsedInverse.data); if (remapped === null) continue; + // A row whose paths were retargeted was journaled against the child's + // PRIVATE store (pre-sharing, or a self-fallback while config.json was + // unreadable): any `sourceTs` it carries is that private clock's, not + // the owner's, so its order among the owner's rows is unknown. + const retargeted = + JSON.stringify(inversePaths(parsedInverse.data)) !== JSON.stringify(inversePaths(remapped)); const inverse = { success: true as const, data: remapped }; if (!inversePaths(inverse.data).every((p) => isInside(ownerMemoryRoot, p))) continue; @@ -271,8 +277,8 @@ export async function migrateSharedMemoryRefinementRows(args: { // failed) has only a journal-local `ts`, incomparable with the // owner's clock-stamped rows: carried as order-unknown rather than // dressed up as a clock value. - ...(row.data.sourceTs !== undefined ? { sourceTs: row.data.sourceTs } : {}), - ...(row.data.orderUnknown === true || row.data.sourceTs === undefined + ...(row.data.sourceTs !== undefined && !retargeted ? { sourceTs: row.data.sourceTs } : {}), + ...(row.data.orderUnknown === true || row.data.sourceTs === undefined || retargeted ? { orderUnknown: true as const } : {}), ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), From fec9766d3a6a936e575f2de28fb03d3aca0d5c0e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 02:47:15 +0000 Subject: [PATCH 69/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-thi?= =?UTF-8?q?rd=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A pending deletion is recovered as "removed by us" only when the strict probe proves the target absent, and a source reappearing identically while its deletion was pending is adopted anew. - Legacy directory rename endpoints map through their adopted descendants when every one landed at its own path as this adoption's copy. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 21 ++++- src/node/services/memoryService.test.ts | 83 +++++++++++++++++++ src/node/services/memoryService.ts | 12 ++- .../refinement/refinementRollback.test.ts | 40 +++++++++ 4 files changed, 150 insertions(+), 6 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index c3d2e645d96..5180734cde5 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -154,10 +154,27 @@ export async function createLegacyPathRemapper(args: { const remapPath = (filePath: string): string => { const relative = path.relative(legacyRoot, path.resolve(filePath)); if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return filePath; - const record = adopted.get(relative.split(path.sep).join("/")); - if (record === undefined || record.pending === true) { + const relPath = relative.split(path.sep).join("/"); + const record = adopted.get(relPath); + if (record === undefined) { + // A directory endpoint (a pre-sharing directory rename): the manifest + // records files only. Mappable when every adopted descendant landed at + // its own relPath in the owner store as this adoption's copy — the + // owner directory then IS the adopted directory. Descendants placed + // elsewhere (conflict imports) or owner-owned make the structural + // move ambiguous: refused. + const descendants = [...adopted].filter(([rel]) => rel.startsWith(`${relPath}/`)); + if ( + descendants.length > 0 && + descendants.every( + ([rel, entry]) => entry.target === rel && entry.created === true && entry.pending !== true + ) + ) { + return path.join(ownerRoot, ...relPath.split("/")); + } throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); } + if (record.pending === true) throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); if (record.created !== true) throw new LegacyPathNotAdoptedError(filePath, "owner-owned"); return path.join(ownerRoot, ...record.target.split("/")); }; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 743b802c7e1..00a11edc5be 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2398,6 +2398,89 @@ describe("MemoryService", () => { expect(tombstone.pendingDeletion).toBeUndefined(); }); + it("does not read owner state at a pending-deletion target as removed by us", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; created?: boolean } + > + )["note.md"]; + // The deletion was recorded pending but failed before the removal; the + // owner replaced the copy with a directory of its own in the meantime. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.mkdir(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile(path.join(ownerRoot, "note.md", "inner.md"), "owner's"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const tombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["note.md"]; + expect(tombstone.deleted).toBe(true); + expect(tombstone.created).not.toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md", "inner.md"), "utf-8")).toBe( + "owner's" + ); + }); + + it("re-adopts a source that reappeared identically while its deletion was pending", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // Crash after the copy's removal, before the tombstone; the downgraded + // build then recreates the source with the same bytes. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pendingDeletion: true } }) + ); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.utimes(path.join(legacyRoot, "note.md"), new Date(), new Date()); + // Removal's handover must restore the copy, not report "nothing to do" + // and delete the only remaining note with the child session. + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pendingDeletion?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pendingDeletion).toBeUndefined(); + }); + it("keeps adopting a legacy note named __proto__ exactly once", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index f84195ee4b3..ba4a8af5bae 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1345,7 +1345,10 @@ export class MemoryService extends EventEmitter { if ( previous?.content === record.content && previous.sidecar === record.sidecar && - previous.pending !== true + previous.pending !== true && + // A deletion under way removed (or is about to remove) the copy: a + // source reappearing with the same bytes must be adopted anew. + previous.pendingDeletion !== true ) { continue; // folded in earlier, nothing changed since } @@ -1570,9 +1573,10 @@ export class MemoryService extends EventEmitter { ([rel, record]) => rel !== relPath && listed.has(rel) && record.target === previous.target ); - // A target already gone while a deletion was pending was removed by - // the interrupted pass, not changed by the owner. - const removedByUs = previous.pendingDeletion === true && current === null; + // A target PROVEN absent (strict probe) while a deletion was pending + // was removed by the interrupted pass, not changed by the owner. A + // directory, symlink or over-cap file there is owner state. + const removedByUs = previous.pendingDeletion === true && targetKind === null; unchangedForTombstone = (unchanged || removedByUs) && successor === undefined; if (successor !== undefined) { // Only a copy still holding the adopted bytes is ours to hand diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index beb650d365d..fb90c8f2517 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -998,6 +998,46 @@ describe("refinementRollback", () => { expect( await fsPromises.readFile(path.join(ownerSessionDir, "memory", "gone.md"), "utf-8") ).toBe("g1\n"); + // A pre-sharing DIRECTORY rename: the manifest records files only, so the + // directory endpoints map through their adopted descendants (all landed + // at their own relPath as this adoption's copies). + await fixture.service.create(fixture.ctx, "/memories/workspace/olddir/a.md", "a\n", "agent"); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/olddir", + "/memories/workspace/newdir", + "agent" + ); + const dirRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "newdir"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "a.md"), "a\n"); + await fsPromises.writeFile( + path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + "newdir/a.md": { content: "x", sidecar: "", target: "newdir/a.md", created: true }, + "olddir/a.md": { + content: "x", + sidecar: "", + target: "olddir/a.md", + created: true, + deleted: true, + }, + }) + ); + const undoneRename = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneRename.success).toBe(true); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "olddir", "a.md"), "utf-8") + ).toBe("a\n"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "newdir"))).toBe(false); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { From 18a8f9962659e10be5fa34bc592f177e3bd5a576 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 03:23:23 +0000 Subject: [PATCH 70/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-fou?= =?UTF-8?q?rth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption: a pending deletion is recovered only for a contained target proven absent; manifest lifecycle flags parse fail-closed; a directory rename endpoint maps only when the owner subtree is exactly the adopted descendants. - The policy epoch boundary reset and carry require an existing config.json. - Tail copies of assistant turn rows without a recorded policy epoch stay unstamped (unknown) instead of taking the closing epoch. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 31 ++--- ...Session.workspaceMemoryPolicyEpoch.test.ts | 21 +++- src/node/services/compactionHandler.test.ts | 23 +++- src/node/services/compactionHandler.ts | 10 +- src/node/services/memoryLegacyAdoption.ts | 116 +++++++++++++++--- src/node/services/memoryService.test.ts | 73 +++++++++++ src/node/services/memoryService.ts | 16 ++- .../refinement/refinementRollback.test.ts | 15 +++ src/node/services/workspaceService.test.ts | 5 +- 9 files changed, 260 insertions(+), 50 deletions(-) diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index d004d4a78b3..d7feff7b049 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1143,15 +1143,13 @@ export class AgentSession { path.join(this.config.sessionsDir, this.workspaceId), options ); - // Strict load: an unreadable config.json would read as the empty default, - // in which this workspace has no records to clear — the reset would - // report success and leave the stale records behind. Throwing makes the - // boundary a retryable partial failure instead. (A genuinely absent file - // holds no records and is the empty default for real.) - const entry = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), - this.workspaceId - ); + // Strict load: an unreadable — or transiently ABSENT — config.json would + // read as the empty default, in which this workspace has no records to + // clear; the reset would report success and leave the stale records + // behind (a destructive boundary reuses epoch -1, so a surviving deny + // would pin the new segment). Throwing makes the boundary a retryable + // partial failure instead; a registered workspace always has a config. + const entry = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) { this.workspaceMemoryWritable = undefined; return; @@ -1185,10 +1183,7 @@ export class AgentSession { // boundary reuses epoch -1, and a surviving `-1: false` would pin the // new segment to the stored-false fast path once the mirror is cleared // below — so the mirror is cleared only after the durable state agrees. - const after = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), - this.workspaceId - ); + const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); const stale = after !== null && (options === undefined @@ -1229,10 +1224,7 @@ export class AgentSession { let carried: boolean | undefined; let failure: string | undefined; try { - const entry = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), - this.workspaceId - ); + const entry = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); const closingBefore = entry === null ? undefined : workspaceMemoryWritableForEpoch(entry.workspace, closingEpoch); if (closingBefore === undefined) return; @@ -1252,10 +1244,7 @@ export class AgentSession { }); // Consumed by another backend's boundary meanwhile: nothing to carry. if (carried === undefined) return; - const after = findWorkspaceEntry( - this.config.loadConfigOrDefault({ throwOnError: true }), - this.workspaceId - ); + const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); if ( after === null || workspaceMemoryWritableForEpoch(after.workspace, nextEpoch) !== carried diff --git a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts index 3e21d1fa877..598b30a78fe 100644 --- a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts +++ b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts @@ -153,7 +153,8 @@ describe("AgentSession workspace memory policy epoch boundary", () => { }); test("reset clears the mirror only once the durable clear is proven", async () => { - const { session, internals, records, setRecords, swallowNextWrite } = await createSession(); + const { session, internals, records, setRecords, swallowNextWrite, config } = + await createSession(); // Destructive boundary: every record goes, then the mirror. session.recordWorkspaceMemoryWritable(false); await setRecords({ "-1": false, "5": true }); @@ -173,6 +174,24 @@ describe("AgentSession workspace memory policy epoch boundary", () => { expect(records()).toEqual({ "-1": false }); expect(session.workspaceMemoryWritableMirror()).toBe(false); + // An ABSENT config.json is not the empty default: a registered workspace + // always has one, so its absence is transient — the reset must fail + // (retryable) rather than clear the mirror over records it never saw. + session.recordWorkspaceMemoryWritable(false); + await setRecords({ "-1": false }); + const configPath = path.join(config.rootDir, "config.json"); + const savedConfig = await fsPromises.readFile(configPath); + await fsPromises.rm(configPath); + try { + expect( + await internals.resetWorkspaceMemoryWritable().then(() => null, getErrorMessage) + ).toMatch(/absent/); + } finally { + await fsPromises.writeFile(configPath, savedConfig); + } + expect(session.workspaceMemoryWritableMirror()).toBe(false); + expect(records()).toEqual({ "-1": false }); + // Compaction boundary, fenced to the closing epoch: the same proof. await setRecords({ "3": false, "8": true }); swallowNextWrite(); diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 9369c56e621..d29eeb32367 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1895,17 +1895,25 @@ describe("CompactionHandler", () => { // boundary's epoch — the chain stays visible to the policy conjunction. const boundarySequence = epoch[0].metadata?.historySequence; if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); - // An assistant row that recorded its policy under an OLDER epoch (a - // turn that straddled a boundary) keeps that epoch on its copy. + // An assistant TURN row (it carries the request bound) that recorded + // its policy under an OLDER epoch (a turn that straddled a boundary) + // keeps that epoch on its copy; a turn row WITHOUT a recorded policy + // (an older build's) stays unstamped — unknown, never vouched for. await seedHistory( createMuxMessage("u2", "user", "second question"), createMuxMessage("a2", "assistant", "second answer", { + requestHistorySequence: boundarySequence + 1, workspaceMemoryPolicyEpoch: -1, }), createMuxMessage("u3", "user", "third question"), createMuxMessage("a3", "assistant", "third answer", { + requestHistorySequence: boundarySequence + 3, workspaceMemoryPolicyEpoch: boundarySequence, }), + createMuxMessage("u4", "user", "old-build question"), + createMuxMessage("a4", "assistant", "old-build answer", { + requestHistorySequence: boundarySequence + 5, + }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1913,7 +1921,16 @@ describe("CompactionHandler", () => { if (!secondEpoch.success) throw new Error(secondEpoch.error); expect( secondEpoch.data.slice(1).map((copy) => copy.metadata?.rlmPreservedTailSourcePolicyEpoch) - ).toEqual([-1, -1, boundarySequence, -1, boundarySequence, boundarySequence]); + ).toEqual([ + -1, + -1, + boundarySequence, + -1, + boundarySequence, + boundarySequence, + boundarySequence, + undefined, + ]); }); it("rewrites MCP snapshot invoking IDs to the copy IDs of LATER tail rows", async () => { diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index dba5a669525..355c031c95c 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1678,10 +1678,18 @@ export class CompactionHandler { // under the epoch this compaction closes. Copies from before the field // existed carry nothing forward — no policy record ever existed for // their epochs. + // An assistant TURN row (it carries the request bound) without a recorded + // policy epoch was produced by a build that did not maintain the policy: + // its copy stays unstamped, which the policy sink reads as unknown + // (deny) — stamping it with the closing epoch would vouch for a policy + // nobody recorded. Non-turn assistant rows (payloads, summaries) and + // user rows belong to the closing epoch. const sourcePolicyEpoch = source?.rlmPreservedTailCopy === true ? source.rlmPreservedTailSourcePolicyEpoch - : (source?.workspaceMemoryPolicyEpoch ?? closingPolicyEpoch); + : row.role === "assistant" && typeof source?.requestHistorySequence === "number" + ? source.workspaceMemoryPolicyEpoch + : closingPolicyEpoch; return { ...row, id: copyId, diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 5180734cde5..b82e93ecee6 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -7,6 +7,7 @@ * rollback engine: refinement rows journaled before the upgrade address the * legacy files, while the note the user sees since is the owner copy. */ +import type { Dirent } from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import type { RefinementInverse } from "@/common/types/refinement"; @@ -58,14 +59,36 @@ export interface LegacyAdoptionRecord { deleted?: boolean; } -function isLegacyAdoptionRecord(value: unknown): value is LegacyAdoptionRecord { - if (typeof value !== "object" || value === null) return false; +/** + * Parse one manifest record. Lifecycle flags are raw JSON: a value that is + * neither absent nor boolean fails CLOSED — `pending`/`pendingDeletion` read + * as set (the pass is redone), `created`/`deleted` as unset (no destructive + * provenance; the source is reconciled as a plain unlisted note) — so a + * corrupted flag can never make an interrupted pass look settled. + */ +function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null { + if (typeof value !== "object" || value === null) return null; const record = value as Record; - return ( - typeof record.content === "string" && - typeof record.sidecar === "string" && - typeof record.target === "string" - ); + if ( + typeof record.content !== "string" || + typeof record.sidecar !== "string" || + typeof record.target !== "string" + ) { + return null; + } + const flag = (raw: unknown, malformed: boolean): boolean | undefined => + raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; + return { + content: record.content, + sidecar: record.sidecar, + target: record.target, + created: flag(record.created, false), + pending: flag(record.pending, true), + pendingDeletion: flag(record.pendingDeletion, true), + deleted: flag(record.deleted, false), + replacementContent: + typeof record.replacementContent === "string" ? record.replacementContent : undefined, + }; } /** @@ -98,15 +121,50 @@ export async function readLegacyAdoptionManifest( const parsed: unknown = JSON.parse(raw); if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); return new Map( - Object.entries(parsed).filter((entry): entry is [string, LegacyAdoptionRecord] => - isLegacyAdoptionRecord(entry[1]) - ) + Object.entries(parsed).flatMap(([relPath, raw]) => { + const record = parseLegacyAdoptionRecord(raw); + return record === null ? [] : [[relPath, record] as const]; + }) ); } catch { return new Map(); } } +/** + * Every non-directory entry (files, symlinks, anything) under `absDir`, + * recursively, as relPaths prefixed with `dirRel`; empty when the directory + * is absent. Throws on any other traversal failure (the caller then refuses + * rather than guess). + */ +async function listEntriesUnder(absDir: string, dirRel: string): Promise> { + const found = new Set(); + const walk = async (abs: string, rel: string): Promise => { + let entries: Dirent[]; + try { + entries = await fsPromises.readdir(abs, { withFileTypes: true }); + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + if (code === "ENOENT" || code === "ENOTDIR") return; + throw error; + } + for (const entry of entries) { + const childRel = `${rel}/${entry.name}`; + if (entry.isDirectory() && !entry.isSymbolicLink()) { + await walk(path.join(abs, entry.name), childRel); + } else { + found.add(childRel); + } + } + }; + await walk(absDir, dirRel); + return found; +} + +function setsEqual(a: ReadonlySet, b: ReadonlySet): boolean { + return a.size === b.size && [...a].every((value) => b.has(value)); +} + /** Thrown for a legacy path the shared store does not represent (see below). */ export class LegacyPathNotAdoptedError extends Error { constructor(legacyPath: string, reason: "not-adopted" | "owner-owned") { @@ -151,6 +209,36 @@ export async function createLegacyPathRemapper(args: { path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME), { strict: args.strict === true } ); + // Directory endpoints (pre-sharing directory renames) map only when the + // owner's on-disk subtree is EXACTLY the adopted descendants: a structural + // rename moves whatever is there, so an owner note added beside the + // adopted copies (no refinement row of its own) would travel along + // unnoticed. Precomputed for every directory prefix the manifest knows. + const ownerSubtreeExact = new Map(); + const directoryPrefixes = new Set(); + for (const rel of adopted.keys()) { + const parts = rel.split("/"); + for (let depth = 1; depth < parts.length; depth++) { + directoryPrefixes.add(parts.slice(0, depth).join("/")); + } + } + for (const dirRel of directoryPrefixes) { + const descendants = [...adopted].filter(([rel]) => rel.startsWith(`${dirRel}/`)); + const oneToOne = descendants.every( + ([rel, entry]) => entry.target === rel && entry.created === true && entry.pending !== true + ); + const expected = new Set( + descendants.filter(([, entry]) => entry.deleted !== true).map(([rel]) => rel) + ); + ownerSubtreeExact.set( + dirRel, + oneToOne && + setsEqual( + expected, + await listEntriesUnder(path.join(ownerRoot, ...dirRel.split("/")), dirRel) + ) + ); + } const remapPath = (filePath: string): string => { const relative = path.relative(legacyRoot, path.resolve(filePath)); if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return filePath; @@ -163,13 +251,7 @@ export async function createLegacyPathRemapper(args: { // owner directory then IS the adopted directory. Descendants placed // elsewhere (conflict imports) or owner-owned make the structural // move ambiguous: refused. - const descendants = [...adopted].filter(([rel]) => rel.startsWith(`${relPath}/`)); - if ( - descendants.length > 0 && - descendants.every( - ([rel, entry]) => entry.target === rel && entry.created === true && entry.pending !== true - ) - ) { + if (ownerSubtreeExact.get(relPath) === true) { return path.join(ownerRoot, ...relPath.split("/")); } throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 00a11edc5be..47239de216b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2441,6 +2441,79 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(ownerRoot, "note.md", "inner.md"), "utf-8")).toBe( "owner's" ); + // Same for a containment failure: an escaping symlink at the target is + // owner state, not proof of absence. + await fsPromises.writeFile(path.join(legacyRoot, "link.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const linkPrior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["link.md"] as Record; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "link.md")); + await fsPromises.rm(path.join(ownerRoot, "link.md")); + await fsPromises.symlink( + path.join(fixture.xumHome, "outside.md"), + path.join(ownerRoot, "link.md") + ); + await fsPromises.writeFile(path.join(fixture.xumHome, "outside.md"), "outside"); + const current = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + unknown + >; + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ ...current, "link.md": { ...linkPrior, pendingDeletion: true } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const linkTombstone = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { deleted?: boolean; created?: boolean } + > + )["link.md"]; + expect(linkTombstone.deleted).toBe(true); + expect(linkTombstone.created).not.toBe(true); + expect((await fsPromises.lstat(path.join(ownerRoot, "link.md"))).isSymbolicLink()).toBe(true); + }); + + it("reads malformed manifest lifecycle flags fail-closed", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const prior = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record + )["note.md"] as Record; + // An interrupted adoption's `pending` corrupted to a string: the copy + // was never written. The record must not read as settled — removal's + // handover reconstructs the copy instead of reporting completion. + await fsPromises.rm(path.join(ownerRoot, "note.md")); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ "note.md": { ...prior, pending: "true" } }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const settled = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { created?: boolean; pending?: boolean } + > + )["note.md"]; + expect(settled).toMatchObject({ created: true }); + expect(settled.pending).toBeUndefined(); }); it("re-adopts a source that reappeared identically while its deletion was pending", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index ba4a8af5bae..1075cad07ea 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1515,11 +1515,13 @@ export class MemoryService extends EventEmitter { // once the filesystem recovers. Keep the entry (and the pass // incomplete) so the next access reconciles it. let targetKind: MemoryEntryKind; + let targetContained = false; try { - targetKind = (await store.assertContained(previous.target).then( + targetContained = await store.assertContained(previous.target).then( () => true, () => false - )) + ); + targetKind = targetContained ? await store.kind(previous.target, { strict: true }) : null; } catch (error) { @@ -1573,10 +1575,12 @@ export class MemoryService extends EventEmitter { ([rel, record]) => rel !== relPath && listed.has(rel) && record.target === previous.target ); - // A target PROVEN absent (strict probe) while a deletion was pending - // was removed by the interrupted pass, not changed by the owner. A - // directory, symlink or over-cap file there is owner state. - const removedByUs = previous.pendingDeletion === true && targetKind === null; + // A target PROVEN absent (contained path, strict probe ENOENT) while + // a deletion was pending was removed by the interrupted pass, not + // changed by the owner. A directory, symlink, escaping component or + // over-cap file there is owner state. + const removedByUs = + previous.pendingDeletion === true && targetContained && targetKind === null; unchangedForTombstone = (unchanged || removedByUs) && successor === undefined; if (successor !== undefined) { // Only a copy still holding the adopted bytes is ours to hand diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index fb90c8f2517..817bd01a58f 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1027,6 +1027,21 @@ describe("refinementRollback", () => { }, }) ); + // An owner note added beside the adopted copies (no refinement row of + // its own) would travel with a structural rename: refused until the + // subtree is exactly the adopted descendants again. + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "owner.md"), "o\n"); + const extraFile = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(extraFile.success).toBe(false); + expect(extraFile.success ? "" : extraFile.error).toContain( + "not folded into the shared workspace store" + ); + await fsPromises.rm(path.join(ownerSessionDir, "memory", "newdir", "owner.md")); const undoneRename = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: dirRenameRow.id, diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index ccea9dcefb5..5cc704144a3 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -7563,8 +7563,11 @@ describe("WorkspaceService truncateHistory goal acknowledgment", () => { }); test("destructive clear waits for startup monitor recovery discovery", async () => { - const { historyService, workspaceService, cleanup } = await createServices(); + const { config, historyService, workspaceService, cleanup } = await createServices(); const workspaceId = "clear-waits-for-monitor-recovery"; + // A destructive clear proves its policy reset against an EXISTING + // config.json (an absent one is a transient state, not the empty default). + await config.editConfig((cfg) => cfg); const recovery = createDeferred(); const internal = workspaceService as unknown as { bashMonitorRecoveryPromise: Promise; From 13e45254f659a3e534c537438ca356775dd0525d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 04:21:09 +0000 Subject: [PATCH 71/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-fif?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption manifest moves out of the model-writable legacy root (`/memory`, a downgraded build's /memories/workspace whose grammar admits dotfiles) to `/memory-adoption-manifest.json`, so a fabricated settled record can no longer drive deletion reconciliation of an owner note. - refinement_rollback resolves shared-memory topology (owner root + peers) per execution from one Config.loadExistingConfigOrThrow() snapshot and refuses the rollback while ownership cannot be proven, instead of degrading to the "self" fallback that omitted the owner root and peer list. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 20 ++- src/node/services/memoryService.test.ts | 119 ++++++++++++++---- src/node/services/memoryService.ts | 13 +- .../refinement/refinementRollback.test.ts | 9 +- .../services/refinement/refinementRollback.ts | 2 +- src/node/services/toolAssembly.ts | 11 +- .../services/tools/refinement_rollback.ts | 43 +++++-- src/node/services/turnRequestBuilder.ts | 53 ++++---- 8 files changed, 194 insertions(+), 76 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index b82e93ecee6..df4129eb312 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -13,11 +13,21 @@ import * as path from "node:path"; import type { RefinementInverse } from "@/common/types/refinement"; /** - * Dotfile inside a sub-agent's legacy `memory` dir recording, per relPath, the - * sha256 of the content already copied into the shared store - * (adoptLegacyPrivateStore). Dotfiles are invisible to every build's listing. + * File in the sub-agent's SESSION dir (beside its legacy `memory` dir, never + * inside it) recording, per relPath, the sha256 of the content already copied + * into the shared store (adoptLegacyPrivateStore). Outside the legacy root on + * purpose: everything under `/memory` is the model-writable + * `/memories/workspace` namespace of a downgraded build (the path grammar + * admits dotfiles), and this manifest's `created`/`target`/hash fields are + * trusted as provenance — a fabricated settled record with an absent source + * would make deletion reconciliation remove a matching owner note. The + * session dir itself is not addressable through any memory path. */ -export const LEGACY_ADOPTION_MANIFEST_FILE_NAME = ".adopted-into-shared-store.json"; +export const LEGACY_ADOPTION_MANIFEST_FILE_NAME = "memory-adoption-manifest.json"; + +export function legacyAdoptionManifestPath(childSessionDir: string): string { + return path.join(childSessionDir, LEGACY_ADOPTION_MANIFEST_FILE_NAME); +} /** * One adopted legacy file: content hash, child sidecar fingerprint, owner-store @@ -206,7 +216,7 @@ export async function createLegacyPathRemapper(args: { const legacyRoot = path.join(path.resolve(args.childSessionDir), "memory"); const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); const adopted = await readLegacyAdoptionManifest( - path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME), + legacyAdoptionManifestPath(path.resolve(args.childSessionDir)), { strict: args.strict === true } ); // Directory endpoints (pre-sharing directory renames) map only when the diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 47239de216b..2e6d2458e91 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -19,6 +19,7 @@ import { type PinnedFileMutation, } from "./memoryService"; import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; +import { legacyAdoptionManifestPath } from "./memoryLegacyAdoption"; import { MemoryRefinementActionSchema, REFINEMENT_CAPTURE_MAX_FILES, @@ -1694,7 +1695,7 @@ describe("MemoryService", () => { } expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); // Same for the adoption manifest: unreadable (not missing) aborts. - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const savedManifest = await fsPromises.readFile(manifestPath); await fsPromises.rm(manifestPath); await fsPromises.mkdir(manifestPath); @@ -1739,7 +1740,7 @@ describe("MemoryService", () => { await fixture.service.listIndexEntries({ ...fixture.ctx }); expect(await fsPromises.readFile(importedCopy, "utf-8")).toBe("child's a"); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; // The old record stays as a tombstone (rollbacks of the child's // pre-sharing rows for a.md still need its mapping). @@ -1787,7 +1788,7 @@ describe("MemoryService", () => { ); await fixture.service.listIndexEntries({ ...fixture.ctx }); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(manifest["imported/ws-child/a.md"].created).not.toBe(true); // ...and the obsolete record's tombstone drops its destructive @@ -2045,7 +2046,7 @@ describe("MemoryService", () => { // still know this adoption created the copy... await fixture.service.listIndexEntries({ ...fixture.ctx }); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(manifest["note.md"]).toMatchObject({ created: true }); expect(manifest["note.md"].pending).toBeUndefined(); @@ -2083,7 +2084,7 @@ describe("MemoryService", () => { } expect(await pathExists(target)).toBe(true); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(Object.keys(manifest)).toEqual(["note.md"]); // Recovered: the retained provenance lets the copy follow its source out. @@ -2142,10 +2143,7 @@ describe("MemoryService", () => { expect( Object.keys( JSON.parse( - await fsPromises.readFile( - path.join(legacyRoot, ".adopted-into-shared-store.json"), - "utf-8" - ) + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record ) ).toEqual(["note.md"]); @@ -2189,7 +2187,7 @@ describe("MemoryService", () => { expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); // Readable again: the pin folds into the same copy. @@ -2230,7 +2228,7 @@ describe("MemoryService", () => { await fsPromises.rm(path.join(legacyRoot, "note.md")); await fixture.service.listIndexEntries({ ...fixture.ctx }); expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const tombstoned = JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< string, { deleted?: boolean; target: string } @@ -2256,6 +2254,42 @@ describe("MemoryService", () => { expect(readopted["note.md"].deleted).toBeUndefined(); }); + it("ignores a manifest a downgraded child wrote into its model-writable legacy root", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fixture.service.create( + fixture.ctx, + "/memories/workspace/note.md", + "owner note", + "agent" + ); + // The downgraded build's memory tool serves `/memory` as + // /memories/workspace and its path grammar admits dotfiles: a model + // there can plant a settled record claiming the owner's note as this + // adoption's creation whose source is already gone. Read as provenance, + // deletion reconciliation would remove the owner's note. + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile( + path.join(legacyRoot, ".adopted-into-shared-store.json"), + JSON.stringify({ + "note.md": { + content: sha256Hex("owner note"), + sidecar: "", + target: "note.md", + created: true, + }, + }) + ); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "owner note" + ); + // Nothing listed in the legacy root: the real manifest was never written. + expect(await pathExists(legacyAdoptionManifestPath(path.dirname(legacyRoot)))).toBe(false); + }); + it("recovers an interrupted in-place replacement without duplicating the note", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2268,7 +2302,7 @@ describe("MemoryService", () => { // after recording its pending state but before writing the bytes — // the on-disk state that leaves: the PRIOR record marked pending, the // owner copy still holding the old bytes. - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const prior = ( JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< string, @@ -2290,7 +2324,7 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v2"); expect(await pathExists(path.join(ownerRoot, "imported", "ws-child", "note.md"))).toBe(false); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(Object.keys(manifest)).toEqual(["note.md"]); expect(manifest["note.md"]).toMatchObject({ target: "note.md", created: true }); @@ -2364,7 +2398,7 @@ describe("MemoryService", () => { await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); await fixture.service.listIndexEntries({ ...fixture.ctx }); - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const prior = ( JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< string, @@ -2406,7 +2440,7 @@ describe("MemoryService", () => { await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); await fixture.service.listIndexEntries({ ...fixture.ctx }); - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const prior = ( JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< string, @@ -2489,7 +2523,7 @@ describe("MemoryService", () => { await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); await fixture.service.listIndexEntries({ ...fixture.ctx }); - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const prior = ( JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record )["note.md"] as Record; @@ -2524,7 +2558,7 @@ describe("MemoryService", () => { await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); await fixture.service.listIndexEntries({ ...fixture.ctx }); - const manifestPath = path.join(legacyRoot, ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); const prior = ( JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record )["note.md"] as Record; @@ -2566,7 +2600,7 @@ describe("MemoryService", () => { "proto notes" ); const manifest = JSON.parse( - await fsPromises.readFile(path.join(legacyRoot, ".adopted-into-shared-store.json"), "utf-8") + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") ) as Record; expect(Object.keys(manifest)).toEqual(["__proto__"]); // A fresh process (empty memo) finds the record and leaves the clock alone. @@ -2778,9 +2812,7 @@ describe("MemoryService", () => { .map((e) => e.relPath) ).toEqual(["clash.md"]); expect(await pathExists(path.join(outside, "clash.md"))).toBe(false); - expect(await pathExists(path.join(legacyRoot, ".adopted-into-shared-store.json"))).toBe( - false - ); + expect(await pathExists(legacyAdoptionManifestPath(path.dirname(legacyRoot)))).toBe(false); await fsPromises.unlink(path.join(ownerRoot, "imported", "ws-child")); await fsPromises.rm(legacyRoot, { recursive: true }); await fsPromises.rm(path.join(ownerRoot, "clash.md")); @@ -3540,7 +3572,7 @@ describe("MemoryService", () => { const [ownerCreate] = await readRefinementEvents(ownerSessionDir); // The child's (pre-sharing) manifest is unreadable: its adopted rows // cannot be consulted, so the owner's rollback must not proceed blind. - const manifestPath = path.join(childSessionDir, "memory", ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); await fsPromises.mkdir(manifestPath, { recursive: true }); const refused = await rollbackRefinement({ sessionDir: ownerSessionDir, @@ -3606,7 +3638,7 @@ describe("MemoryService", () => { const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); await fixture.service.create(fixture.ctx, "/memories/workspace/m.md", "v1", "agent"); - const manifestPath = path.join(childSessionDir, "memory", ".adopted-into-shared-store.json"); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); await fsPromises.mkdir(manifestPath, { recursive: true }); expect( await migrateSharedMemoryRefinementRows({ @@ -3631,7 +3663,7 @@ describe("MemoryService", () => { createRefinementRollbackTool({ workspaceId: "ws-child", sessionDir: childSessionDir, - sharedWorkspaceMemorySessionDir: ownerSessionDir, + sharedWorkspaceMemory: () => ({ ownerSessionDir, peerSessionDirs: [] }), memory: { service: fixture.service, ctx: fixture.ctx, access }, }); const run = async (access: MemoryScopeAccess) => @@ -3653,7 +3685,7 @@ describe("MemoryService", () => { const foreignTool = createRefinementRollbackTool({ workspaceId: "ws-child", sessionDir: childSessionDir, - sharedWorkspaceMemorySessionDir: ownerSessionDir, + sharedWorkspaceMemory: () => ({ ownerSessionDir, peerSessionDirs: [] }), memory: { service: fixture.service, ctx: { ...fixture.ctx, workspaceId: "ws-solo" }, @@ -3684,7 +3716,7 @@ describe("MemoryService", () => { await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "legacy.md"), "v2"); await fsPromises.writeFile( - path.join(legacyRoot, ".adopted-into-shared-store.json"), + legacyAdoptionManifestPath(path.dirname(legacyRoot)), JSON.stringify({ "legacy.md": { content: "x", sidecar: "", target: "legacy.md", created: true }, }) @@ -3728,6 +3760,41 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(legacyRoot, "legacy.md"), "utf-8")).toBe("v2"); }); + it("the refinement_rollback tool refuses while shared-memory ownership cannot be proven", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const [row] = await readRefinementEvents(childSessionDir); + // config.json mid-rewrite at execution time: the topology resolver + // throws instead of degrading to "the child owns its notebook" (which + // would drop the owner root and the peer list from the rollback). + const tool = createRefinementRollbackTool({ + workspaceId: "ws-child", + sessionDir: childSessionDir, + sharedWorkspaceMemory: () => { + throw new Error("config.json is absent"); + }, + memory: { + service: fixture.service, + ctx: fixture.ctx, + access: { global: "readwrite", project: "readwrite", workspace: "readwrite" }, + }, + }); + const refused = (await tool.execute!( + { id: row.id, reason: "test" }, + mockToolCallOptions + )) as { + success: boolean; + error?: string; + }; + expect(refused.success).toBe(false); + expect(refused.error).toContain("config.json is absent"); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(true); + expect((await readRefinementEvents(childSessionDir)).length).toBe(1); + }); + it("notifyExternalMutation emits one owner-addressed event per touched scope", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 1075cad07ea..d1c322c10c0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -49,7 +49,7 @@ import { } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; import { - LEGACY_ADOPTION_MANIFEST_FILE_NAME, + legacyAdoptionManifestPath, readLegacyAdoptionManifest, type LegacyAdoptionRecord, } from "@/node/services/memoryLegacyAdoption"; @@ -1285,10 +1285,11 @@ export class MemoryService extends EventEmitter { // copy. A traversal failure fails the pass instead (access-time: // retried on the next access; removal: aborted, session intact). const files = await legacy.listFiles({ strict: true }); - // What was already folded in, kept beside the legacy files (a dotfile, - // so neither build lists it): per relPath the content hash, the - // fingerprint of the child-keyed sidecar entry, and where the copy - // landed. Content: without it, a note later edited through the shared + // What was already folded in, kept in the child's session dir OUTSIDE + // the legacy root (which is a downgraded build's model-writable + // namespace; see legacyAdoptionManifestPath): per relPath the content + // hash, the fingerprint of the child-keyed sidecar entry, and where the + // copy landed. Content: without it, a note later edited through the shared // store would be re-imported as a stale duplicate on every backend // start. Sidecar: a downgraded build can change only a pin or usage // stats, which must reach the owner key without the bytes changing. @@ -1296,7 +1297,7 @@ export class MemoryService extends EventEmitter { // consider done (and removal then deletes the child session on that // basis), so a transiently unreadable manifest, sidecar or owner // listing must fail the pass rather than stand in as "empty". - const manifestPath = path.join(legacyRoot, LEGACY_ADOPTION_MANIFEST_FILE_NAME); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); const sidecarEntries = await this.metaService.getEntriesOrThrow(); // The per-scope file cap is a store invariant (create/rename enforce diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 817bd01a58f..972e4766779 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -6,6 +6,7 @@ import * as path from "node:path"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; import { Config } from "@/node/config"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; +import { legacyAdoptionManifestPath } from "@/node/services/memoryLegacyAdoption"; import { MemoryMetaService } from "@/node/services/memoryMeta"; import { MemoryService, type MemoryScopeContext } from "@/node/services/memoryService"; import { TestTempDir } from "@/node/services/tools/testHelpers"; @@ -894,7 +895,7 @@ describe("refinementRollback", () => { await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "sub"), { recursive: true }); await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "sub", "note.md"), "v2\n"); await fsPromises.writeFile( - path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + legacyAdoptionManifestPath(fixture.sessionDir), JSON.stringify({ "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, }) @@ -959,7 +960,7 @@ describe("refinementRollback", () => { const sameRow = await lastRow(fixture.sessionDir); await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "same.md"), "same\n"); await fsPromises.writeFile( - path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + legacyAdoptionManifestPath(fixture.sessionDir), JSON.stringify({ "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, "same.md": { content: "x", sidecar: "", target: "same.md" }, @@ -981,7 +982,7 @@ describe("refinementRollback", () => { await fixture.service.deletePath(fixture.ctx, "/memories/workspace/gone.md", "agent"); const deleteRow = await lastRow(fixture.sessionDir); await fsPromises.writeFile( - path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + legacyAdoptionManifestPath(fixture.sessionDir), JSON.stringify({ "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, "same.md": { content: "x", sidecar: "", target: "same.md" }, @@ -1012,7 +1013,7 @@ describe("refinementRollback", () => { await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "newdir"), { recursive: true }); await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "a.md"), "a\n"); await fsPromises.writeFile( - path.join(fixture.sessionDir, "memory", ".adopted-into-shared-store.json"), + legacyAdoptionManifestPath(fixture.sessionDir), JSON.stringify({ "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, "same.md": { content: "x", sidecar: "", target: "same.md" }, diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index ed69e521b64..853941856d7 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -593,7 +593,7 @@ async function readSharedMemoryPeerRows( ); // A peer sub-agent's pre-sharing rows address ITS legacy private // notebook; the notes live in the shared store now (adoption manifest - // beside the legacy files). Retarget them through that peer's manifest + // in the peer's session dir). Retarget them through that peer's manifest // before the overlap test — the peer's later edit of an adopted note must // surface against the owner's rollback like any shared-store row. The // remapped inverse replaces the recorded one on the returned row, so the diff --git a/src/node/services/toolAssembly.ts b/src/node/services/toolAssembly.ts index 0eefa39b351..fade466139b 100644 --- a/src/node/services/toolAssembly.ts +++ b/src/node/services/toolAssembly.ts @@ -35,7 +35,10 @@ import type { QuickJSRuntimeFactory } from "@/node/services/ptc/quickjsRuntime"; import type { ToolBridge } from "@/node/services/ptc/toolBridge"; import type { PTCExecutionResult } from "@/node/services/ptc/types"; import { sandboxHostService, type SandboxMount } from "@/node/services/sandbox/sandboxHostService"; -import { createRefinementRollbackTool } from "@/node/services/tools/refinement_rollback"; +import { + createRefinementRollbackTool, + type SharedWorkspaceMemoryTopologyResolver, +} from "@/node/services/tools/refinement_rollback"; import { READ_ONLY_ACCESS } from "@/node/services/tools/memory"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; import type { MemoryScopeAccess } from "@/common/constants/memory"; @@ -121,10 +124,8 @@ export interface ApplyToolPolicyAndExperimentsOptions { sandbox?: { workspaceId: string; sessionDir: string; - /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ - sharedWorkspaceMemorySessionDir?: string; - /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ - listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; + /** Shared-notebook topology resolver for refinement_rollback (see its ctx). */ + sharedWorkspaceMemory?: SharedWorkspaceMemoryTopologyResolver; /** Lets refinement_rollback announce its direct-to-disk memory writes. */ memory?: { service: MemoryService; ctx: MemoryScopeContext; access: MemoryScopeAccess }; kernelFileLoader?: KernelFileLoader; diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 509b05fb9b0..6d0971fa9df 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -11,6 +11,7 @@ import { createLegacyPathRemapper, LegacyPathNotAdoptedError, } from "@/node/services/memoryLegacyAdoption"; +import { getErrorMessage } from "@/common/utils/errors"; interface RefinementRollbackToolArgs { id: string; @@ -84,13 +85,29 @@ async function refuseReadOnlyMemoryRollback( return null; } +export interface SharedWorkspaceMemoryTopology { + /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ + ownerSessionDir: string | undefined; + /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ + peerSessionDirs: string[]; +} + +export type SharedWorkspaceMemoryTopologyResolver = () => SharedWorkspaceMemoryTopology; + export function createRefinementRollbackTool(ctx: { workspaceId: string; sessionDir: string; - /** Owner session dir when this workspace is a sub-agent sharing its notebook. */ - sharedWorkspaceMemorySessionDir?: string; - /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ - listSharedWorkspaceMemoryPeerSessionDirs?: () => string[]; + /** + * Task-tree topology of a workspace sharing its notebook, resolved PER + * EXECUTION (membership changes while the tool instance lives) from a + * config snapshot that must prove itself: a throw refuses the rollback. No + * fallback view is acceptable here — ownership read from a missing + * config.json resolves to "self", which would omit the owner root (a + * pre-sharing row's inverse then lands on the hidden legacy notebook + * instead of the owner's adopted copy) and the peer list (conflicting + * sibling rows go unseen). Omitted = the workspace owns its notebook. + */ + sharedWorkspaceMemory?: SharedWorkspaceMemoryTopologyResolver; /** * Memory integration: announces rolled-back memory files so shared-store * readers refresh, and applies the agent's per-scope write policy — a @@ -107,10 +124,22 @@ export function createRefinementRollbackTool(ctx: { { id, reason }: RefinementRollbackToolArgs, { toolCallId } ): Promise => { + let topology: SharedWorkspaceMemoryTopology; + try { + topology = ctx.sharedWorkspaceMemory?.() ?? { + ownerSessionDir: undefined, + peerSessionDirs: [], + }; + } catch (error) { + return { + success: false, + error: `Cannot resolve this workspace's shared-memory ownership right now (${getErrorMessage(error)}); refusing to roll back '${id}'.`, + }; + } if (ctx.memory !== undefined) { const refusal = await refuseReadOnlyMemoryRollback( ctx.sessionDir, - ctx.sharedWorkspaceMemorySessionDir, + topology.ownerSessionDir, id, ctx.memory ); @@ -118,8 +147,8 @@ export function createRefinementRollbackTool(ctx: { } const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, - sharedWorkspaceMemorySessionDir: ctx.sharedWorkspaceMemorySessionDir, - listSharedWorkspaceMemoryPeerSessionDirs: ctx.listSharedWorkspaceMemoryPeerSessionDirs, + sharedWorkspaceMemorySessionDir: topology.ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => topology.peerSessionDirs, id, reason, evidence: { toolName: "refinement_rollback", toolCallId, actor: "agent" }, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index be63ebcb117..bfa694e14ee 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -121,7 +121,10 @@ import { } from "@/common/utils/providers/customProviders"; import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpServerManager"; import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; -import { sharedWorkspaceMemoryPeerSessionDirs } from "@/node/services/memoryWorkspaceOwner"; +import { + resolveWorkspaceMemoryOwnerId, + sharedWorkspaceMemoryPeerSessionDirs, +} from "@/node/services/memoryWorkspaceOwner"; import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; import { READ_ONLY_ACCESS, resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; @@ -2520,27 +2523,34 @@ export class TurnRequestBuilder { // A sub-agent's workspace-scope memory rows point into its task-tree // owner's session dir; rollback must admit that root (and only that), // and announce its direct-to-disk writes through MemoryService so the - // shared store's readers refresh. + // shared store's readers refresh. Resolved per rollback (not per + // turn): tree membership changes as sub-agents are spawned and removed + // while the tool instance lives. Strict AND existence-requiring load, + // from one snapshot for owner and peers: a config.json that is + // unreadable or absent (mid-rewrite) must refuse the rollback (see + // RollbackRefinementOptions), not read as a fresh install in which + // the child owns its notebook — that "self" fallback would omit the + // owner root (a pre-sharing row's inverse then lands on the hidden + // legacy notebook instead of the owner's adopted copy) and the peer + // list (conflicting sibling rows go unseen). const memoryService = this.dependencies.bindings.memoryService; - const memoryOwnerId = - memoryService?.resolveWorkspaceMemoryOwnerId(workspaceId) ?? workspaceId; - const sharedWorkspaceMemorySessionDir = - memoryOwnerId === workspaceId - ? undefined - : path.join(this.dependencies.config.sessionsDir, memoryOwnerId); - // Resolved per rollback (not per turn): tree membership changes as - // sub-agents are spawned and removed while the tool instance lives. - // Strict load: an unreadable config must refuse the rollback (see - // RollbackRefinementOptions), not read as an empty tree. - const listSharedWorkspaceMemoryPeerSessionDirs = + const sessionsDir = this.dependencies.config.sessionsDir; + const sharedWorkspaceMemory = memoryService === undefined ? undefined - : () => - sharedWorkspaceMemoryPeerSessionDirs( - this.dependencies.config.loadConfigOrDefault({ throwOnError: true }), - this.dependencies.config.sessionsDir, - workspaceId - ); + : () => { + const cfg = this.dependencies.config.loadExistingConfigOrThrow(); + const ownerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + return { + ownerSessionDir: + ownerId === workspaceId ? undefined : path.join(sessionsDir, ownerId), + peerSessionDirs: sharedWorkspaceMemoryPeerSessionDirs( + cfg, + sessionsDir, + workspaceId + ), + }; + }; // Built anew for EVERY attempt (prepareModelRequest runs per primary / // fallback request) from the never-mutated policy: refinement_rollback // reads it by reference, and the request.assemble demotion below only @@ -2567,9 +2577,8 @@ export class TurnRequestBuilder { emitNestedToolEvent: emitNestedPtcToolEvent, sandbox: { workspaceId, - sessionDir: path.join(this.dependencies.config.sessionsDir, workspaceId), - sharedWorkspaceMemorySessionDir, - listSharedWorkspaceMemoryPeerSessionDirs, + sessionDir: path.join(sessionsDir, workspaceId), + sharedWorkspaceMemory, memory: attemptSandboxMemory, kernelFileLoader, }, From 7d2a32edd25c3efa82d8e9bb18fd89ceb3f1d331 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 05:34:48 +0000 Subject: [PATCH 72/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-six?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Config.loadExistingConfigOrThrow rejects a config.json whose stamp changed between the pre- and post-read stats (replaced mid-read), not only an absent one, so strict callers retry from one stable snapshot. - readLegacyAdoptionManifest distinguishes absent from malformed for strict callers: bad JSON, a non-object, or a record missing its string fields throws instead of reading as "nothing adopted", so the adoption pass and the removal handover never substitute an empty map for lost provenance. - MemoryService re-checks the removal tombstones AFTER every workspace-scope read whose result leaves the service (view file/dir/root, readFileWithSha, listIndexEntries per-scope buffer, hot-set reads), closing the window in which a tombstone published mid-read still disclosed owner notes. - resolveSharedWorkspaceMemoryTopology (memoryWorkspaceOwner.ts) resolves owner root + peers from one existence-requiring config snapshot; used by TurnRequestBuilder, the refinement_rollback tool (live peer callback), and the debug CLI (which now refuses with exit code 1 instead of assuming self-ownership; `config` test seam). - Test fixes surfaced by the R64 existence-requiring reset: the agent session harness persists an empty config.json (hookService first-rollover CI failure), and the post-compaction attachments mock config gains loadExistingConfigOrThrow. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/cli/debug/refinements.test.ts | 49 ++++++++-- src/cli/debug/refinements.ts | 44 +++++---- src/node/config.test.ts | 29 ++++++ src/node/config/index.ts | 14 ++- ...tSession.postCompactionAttachments.test.ts | 6 +- src/node/services/agentSession.testHarness.ts | 12 +++ src/node/services/memoryLegacyAdoption.ts | 46 ++++++--- src/node/services/memoryService.test.ts | 98 +++++++++++++++++++ src/node/services/memoryService.ts | 39 +++++++- src/node/services/memoryWorkspaceOwner.ts | 31 ++++++ .../services/tools/refinement_rollback.ts | 14 ++- src/node/services/turnRequestBuilder.ts | 29 +----- 12 files changed, 330 insertions(+), 81 deletions(-) diff --git a/src/cli/debug/refinements.test.ts b/src/cli/debug/refinements.test.ts index 7a34c5c8d95..9d1d7a43ef1 100644 --- a/src/cli/debug/refinements.test.ts +++ b/src/cli/debug/refinements.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, spyOn } from "bun:test"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; +import { Config } from "@/node/config"; import { appendRefinementEvent } from "@/node/services/refinement/refinementJournal"; import { TestTempDir } from "@/node/services/tools/testHelpers"; import { refinementsCommand } from "./refinements"; @@ -11,8 +12,14 @@ import { refinementsCommand } from "./refinements"; * created, inside a `/sessions/` layout so the confinement roots * resolve like a real mux home. */ -async function seedFixture(root: string): Promise<{ sessionDir: string; skillFile: string }> { +async function seedFixture( + root: string +): Promise<{ sessionDir: string; skillFile: string; config: Config }> { const sessionDir = path.join(root, "sessions", "ws-cli"); + // Rollback resolves shared-memory ownership from a config.json that must + // exist (an absent one reads as mid-rewrite): persist the empty default. + const config = new Config(root); + await config.editConfig((cfg) => cfg); const skillFile = path.join(root, "checkout", ".mux", "skills", "cli-skill", "SKILL.md"); await fsPromises.mkdir(path.dirname(skillFile), { recursive: true }); await fsPromises.writeFile(skillFile, "---\nname: cli-skill\n---\n", "utf-8"); @@ -24,7 +31,7 @@ async function seedFixture(root: string): Promise<{ sessionDir: string; skillFil inverse: { op: "delete-files", paths: [skillFile] }, evidence: { toolName: "agent_skill_write" }, }); - return { sessionDir, skillFile }; + return { sessionDir, skillFile, config }; } describe("debug refinements command", () => { @@ -37,7 +44,7 @@ describe("debug refinements command", () => { it("lists rows and performs a rollback with lineage output", async () => { using tempDir = new TestTempDir("test-debug-refinements"); - const { sessionDir, skillFile } = await seedFixture(tempDir.path); + const { sessionDir, skillFile, config } = await seedFixture(tempDir.path); const lines: string[] = []; const logSpy = spyOn(console, "log").mockImplementation((line: string) => { lines.push(line); @@ -50,7 +57,7 @@ describe("debug refinements command", () => { const rowId = lines[0].split(" ")[0]; lines.length = 0; - await refinementsCommand("ws-cli", { sessionDir, rollback: rowId }); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: rowId }); // Earlier test files in the same process may have reset exitCode to 0, // so assert "not failing" rather than "never touched". expect(process.exitCode ?? 0).toBe(0); @@ -74,14 +81,14 @@ describe("debug refinements command", () => { it("reports refusals on stderr and sets a failing exit code", async () => { using tempDir = new TestTempDir("test-debug-refinements-refuse"); - const { sessionDir } = await seedFixture(tempDir.path); + const { sessionDir, config } = await seedFixture(tempDir.path); const logSpy = spyOn(console, "log").mockImplementation(() => undefined); const errors: string[] = []; const errorSpy = spyOn(console, "error").mockImplementation((line: string) => { errors.push(line); }); try { - await refinementsCommand("ws-cli", { sessionDir, rollback: "missing-id" }); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: "missing-id" }); expect(process.exitCode).toBe(1); expect(errors.join("\n")).toContain("No refinement row"); } finally { @@ -89,4 +96,34 @@ describe("debug refinements command", () => { errorSpy.mockRestore(); } }); + + it("refuses a rollback while config.json is absent instead of assuming self-ownership", async () => { + using tempDir = new TestTempDir("test-debug-refinements-noconfig"); + const { sessionDir, skillFile, config } = await seedFixture(tempDir.path); + const lines: string[] = []; + const logSpy = spyOn(console, "log").mockImplementation((line: string) => { + lines.push(line); + }); + const errors: string[] = []; + const errorSpy = spyOn(console, "error").mockImplementation((line: string) => { + errors.push(line); + }); + try { + await refinementsCommand("ws-cli", { sessionDir }); + const rowId = lines[0].split(" ")[0]; + await fsPromises.rm(path.join(tempDir.path, "config.json")); + await refinementsCommand("ws-cli", { sessionDir, config, rollback: rowId }); + expect(process.exitCode).toBe(1); + expect(errors.join("\n")).toContain("shared-memory ownership could not be resolved"); + expect( + await fsPromises.access(skillFile).then( + () => true, + () => false + ) + ).toBe(true); + } finally { + logSpy.mockRestore(); + errorSpy.mockRestore(); + } + }); }); diff --git a/src/cli/debug/refinements.ts b/src/cli/debug/refinements.ts index 72ef3b1e1c3..4a20e7136c3 100644 --- a/src/cli/debug/refinements.ts +++ b/src/cli/debug/refinements.ts @@ -1,9 +1,10 @@ import * as path from "path"; -import { defaultConfig } from "@/node/config"; +import { defaultConfig, type Config } from "@/node/config"; import { - resolveWorkspaceMemoryOwnerId, - sharedWorkspaceMemoryPeerSessionDirs, + resolveSharedWorkspaceMemoryTopology, + type SharedWorkspaceMemoryTopology, } from "@/node/services/memoryWorkspaceOwner"; +import { getErrorMessage } from "@/common/utils/errors"; import { MemoryRefinementActionSchema, RollbackRefinementActionSchema, @@ -41,6 +42,8 @@ export interface RefinementsCommandOptions { force?: boolean; /** Test seam: bypass ~/.mux session resolution for fixture sessions. */ sessionDir?: string; + /** Test seam: the config whose task tree resolves shared-memory ownership. */ + config?: Pick; } /** @@ -55,28 +58,31 @@ export async function refinementsCommand( if (opts.rollback !== undefined) { // Sub-agents journal workspace-scope rows that point into the owner's - // session dir; admit that root the same way the in-app tool does. - // Strict: a tolerant read of an unreadable config.json would resolve a - // sub-agent to ITSELF, and the rollback would then mutate its hidden - // legacy notebook (no owner root, no adoption remap) and report success. - const cfg = defaultConfig.loadConfigOrDefault({ throwOnError: true }); - const memoryOwnerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + // session dir; admit that root the same way the in-app tool does, from a + // config that must EXIST and read: a tolerant (or fresh-install) view + // would resolve a sub-agent to ITSELF, and the rollback would then mutate + // its hidden legacy notebook (no owner root, no adoption remap) and + // report success. Throws → the command fails before touching anything. + const config = opts.config ?? defaultConfig; + let topology: SharedWorkspaceMemoryTopology; + try { + topology = resolveSharedWorkspaceMemoryTopology(config, workspaceId); + } catch (error) { + console.error( + `Refusing rollback of '${opts.rollback}': shared-memory ownership could not be resolved (${getErrorMessage(error)})` + ); + process.exitCode = 1; + return; + } const result = await rollbackRefinement({ sessionDir, - sharedWorkspaceMemorySessionDir: - memoryOwnerId === workspaceId - ? undefined - : path.join(defaultConfig.sessionsDir, memoryOwnerId), + sharedWorkspaceMemorySessionDir: topology.ownerSessionDir, // Reloaded per check (plan-time and in-lock), not from the snapshot // above: a live backend may register a new tree member while this // process waits for the shared-store lock, and its rows must count. - // Strict: an unreadable config refuses the rollback (empty tree = guess). + // Same existence-requiring load: an unproven tree refuses the rollback. listSharedWorkspaceMemoryPeerSessionDirs: () => - sharedWorkspaceMemoryPeerSessionDirs( - defaultConfig.loadConfigOrDefault({ throwOnError: true }), - defaultConfig.sessionsDir, - workspaceId - ), + resolveSharedWorkspaceMemoryTopology(config, workspaceId).peerSessionDirs, id: opts.rollback, force: opts.force, evidence: { toolName: "debug-cli", actor: "user" }, diff --git a/src/node/config.test.ts b/src/node/config.test.ts index ea7e250736c..593ade5351c 100644 --- a/src/node/config.test.ts +++ b/src/node/config.test.ts @@ -923,6 +923,35 @@ describe("Config", () => { expect(loaded.projects.size).toBe(0); }); + describe("loadExistingConfigOrThrow", () => { + it("throws for an absent config.json even though strict loads read it as empty", () => { + const config = new Config(tempDir); + expect(() => config.loadConfigOrDefault({ throwOnError: true })).not.toThrow(); + expect(() => config.loadExistingConfigOrThrow()).toThrow(/absent/); + }); + + it("rejects a config.json replaced during the read instead of returning a torn view", () => { + const configFile = path.join(tempDir, "config.json"); + fs.writeFileSync(configFile, JSON.stringify({ defaultProjectDir: "/tmp" })); + const config = new Config(tempDir); + expect(config.loadExistingConfigOrThrow().projects.size).toBe(0); + // Another backend's atomic rewrite lands between the pre-read stamp + // and the post-read one (a different size guarantees a new stamp). + const original = config.loadConfigOrDefault.bind(config); + const load = spyOn(config, "loadConfigOrDefault").mockImplementation((options) => { + const loaded = original(options); + fs.writeFileSync(configFile, JSON.stringify({ defaultProjectDir: "/tmp/replaced" })); + return loaded; + }); + try { + expect(() => config.loadExistingConfigOrThrow()).toThrow(/replaced during the read/); + } finally { + load.mockRestore(); + } + expect(() => config.loadExistingConfigOrThrow()).not.toThrow(); + }); + }); + it("keeps the canonical legacy identity when a secondary alias file is unreadable in lenient loads", async () => { // An id-less legacy entry with a HEALTHY canonical (generated-legacy) // metadata file and an unreadable basename-backed second candidate: diff --git a/src/node/config/index.ts b/src/node/config/index.ts index 2303821b5d1..45d9a4f6aec 100644 --- a/src/node/config/index.ts +++ b/src/node/config/index.ts @@ -1336,15 +1336,23 @@ export class Config { * empty, valid config), which fail-closed callers — workspace removal's * shared-memory handover, the workspace-memory policy accumulator — must * not mistake for "this workspace is not registered" while the file is - * merely mid-rewrite. Stat before and after the read: a file present at - * both is taken as present during it. + * merely mid-rewrite. Stat before and after the read, and require the SAME + * stamp at both: a file present at both with one stamp is taken as present + * and unchanged during the read, while a replacement landing in between + * (another backend's atomic rewrite) means the bytes read may belong to + * neither snapshot's topology — the caller retries from one stable + * snapshot rather than act on a torn view. */ loadExistingConfigOrThrow(): ProjectsConfig { const before = this.configFileStamp(); const config = this.loadConfigOrDefault({ throwOnError: true }); - if (before === "missing" || this.configFileStamp() === "missing") { + const after = this.configFileStamp(); + if (before === "missing" || after === "missing") { throw new Error(`config.json is absent at ${this.configFile}`); } + if (before !== after) { + throw new Error(`config.json at ${this.configFile} was replaced during the read`); + } return config; } diff --git a/src/node/services/agentSession.postCompactionAttachments.test.ts b/src/node/services/agentSession.postCompactionAttachments.test.ts index be4ef966429..3dd9046b86e 100644 --- a/src/node/services/agentSession.postCompactionAttachments.test.ts +++ b/src/node/services/agentSession.postCompactionAttachments.test.ts @@ -141,8 +141,12 @@ function createSessionForHistory(historyService: HistoryService, sessionDir: str sessionsDir: path.dirname(sessionDir), srcDir: "/tmp", // A context boundary resets the durable memory-policy accumulator, which - // looks this workspace up in the config snapshot. + // looks this workspace up in the config snapshot (existence-requiring + // load; the mock delegates so the strict-mode override below propagates). loadConfigOrDefault: mock(() => ({ projects: new Map() })), + loadExistingConfigOrThrow(this: Config) { + return this.loadConfigOrDefault({ throwOnError: true }); + }, } as unknown as Config; return new AgentSession({ diff --git a/src/node/services/agentSession.testHarness.ts b/src/node/services/agentSession.testHarness.ts index 08b90cef519..2a12003b25e 100644 --- a/src/node/services/agentSession.testHarness.ts +++ b/src/node/services/agentSession.testHarness.ts @@ -197,6 +197,18 @@ export async function createAgentSessionHarness( const historyService = options.historyService ?? testHistory!.historyService; const config = options.config ?? testHistory?.config ?? createAgentSessionTestConfig(); const cleanup = testHistory?.cleanup ?? (() => Promise.resolve()); + // A registered workspace always has a config.json in production, and the + // session's compaction/reset boundaries require one to exist (an absent + // file reads as mid-rewrite, not as a fresh install). Persist the empty + // default so harness sessions do not fail those boundaries spuriously. + // Some suites pass a partial mock config (cast) without these methods. + if ( + typeof config.configFileStamp === "function" && + typeof config.editConfig === "function" && + config.configFileStamp() === "missing" + ) { + await config.editConfig((cfg) => cfg); + } const { aiEmitter, aiService } = options.aiService ? { aiEmitter: options.aiEmitter ?? new EventEmitter(), aiService: options.aiService } : createMockAiService({ diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index df4129eb312..eb8ac743325 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -102,13 +102,17 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null } /** - * Self-healing read of the adoption manifest: a missing or malformed file - * reads as "nothing adopted" (malformed content IS the file's state; the next - * pass rewrites it). An UNREADABLE file (EACCES, EIO) says nothing about that - * state: tolerant callers read it as empty too, `strict` callers throw — the - * removal handover decides what may be deleted from the manifest, and an - * empty substitute would delete the child session with the only provenance - * for a stale owner copy. A Map, not a plain object: a legacy note may + * Read of the adoption manifest. A MISSING file reads as "nothing adopted" + * for every caller. Tolerant callers also read an unreadable (EACCES, EIO) + * or malformed file — bad JSON, a non-object, a record missing its string + * fields — as empty (self-healing: the next pass rewrites it). `strict` + * callers throw on all of those: the adoption pass and the removal handover + * decide what may be deleted on the manifest's authority, and an empty + * substitute would drop provenance — a malformed record whose downgraded + * source is already gone can no longer be reconciled (its target is in the + * bad record), and removal would delete the child session while the + * adoption-created owner copy stays visible for good. A Map, not a plain + * object: a legacy note may * legitimately be named `__proto__` (any store-valid relPath), and assigning * that key on an ordinary object hits the prototype setter instead of * creating an entry the serialization would carry — the note would then be @@ -127,18 +131,28 @@ export async function readLegacyAdoptionManifest( if (options?.strict === true && code !== "ENOENT" && code !== "ENOTDIR") throw error; return new Map(); } + const malformed = (detail: string): Map => { + if (options?.strict === true) { + throw new Error(`the legacy adoption manifest at ${manifestPath} is malformed (${detail})`); + } + return new Map(); + }; + let parsed: unknown; try { - const parsed: unknown = JSON.parse(raw); - if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) return new Map(); - return new Map( - Object.entries(parsed).flatMap(([relPath, raw]) => { - const record = parseLegacyAdoptionRecord(raw); - return record === null ? [] : [[relPath, record] as const]; - }) - ); + parsed = JSON.parse(raw); } catch { - return new Map(); + return malformed("not JSON"); + } + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return malformed("not an object"); + } + const entries: Array<[string, LegacyAdoptionRecord]> = []; + for (const [relPath, value] of Object.entries(parsed)) { + const record = parseLegacyAdoptionRecord(value); + if (record === null) return malformed(`record '${relPath}'`); + entries.push([relPath, record]); } + return new Map(entries); } /** diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 2e6d2458e91..c3559b3316e 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1406,6 +1406,87 @@ describe("MemoryService", () => { if (!refused.success) expect(refused.error).toContain("was removed"); }); + it("withholds a read whose workspace was tombstoned after the pre-read check", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + await fixture.service.create(fixture.ctx, "/memories/global/g.md", "global", "agent"); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + // Another backend's removal lands between the check that opened the + // store and the read itself: every path that exposes the store's bytes + // or listing re-checks before returning them. + const service = fixture.service as unknown as { + openWorkspaceStore: (...args: unknown[]) => Promise; + }; + const original = service.openWorkspaceStore.bind(fixture.service); + const tombstoneAfterOpen = () => + spyOn(service, "openWorkspaceStore").mockImplementationOnce(async (...args) => { + await original(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const untombstone = () => fsPromises.rm(tombstonePath, { force: true }); + + tombstoneAfterOpen(); + const file = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(file.success).toBe(false); + if (!file.success) expect(file.error).toContain("was removed"); + await untombstone(); + + tombstoneAfterOpen(); + const dir = await fixture.service.view(fixture.ctx, "/memories/workspace"); + expect(dir.success).toBe(false); + if (!dir.success) expect(dir.error).toContain("was removed"); + await untombstone(); + + tombstoneAfterOpen(); + const root = await fixture.service.view(fixture.ctx, "/memories"); + expect(root.success).toBe(true); + if (root.success) { + expect(root.output).toContain("unavailable"); + expect(root.output).not.toContain("n.md"); + } + await untombstone(); + + tombstoneAfterOpen(); + const entries = await fixture.service.listIndexEntries(fixture.ctx); + expect(entries.map((entry) => entry.scope)).toEqual(["global"]); + await untombstone(); + + tombstoneAfterOpen(); + const ui = await fixture.service.readFileWithSha(fixture.ctx, "/memories/workspace/n.md"); + expect(ui.success).toBe(false); + await untombstone(); + + // Hot-set reads happen after the index enumeration passed: the + // tombstone landing before the file read drops the item. + const hotBefore = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: (text) => Promise.resolve(text.length), + }); + expect(hotBefore.some((item) => item.path === "/memories/workspace/n.md")).toBe(true); + // Interleaving: the tombstone lands after listIndexEntries built the + // candidate list and before the hot-set file reads. + const originalList = fixture.service.listIndexEntries.bind(fixture.service); + const listIndex = spyOn(fixture.service, "listIndexEntries").mockImplementationOnce( + async (ctx) => { + const result = await originalList(ctx); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + return result; + } + ); + try { + const hot = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: (text) => Promise.resolve(text.length), + }); + expect(hot.some((item) => item.path === "/memories/workspace/n.md")).toBe(false); + expect(hot.some((item) => item.path === "/memories/global/g.md")).toBe(true); + } finally { + listIndex.mockRestore(); + await untombstone(); + } + }); + it("refuses a pin toggle once the owner it was bound to is tombstoned", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -1710,6 +1791,23 @@ describe("MemoryService", () => { await fsPromises.writeFile(manifestPath, savedManifest); } expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + // Malformed (not missing) aborts too: a bad record whose source is + // gone could not be reconciled, and an empty substitute would drop the + // provenance for its owner copy while removal deletes the child. + for (const [label, body] of [ + ["not JSON", "{nope"], + ["not an object", "[]"], + ["record 'note.md'", JSON.stringify({ "note.md": { content: 1 } })], + ] as const) { + await fsPromises.writeFile(manifestPath, body); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toContain(`malformed (${label})`); + expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(false); + } + await fsPromises.writeFile(manifestPath, savedManifest); await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); expect(await pathExists(path.join(ownerRoot, "late.md"))).toBe(true); }); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index d1c322c10c0..14b245259f0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1776,6 +1776,24 @@ export class MemoryService extends EventEmitter { } } + /** + * Post-read gate for workspace-scope reads. openWorkspaceStore checks the + * tombstones BEFORE the read; another backend can publish the acting + * workspace's (or the owner's) removal tombstone while the read is in + * flight, and the bytes would then be exposed on behalf of a workspace that + * no longer exists. Re-checked after every read whose result leaves the + * service (view, listings, index/hot-set builds, UI reads), before the + * result is returned. Other scopes are never shared and have no tombstone. + */ + private async assertWorkspaceReadExposable( + ctx: MemoryScopeContext, + scope: MemoryScope, + store: MemoryStore + ): Promise { + if (scope !== "workspace") return; + await this.assertWorkspaceStoreReadable(ctx, store); + } + private requireFilePath(parsed: ParsedMemoryPath, virtualPath: string): MemoryScope { if (parsed.scope === null || parsed.relPath === "") { throw new MemoryCommandError( @@ -2230,6 +2248,7 @@ export class MemoryService extends EventEmitter { // Read-only: never create roots just to list (missing ⇒ empty). await store.assertRootSafe(); const files = await store.listFiles(); + await this.assertWorkspaceReadExposable(ctx, scope, store); sections.push(...renderTree(files, MEMORY_VIEW_MAX_DEPTH - 1, " ")); } catch (error) { // Self-healing: an unavailable scope must not break the whole view. @@ -2246,6 +2265,7 @@ export class MemoryService extends EventEmitter { // write — but the scope itself always exists in the protocol. if (kind === "dir" || (kind === null && parsed.relPath === "")) { const files = await store.listFiles(); + await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); const prefix = parsed.relPath === "" ? "" : `${parsed.relPath}/`; const scopedFiles = files .filter((file) => file.startsWith(prefix)) @@ -2261,6 +2281,9 @@ export class MemoryService extends EventEmitter { } const content = await this.readBoundedTextFile(store, parsed.relPath, virtualPath); + // Before recordUsage: its refusal is swallowed (usage is best-effort), + // so it cannot stand in for this gate. + await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); const output = renderFileView(content, options); await this.recordUsage(ctx, parsed.scope, parsed.relPath, { write: false }); return { success: true, output }; @@ -2822,6 +2845,7 @@ export class MemoryService extends EventEmitter { const scope = this.requireFilePath(parsed, virtualPath); const store = await this.resolveStore(ctx, scope, parsed.relPath); const content = await this.readTextFileForEdit(store, parsed.relPath, virtualPath); + await this.assertWorkspaceReadExposable(ctx, scope, store); // Deliberately NOT recorded as a use: this is a human browsing the // Memory tab/settings, and usage stats must reflect agent reads only so // UI browsing never inflates hot-set ranking. (UI saves still count — @@ -2921,6 +2945,10 @@ export class MemoryService extends EventEmitter { async listIndexEntries(ctx: MemoryScopeContext): Promise { const entries: MemoryIndexEntry[] = []; for (const scope of MEMORY_SCOPES) { + // Per-scope buffer: the scope's entries join the result only once the + // post-read gate below passed, so a tombstone published mid-enumeration + // drops the whole scope rather than a prefix of it. + const scopeEntries: MemoryIndexEntry[] = []; try { const store = this.getStore(ctx, scope); // Prompt context is a read of the (possibly shared) store: a removed @@ -2975,8 +3003,10 @@ export class MemoryService extends EventEmitter { } catch { // Unreadable file: list it without a description. } - entries.push({ path: toVirtualPath(scope, relPath), scope, relPath, description }); + scopeEntries.push({ path: toVirtualPath(scope, relPath), scope, relPath, description }); } + await this.assertWorkspaceReadExposable(ctx, scope, store); + entries.push(...scopeEntries); } catch (error) { log.debug("[MemoryService] skipping scope in memory index", { scope, error }); } @@ -3015,17 +3045,20 @@ export class MemoryService extends EventEmitter { countTokens: options.countTokens, tokenBudgetActive: options.tokenBudgetActive, onlyContextNotes: options.onlyContextNotes, - readFile: (virtualPath) => { + readFile: async (virtualPath) => { const parsed = parseMemoryPath(virtualPath); const scope = this.requireFilePath(parsed, virtualPath); // Paths come from listIndexEntries (already enumerated under the scope // roots), so no extra containment walk is needed for these reads. // Bounded prefix: selection truncates to MEMORY_HOT_SET_MAX_ITEM_BYTES // anyway; +1 byte preserves its over-budget (truncation marker) check. - return this.getStore(ctx, scope).readFilePrefix( + const store = this.getStore(ctx, scope); + const content = await store.readFilePrefix( parsed.relPath, MEMORY_HOT_SET_MAX_ITEM_BYTES + 1 ); + await this.assertWorkspaceReadExposable(ctx, scope, store); + return content; }, }); } diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index 1996ca62b7b..b6688e2465c 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -115,3 +115,34 @@ export function sharedWorkspaceMemoryPeerSessionDirs( } return peers; } + +/** Owner root and live peers of a workspace sharing its notebook (rollback). */ +export interface SharedWorkspaceMemoryTopology { + /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ + ownerSessionDir: string | undefined; + /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ + peerSessionDirs: string[]; +} + +/** + * The rollback topology from ONE config snapshot that must prove itself: + * `loadExistingConfigOrThrow` throws when config.json is unreadable OR + * absent (mid-rewrite), so callers refuse instead of degrading to the + * fresh-install view in which the workspace owns its notebook — that "self" + * fallback would omit the owner root (a pre-sharing row's inverse then lands + * on the hidden legacy notebook instead of the owner's adopted copy) and the + * peer list (conflicting sibling rows go unseen). Peers are re-resolved per + * check by the engine (a member registered while waiting for the store lock + * must count), so callers hand it a callback that calls this again. + */ +export function resolveSharedWorkspaceMemoryTopology( + config: Pick, + workspaceId: string +): SharedWorkspaceMemoryTopology { + const cfg = config.loadExistingConfigOrThrow(); + const ownerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); + return { + ownerSessionDir: ownerId === workspaceId ? undefined : path.join(config.sessionsDir, ownerId), + peerSessionDirs: sharedWorkspaceMemoryPeerSessionDirs(cfg, config.sessionsDir, workspaceId), + }; +} diff --git a/src/node/services/tools/refinement_rollback.ts b/src/node/services/tools/refinement_rollback.ts index 6d0971fa9df..be4169eed4a 100644 --- a/src/node/services/tools/refinement_rollback.ts +++ b/src/node/services/tools/refinement_rollback.ts @@ -12,6 +12,7 @@ import { LegacyPathNotAdoptedError, } from "@/node/services/memoryLegacyAdoption"; import { getErrorMessage } from "@/common/utils/errors"; +import type { SharedWorkspaceMemoryTopology } from "@/node/services/memoryWorkspaceOwner"; interface RefinementRollbackToolArgs { id: string; @@ -85,13 +86,6 @@ async function refuseReadOnlyMemoryRollback( return null; } -export interface SharedWorkspaceMemoryTopology { - /** Owner session dir when the workspace is a sub-agent sharing its notebook. */ - ownerSessionDir: string | undefined; - /** Other live task-tree members' session dirs (see RollbackRefinementOptions). */ - peerSessionDirs: string[]; -} - export type SharedWorkspaceMemoryTopologyResolver = () => SharedWorkspaceMemoryTopology; export function createRefinementRollbackTool(ctx: { @@ -124,6 +118,9 @@ export function createRefinementRollbackTool(ctx: { { id, reason }: RefinementRollbackToolArgs, { toolCallId } ): Promise => { + // Resolved once here for the owner root (stable for the session's + // lifetime) and the policy gate; peers are re-resolved by the engine per + // check through the same resolver (a throw there refuses too). let topology: SharedWorkspaceMemoryTopology; try { topology = ctx.sharedWorkspaceMemory?.() ?? { @@ -148,7 +145,8 @@ export function createRefinementRollbackTool(ctx: { const result = await rollbackRefinement({ sessionDir: ctx.sessionDir, sharedWorkspaceMemorySessionDir: topology.ownerSessionDir, - listSharedWorkspaceMemoryPeerSessionDirs: () => topology.peerSessionDirs, + listSharedWorkspaceMemoryPeerSessionDirs: () => + ctx.sharedWorkspaceMemory?.().peerSessionDirs ?? [], id, reason, evidence: { toolName: "refinement_rollback", toolCallId, actor: "agent" }, diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index bfa694e14ee..5d11e90816e 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -121,10 +121,7 @@ import { } from "@/common/utils/providers/customProviders"; import type { MCPServerManager, MCPWorkspaceStats } from "@/node/services/mcpServerManager"; import { type MemoryService, type MemorySessionContext } from "@/node/services/memoryService"; -import { - resolveWorkspaceMemoryOwnerId, - sharedWorkspaceMemoryPeerSessionDirs, -} from "@/node/services/memoryWorkspaceOwner"; +import { resolveSharedWorkspaceMemoryTopology } from "@/node/services/memoryWorkspaceOwner"; import { memoryScopeContextFromToolConfig } from "@/node/services/tools/memory"; import type { TaskService } from "@/node/services/taskService"; import { READ_ONLY_ACCESS, resolveMemoryAccessPolicy } from "@/node/services/tools/memory"; @@ -2525,32 +2522,14 @@ export class TurnRequestBuilder { // and announce its direct-to-disk writes through MemoryService so the // shared store's readers refresh. Resolved per rollback (not per // turn): tree membership changes as sub-agents are spawned and removed - // while the tool instance lives. Strict AND existence-requiring load, - // from one snapshot for owner and peers: a config.json that is - // unreadable or absent (mid-rewrite) must refuse the rollback (see - // RollbackRefinementOptions), not read as a fresh install in which - // the child owns its notebook — that "self" fallback would omit the - // owner root (a pre-sharing row's inverse then lands on the hidden - // legacy notebook instead of the owner's adopted copy) and the peer - // list (conflicting sibling rows go unseen). + // while the tool instance lives; a topology that cannot be proven + // refuses the rollback (see resolveSharedWorkspaceMemoryTopology). const memoryService = this.dependencies.bindings.memoryService; const sessionsDir = this.dependencies.config.sessionsDir; const sharedWorkspaceMemory = memoryService === undefined ? undefined - : () => { - const cfg = this.dependencies.config.loadExistingConfigOrThrow(); - const ownerId = resolveWorkspaceMemoryOwnerId(cfg, workspaceId); - return { - ownerSessionDir: - ownerId === workspaceId ? undefined : path.join(sessionsDir, ownerId), - peerSessionDirs: sharedWorkspaceMemoryPeerSessionDirs( - cfg, - sessionsDir, - workspaceId - ), - }; - }; + : () => resolveSharedWorkspaceMemoryTopology(this.dependencies.config, workspaceId); // Built anew for EVERY attempt (prepareModelRequest runs per primary / // fallback request) from the never-mutated policy: refinement_rollback // reads it by reference, and the request.assemble demotion below only From eb700489930b436dde41e85e829c8b1f6ba7c3c5 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 06:24:06 +0000 Subject: [PATCH 73/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-sev?= =?UTF-8?q?enth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adoption manifest: a present non-string `replacementContent` makes the record malformed (strict reads throw), so an interrupted in-place replacement can never be tombstoned as owner-owned and lose provenance. - Rollback peer rows: originals of copies already migrated into the acting journal (aborted removal, child still registered) are skipped, so an owner can roll back a migrated copy without the child's orderUnknown original reporting divergence against it. - Inverse blob resweep ranks migrated rows without a source time behind every dated row, so old migrated history can no longer evict the owner's recent rollback payloads. - Consolidation pin protection is enforced inside MemoryService's mutation lock (`rejectPinned`) against the owner the command's store is bound to; the tool's pre-check remains only as early feedback. - Preserved-tail copies get their policy epoch by coverage: a first-time copy carries the recorded epoch of the assistant turn row that answered it; rows after the last turn row (an accepted batch no assistant answered) and rows answered by a turn without a record stay unstamped (policy-unknown). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/compactionHandler.test.ts | 25 ++++--- src/node/services/compactionHandler.ts | 67 ++++++++++++------- src/node/services/memoryConsolidation.test.ts | 42 ++++++++++++ src/node/services/memoryConsolidation.ts | 6 ++ src/node/services/memoryLegacyAdoption.ts | 12 +++- src/node/services/memoryService.test.ts | 54 +++++++++++++++ src/node/services/memoryService.ts | 43 +++++++++++- .../refinement/refinementJournal.test.ts | 47 +++++++++++++ .../services/refinement/refinementJournal.ts | 21 ++++-- .../services/refinement/refinementRollback.ts | 13 ++++ src/node/services/tools/memory.ts | 13 +++- 11 files changed, 298 insertions(+), 45 deletions(-) diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index d29eeb32367..48eb088739d 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1839,10 +1839,14 @@ describe("CompactionHandler", () => { onCompactionComplete, }); + // A real turn row: it carries its request bound and the epoch it + // recorded the workspace-memory policy under (none before any boundary). const tailAssistant = createMuxMessage("a1", "assistant", "tail answer", { model: "claude-x", usage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 }, contextUsage: { inputTokens: 500, outputTokens: 100, totalTokens: 600 }, + requestHistorySequence: 2, + workspaceMemoryPolicyEpoch: -1, }); await seedHistory( createMuxMessage("u0", "user", "old head question"), @@ -1880,8 +1884,8 @@ describe("CompactionHandler", () => { expect(copy.metadata?.contextUsage).toBeUndefined(); // Copies must never masquerade as boundaries. expect(copy.metadata?.compactionBoundary).toBeUndefined(); - // The epoch the row was produced under (none before this boundary): - // its workspace-memory write policy governs the copy. + // The epoch the turn that covered the row recorded its policy under + // (none before this boundary): that policy governs the copy. expect(copy.metadata?.rlmPreservedTailSourcePolicyEpoch).toBe(-1); } // Informational metadata survives. @@ -1895,10 +1899,13 @@ describe("CompactionHandler", () => { // boundary's epoch — the chain stays visible to the policy conjunction. const boundarySequence = epoch[0].metadata?.historySequence; if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); - // An assistant TURN row (it carries the request bound) that recorded - // its policy under an OLDER epoch (a turn that straddled a boundary) - // keeps that epoch on its copy; a turn row WITHOUT a recorded policy - // (an older build's) stays unstamped — unknown, never vouched for. + // A user row is covered by the assistant TURN row that answered it and + // carries THAT turn's recorded epoch: an OLDER one for a turn that + // straddled a boundary (u2/a2); none for a turn row WITHOUT a recorded + // policy (an older build's, u4/a4) — unknown, never vouched for; and + // none for an accepted batch no assistant ever answered (u5: stream + // never started or crashed first), whose repo-controlled content + // nobody vetted. await seedHistory( createMuxMessage("u2", "user", "second question"), createMuxMessage("a2", "assistant", "second answer", { @@ -1914,6 +1921,7 @@ describe("CompactionHandler", () => { createMuxMessage("a4", "assistant", "old-build answer", { requestHistorySequence: boundarySequence + 5, }), + createMuxMessage("u5", "user", "unanswered question"), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1924,11 +1932,12 @@ describe("CompactionHandler", () => { ).toEqual([ -1, -1, - boundarySequence, + -1, -1, boundarySequence, boundarySequence, - boundarySequence, + undefined, + undefined, undefined, ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 355c031c95c..14730735397 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1267,10 +1267,7 @@ export class CompactionHandler { } ); const idMap = new Map(params.tail.map((row) => [row.id, createPreservedTailCopyMessageId()])); - // Same closing epoch the completion metadata reports below. - const closingPolicyEpoch = latestContextBoundaryHistorySequence(params.messages) ?? -1; - const copies = params.tail.map((row) => { - const copy = this.buildPreservedTailCopy(row, idMap, closingPolicyEpoch); + const copies = this.buildCoveredTailCopies(params.tail, idMap).map((copy) => { // Continuous compaction prunes the just-finished answer too. Keep recent pages // visible below the boundary while retaining RLM's usage/snapshot sanitizer. copy.metadata = { ...copy.metadata, uiVisible: true }; @@ -1505,8 +1502,7 @@ export class CompactionHandler { const preservedTailCopies = this.buildPreservedTailCopies( messages, compactionRequestMessageId, - summaryMessage.id, - previousBoundaryHistorySequence ?? -1 + summaryMessage.id ); const persistenceResult = @@ -1592,8 +1588,7 @@ export class CompactionHandler { private buildPreservedTailCopies( messages: MuxMessage[], compactionRequestMessageId: string, - summaryMessageId: string, - closingPolicyEpoch: number + summaryMessageId: string ): MuxMessage[] { const requestIndex = messages.findIndex((message) => message.id === compactionRequestMessageId); if (requestIndex === -1) { @@ -1633,7 +1628,34 @@ export class CompactionHandler { for (const row of tailRows) { idMap.set(row.id, createPreservedTailCopyMessageId()); } - return tailRows.map((row) => this.buildPreservedTailCopy(row, idMap, closingPolicyEpoch)); + return this.buildCoveredTailCopies(tailRows, idMap); + } + + /** + * Copies of a whole tail with their policy epochs assigned by COVERAGE. A + * first-time copy's epoch is the one the assistant TURN row that answered + * it recorded its policy under (the nearest later turn row in the tail): + * that turn is what consumed the row's content under a recorded policy. + * A nearest turn row WITHOUT a record (an older build's) leaves the rows + * it answered unknown, and rows after the last turn row — an accepted + * user/prelude batch whose stream never started or crashed before its + * assistant row landed — have no policy at all. Both stay unstamped, which + * the policy sink reads as unknown (deny): stamping them with the closing + * epoch would present repo-controlled input nobody vetted as covered, and + * the next turn could harvest output conditioned on it into the shared + * notebook. Copies of copies keep their original epoch regardless. + */ + private buildCoveredTailCopies(tailRows: MuxMessage[], idMap: Map): MuxMessage[] { + const copies: MuxMessage[] = new Array(tailRows.length); + let covering: number | undefined; + for (let i = tailRows.length - 1; i >= 0; i--) { + const row = tailRows[i]; + if (row.role === "assistant" && typeof row.metadata?.requestHistorySequence === "number") { + covering = row.metadata.workspaceMemoryPolicyEpoch; + } + copies[i] = this.buildPreservedTailCopy(row, idMap, covering); + } + return copies; } /** @@ -1649,7 +1671,7 @@ export class CompactionHandler { private buildPreservedTailCopy( row: MuxMessage, idMap: Map, - closingPolicyEpoch: number + coveringPolicyEpoch: number | undefined ): MuxMessage { // IDs are preassigned for the whole tail (see caller) so forward-pointing // references (snapshot row → later invoking user row) rewrite correctly. @@ -1671,25 +1693,18 @@ export class CompactionHandler { // The epoch whose workspace-memory write policy governs this row: a copy // of a copy keeps its ORIGINAL epoch (the chain must stay visible to the - // policy conjunction); an assistant row keeps the epoch its policy was - // recorded under (a turn that started before a destructive reset and - // landed after the new boundary belongs to the OLD epoch, whose policy - // the new one cannot vouch for); any other first-time copy was produced - // under the epoch this compaction closes. Copies from before the field - // existed carry nothing forward — no policy record ever existed for - // their epochs. - // An assistant TURN row (it carries the request bound) without a recorded - // policy epoch was produced by a build that did not maintain the policy: - // its copy stays unstamped, which the policy sink reads as unknown - // (deny) — stamping it with the closing epoch would vouch for a policy - // nobody recorded. Non-turn assistant rows (payloads, summaries) and - // user rows belong to the closing epoch. + // policy conjunction; copies from before the field existed carry nothing + // forward — no policy record ever existed for their epochs). A first-time + // copy carries the epoch of the assistant TURN row covering it (see + // buildCoveredTailCopies): for a turn row that is its own recorded epoch + // — a turn that started before a destructive reset and landed after the + // new boundary belongs to the OLD epoch, whose policy the new one cannot + // vouch for — and a turn row without a record (an older build's) stays + // unstamped, read as unknown (deny), never vouched for. const sourcePolicyEpoch = source?.rlmPreservedTailCopy === true ? source.rlmPreservedTailSourcePolicyEpoch - : row.role === "assistant" && typeof source?.requestHistorySequence === "number" - ? source.workspaceMemoryPolicyEpoch - : closingPolicyEpoch; + : coveringPolicyEpoch; return { ...row, id: copyId, diff --git a/src/node/services/memoryConsolidation.test.ts b/src/node/services/memoryConsolidation.test.ts index 2acadf5230d..9f1a60d3645 100644 --- a/src/node/services/memoryConsolidation.test.ts +++ b/src/node/services/memoryConsolidation.test.ts @@ -313,6 +313,48 @@ describe("consolidation memory tool rails", () => { ).toContain("polished"); }); + it("enforces pin protection inside the mutation against the store the command binds to", async () => { + using fixture = await createFixture(); + const sessionMemory = path.join(fixture.xumHome, "sessions", fixture.ctx.workspaceId, "memory"); + await fsPromises.mkdir(sessionMemory, { recursive: true }); + await fsPromises.writeFile(path.join(sessionMemory, "pinned.md"), "keep me\n"); + await fixture.metaService.setPinned( + memoryLogicalKey("workspace", "pinned.md", { + projectPath: fixture.ctx.projectPath, + workspaceId: fixture.ctx.workspaceId, + }), + true + ); + // The tool's pre-check resolves the shared-store owner on its own; make + // that resolution disagree with the command's (as a transiently unreadable + // config.json can, falling back to a different store) so the pre-check + // looks at the wrong sidecar key and passes. The command itself must + // still refuse: its check runs in-lock against the owner its store is + // bound to. + const resolve = spyOn(fixture.memoryService, "resolveWorkspaceMemoryOwnerId"); + resolve.mockImplementationOnce(() => "ws-elsewhere"); + try { + const deletion = await execute(fixture.tool, { + command: "delete", + path: "/memories/workspace/pinned.md", + }); + expect(deletion.success).toBe(false); + if (!deletion.success) expect(deletion.error).toContain("pinned"); + expect(await pathExists(path.join(sessionMemory, "pinned.md"))).toBe(true); + resolve.mockImplementationOnce(() => "ws-elsewhere"); + const rename = await execute(fixture.tool, { + command: "rename", + old_path: "/memories/workspace/pinned.md", + new_path: "/memories/workspace/moved.md", + }); + expect(rename.success).toBe(false); + if (!rename.success) expect(rename.error).toContain("pinned"); + expect(await pathExists(path.join(sessionMemory, "pinned.md"))).toBe(true); + } finally { + resolve.mockRestore(); + } + }); + it("rejects deleting or renaming a directory that contains a pinned file", async () => { using fixture = await createFixture(); const nestedDir = path.join(fixture.globalMemoryDir, "nested"); diff --git a/src/node/services/memoryConsolidation.ts b/src/node/services/memoryConsolidation.ts index cf86bd37f12..cf84b904989 100644 --- a/src/node/services/memoryConsolidation.ts +++ b/src/node/services/memoryConsolidation.ts @@ -244,6 +244,11 @@ export function createConsolidationMemoryTool(args: { // Deletes/renames may target a directory (MemoryService removes // recursively), so reject when the path itself OR anything under it is // pinned — otherwise `delete dir/` would silently destroy dir/pinned.md. + // This pre-check gives the model (and dry-run staging) early feedback; + // the AUTHORITATIVE check runs inside MemoryService's mutation lock + // against the owner the command's store is actually bound to + // (`rejectPinned` below): the owner resolved here can differ from the + // command's when config.json is transiently unreadable in between. if (target.command === "delete" || target.command === "rename") { const { scope, relPath } = parseMemoryPath(target.path); assert( @@ -331,6 +336,7 @@ export function createConsolidationMemoryTool(args: { const result = await executeMemoryCommand(memoryService, ctx, input, () => null, toolCallId, { expectedTargetFingerprint: args.expectedTargetFingerprints?.get(toolCallId), abortSignal: args.abortSignal, + rejectPinned: true, }); journal.push({ ...target, diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index eb8ac743325..f183014b7a8 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -86,6 +86,15 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null ) { return null; } + // A present but non-string replacement hash is a malformed RECORD (not a + // flag to fail closed on): without it, a replacement pass that crashed + // after writing the new owner bytes leaves a copy reconciliation cannot + // recognize as this adoption's — a later source deletion would tombstone + // it as owner-owned and removal would report a complete handover while the + // adoption-created note stays visible without provenance. + if (record.replacementContent !== undefined && typeof record.replacementContent !== "string") { + return null; + } const flag = (raw: unknown, malformed: boolean): boolean | undefined => raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; return { @@ -96,8 +105,7 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null pending: flag(record.pending, true), pendingDeletion: flag(record.pendingDeletion, true), deleted: flag(record.deleted, false), - replacementContent: - typeof record.replacementContent === "string" ? record.replacementContent : undefined, + replacementContent: record.replacementContent, }; } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index c3559b3316e..158f1021471 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1798,6 +1798,12 @@ describe("MemoryService", () => { ["not JSON", "{nope"], ["not an object", "[]"], ["record 'note.md'", JSON.stringify({ "note.md": { content: 1 } })], + [ + "record 'late.md'", + JSON.stringify({ + "late.md": { content: "x", sidecar: "", target: "late.md", replacementContent: 5 }, + }), + ], ] as const) { await fsPromises.writeFile(manifestPath, body); expect( @@ -3287,6 +3293,54 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); }); + it("an owner rollback of a migrated copy ignores the still-registered child's original row", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + }, + }); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + // Removal aborted after the pre-teardown pass: the child stays + // registered (a peer of the owner) with its original row in place. + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom?.startsWith("ws-child:") === true + )!; + const rolledBack = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: copy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(rolledBack.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + it("follows a row rolled back between the two handover passes with its rollback row", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 14b245259f0..73e9fa2af50 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -881,6 +881,37 @@ export class MemoryService extends EventEmitter { }); } + /** + * Consolidation's pin protection (pinned files are editable but never + * deleted/renamed; a directory counts when anything under it is pinned), + * evaluated INSIDE the mutation lock against the owner the command's store + * is bound to: logicalKeyFor and getStore share this command's owner + * resolution (ownerWorkspaceIdFor), so the key checked is the key of the + * file about to be removed. A guard run before the command against a + * separately resolved owner (the private-store fallback while config.json + * was unreadable) would check the wrong sidecar entries and let an + * owner-pinned note go. Strict sidecar read: an unreadable pin file must + * refuse, not read as "nothing pinned". + */ + private async assertNotPinnedForRemoval( + ctx: MemoryScopeContext, + scope: MemoryScope, + relPath: string, + virtualPath: string + ): Promise { + const key = this.logicalKeyFor(ctx, scope, relPath); + if (key === null) return; + const subtreePrefix = `${key}/`; + for (const [entryKey, entry] of await this.metaService.getEntriesOrThrow()) { + if (entry.pinned !== true) continue; + if (entryKey === key || entryKey.startsWith(subtreePrefix)) { + throw new MemoryCommandError( + `${virtualPath} is pinned by the user (directly or via a pinned file inside it); pinned files may be edited but never deleted or renamed.` + ); + } + } + } + private async recordUsage( ctx: MemoryScopeContext, scope: MemoryScope, @@ -2665,7 +2696,8 @@ export class MemoryService extends EventEmitter { actor: MemoryActor, toolCallId?: string, expectedFingerprint?: string, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + options?: { rejectPinned?: boolean } ): Promise { return this.runCommand(ctx, async () => { const parsed = parseMemoryPath(virtualPath); @@ -2676,6 +2708,9 @@ export class MemoryService extends EventEmitter { if (kind === null) { throw new MemoryCommandError(`No memory file or directory at ${virtualPath}`); } + if (options?.rejectPinned === true) { + await this.assertNotPinnedForRemoval(ctx, scope, parsed.relPath, virtualPath); + } // r55: staged refine deletes were approved against the target's // staging-time state — a target edited between staging and apply // must refuse rather than silently destroying the newer contents. @@ -2725,7 +2760,8 @@ export class MemoryService extends EventEmitter { newVirtualPath: string, actor: MemoryActor, toolCallId?: string, - abortSignal?: AbortSignal + abortSignal?: AbortSignal, + options?: { rejectPinned?: boolean } ): Promise { return this.runCommand(ctx, async () => { const oldParsed = parseMemoryPath(oldVirtualPath); @@ -2745,6 +2781,9 @@ export class MemoryService extends EventEmitter { if (oldKind === null) { throw new MemoryCommandError(`No memory file or directory at ${oldVirtualPath}`); } + if (options?.rejectPinned === true) { + await this.assertNotPinnedForRemoval(ctx, scope, oldParsed.relPath, oldVirtualPath); + } // Pre-flight (mirrored in validateMutation): store.rename would mkdir // the destination parent INSIDE the source before the filesystem // rejects the move — refuse cleanly instead of polluting the source. diff --git a/src/node/services/refinement/refinementJournal.test.ts b/src/node/services/refinement/refinementJournal.test.ts index 76145747e59..72949c2c1a5 100644 --- a/src/node/services/refinement/refinementJournal.test.ts +++ b/src/node/services/refinement/refinementJournal.test.ts @@ -60,6 +60,53 @@ describe("reclaimExcessRefinementInverseBlobs", () => { deleteSpy.mockRestore(); }); + test("the recovery sweep ranks unordered migrated rows behind the owner's dated rows", async () => { + using tmp = new DisposableTempDir("refinement-journal-test"); + const journal = new DurableEventJournal(tmp.path); + // Owner rows carry the shared store clock as sourceTs; the migrated row + // (a removed child's pre-sharing history, retargeted, private clock + // dropped) is appended LAST — its envelope `ts` is the newest of all. + const appendRow = async ( + content: string, + extra: { sourceTs?: number; migratedFrom?: string; orderUnknown?: true } + ): Promise => + await journal.withBlobLock(async () => { + const { ref } = await journal.blobs.put(content); + await journal.append({ + workspaceId: "ws-owner", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/n.md" }, + inverse: { op: "restore-files", files: [{ path: "/m/n.md", blobRef: ref }] }, + evidence: { workspaceId: "ws-owner", toolName: "test" }, + ...extra, + }, + }); + return ref; + }); + const ownerOld = await appendRow("owner-old", { sourceTs: 10 }); + const ownerNew = await appendRow("owner-new", { sourceTs: 20 }); + const migrated = await appendRow("child-legacy", { + migratedFrom: "ws-child:row-1", + orderUnknown: true, + }); + // Each payload charged at 0.4x quota: only two survive the sweep. + const size = spyOn(journal.blobs, "size").mockResolvedValue( + Math.ceil(REFINEMENT_INVERSE_BLOB_QUOTA_BYTES * 0.4) + ); + try { + await reclaimExcessRefinementInverseBlobs(journal, [], { resweep: true }); + } finally { + size.mockRestore(); + } + // The owner's genuinely recent payloads stay; the unordered migrated + // history is evicted first despite its newer append time. + expect(await journal.blobs.has(ownerOld)).toBe(true); + expect(await journal.blobs.has(ownerNew)).toBe(true); + expect(await journal.blobs.has(migrated)).toBe(false); + }); + test("a payload hash shared with another event kind survives eviction", async () => { using tmp = new DisposableTempDir("refinement-journal-test"); const journal = new DurableEventJournal(tmp.path); diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index bfdaf493920..1b47b09e88e 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -242,14 +242,25 @@ export async function reclaimExcessRefinementInverseBlobs( // Recovery sweep: walk refinement rows newest-first — by SOURCE time // (`data.sourceTs ?? ts`, append sequence as tie-breaker), so migrated // rows sit at their real chronological position — and re-derive the - // retained set. Rows never recorded payload sizes, so stat the blobs; - // a missing blob was already evicted (or never landed) — skip it. + // retained set. A migrated row WITHOUT a source time (pre-sharing or + // private-clock history; the migration omits the incomparable value and + // marks it orderUnknown) has only its append-time `ts`, which would rank + // that old history as the newest and let it evict the owner's genuinely + // recent payloads: rank it behind every dated row instead (evicted + // first). Rows never recorded payload sizes, so stat the blobs; a + // missing blob was already evicted (or never landed) — skip it. + const retentionTs = (event: { + ts: number; + data: { sourceTs?: number; migratedFrom?: string }; + }): number => + event.data.sourceTs ?? + (event.data.migratedFrom !== undefined ? Number.NEGATIVE_INFINITY : event.ts); const events = (await journal.read()) .filter((event) => event.kind === "refinement") .sort((left, right) => { - const leftTs = left.data.sourceTs ?? left.ts; - const rightTs = right.data.sourceTs ?? right.ts; - return leftTs !== rightTs ? leftTs - rightTs : left.seq - right.seq; + const leftTs = retentionTs(left); + const rightTs = retentionTs(right); + return leftTs !== rightTs ? (leftTs < rightTs ? -1 : 1) : left.seq - right.seq; }); entries = []; for (let i = events.length - 1; i >= 0; i--) { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 853941856d7..f4fa7b82717 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -585,6 +585,16 @@ async function readSharedMemoryPeerRows( ); const ownerSessionDir = path.dirname(sharedRoot); const actingWorkspaceId = path.basename(path.resolve(opts.sessionDir)); + // The same duplicate seen from the other side: a removal that aborted after + // its pre-teardown pass leaves THIS (owner) journal holding copies of a + // still-registered child's rows. Read as that child's peer rows, the + // originals would count as separate later mutations of the very notes the + // copies describe — and a retargeted original is `orderUnknown`, so the + // copy could never be rolled back until the removal finally succeeds. + const migratedHere = new Set(); + for (const row of await listRefinements(opts.sessionDir)) { + if (row.data.migratedFrom !== undefined) migratedHere.add(row.data.migratedFrom); + } const peerRows: RefinementEvent[] = []; for (const peerDir of peerDirs) { assert( @@ -618,6 +628,7 @@ async function readSharedMemoryPeerRows( ); } } + const peerWorkspaceId = path.basename(path.resolve(peerDir)); for (const row of await listRefinements(peerDir)) { if (row.data.kind !== "memory") continue; // A removal that aborted after its pre-teardown pass leaves the owner @@ -625,6 +636,8 @@ async function readSharedMemoryPeerRows( // ":") while this session lives on. They are // this journal's rows seen twice, not later peer edits. if (row.data.migratedFrom?.startsWith(`${actingWorkspaceId}:`) === true) continue; + // ...and the originals of copies this journal already holds (above). + if (migratedHere.has(`${peerWorkspaceId}:${row.id}`)) continue; const original = RefinementInverseSchema.safeParse(row.data.inverse); const parsed = parseRemappedInverse(row, remap); if (parsed === null || !original.success) continue; diff --git a/src/node/services/tools/memory.ts b/src/node/services/tools/memory.ts index 192dc33f3f3..9d9d48138ae 100644 --- a/src/node/services/tools/memory.ts +++ b/src/node/services/tools/memory.ts @@ -257,6 +257,13 @@ export async function executeMemoryCommand( * I/O unblocks. Ignored by reads. */ abortSignal?: AbortSignal; + /** + * Consolidation's pin protection (pinned files are editable but never + * deleted/renamed), enforced by MemoryService INSIDE its target mutation + * lock against the owner the command's store is bound to — see + * MemoryService.assertNotPinnedForRemoval. Ignored by other commands. + */ + rejectPinned?: boolean; } ): Promise { try { @@ -336,7 +343,8 @@ export async function executeMemoryCommand( "agent", toolCallId, options?.expectedTargetFingerprint, - options?.abortSignal + options?.abortSignal, + { rejectPinned: options?.rejectPinned } )) ); } @@ -355,7 +363,8 @@ export async function executeMemoryCommand( input.new_path, "agent", toolCallId, - options?.abortSignal + options?.abortSignal, + { rejectPinned: options?.rejectPinned } )) ); } From 5a6f3a4de7e7faf7a2453ce29fd1d6b463cc73ab Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 07:02:03 +0000 Subject: [PATCH 74/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-eig?= =?UTF-8?q?hth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Preserved-tail copies are stamped by REQUEST coverage, mirroring the harvest gate: an assistant turn row covers exactly the last user row at or below its requestHistorySequence plus that row's requestPreludeMessageIds (exact ids). A user row another backend or a sub-agent report appended between a turn's anchor and its assistant row stays unstamped (unknown); token-budget control rows and non-turn assistant rows keep the closing epoch; copies of copies keep their original epoch. - Rollback re-derives the legacy path remapping (the adopted-directory exact subtree proof) INSIDE the target lock and refuses when it no longer yields the plan-time paths, so an owner note added while waiting for the lock can no longer travel with a legacy directory rename. - Peer-row deduplication suppresses a still-registered child's original only when the local migrated copy is still a usable memory row (parseable inverse); an unusable copy leaves the intact original as the conflict record. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/compactionHandler.test.ts | 51 +++++--- src/node/services/compactionHandler.ts | 119 ++++++++++++------ src/node/services/memoryService.test.ts | 63 ++++++++++ .../refinement/refinementRollback.test.ts | 21 ++++ .../services/refinement/refinementRollback.ts | 41 +++++- 5 files changed, 235 insertions(+), 60 deletions(-) diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 48eb088739d..9fef33e9405 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1899,27 +1899,36 @@ describe("CompactionHandler", () => { // boundary's epoch — the chain stays visible to the policy conjunction. const boundarySequence = epoch[0].metadata?.historySequence; if (typeof boundarySequence !== "number") throw new Error("boundary lacks a sequence"); - // A user row is covered by the assistant TURN row that answered it and - // carries THAT turn's recorded epoch: an OLDER one for a turn that - // straddled a boundary (u2/a2); none for a turn row WITHOUT a recorded - // policy (an older build's, u4/a4) — unknown, never vouched for; and - // none for an accepted batch no assistant ever answered (u5: stream - // never started or crashed first), whose repo-controlled content - // nobody vetted. + // A user row is covered by the assistant TURN row whose request bound + // anchors on it (the LAST user row at or below the bound) or that lists + // it as a prelude snapshot, and carries THAT turn's recorded epoch: an + // OLDER one for a turn that straddled a boundary (p2/u2/a2); none for a + // turn row WITHOUT a recorded policy (an older build's, u4/a4) — + // unknown, never vouched for; none for an accepted batch no assistant + // ever answered (u5: stream never started or crashed first); and none + // for another backend's row that landed between a turn's anchor and its + // assistant row (ux: the turn never consumed it, so "nearest later + // assistant" would vouch for repo-controlled content nobody vetted). + // Sequences: the boundary sits at boundarySequence, the two first-epoch + // copies at +1/+2, so the rows seeded here start at +3. await seedHistory( - createMuxMessage("u2", "user", "second question"), + createMuxMessage("p2", "user", "prelude snapshot"), + createMuxMessage("u2", "user", "second question", { + requestPreludeMessageIds: ["p2"], + }), createMuxMessage("a2", "assistant", "second answer", { - requestHistorySequence: boundarySequence + 1, + requestHistorySequence: boundarySequence + 4, // anchors on u2 workspaceMemoryPolicyEpoch: -1, }), createMuxMessage("u3", "user", "third question"), + createMuxMessage("ux", "user", "foreign backend's batch"), createMuxMessage("a3", "assistant", "third answer", { - requestHistorySequence: boundarySequence + 3, + requestHistorySequence: boundarySequence + 6, // anchors on u3, not ux workspaceMemoryPolicyEpoch: boundarySequence, }), createMuxMessage("u4", "user", "old-build question"), createMuxMessage("a4", "assistant", "old-build answer", { - requestHistorySequence: boundarySequence + 5, + requestHistorySequence: boundarySequence + 9, // anchors on u4 }), createMuxMessage("u5", "user", "unanswered question"), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) @@ -1930,15 +1939,17 @@ describe("CompactionHandler", () => { expect( secondEpoch.data.slice(1).map((copy) => copy.metadata?.rlmPreservedTailSourcePolicyEpoch) ).toEqual([ - -1, - -1, - -1, - -1, - boundarySequence, - boundarySequence, - undefined, - undefined, - undefined, + -1, // copy(u1) + -1, // copy(a1) + -1, // p2 (prelude of u2) + -1, // u2 + -1, // a2 + boundarySequence, // u3 + undefined, // ux + boundarySequence, // a3 + undefined, // u4 + undefined, // a4 + undefined, // u5 ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 14730735397..ff48ddb21ef 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -21,7 +21,9 @@ import { type CompactionFollowUpRequest, type CompactionSummaryMetadata, type MuxMessage, + isTokenBudgetInternalMessage, } from "@/common/types/message"; +import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { createCompactionSummaryMessageId } from "@/node/services/utils/messageIds"; import type { TelemetryService } from "@/node/services/telemetryService"; import { @@ -1267,15 +1269,19 @@ export class CompactionHandler { } ); const idMap = new Map(params.tail.map((row) => [row.id, createPreservedTailCopyMessageId()])); - const copies = this.buildCoveredTailCopies(params.tail, idMap).map((copy) => { - // Continuous compaction prunes the just-finished answer too. Keep recent pages - // visible below the boundary while retaining RLM's usage/snapshot sanitizer. - copy.metadata = { ...copy.metadata, uiVisible: true }; - // User Esc keeps its interrupted marker and next-send continuation sentinel. - // Only our internal stop has an explicit durable Continue replacing it. - if (params.pendingFollowUp) delete copy.metadata.partial; - return copy; - }); + // Same closing epoch the completion metadata reports below. + const closingPolicyEpoch = latestContextBoundaryHistorySequence(params.messages) ?? -1; + const copies = this.buildCoveredTailCopies(params.tail, idMap, closingPolicyEpoch).map( + (copy) => { + // Continuous compaction prunes the just-finished answer too. Keep recent pages + // visible below the boundary while retaining RLM's usage/snapshot sanitizer. + copy.metadata = { ...copy.metadata, uiVisible: true }; + // User Esc keeps its interrupted marker and next-send continuation sentinel. + // Only our internal stop has an explicit durable Continue replacing it. + if (params.pendingFollowUp) delete copy.metadata.partial; + return copy; + } + ); return { boundary, copies }; } @@ -1502,7 +1508,8 @@ export class CompactionHandler { const preservedTailCopies = this.buildPreservedTailCopies( messages, compactionRequestMessageId, - summaryMessage.id + summaryMessage.id, + previousBoundaryHistorySequence ?? -1 ); const persistenceResult = @@ -1588,7 +1595,8 @@ export class CompactionHandler { private buildPreservedTailCopies( messages: MuxMessage[], compactionRequestMessageId: string, - summaryMessageId: string + summaryMessageId: string, + closingPolicyEpoch: number ): MuxMessage[] { const requestIndex = messages.findIndex((message) => message.id === compactionRequestMessageId); if (requestIndex === -1) { @@ -1628,34 +1636,68 @@ export class CompactionHandler { for (const row of tailRows) { idMap.set(row.id, createPreservedTailCopyMessageId()); } - return this.buildCoveredTailCopies(tailRows, idMap); + return this.buildCoveredTailCopies(tailRows, idMap, closingPolicyEpoch); } /** - * Copies of a whole tail with their policy epochs assigned by COVERAGE. A - * first-time copy's epoch is the one the assistant TURN row that answered - * it recorded its policy under (the nearest later turn row in the tail): - * that turn is what consumed the row's content under a recorded policy. - * A nearest turn row WITHOUT a record (an older build's) leaves the rows - * it answered unknown, and rows after the last turn row — an accepted - * user/prelude batch whose stream never started or crashed before its - * assistant row landed — have no policy at all. Both stay unstamped, which - * the policy sink reads as unknown (deny): stamping them with the closing - * epoch would present repo-controlled input nobody vetted as covered, and - * the next turn could harvest output conditioned on it into the shared - * notebook. Copies of copies keep their original epoch regardless. + * Copies of a whole tail with their policy epochs assigned by request + * COVERAGE — the same batch rule as the harvest gate + * (memoryConsolidationService epochHarvestRefusal). An assistant TURN row + * (it carries `requestHistorySequence`, the last history sequence its + * request was built from) consumed exactly one batch under the policy it + * recorded: the LAST user row at or below that bound plus the rows that + * user row lists in `requestPreludeMessageIds`, matched by exact id. Only + * those user rows carry the turn's recorded epoch; the turn row carries it + * itself. Not "the nearest later assistant": with several backends on one + * chat.jsonl — or a sub-agent report row appended while the parent's + * request was preparing — a user row can land between a turn's anchor and + * its assistant row without that turn ever having seen it, and stamping it + * would present unvetted (model- or repo-controlled) content to the next + * epoch as covered. Everything else stays unstamped — a user row no turn + * covered (an accepted batch whose stream never started or crashed first), + * one answered by a turn without a recorded policy (an older build's) — + * which the policy sink reads as unknown (deny). Token-budget control rows + * (backend template text, no agent or repository content) need no turn and + * belong to the closing epoch, as do non-turn assistant rows (payloads, + * summaries). Copies of copies keep their original epoch regardless. */ - private buildCoveredTailCopies(tailRows: MuxMessage[], idMap: Map): MuxMessage[] { - const copies: MuxMessage[] = new Array(tailRows.length); - let covering: number | undefined; - for (let i = tailRows.length - 1; i >= 0; i--) { - const row = tailRows[i]; - if (row.role === "assistant" && typeof row.metadata?.requestHistorySequence === "number") { - covering = row.metadata.workspaceMemoryPolicyEpoch; + private buildCoveredTailCopies( + tailRows: MuxMessage[], + idMap: Map, + closingPolicyEpoch: number + ): MuxMessage[] { + const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + for (const message of tailRows) { + const sequence = message.metadata?.historySequence; + if (message.role === "user" && typeof sequence === "number") { + userRows.push({ message, sequence }); + } + } + const coveredEpochById = new Map(); + for (const message of tailRows) { + if (message.role !== "assistant") continue; + const bound = message.metadata?.requestHistorySequence; + const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; + if (typeof bound !== "number" || typeof policyEpoch !== "number") continue; + const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; + if (anchor === undefined) continue; + for (const id of [ + anchor.id, + ...getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds), + ]) { + if (!coveredEpochById.has(id)) coveredEpochById.set(id, policyEpoch); } - copies[i] = this.buildPreservedTailCopy(row, idMap, covering); } - return copies; + return tailRows.map((row) => { + const isTurnRow = + row.role === "assistant" && typeof row.metadata?.requestHistorySequence === "number"; + const epoch = isTurnRow + ? row.metadata?.workspaceMemoryPolicyEpoch + : row.role !== "user" || isTokenBudgetInternalMessage(row) + ? closingPolicyEpoch + : coveredEpochById.get(row.id); + return this.buildPreservedTailCopy(row, idMap, epoch); + }); } /** @@ -1695,12 +1737,11 @@ export class CompactionHandler { // of a copy keeps its ORIGINAL epoch (the chain must stay visible to the // policy conjunction; copies from before the field existed carry nothing // forward — no policy record ever existed for their epochs). A first-time - // copy carries the epoch of the assistant TURN row covering it (see - // buildCoveredTailCopies): for a turn row that is its own recorded epoch - // — a turn that started before a destructive reset and landed after the - // new boundary belongs to the OLD epoch, whose policy the new one cannot - // vouch for — and a turn row without a record (an older build's) stays - // unstamped, read as unknown (deny), never vouched for. + // copy carries the epoch buildCoveredTailCopies assigned it: for a turn + // row its own recorded epoch — a turn that started before a destructive + // reset and landed after the new boundary belongs to the OLD epoch, whose + // policy the new one cannot vouch for — and a turn row without a record + // (an older build's) stays unstamped, read as unknown (deny). const sourcePolicyEpoch = source?.rlmPreservedTailCopy === true ? source.rlmPreservedTailSourcePolicyEpoch diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 158f1021471..59c06feea04 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -3341,6 +3341,69 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); }); + it("keeps a still-registered child's original row when its migrated copy is unusable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory (a rename row carries no post-state hash, so + // a later edit beneath the destination is visible ONLY as a row), the + // child edits a file under the destination, then a removal of the child + // aborts after migrating the child's row (the child stays registered). + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + // The copy's inverse is corrupted on disk (the row survives the + // self-healing read; only its inverse no longer parses). + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const lines = (await fsPromises.readFile(journalPath, "utf-8")).split("\n"); + let corrupted = 0; + const rewritten = lines.map((line) => { + if (!line.includes('"migratedFrom":"ws-child:')) return line; + const row = JSON.parse(line) as { data: { inverse: unknown } }; + row.data.inverse = { op: "bogus" }; + corrupted++; + return JSON.stringify(row); + }); + expect(corrupted).toBe(1); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + // Rolling back the owner's rename would move the child's newer content + // along: the child's intact original must still surface as the conflict + // the unusable copy can no longer report. + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: ownerRename.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("Refusing rollback"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }); + it("follows a row rolled back between the two handover passes with its rollback row", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 972e4766779..10f62fd98b2 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1043,6 +1043,27 @@ describe("refinementRollback", () => { "not folded into the shared workspace store" ); await fsPromises.rm(path.join(ownerSessionDir, "memory", "newdir", "owner.md")); + // The same note landing while the rollback waits for the target lock: + // the plan-time directory proof is re-derived under the lock. + const lateExtraFile = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dirRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + testOnlyBeforeTargetLock: async () => { + await fsPromises.writeFile( + path.join(ownerSessionDir, "memory", "newdir", "late.md"), + "l\n" + ); + }, + }); + expect(lateExtraFile.success).toBe(false); + expect(lateExtraFile.success ? "" : lateExtraFile.error).toContain( + "not folded into the shared workspace store" + ); + expect(await pathExists(path.join(ownerSessionDir, "memory", "newdir", "late.md"))).toBe(true); + expect(await pathExists(path.join(ownerSessionDir, "memory", "olddir"))).toBe(false); + await fsPromises.rm(path.join(ownerSessionDir, "memory", "newdir", "late.md")); const undoneRename = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: dirRenameRow.id, diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index f4fa7b82717..bd3dba33d3e 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -591,9 +591,19 @@ async function readSharedMemoryPeerRows( // originals would count as separate later mutations of the very notes the // copies describe — and a retargeted original is `orderUnknown`, so the // copy could never be rolled back until the removal finally succeeds. + // Only a copy that still REPRESENTS its source stands in for it: a copy + // whose inverse is unparseable contributes nothing to divergence + // (collectDivergence skips it), so suppressing its intact original would + // let this rollback overwrite the peer mutation that original records. const migratedHere = new Set(); for (const row of await listRefinements(opts.sessionDir)) { - if (row.data.migratedFrom !== undefined) migratedHere.add(row.data.migratedFrom); + if ( + row.data.migratedFrom !== undefined && + row.data.kind === "memory" && + RefinementInverseSchema.safeParse(row.data.inverse).success + ) { + migratedHere.add(row.data.migratedFrom); + } } const peerRows: RefinementEvent[] = []; for (const peerDir of peerDirs) { @@ -1131,6 +1141,35 @@ export async function rollbackRefinement( } } } + // The remapper's directory proof (an adopted directory maps only while + // the owner's subtree is EXACTLY the adopted descendants) was computed + // before this lock: an owner note added beside the copies while the + // rollback waited would travel along with a legacy directory rename. + // Re-derive the mapping under the lock and require it to be identical; + // a mapping that no longer holds refuses like at plan time. + if (remap !== identityRemapper) { + assert( + opts.sharedWorkspaceMemorySessionDir !== undefined, + "a legacy remapper exists only for a sub-agent sharing its notebook" + ); + let relocked: RefinementInverse; + try { + relocked = ( + await createLegacyPathRemapper({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + }) + ).inverse(parsedInverse.data); + } catch (error) { + if (!(error instanceof LegacyPathNotAdoptedError)) throw error; + throw new RollbackError(`Refusing rollback of '${opts.id}': ${error.message}`); + } + if (JSON.stringify(inversePaths(relocked)) !== JSON.stringify(inversePaths(inverse))) { + throw new RollbackError( + `Refusing rollback of '${opts.id}': the shared-store mapping of its recorded paths changed while waiting for the target lock` + ); + } + } // Re-verify INSIDE the lock, immediately before mutating: a writer that // won the lock first has already landed, and its change must surface as // divergence rather than be overwritten. The journals are re-read here From efc46a87a701267033274cf39d6665f7dd656865 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 07:30:40 +0000 Subject: [PATCH 75/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20sixty-nin?= =?UTF-8?q?th=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The prior-turn detection feeding the unknown-history rule of the workspace-memory write policy exempts token-budget control rows (rollover lead-in, budget warning), like the harvest gate does, so a fresh epoch is no longer denied by backend template text. Extracted as epochHasPriorTurnRows (compactionBoundary.ts) with unit tests. - Preserved-tail copies of an assistant row that recorded its policy epoch keep that stamp even when the request bound is missing or malformed (a malformed stamp stays unstamped); only a row with neither stamp nor bound is a synthetic payload row of the closing epoch. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 53 +++++++++++++++++++ .../utils/messages/compactionBoundary.ts | 31 +++++++++++ src/node/services/compactionHandler.test.ts | 14 +++++ src/node/services/compactionHandler.ts | 38 +++++++++---- src/node/services/turnRequestBuilder.ts | 19 ++----- 5 files changed, 132 insertions(+), 23 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index dc0230a8825..21a732d8dc1 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import { + epochHasPriorTurnRows, findLatestCompactionBoundaryIndex, findLatestContextBoundaryIndex, hasProviderEligibleMessages, @@ -378,3 +379,55 @@ describe("sliceMessagesFromLatestCompactionBoundary", () => { expect(sliced.map((msg) => msg.id)).toEqual(["u0", "summary-malformed", "u1"]); }); }); + +describe("epochHasPriorTurnRows", () => { + const current = new Set(["u-now", "p-now"]); + const withRows = (...rows: Array>) => + epochHasPriorTurnRows( + rows.map((args) => createMuxMessage(...args)), + current + ); + + it("counts an earlier user turn but not the batch being started", () => { + expect(withRows(["u-now", "user", "now"], ["p-now", "user", "prelude"])).toBe(false); + expect(withRows(["u-old", "user", "earlier"], ["u-now", "user", "now"])).toBe(true); + expect(withRows(["a-old", "assistant", "answer"], ["u-now", "user", "now"])).toBe(false); + }); + + it("ignores rows that are no turn of the epoch: compaction requests, tail copies, token-budget internals", () => { + expect( + withRows( + [ + "req", + "user", + "/compact", + { muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} } }, + ], + ["u-now", "user", "now"] + ) + ).toBe(false); + expect( + withRows( + ["copy", "user", "earlier", { rlmPreservedTailCopy: true }], + ["u-now", "user", "now"] + ) + ).toBe(false); + expect( + withRows( + [ + "lead", + "user", + "lead-in", + { muxMetadata: { type: "context-window-lead-in", rolloverId: "r1" } }, + ], + [ + "warn", + "user", + "warning", + { muxMetadata: { type: "context-budget-warning", contextTokens: 1, maxTokens: 2 } }, + ], + ["u-now", "user", "now"] + ) + ).toBe(false); + }); +}); diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 1d391bbad9a..aacf01a6350 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -7,6 +7,7 @@ import { isPositiveInteger } from "@/common/utils/numbers"; import { hasProviderReplayableContent } from "@/common/utils/messages/providerEligibility"; import type { MuxMessage } from "@/common/types/message"; +import { isTokenBudgetInternalMessage } from "@/common/types/message"; export { CONTEXT_BOUNDARY_KINDS }; @@ -194,3 +195,33 @@ export function sliceMessagesForProviderFromLatestContextBoundary( ? messages.slice(boundaryIndex + 1) : messages.slice(boundaryIndex); } + +/** + * Whether the active epoch already holds a turn other than the one being + * started (`currentBatch`: the request's user row plus its prelude snapshot + * ids). Feeds the unknown-history rule of the workspace-memory write policy + * (WorkspaceService.recordWorkspaceMemoryWritable): an epoch with prior turns + * this process never recorded a policy for cannot be vouched for. Not turns: + * compaction request rows (they open an epoch), RLM keep-recent copies (the + * previous epoch's turns re-appended after the boundary; the harvest gate + * skips them too, and counting them would make another backend's first turn + * of the new epoch — racing the compacting backend's asynchronous policy + * carry — record an unknown-history deny for an all-writable epoch), and + * token-budget control rows (rollover lead-in, budget warning: backend + * template text prepended to the turn they precede; the harvest gate exempts + * them for the same reason, and counting one would record a false + * unknown-history deny for a fresh, otherwise writable epoch). + */ +export function epochHasPriorTurnRows( + activeContextMessages: readonly MuxMessage[], + currentBatch: ReadonlySet +): boolean { + return activeContextMessages.some( + (message) => + message.role === "user" && + !currentBatch.has(message.id) && + message.metadata?.muxMetadata?.type !== "compaction-request" && + message.metadata?.rlmPreservedTailCopy !== true && + !isTokenBudgetInternalMessage(message) + ); +} diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 9fef33e9405..71433c34b8e 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1931,6 +1931,17 @@ describe("CompactionHandler", () => { requestHistorySequence: boundarySequence + 9, // anchors on u4 }), createMuxMessage("u5", "user", "unanswered question"), + // A row that recorded its policy under a foreign epoch but lost its + // request bound keeps that stamp (never the closing epoch); a present + // but malformed stamp stays unstamped; a synthetic payload row with + // neither belongs to the closing epoch. + createMuxMessage("a6", "assistant", "foreign-epoch answer without a bound", { + workspaceMemoryPolicyEpoch: -1, + }), + createMuxMessage("a7", "assistant", "corrupted stamp", { + workspaceMemoryPolicyEpoch: null as unknown as number, + }), + createMuxMessage("a8", "assistant", "synthetic payload"), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1950,6 +1961,9 @@ describe("CompactionHandler", () => { undefined, // u4 undefined, // a4 undefined, // u5 + -1, // a6 (recorded stamp kept despite the missing bound) + undefined, // a7 (malformed stamp) + boundarySequence, // a8 (no stamp, no bound: synthetic payload row) ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index ff48ddb21ef..3c28c5d8b6c 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1658,8 +1658,10 @@ export class CompactionHandler { * one answered by a turn without a recorded policy (an older build's) — * which the policy sink reads as unknown (deny). Token-budget control rows * (backend template text, no agent or repository content) need no turn and - * belong to the closing epoch, as do non-turn assistant rows (payloads, - * summaries). Copies of copies keep their original epoch regardless. + * belong to the closing epoch, as do assistant rows that carry no policy + * stamp and no bound (synthetic payloads, summaries). An assistant row + * WITH a recorded stamp keeps it even when its bound is missing or + * malformed. Copies of copies keep their original epoch regardless. */ private buildCoveredTailCopies( tailRows: MuxMessage[], @@ -1689,13 +1691,31 @@ export class CompactionHandler { } } return tailRows.map((row) => { - const isTurnRow = - row.role === "assistant" && typeof row.metadata?.requestHistorySequence === "number"; - const epoch = isTurnRow - ? row.metadata?.workspaceMemoryPolicyEpoch - : row.role !== "user" || isTokenBudgetInternalMessage(row) - ? closingPolicyEpoch - : coveredEpochById.get(row.id); + let epoch: number | undefined; + if (row.role === "assistant") { + // A recorded stamp is kept whatever the bound: a row that recorded + // its policy (possibly a read-only one, under a foreign epoch whose + // deny a destructive reset since discarded) but lost or corrupted its + // bound must not be reclassified as a synthetic non-turn row and + // handed the closing epoch — that would present it to the next epoch + // as current-policy content. History is raw JSON: a stamp that is + // present but not a number stays unstamped (unknown). Only a row + // with no stamp at all is a synthetic payload/summary row (closing + // epoch) — or, with a bound, an older build's turn (unknown). + const stamp: unknown = row.metadata?.workspaceMemoryPolicyEpoch; + epoch = + stamp !== undefined + ? typeof stamp === "number" + ? stamp + : undefined + : typeof row.metadata?.requestHistorySequence === "number" + ? undefined + : closingPolicyEpoch; + } else if (row.role !== "user" || isTokenBudgetInternalMessage(row)) { + epoch = closingPolicyEpoch; + } else { + epoch = coveredEpochById.get(row.id); + } return this.buildPreservedTailCopy(row, idMap, epoch); }); } diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 5d11e90816e..6d583f7b473 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -37,7 +37,10 @@ import type { SendMessageError } from "@/common/types/errors"; import type { GoalRecordV1 } from "@/common/types/goal"; import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/types/message"; import { createMuxMessage } from "@/common/types/message"; -import { latestContextBoundaryHistorySequence } from "@/common/utils/messages/compactionBoundary"; +import { + epochHasPriorTurnRows, + latestContextBoundaryHistorySequence, +} from "@/common/utils/messages/compactionBoundary"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; import { secretsToRecord } from "@/common/types/secrets"; @@ -1521,19 +1524,7 @@ export class TurnRequestBuilder { ...getRequestPreludeMessageIds(latestUserMessage.metadata?.requestPreludeMessageIds), ] ); - // RLM keep-recent copies are the previous epoch's turns re-appended - // after the boundary (compactionHandler), not turns of this epoch: the - // harvest gate skips them too, and counting them would make another - // backend's first turn of the new epoch — racing the compacting - // backend's asynchronous policy carry — record an unknown-history - // deny for an all-writable epoch. - return activeContextMessages.some( - (message) => - message.role === "user" && - !currentBatch.has(message.id) && - message.metadata?.muxMetadata?.type !== "compaction-request" && - message.metadata?.rlmPreservedTailCopy !== true - ); + return epochHasPriorTurnRows(activeContextMessages, currentBatch); })(); // The compaction epoch this turn's policy accumulates over: the latest // durable boundary's history sequence (any kind), -1 before any boundary From c433acf7f3e62c8473ba2627ae906ce2b5c2d03b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 08:06:48 +0000 Subject: [PATCH 76/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventiet?= =?UTF-8?q?h=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Policy accumulator: every carried tail epoch must be represented (a record under its own key, a record under this epoch, or the marker); one recorded grant no longer masks a sibling epoch with no record anywhere. - Preserved-tail copies: an assistant row with no stamp but a present (malformed) request bound stays unstamped; only a row with neither is a synthetic payload row of the closing epoch. - Row migration: an owner-side copy counts as "already copied" only while it is a usable memory row (parseable action + inverse); a retried removal re-copies the intact source instead of deleting the child behind a corrupted record. - Harvest gate: an assistant row carrying a request bound but no policy stamp refuses the harvest outright (its user row may be gone with a reset). - memory view: the post-read tombstone re-check runs after usage recording (the last await, which waits for the owner-store lock a removal holds). - Merged origin/main; the prior-turn helper test now matches main's context-budget-warning shape (CI Static Checks on the merge ref). --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 9 ++- src/node/services/compactionHandler.test.ts | 4 ++ src/node/services/compactionHandler.ts | 13 ++-- .../memoryConsolidationService.test.ts | 39 +++++++++++ .../services/memoryConsolidationService.ts | 30 +++++---- src/node/services/memoryService.test.ts | 64 +++++++++++++++++++ src/node/services/memoryService.ts | 9 ++- .../refinement/sharedMemoryRowMigration.ts | 17 +++-- src/node/services/workspaceService.test.ts | 15 +++++ src/node/services/workspaceService.ts | 20 ++++-- 10 files changed, 189 insertions(+), 31 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index 21a732d8dc1..e56effb1104 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -424,7 +424,14 @@ describe("epochHasPriorTurnRows", () => { "warn", "user", "warning", - { muxMetadata: { type: "context-budget-warning", contextTokens: 1, maxTokens: 2 } }, + { + muxMetadata: { + type: "context-budget-warning", + contextTokens: 1, + maxTokens: 2, + budgetTokens: 2, + }, + }, ], ["u-now", "user", "now"] ) diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 71433c34b8e..3de6d3846d2 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1942,6 +1942,9 @@ describe("CompactionHandler", () => { workspaceMemoryPolicyEpoch: null as unknown as number, }), createMuxMessage("a8", "assistant", "synthetic payload"), + createMuxMessage("a9", "assistant", "old-build turn with a corrupted bound", { + requestHistorySequence: null as unknown as number, + }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1964,6 +1967,7 @@ describe("CompactionHandler", () => { -1, // a6 (recorded stamp kept despite the missing bound) undefined, // a7 (malformed stamp) boundarySequence, // a8 (no stamp, no bound: synthetic payload row) + undefined, // a9 (no stamp, malformed bound: not a synthetic row) ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 3c28c5d8b6c..93e0109fc0c 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -1700,17 +1700,20 @@ export class CompactionHandler { // handed the closing epoch — that would present it to the next epoch // as current-policy content. History is raw JSON: a stamp that is // present but not a number stays unstamped (unknown). Only a row - // with no stamp at all is a synthetic payload/summary row (closing - // epoch) — or, with a bound, an older build's turn (unknown). + // with no stamp AND no bound at all is a synthetic payload/summary + // row (closing epoch); a row with a bound — present in any form, + // malformed included: it may be an older build's turn whose policy + // was never recorded — stays unstamped (unknown). const stamp: unknown = row.metadata?.workspaceMemoryPolicyEpoch; + const bound: unknown = row.metadata?.requestHistorySequence; epoch = stamp !== undefined ? typeof stamp === "number" ? stamp : undefined - : typeof row.metadata?.requestHistorySequence === "number" - ? undefined - : closingPolicyEpoch; + : bound === undefined + ? closingPolicyEpoch + : undefined; } else if (row.role !== "user" || isTokenBudgetInternalMessage(row)) { epoch = closingPolicyEpoch; } else { diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index f9766bc29ab..abc5b457e00 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1466,6 +1466,45 @@ describe("MemoryConsolidationService", () => { expect(record?.error).toContain("never recorded"); }); + it("refuses an unstamped turn row even when no user row of its own survives", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // An older/downgraded backend started a turn before a destructive reset + // and its assistant landed afterwards: the row keeps its request bound + // but has no policy stamp, and its user row is gone with the reset — so + // no uncovered user row would surface it. The row itself must refuse. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("orphan-reply", "assistant", "Produced under an unknown policy.", { + requestHistorySequence: 0, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + expect(fixture.modelCalls).toHaveLength(0); + const record = (await fixture.service.getStatus("ws-dream")).latestHarvestRecord; + expect(record?.status).toBe("failed"); + expect(record?.error).toContain("never recorded"); + }); + it("covers a turn's request prelude rows by id, not by adjacency", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); let previousBoundaryHistorySequence: number | undefined; diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index dac4daaad93..9a50d60e8af 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -352,15 +352,14 @@ class HarvestRefusedError extends Error { * turn's latest user message) and this turn's own row would be left without * a turn of its own — which is exactly what surfaces here as uncovered. Rows * are matched by exact id, never by adjacency, so no interleaved foreign row - * can ride along. Assistant rows without the bound or without the epoch - * stamp (a build that does not maintain the policy — e.g. turns run by a - * downgraded build mid-epoch, which also left the durable accumulator - * untouched; synthetic payload/summary rows that are no turn) cover nothing - * (fail closed); a stamp that is present but malformed refuses like a foreign - * epoch's. Token-budget control rows (rollover lead-in, budget - * warning) need no turn: backend template text appended in the same durable - * batch as the turn they precede, carrying neither agent nor repository - * content. + * can ride along. Assistant rows without the bound and without the epoch + * stamp (synthetic payload/summary rows that are no turn) cover nothing; a + * stamp that is present but malformed refuses like a foreign + * epoch's; a row carrying a bound but no stamp is such a turn with no record + * at all and refuses too (its user row may be gone with a reset). Token-budget + * control rows (rollover lead-in, budget warning) need no turn: backend + * template text appended in the same durable batch as the turn they precede, + * carrying neither agent nor repository content. */ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: number): string | null { const userRows: Array<{ message: MuxMessage; sequence: number }> = []; @@ -373,8 +372,17 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb for (const message of messages) { if (message.role !== "assistant") continue; const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; - // No stamp at all: a row that is no turn of this build (covers nothing). - if (policyEpoch === undefined) continue; + if (policyEpoch === undefined) { + // No stamp: a synthetic payload/summary row (no request bound either) + // covers nothing. A row WITH a bound — present in any form — is a turn + // whose policy was never recorded (an older or downgraded build's, + // possibly started before a destructive reset that removed its user + // row, so no uncovered row would surface it below): refuse. + if (message.metadata?.requestHistorySequence !== undefined) { + return "the compacted epoch holds a turn whose memory policy was never recorded; harvest refused (fail closed)"; + } + continue; + } // History rows are raw JSON: a stamp that is present but not the integer // equal to the closing epoch — another epoch's, or a corrupted value such // as null — proves no policy for this epoch and refuses the harvest. diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 59c06feea04..6d206ce45b4 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1433,6 +1433,25 @@ describe("MemoryService", () => { if (!file.success) expect(file.error).toContain("was removed"); await untombstone(); + // The usage record waits for the owner-store lock, which a removal + // holds while it publishes the tombstone: landing there, after the + // bytes were read, must still withhold them. + const usageService = fixture.service as unknown as { + recordUsage: (...args: unknown[]) => Promise; + }; + const originalUsage = usageService.recordUsage.bind(fixture.service); + const usage = spyOn(usageService, "recordUsage").mockImplementationOnce(async (...args) => { + await originalUsage(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + }); + const lateFile = await fixture.service.view(fixture.ctx, "/memories/workspace/n.md"); + expect(usage).toHaveBeenCalledTimes(1); + expect(lateFile.success).toBe(false); + if (!lateFile.success) expect(lateFile.error).toContain("was removed"); + usage.mockRestore(); + await untombstone(); + tombstoneAfterOpen(); const dir = await fixture.service.view(fixture.ctx, "/memories/workspace"); expect(dir.success).toBe(false); @@ -3341,6 +3360,51 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); }); + it("a retried removal re-copies a row whose earlier owner-side copy is unusable", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const childRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(1); + // The copy's inverse is corrupted on disk before the removal is retried. + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${childRow.id}"`)) return line; + const row = JSON.parse(line) as { data: { inverse: unknown } }; + row.data.inverse = { op: "bogus" }; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + // Not "already copied": the intact source is copied again, and the new + // copy is the one the owner can roll back. + expect(await migrate()).toBe(1); + const copies = (await readRefinementEvents(ownerSessionDir)).filter( + (row) => row.data.migratedFrom === `ws-child:${childRow.id}` + ); + expect(copies).toHaveLength(2); + const usable = copies.find((row) => (row.data.inverse as { op: string }).op !== "bogus")!; + const undo = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: usable.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(undo.success).toBe(true); + expect(await pathExists(path.join(ownerSessionDir, "memory", "n.md"))).toBe(false); + // A third pass sees the usable copy: nothing more to do. + expect(await migrate()).toBe(0); + }); + it("keeps a still-registered child's original row when its migrated copy is unusable", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 73e9fa2af50..b532aa7420b 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -2312,11 +2312,14 @@ export class MemoryService extends EventEmitter { } const content = await this.readBoundedTextFile(store, parsed.relPath, virtualPath); - // Before recordUsage: its refusal is swallowed (usage is best-effort), - // so it cannot stand in for this gate. - await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); const output = renderFileView(content, options); await this.recordUsage(ctx, parsed.scope, parsed.relPath, { write: false }); + // AFTER recordUsage — the last await before the content leaves: a + // read-side usage record waits for the owner-store lock, which a + // removal holds for its handover before publishing the tombstone and + // releasing; recordUsage's own refusal is swallowed (usage is + // best-effort), so it cannot stand in for this gate. + await this.assertWorkspaceReadExposable(ctx, parsed.scope, store); return { success: true, output }; }); } diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index bf3393f340e..fe32a66412f 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -153,12 +153,21 @@ export async function migrateSharedMemoryRefinementRows(args: { // identity on the owner side. await ownerJournal.withBlobLock(async () => { const ownerRows = await listRefinements(args.ownerSessionDir); - // Source identity → owner-journal id of its copy (earlier passes and this one). + // Source identity → owner-journal id of its copy (earlier passes and this + // one). Only a copy that is still a USABLE memory row counts — parseable + // action (memory or rollback) and inverse: a copy whose persisted state + // is corrupted would otherwise make a retried removal skip its intact + // source, delete the child session, and leave the owner with nothing + // but an unusable rollback record. Such a source is copied again (the + // corrupted row stays behind as an audit record). const ownerIdBySource = new Map(); for (const ownerRow of ownerRows) { - if (ownerRow.data.migratedFrom !== undefined) { - ownerIdBySource.set(ownerRow.data.migratedFrom, ownerRow.id); - } + if (ownerRow.data.migratedFrom === undefined || ownerRow.data.kind !== "memory") continue; + const usable = + RefinementInverseSchema.safeParse(ownerRow.data.inverse).success && + (MemoryRefinementActionSchema.safeParse(ownerRow.data.action).success || + RollbackRefinementActionSchema.safeParse(ownerRow.data.action).success); + if (usable) ownerIdBySource.set(ownerRow.data.migratedFrom, ownerRow.id); } // Owner rows already rolled back (by anyone): a second rollback row for // the same target would corrupt the lineage the rollback engine walks. diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index e728dc76a8b..7072a74c60d 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9591,6 +9591,21 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persistedFor(24)).toBe(false); + // A chain of tail compactions names several epochs: one recorded grant + // does not vouch for a sibling epoch with no record anywhere. + await realConfig.editConfig((cfg) => { + const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; + entry.workspaceMemoryWritableByEpoch = { "-1": true }; + return cfg; + }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 26, + carriedPolicyEpochs: [-1, 5], + }) + ).toBe(true); + expect(persistedFor(26)).toBe(false); // A tail copy whose source epoch is unknown (persisted before the field // existed) carries a policy nobody can look up: denied, like unknown // history, even for an otherwise writable first turn. diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 1f560025bf8..c00028dbc55 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4421,16 +4421,22 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { const storedFor = (entry: WorkspaceConfigEntry): boolean | undefined => workspaceMemoryWritableForEpoch(entry, policyEpoch); const stored = storedFor(before.workspace); - // Carried epochs whose policy was never recorded anywhere — no record - // under any carried key, none under this epoch's (where a completed - // carry would have moved it), no marker — are unknown history too: the - // tail copies ARE turns of those epochs (excluded from epochHasPriorTurns - // by design), e.g. the first tail compaction after upgrading a chat. + // A carried epoch whose policy was never recorded anywhere — no record + // under ITS key, none under this epoch's (where a completed carry would + // have moved it), no marker — is unknown history too: the tail copies ARE + // turns of that epoch (excluded from epochHasPriorTurns by design), e.g. + // the first tail compaction after upgrading a chat. EVERY carried epoch + // must be represented: with a chain like [-1, 5], a recorded -1 says + // nothing about 5, and one recorded grant must not mask the epoch whose + // record is missing. (The first turn recording a deny for this reason + // persists it under this epoch's key, so a later turn that finds a + // record here inherits the verdict rather than re-deriving it.) const carriedUnrecorded = - carriedPolicyEpochs.length > 0 && stored === undefined && !denyMarker && - carriedFor(before.workspace) === undefined; + carriedPolicyEpochs.some( + (epoch) => workspaceMemoryWritableForEpoch(before.workspace, epoch) === undefined + ); const unknownHistory = (stored === undefined && mirror === undefined && options.epochHasPriorTurns) || options.carriedPolicyUnknown === true || From 042664b10c909254601e5876a9a505d006a29ec3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 08:59:38 +0000 Subject: [PATCH 77/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-f?= =?UTF-8?q?irst=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption lists dot-entries too (the path grammar admits them, so a downgraded child's `.note` is a real note); unrepresentable stray dot-entries (binary `.DS_Store`) neither move nor hold up removal. - listHotMemories re-checks the removal tombstones once selection (token counting) is done and drops the workspace items when revoked. - Deny marker: a present non-boolean `wildcard` is a malformed record (deny), not "false". - A persisted `sourceTs` outside the store clock's domain (zero, negative, fraction, unsafe integer) is order-unknown in rollback conflict detection, copied as no clock by row migration, and ranked last by the blob resweep (isValidSourceClock). - The post-harvest owner sweep runs as the acting child: refused while the child's removal is cancelled, and its run controller is registered under the child id so the child's removal drain aborts it. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/durableEvent.ts | 12 +++ .../memoryConsolidationService.test.ts | 40 +++++++ .../services/memoryConsolidationService.ts | 59 +++++++--- src/node/services/memoryService.test.ts | 101 +++++++++++++++++- src/node/services/memoryService.ts | 38 +++++-- .../services/refinement/refinementJournal.ts | 8 +- .../services/refinement/refinementRollback.ts | 22 +++- .../refinement/sharedMemoryRowMigration.ts | 8 +- .../services/workspaceMemoryDenyMarker.ts | 4 + src/node/services/workspaceService.test.ts | 15 +++ 10 files changed, 276 insertions(+), 31 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 0b3577c0679..8b50255ea46 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -99,6 +99,9 @@ export const RefinementDataSchema = z.object({ * mutation lock by every mutation (workspaceMemoryRevision.ts), so rows in * an owner's and its sub-agents' journals — whose `ts`/`seq` are not * comparable — still order totally; migrated rows keep their source value. + * Persisted rows are raw JSON: only a value isValidSourceClock accepts is + * order evidence; a present value outside that domain reads as order + * unknown (refinementRollback.ts), never as "earlier than everything". */ sourceTs: z.number().optional(), /** @@ -198,3 +201,12 @@ type DistributiveOmit = T extends unknown ? Omit export type DurableEventDraft = DistributiveOmit & { id?: string; }; + +/** + * A usable shared-store clock value: the clock is `max(Date.now(), prev + 1)` + * (workspaceMemoryRevision.ts), so a genuine value is a positive safe integer. + * Zero, negatives, fractions, unsafe integers and non-numbers are corruption. + */ +export function isValidSourceClock(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value > 0; +} diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index abc5b457e00..cace08b452e 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -462,6 +462,46 @@ describe("MemoryConsolidationService", () => { await fixture.service.cancelInFlightConsolidation("ws-dream"); }); + it("a sub-agent's removal drain aborts the owner-keyed run made on its behalf", async () => { + // A child's trigger (and the post-harvest sweep) runs the OWNER's + // consolidation; a cancelled child harvest deliberately falls through to + // that sweep. The child's removal drain must still reach the owner-keyed + // run, or it would keep mutating the shared notebook after removal. + let streamStarted!: () => void; + const started = new Promise((resolve) => (streamStarted = resolve)); + using fixture = await createFixture({ + modelFactory: () => + new MockLanguageModelV3({ + doStream: (options) => { + streamStarted(); + return Promise.resolve({ + stream: new ReadableStream({ + start(controller) { + options.abortSignal?.addEventListener("abort", () => { + controller.error(new Error("request aborted")); + }); + }, + }), + }); + }, + }), + }); + await fixture.addWorkspace("ws-sub", { parentWorkspaceId: "ws-dream" }); + const run = fixture.service.maybeRun("ws-sub", "manual"); + await started; + await fixture.service.cancelInFlightConsolidation("ws-sub"); + const result = await run; + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("stream failed"); + // Locally cancelled child: neither its own trigger nor an owner run made + // on its behalf may start while teardown is under way. + const refused = await fixture.service.maybeRun("ws-dream", "manual", { + actingWorkspaceId: "ws-sub", + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("being removed"); + }); + it("runs, persists the journal record, and reports it via getRecord", async () => { using fixture = await createFixture(); const result = await fixture.service.maybeRun("ws-dream", "compaction"); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 9a50d60e8af..d832e37701e 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -711,26 +711,48 @@ export class MemoryConsolidationService extends EventEmitter { return Effect.promise(() => isWorkspaceRemovalTombstoned(this.config.rootDir, workspaceId)); } - /** Register one run's removal controller; disposed when the run settles. */ - private trackRunController(workspaceId: string): { + /** + * Register one run's removal controller; disposed when the run settles. + * A run made on a sub-agent's behalf over the OWNER's store (a redirected + * trigger, the post-harvest sweep) is registered under the acting child's + * id too: the child's removal drain (cancelInFlightConsolidation) must abort + * it — a cancelled child harvest deliberately falls through to this sweep, + * which could otherwise keep mutating the shared notebook for its whole + * run after the removal's bounded drain returned. + */ + private trackRunController( + workspaceId: string, + actingWorkspaceId?: string + ): { controller: AbortController; dispose: () => void; } { const controller = new AbortController(); + const ids = [ + workspaceId, + ...(actingWorkspaceId !== undefined && actingWorkspaceId !== workspaceId + ? [actingWorkspaceId] + : []), + ]; // r61: a run registered after teardown began (follow-on sweep/recovery // racing the entry checks) must start already aborted. - if (this.removalCancelled.has(workspaceId)) { + if (ids.some((id) => this.removalCancelled.has(id))) { controller.abort(); } - const set = this.runControllers.get(workspaceId) ?? new Set(); - set.add(controller); - this.runControllers.set(workspaceId, set); + const sets = ids.map((id) => { + const set = this.runControllers.get(id) ?? new Set(); + set.add(controller); + this.runControllers.set(id, set); + return [id, set] as const; + }); return { controller, dispose: () => { - set.delete(controller); - if (set.size === 0 && this.runControllers.get(workspaceId) === set) { - this.runControllers.delete(workspaceId); + for (const [id, set] of sets) { + set.delete(controller); + if (set.size === 0 && this.runControllers.get(id) === set) { + this.runControllers.delete(id); + } } }, }; @@ -927,7 +949,11 @@ export class MemoryConsolidationService extends EventEmitter { // cancelInFlightConsolidation(child) marks only the child id, and an // owner-keyed run reserved after that would neither be refused by the // owner's check below nor be cancellable by the child's removal drain. - if (this.removalCancelled.has(workspaceId)) { + if ( + this.removalCancelled.has(workspaceId) || + (options.actingWorkspaceId !== undefined && + this.removalCancelled.has(options.actingWorkspaceId)) + ) { return Err("workspace is being removed; consolidation refused"); } const ownerWorkspaceId = this.memoryService.resolveWorkspaceMemoryOwnerId(workspaceId); @@ -961,7 +987,7 @@ export class MemoryConsolidationService extends EventEmitter { // would otherwise also pass the check and start a second concurrent run // over the same directories. runPromise starts the fiber synchronously, // and the map is populated before this frame yields either way. - const removal = this.trackRunController(workspaceId); + const removal = this.trackRunController(workspaceId, options.actingWorkspaceId); const run = Effect.runPromise( this.runLockedEffect(workspaceId, trigger, options, removal.controller.signal) ); @@ -1278,10 +1304,13 @@ export class MemoryConsolidationService extends EventEmitter { } // A sub-agent's inbox lives in the owner's store: wait on and run the - // OWNER's consolidation (see maybeRun) so the harvest is actually swept. + // OWNER's consolidation (see maybeRun) so the harvest is actually swept — + // still as the acting CHILD, so the child's in-process cancellation, + // durable tombstone and removal drain bind the owner-keyed run too. return yield* Effect.promise(() => self.runCompactionSweepAfterHarvest( - self.memoryService.resolveWorkspaceMemoryOwnerId(metadata.workspaceId) + self.memoryService.resolveWorkspaceMemoryOwnerId(metadata.workspaceId), + metadata.workspaceId ) ); }); @@ -1411,7 +1440,8 @@ export class MemoryConsolidationService extends EventEmitter { } private async runCompactionSweepAfterHarvest( - workspaceId: string + workspaceId: string, + actingWorkspaceId: string ): Promise> { for (;;) { const active = this.inFlight.get(workspaceId); @@ -1424,6 +1454,7 @@ export class MemoryConsolidationService extends EventEmitter { const result = await this.maybeRun(workspaceId, trigger, { skipWorkspaceDebounce: true, skipHarvestRecovery: true, + ...(actingWorkspaceId !== workspaceId ? { actingWorkspaceId } : {}), }); if (!result.success && result.error === "a consolidation run is already in flight") { continue; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 6d206ce45b4..3a34b4b3470 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1504,6 +1504,22 @@ describe("MemoryService", () => { listIndex.mockRestore(); await untombstone(); } + // Selection keeps awaiting token counts after the file reads: a + // tombstone landing there still withholds the workspace items. + let counted = 0; + const hotAfterCount = await fixture.service.listHotMemories(fixture.ctx, { + countTokens: async (text) => { + if (counted++ === 0) { + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + } + return text.length; + }, + }); + expect(counted).toBeGreaterThan(0); + expect(hotAfterCount.some((item) => item.path === "/memories/workspace/n.md")).toBe(false); + expect(hotAfterCount.some((item) => item.path === "/memories/global/g.md")).toBe(true); + await untombstone(); }); it("refuses a pin toggle once the owner it was bound to is tombstoned", async () => { @@ -1752,6 +1768,33 @@ describe("MemoryService", () => { ]); }); + it("adopts addressable dot-entry notes and ignores unrepresentable stray dot-entries", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + // The path grammar admits dotfiles, so a downgraded child may hold a + // real note at `.note` that no listing ever showed; a stray binary + // dot-entry (Finder's `.DS_Store`) was never a note on any build. + await fsPromises.mkdir(path.join(legacyRoot, ".hidden"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, ".note"), "dot note"); + await fsPromises.writeFile(path.join(legacyRoot, ".hidden", "n.md"), "nested dot note"); + await fsPromises.writeFile( + path.join(legacyRoot, ".DS_Store"), + Buffer.from([0, 0, 1, 255, 254]) + ); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect(await fsPromises.readFile(path.join(ownerRoot, ".note"), "utf-8")).toBe("dot note"); + expect(await fsPromises.readFile(path.join(ownerRoot, ".hidden", "n.md"), "utf-8")).toBe( + "nested dot note" + ); + expect(await pathExists(path.join(ownerRoot, ".DS_Store"))).toBe(false); + // Still addressable through the shared store, like on the old build. + const viewed = await fixture.service.view(fixture.ctx, "/memories/workspace/.note"); + expect(viewed.success).toBe(true); + if (viewed.success) expect(viewed.output).toContain("dot note"); + }); + it("adopts the legacy notebook for removal without any prior access, and throws instead of deferring", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2409,8 +2452,14 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( "owner note" ); - // Nothing listed in the legacy root: the real manifest was never written. - expect(await pathExists(legacyAdoptionManifestPath(path.dirname(legacyRoot)))).toBe(false); + // The planted file is just an (addressable) dot-entry note of the child: + // adopted as such, never read as provenance — the real manifest records + // that note and knows nothing of `note.md`. + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + expect(Object.keys(manifest)).toEqual([".adopted-into-shared-store.json"]); + expect(manifest[".adopted-into-shared-store.json"]).toMatchObject({ created: true }); }); it("recovers an interrupted in-place replacement without duplicating the note", async () => { @@ -3405,6 +3454,54 @@ describe("MemoryService", () => { expect(await migrate()).toBe(0); }); + it("treats a malformed store clock as order-unknown instead of 'earlier than everything'", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + // Owner renames a directory; the child then edits a file beneath the + // destination. The child row's clock is corrupted to -1 on disk: trusted, + // it would sort BEFORE the rename and the rollback would move the + // child's newer content silently. + await fixture.service.create(ownerCtx, "/memories/workspace/notes/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/notes", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + const journalPath = path.join(childSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes('"sourceTs"')) return line; + const row = JSON.parse(line) as { data: { sourceTs: number } }; + row.data.sourceTs = -1; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + id: ownerRename.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("Refusing rollback"); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }); + it("keeps a still-registered child's original row when its migrated copy is unusable", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index b532aa7420b..4f4a6ad6b6e 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -384,8 +384,12 @@ interface MemoryStore { * unreadable directory lists as empty; the walk stops past the per-scope * cap); `strict` throws on any traversal failure and is unbounded, for * callers whose decision must not rest on a possibly partial listing. + * Dot-entries are omitted unless `includeDotfiles`: listings and the index + * hide them, but the path grammar admits them, so a note such as `.note` + * is addressable — the legacy adoption pass must see it or removal would + * delete the only copy. */ - listFiles(options?: { strict?: boolean }): Promise; + listFiles(options?: { strict?: boolean; includeDotfiles?: boolean }): Promise; /** * Kind of an entry, null when absent. Tolerant by default (any stat failure * reads as absent); `strict` throws unless the absence is proven (ENOENT / @@ -451,7 +455,9 @@ async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Pr .stat(legacyRoot) .then((stat) => String(stat.mtimeMs)) .catch(() => "missing"); - const files = await new LocalMemoryStore(legacyRoot).listFiles().catch(() => []); + const files = await new LocalMemoryStore(legacyRoot) + .listFiles({ includeDotfiles: true }) + .catch(() => []); const fileStamps = await Promise.all( files.map(async (relPath) => { const stamp = await fsPromises @@ -529,7 +535,7 @@ class LocalMemoryStore implements MemoryStore { await fsPromises.mkdir(this.physicalRoot, { recursive: true }); } - async listFiles(options?: { strict?: boolean }): Promise { + async listFiles(options?: { strict?: boolean; includeDotfiles?: boolean }): Promise { const results: string[] = []; const walk = async (dirRel: string): Promise => { // Bounded walk: files may have been edited outside MemoryService. +1 lets @@ -562,7 +568,7 @@ class LocalMemoryStore implements MemoryStore { for (const entry of entries) { // Per-entry cap: a single flat directory can exceed the cap on its own. if (options?.strict !== true && results.length > MEMORY_MAX_FILES_PER_SCOPE) return; - if (entry.name.startsWith(".")) continue; + if (options?.includeDotfiles !== true && entry.name.startsWith(".")) continue; const childRel = dirRel === "" ? entry.name : `${dirRel}/${entry.name}`; if (entry.isDirectory()) { await walk(childRel); @@ -1315,7 +1321,9 @@ export class MemoryService extends EventEmitter { // to adopt" (skipped stays 0) and removal would then delete its only // copy. A traversal failure fails the pass instead (access-time: // retried on the next access; removal: aborted, session intact). - const files = await legacy.listFiles({ strict: true }); + // Dot-entries included: no listing shows them, but the path grammar + // admits them, so `.note` may be a real note of the downgraded child. + const files = await legacy.listFiles({ strict: true, includeDotfiles: true }); // What was already folded in, kept in the child's session dir OUTSIDE // the legacy root (which is a downgraded build's model-writable // namespace; see legacyAdoptionManifestPath): per relPath the content @@ -1356,6 +1364,10 @@ export class MemoryService extends EventEmitter { .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) .catch(() => null); if (content === null || content.includes("\uFFFD")) { + // An unrepresentable dot-entry (a stray `.DS_Store`, a binary) was + // never a note on any build — no listing showed it — so it neither + // moves nor holds up removal; an unrepresentable LISTED note does. + if (relPath.split("/").some((segment) => segment.startsWith("."))) continue; skipped++; continue; } @@ -3082,7 +3094,7 @@ export class MemoryService extends EventEmitter { lastAccessedAt: stats?.lastAccessedAt ?? null, }; }); - return selectHotMemories({ + const selected = await selectHotMemories({ candidates, countTokens: options.countTokens, tokenBudgetActive: options.tokenBudgetActive, @@ -3103,6 +3115,20 @@ export class MemoryService extends EventEmitter { return content; }, }); + // Selection keeps awaiting (token counting, repeatedly) after the last + // per-file gate: a tombstone published meanwhile must still withhold the + // buffered owner notes. Final check once selection is done; the workspace + // items are dropped (the scope reads as unavailable, like in the index). + const isWorkspaceItem = (item: MemoryHotSetItem): boolean => + parseMemoryPath(item.path).scope === "workspace"; + if (selected.some(isWorkspaceItem)) { + try { + await this.assertWorkspaceStoreReadable(ctx, this.getStore(ctx, "workspace")); + } catch { + return selected.filter((item) => !isWorkspaceItem(item)); + } + } + return selected; } } diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index 1b47b09e88e..ffedca07063 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -18,6 +18,7 @@ import { createHash } from "node:crypto"; import assert from "@/common/utils/assert"; +import { isValidSourceClock } from "@/common/types/durableEvent"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES, REFINEMENT_INVERSE_QUOTA_MIN_CHARGE_BYTES, @@ -253,8 +254,11 @@ export async function reclaimExcessRefinementInverseBlobs( ts: number; data: { sourceTs?: number; migratedFrom?: string }; }): number => - event.data.sourceTs ?? - (event.data.migratedFrom !== undefined ? Number.NEGATIVE_INFINITY : event.ts); + isValidSourceClock(event.data.sourceTs) + ? event.data.sourceTs + : event.data.migratedFrom !== undefined + ? Number.NEGATIVE_INFINITY + : event.ts; const events = (await journal.read()) .filter((event) => event.kind === "refinement") .sort((left, right) => { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index bd3dba33d3e..8bce84ad47c 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -29,7 +29,7 @@ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; -import type { DurableEvent } from "@/common/types/durableEvent"; +import { isValidSourceClock, type DurableEvent } from "@/common/types/durableEvent"; import { MemoryRefinementActionSchema, RefinementInverseSchema, @@ -535,16 +535,27 @@ interface InverseContentReader { * to the owner's own rows. Same-instant ties fall back to append sequence. */ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { - const rowTs = row.data.sourceTs ?? row.ts; - const otherTs = other.data.sourceTs ?? other.ts; + const rowTs = isValidSourceClock(row.data.sourceTs) ? row.data.sourceTs : row.ts; + const otherTs = isValidSourceClock(other.data.sourceTs) ? other.data.sourceTs : other.ts; return rowTs > otherTs || (rowTs === otherTs && row.seq > other.seq); } +/** + * A persisted `sourceTs` outside the clock's domain (zero, negative, a + * fraction, an unsafe integer): corruption, never order evidence — trusted, + * a later mutation corrupted to `-1` would sort before an older rename target + * and its file would move silently. + */ +function hasMalformedSourceClock(row: RefinementEvent): boolean { + return row.data.sourceTs !== undefined && !isValidSourceClock(row.data.sourceTs); +} + /** * Whether the order of two rows cannot be established: a row journaled while * the shared store's clock write failed (`orderUnknown`) has only - * journal-local `ts`/`seq`, incomparable with other journals' rows. Callers - * fail closed — such a pair conflicts in either direction (force overrides). + * journal-local `ts`/`seq`, incomparable with other journals' rows — as does + * a row whose clock value is malformed. Callers fail closed — such a pair + * conflicts in either direction (force overrides). */ function orderUnknown( row: RefinementEvent, @@ -552,6 +563,7 @@ function orderUnknown( targetRetargeted: boolean ): boolean { if (row.data.orderUnknown === true || target.data.orderUnknown === true) return true; + if (hasMalformedSourceClock(row) || hasMalformedSourceClock(target)) return true; // A retargeted target (see wasRetargeted) vs. a row of another journal. return targetRetargeted && row.workspaceId !== target.workspaceId; } diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index fe32a66412f..5473bc79e85 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -1,5 +1,6 @@ import * as path from "node:path"; import assert from "@/common/utils/assert"; +import { isValidSourceClock } from "@/common/types/durableEvent"; import { MemoryRefinementActionSchema, RefinementEvidenceSchema, @@ -286,8 +287,11 @@ export async function migrateSharedMemoryRefinementRows(args: { // failed) has only a journal-local `ts`, incomparable with the // owner's clock-stamped rows: carried as order-unknown rather than // dressed up as a clock value. - ...(row.data.sourceTs !== undefined && !retargeted ? { sourceTs: row.data.sourceTs } : {}), - ...(row.data.orderUnknown === true || row.data.sourceTs === undefined || retargeted + ...(isValidSourceClock(row.data.sourceTs) && !retargeted + ? { sourceTs: row.data.sourceTs } + : {}), + // A malformed clock value is copied as no clock at all (order unknown). + ...(row.data.orderUnknown === true || !isValidSourceClock(row.data.sourceTs) || retargeted ? { orderUnknown: true as const } : {}), ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 06626db06c1..a1b261832bc 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -112,6 +112,10 @@ async function readMarkerRecord( ) { return null; } + // `wildcard` may be omitted (false) but, when present, must be a boolean: + // a corrupted `"true"` read as false would drop the only surviving deny + // of an epoch the `epochs` list does not name. + if (wildcard !== undefined && typeof wildcard !== "boolean") return null; return { epochs, wildcard: wildcard === true }; } catch { return null; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 7072a74c60d..64390e8ee02 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9452,6 +9452,21 @@ describe("WorkspaceService initialize", () => { await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); + // A present but non-boolean wildcard is corruption, not "false": read as + // malformed (deny for every epoch) rather than dropping the only + // surviving deny of an epoch the list does not name. + await fsPromises.writeFile( + workspaceMemoryDenyMarkerPath(sessionDir), + JSON.stringify({ epochs: [3], wildcard: "true" }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); + await fsPromises.writeFile( + workspaceMemoryDenyMarkerPath(sessionDir), + JSON.stringify({ epochs: [3] }) + ); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 3 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); // Unreadable is not malformed: a marker that cannot be read may hold a // newer epoch's deny, so the fenced clear refuses instead of deleting it. await writeWorkspaceMemoryDenyMarker(sessionDir, 9); From 85c56002a8a4cb87b6612d32833e09bde6e62d6b Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 09:31:55 +0000 Subject: [PATCH 78/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-s?= =?UTF-8?q?econd=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - workspaceMemoryRevision re-checks the acting workspace's and owner's removal tombstones after the token's reads and reports "revoked" for a tombstone published while they were in flight, so a cached memory context cannot keep serving owner notes to a removed child. - Legacy adoption records the owner file's generation (`targetStamp`: ino:size:mtimeNs right after this adoption's write) and deletion reconciliation requires that generation, not just the adopted bytes: an owner note deleted and recreated (or edited and restored) to identical bytes is the owner's and is preserved (tombstoned as owner-owned). The far side of an interrupted in-place replacement (bytes equal to the pending replacement hash) is still recognized as this adoption's write. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 25 ++++++++++ src/node/services/memoryService.test.ts | 58 +++++++++++++++++++++++ src/node/services/memoryService.ts | 58 ++++++++++++++++++++--- 3 files changed, 135 insertions(+), 6 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index f183014b7a8..8d8e39a0644 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -44,6 +44,16 @@ export interface LegacyAdoptionRecord { target: string; created?: boolean; pending?: boolean; + /** + * Identity of the owner file this adoption wrote (`ino:size:mtimeNs` right + * after the write). Deletion reconciliation requires the copy to be THIS + * generation of the file, not merely to hold the adopted bytes: an owner + * who deleted and recreated (or edited and restored) the note to identical + * bytes owns the new file, and a byte match alone would let a downgraded + * child's source deletion remove it. Absent (write before stamping, or the + * stamp could not be taken): never unchanged — the copy is preserved. + */ + targetStamp?: string; /** * Hash of the bytes an in-place replacement is about to write (set on the * pending prior record, cleared once the pass completes). With `content` @@ -95,6 +105,7 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null if (record.replacementContent !== undefined && typeof record.replacementContent !== "string") { return null; } + if (record.targetStamp !== undefined && typeof record.targetStamp !== "string") return null; const flag = (raw: unknown, malformed: boolean): boolean | undefined => raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; return { @@ -106,6 +117,7 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null pendingDeletion: flag(record.pendingDeletion, true), deleted: flag(record.deleted, false), replacementContent: record.replacementContent, + targetStamp: record.targetStamp, }; } @@ -163,6 +175,19 @@ export async function readLegacyAdoptionManifest( return new Map(entries); } +/** + * The file identity a LegacyAdoptionRecord.targetStamp records; null when the + * file cannot be stat'ed (the record then carries no stamp: preserved). + */ +export async function adoptionTargetStamp(absPath: string): Promise { + try { + const stat = await fsPromises.lstat(absPath, { bigint: true }); + return `${stat.ino}:${stat.size}:${stat.mtimeNs}`; + } catch { + return null; + } +} + /** * Every non-directory entry (files, symlinks, anything) under `absDir`, * recursively, as relPaths prefixed with `dirRel`; empty when the directory diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 3a34b4b3470..6cfb7b20a2c 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1386,6 +1386,36 @@ describe("MemoryService", () => { ).toBe(false); }); + it("reports revocation for a tombstone published while the revision token was being built", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "shared", "agent"); + const before = await fixture.service.workspaceMemoryRevision("ws-child"); + expect(before).not.toBe("revoked"); + // Removal lands between the entry check and the token's reads: the + // unchanged pre-removal token would let a cached context keep serving + // the owner's notes to the removed child's next request. + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + const service = fixture.service as unknown as { + buildWorkspaceMemoryRevisionToken: (...args: unknown[]) => Promise; + }; + const original = service.buildWorkspaceMemoryRevisionToken.bind(fixture.service); + const build = spyOn(service, "buildWorkspaceMemoryRevisionToken").mockImplementationOnce( + async (...args) => { + const token = await original(...args); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + return token; + } + ); + try { + expect(await fixture.service.workspaceMemoryRevision("ws-child")).toBe("revoked"); + expect(build).toHaveBeenCalledTimes(1); + } finally { + build.mockRestore(); + } + }); + it("refuses a read whose workspace was tombstoned while the legacy adoption pass ran", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2462,6 +2492,34 @@ describe("MemoryService", () => { expect(manifest[".adopted-into-shared-store.json"]).toMatchObject({ created: true }); }); + it("preserves an owner note recreated with the adopted bytes when the legacy source is deleted", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + // ABA on the owner side: the owner deletes the adopted copy and later + // writes a note of its own at the same path with the same bytes (or + // edits and restores it). The bytes match the record; the file is not + // this adoption's copy any more. + await fixture.service.deletePath(ownerCtx, "/memories/workspace/note.md", "agent"); + await fixture.service.create(ownerCtx, "/memories/workspace/note.md", "v1", "agent"); + // The downgraded child then deletes its source. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v1"); + const manifest = JSON.parse( + await fsPromises.readFile(legacyAdoptionManifestPath(path.dirname(legacyRoot)), "utf-8") + ) as Record; + // Tombstoned as owner-owned: no destructive provenance survives. + expect(manifest["note.md"]).toMatchObject({ deleted: true, created: false }); + }); + it("recovers an interrupted in-place replacement without duplicating the note", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 4f4a6ad6b6e..a0a932a5d0b 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -49,6 +49,7 @@ import { } from "@/node/services/refinement/targetMutationLocks"; import { memoryLogicalKey, type MemoryMetaService } from "@/node/services/memoryMeta"; import { + adoptionTargetStamp, legacyAdoptionManifestPath, readLegacyAdoptionManifest, type LegacyAdoptionRecord, @@ -1424,6 +1425,17 @@ export class MemoryService extends EventEmitter { if (priorContent === content) { target = { relPath: previous.target, write: false }; record.created = previous.created === true; + // A pending record is a copy this adoption wrote but could not + // finish recording (crash or sidecar failure after the write): + // the file holding exactly those bytes now is that write, so its + // current identity is the generation to bind. A settled record + // keeps the stamp it recorded — identical bytes in a different + // generation are the owner's (deletion then preserves). + record.targetStamp = + previous.targetStamp ?? + (previous.pending === true + ? ((await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined) + : undefined); } else if ( previous.created === true && priorContent !== null && @@ -1481,6 +1493,9 @@ export class MemoryService extends EventEmitter { if (target.replaces !== true) remainingCapacity--; imported++; record.created = true; + // The generation of the file just written (see targetStamp). + record.targetStamp = + (await adoptionTargetStamp(store.physicalPath(target.relPath))) ?? undefined; } record.target = target.relPath; // Pins/stats were keyed by the child: fold them into the owner key. @@ -1606,10 +1621,24 @@ export class MemoryService extends EventEmitter { } } } - // Either side of an interrupted in-place replacement counts as ours. + // Ours only while it is THIS generation of the file (targetStamp, + // taken right after this adoption's write): identical bytes in a + // file the owner deleted and recreated, or edited and restored, are + // the owner's, and a record without a stamp preserves. The one + // exception is the far side of an interrupted in-place replacement: + // the pending record names the bytes about to be written, and a + // file holding exactly those (never before in the store) is that + // write, whose stamp the crash kept from being recorded. + const currentHash = current === null ? null : sha256Hex(current); const unchanged = - current !== null && - [previous.content, previous.replacementContent].includes(sha256Hex(current)); + currentHash !== null && + ((previous.pending === true && + previous.replacementContent !== undefined && + currentHash === previous.replacementContent) || + (currentHash === previous.content && + previous.targetStamp !== undefined && + (await adoptionTargetStamp(store.physicalPath(previous.target))) === + previous.targetStamp)); // A listed note may now point at this very target (the downgraded // build renamed `a.md` to the path its conflict copy was adopted // under, and the new record reused the identical file): the target @@ -1632,6 +1661,7 @@ export class MemoryService extends EventEmitter { // successor keeps its own (non-created) provenance. if (unchanged && successor[1].created !== true) { successor[1].created = true; + successor[1].targetStamp = previous.targetStamp; manifestDirty = true; } } else if (unchanged) { @@ -2132,9 +2162,25 @@ export class MemoryService extends EventEmitter { // Access revoked (acting workspace or owner tombstoned by any backend): // a distinct token so a cached context built from the owner's notes is // invalidated and the rebuild (listIndexEntries) then excludes the store. - for (const guarded of new Set([workspaceId, owner])) { - if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return "revoked"; - } + const revoked = async (): Promise => { + for (const guarded of new Set([workspaceId, owner])) { + if (await isWorkspaceRemovalTombstoned(this.config.rootDir, guarded)) return true; + } + return false; + }; + if (await revoked()) return "revoked"; + const token = await this.buildWorkspaceMemoryRevisionToken(workspaceId, owner); + // Re-checked AFTER the token's reads: a tombstone published while they + // were in flight is not part of the token, so the unchanged pre-removal + // token would let a cached context keep serving the owner's notes to the + // removed workspace's next request. + return (await revoked()) ? "revoked" : token; + } + + private async buildWorkspaceMemoryRevisionToken( + workspaceId: string, + owner: string + ): Promise { const ownerSessionDir = path.join(this.config.sessionsDir, owner); const revision = await readWorkspaceMemoryRevision(ownerSessionDir); // The clock is what this build's writers advance. A downgraded build From d0865825309c083d89104acd719217533b17384d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 10:36:13 +0000 Subject: [PATCH 79/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-t?= =?UTF-8?q?hird=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Legacy adoption: an unrepresentable dot-entry counts as skipped like any listed note (`.note` is addressable; a stray `.DS_Store` costs a forced removal instead of a silently complete handover). - Refinement rows carry `originJournal`/`originSeq`: migrated copies of one child order by their source position, so the owner can unwind the child's own overlapping history LIFO without force; copies without provenance stay order-unknown. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/durableEvent.ts | 45 ++++ src/node/services/memoryService.test.ts | 215 +++++++++++++++++- src/node/services/memoryService.ts | 11 +- .../services/refinement/refinementJournal.ts | 9 + .../services/refinement/refinementRollback.ts | 34 ++- .../refinement/sharedMemoryRowMigration.ts | 9 +- 6 files changed, 311 insertions(+), 12 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 8b50255ea46..2771f0242bd 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -93,6 +93,20 @@ export const RefinementDataSchema = z.object({ * (sharedMemoryRowMigration.ts); lets a retried migration skip it. */ migratedFrom: z.string().optional(), + /** + * Where a migrated row was ORIGINALLY appended: the journal (workspace id) + * whose append sequence positions it, and its `seq` there. Two rows of one + * origin were serialized by that store's mutation lock (clock + append run + * inside it), so their origin sequence IS their mutation order — even when + * neither carries a usable clock value (pre-sharing history, a failed clock + * write). Copies are appended to the owner journal later than they happened + * and in migration order, so their own `seq` is no order evidence; a + * copy-of-copy keeps the first origin. Absent on a copy (older builds, + * corruption) = order unknown, never the copying journal's position. + * Native rows need no fields: their origin is (`workspaceId`, `seq`). + */ + originJournal: z.string().optional(), + originSeq: z.number().optional(), /** * Cross-session order key for rollback conflict detection (`sourceTs ?? ts`): * a shared workspace store's monotonic clock, advanced under the store's @@ -210,3 +224,34 @@ export type DurableEventDraft = DistributiveOmit 0; } + +/** Where a refinement row was originally appended (see `originJournal`). */ +export interface RefinementRowOrigin { + journal: string; + seq: number; +} + +/** + * The journal position that orders a refinement row against rows of the same + * origin (RefinementDataSchema.originJournal). A native row is positioned by + * its own journal; a migrated copy only by a carried, well-formed origin — + * anything else about a copy is no order evidence (null). + */ +export function refinementRowOrigin(row: { + workspaceId: string; + seq: number; + data: { migratedFrom?: string; originJournal?: unknown; originSeq?: unknown }; +}): RefinementRowOrigin | null { + if (row.data.migratedFrom === undefined) return { journal: row.workspaceId, seq: row.seq }; + const { originJournal, originSeq } = row.data; + if ( + typeof originJournal !== "string" || + originJournal === "" || + typeof originSeq !== "number" || + !Number.isSafeInteger(originSeq) || + originSeq < 0 + ) { + return null; + } + return { journal: originJournal, seq: originSeq }; +} diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 6cfb7b20a2c..2a7bd28cd1b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1798,14 +1798,16 @@ describe("MemoryService", () => { ]); }); - it("adopts addressable dot-entry notes and ignores unrepresentable stray dot-entries", async () => { + it("adopts addressable dot-entry notes and refuses the handover over unrepresentable ones", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); // The path grammar admits dotfiles, so a downgraded child may hold a - // real note at `.note` that no listing ever showed; a stray binary - // dot-entry (Finder's `.DS_Store`) was never a note on any build. + // real note at `.note` that no listing ever showed — including one + // whose text `create` accepted but the lossy-decode gate cannot vouch + // for. Such an entry must hold up removal like a listed note would + // (r73), not be written off as a stray `.DS_Store`. await fsPromises.mkdir(path.join(legacyRoot, ".hidden"), { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, ".note"), "dot note"); await fsPromises.writeFile(path.join(legacyRoot, ".hidden", "n.md"), "nested dot note"); @@ -1813,12 +1815,20 @@ describe("MemoryService", () => { path.join(legacyRoot, ".DS_Store"), Buffer.from([0, 0, 1, 255, 254]) ); - await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); + expect( + await fixture.service + .adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner") + .then(() => null, getErrorMessage) + ).toMatch(/1 legacy workspace memory note\(s\)/); + // The representable dot-entries were folded in by that same pass. expect(await fsPromises.readFile(path.join(ownerRoot, ".note"), "utf-8")).toBe("dot note"); expect(await fsPromises.readFile(path.join(ownerRoot, ".hidden", "n.md"), "utf-8")).toBe( "nested dot note" ); expect(await pathExists(path.join(ownerRoot, ".DS_Store"))).toBe(false); + // Removing the stray entry lets a retried (non-forced) handover complete. + await fsPromises.rm(path.join(legacyRoot, ".DS_Store")); + await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); // Still addressable through the shared store, like on the old build. const viewed = await fixture.service.view(fixture.ctx, "/memories/workspace/.note"); expect(viewed.success).toBe(true); @@ -3837,6 +3847,203 @@ describe("MemoryService", () => { expect(await pathExists(shared)).toBe(false); }); + it("unwinds a removed child's own overlapping history LIFO through the owner journal", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + // Two pre-sharing rows over one note: a create, then an edit. Both are + // retargeted on migration (order-unknown against the OWNER's rows), but + // their order against EACH OTHER is the child journal's sequence. + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v2") }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(2); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const copyOf = (source: { id: string }) => + ownerRows.find((row) => row.data.migratedFrom === `ws-child:${source.id}`)!; + const createCopy = copyOf(childCreate); + const editCopy = copyOf(childEdit); + for (const [copy, source] of [ + [createCopy, childCreate], + [editCopy, childEdit], + ] as const) { + expect(copy.data.orderUnknown).toBe(true); + expect(copy.data.originJournal).toBe("ws-child"); + expect(copy.data.originSeq).toBe(source.seq); + } + // The older copy is not the newest mutation of the note: refused. + const stale = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(stale.success).toBe(false); + expect(stale.success ? "" : stale.error).toContain("later refinement row"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + // Newest first, no force needed... + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // ...then the create. + const second = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(second.success).toBe(true); + expect(await pathExists(ownerCopy)).toBe(false); + }); + + it("orders re-copied and provenance-less migrated rows by source position, not migration order", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(2); + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewriteCopyOf = async ( + source: { id: string }, + edit: (data: Record) => void + ) => { + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${source.id}"`)) return line; + const row = JSON.parse(line) as { data: Record }; + edit(row.data); + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + }; + // The CREATE's copy is corrupted before the removal is retried: the + // retry re-copies it, so the OLDER mutation now has the higher + // owner-journal seq (and a later append time). + await rewriteCopyOf(childCreate, (data) => { + data.inverse = { op: "bogus" }; + }); + expect(await migrate()).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const editCopy = ownerRows.find( + (row) => row.data.migratedFrom === `ws-child:${childEdit.id}` + )!; + const createCopy = ownerRows.find( + (row) => + row.data.migratedFrom === `ws-child:${childCreate.id}` && + (row.data.inverse as { op: string }).op !== "bogus" + )!; + expect(createCopy.seq).toBeGreaterThan(editCopy.seq); + expect(createCopy.data.originSeq!).toBeLessThan(editCopy.data.originSeq!); + // Migration order is not mutation order: the edit is still the newest + // mutation of the note and rolls back without force. + const first = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: editCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(first.success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // A copy WITHOUT a carried origin (an older build's migration, or the + // fields lost to corruption) has no position: its order against every + // other row stays unknown, so the create's rollback needs force. + await rewriteCopyOf(childEdit, (data) => { + delete data.originJournal; + delete data.originSeq; + }); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + "order relative to this row is unknown" + ); + const forced = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: createCopy.id, + force: true, + evidence: { toolName: "test", actor: "user" }, + }); + expect(forced.success).toBe(true); + expect(await pathExists(ownerCopy)).toBe(false); + }); + it("orders owner and child rows of the shared store by one store clock, advanced by rollbacks too", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index a0a932a5d0b..ac9183ac4d0 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1120,7 +1120,7 @@ export class MemoryService extends EventEmitter { * DOWNGRADED build reads (and writes) this child's notebook, so the notes * stay visible across upgrade↔downgrade (the child-keyed sidecar entries * stay for the same reason) and files the import cannot carry - * (binary/oversize, dotfiles, doubly conflicting) are never moved anywhere. + * (binary/oversize, doubly conflicting) are never moved anywhere. * The copy is idempotent — identical files are skipped, differing ones land * under imported// — so notes edited during a downgrade are folded in * again on the next upgrade. Writes made through the shared store meanwhile @@ -1365,10 +1365,11 @@ export class MemoryService extends EventEmitter { .then(() => this.readBoundedTextFile(legacy, relPath, relPath)) .catch(() => null); if (content === null || content.includes("\uFFFD")) { - // An unrepresentable dot-entry (a stray `.DS_Store`, a binary) was - // never a note on any build — no listing showed it — so it neither - // moves nor holds up removal; an unrepresentable LISTED note does. - if (relPath.split("/").some((segment) => segment.startsWith("."))) continue; + // Dot-entries too (r73): `.note` is addressable, so a real note + // there may hold text `create` permitted (U+FFFD included) or be + // transiently unreadable — exempting dot-entries would report a + // complete handover and let removal take the only copy. A stray + // `.DS_Store` costs a forced removal, never a note. skipped++; continue; } diff --git a/src/node/services/refinement/refinementJournal.ts b/src/node/services/refinement/refinementJournal.ts index ffedca07063..b9a9de49fb2 100644 --- a/src/node/services/refinement/refinementJournal.ts +++ b/src/node/services/refinement/refinementJournal.ts @@ -111,6 +111,12 @@ export interface RefinementEmitArgs { sourceTs?: number; /** See the durable event schema: the store clock write failed for this row. */ orderUnknown?: true; + /** + * Original journal position of a migrated row (see the durable event + * schema); only shared-memory row migration sets these, together. + */ + originJournal?: string; + originSeq?: number; /** * "remote" when the mutation ran through a non-local runtime (SSH/Docker). * Such rows carry runtime-namespace paths and are refused by rollback, @@ -387,6 +393,9 @@ export async function appendRefinementEventUnderBlobLock( ...(args.rollbackOf !== undefined ? { rollbackOf: args.rollbackOf } : {}), ...(args.sourceTs !== undefined ? { sourceTs: args.sourceTs } : {}), ...(args.orderUnknown === true ? { orderUnknown: true } : {}), + ...(args.originJournal !== undefined && args.originSeq !== undefined + ? { originJournal: args.originJournal, originSeq: args.originSeq } + : {}), ...(args.runtime !== undefined ? { runtime: args.runtime } : {}), }, }); diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 8bce84ad47c..49420ec2139 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -29,7 +29,11 @@ import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; import assert from "@/common/utils/assert"; -import { isValidSourceClock, type DurableEvent } from "@/common/types/durableEvent"; +import { + isValidSourceClock, + refinementRowOrigin, + type DurableEvent, +} from "@/common/types/durableEvent"; import { MemoryRefinementActionSchema, RefinementInverseSchema, @@ -528,13 +532,37 @@ interface InverseContentReader { * Collect divergence complaints for rolling back `target` given the current * filesystem + journal state. Empty array = safe to apply. */ +/** + * Two rows first appended to the same journal (a copy at its source position, + * see refinementRowOrigin): that store's mutation lock serialized clock and + * append, so the origin sequence is their mutation order whatever their clock + * values — pre-sharing history and clock-failed rows included. A copy without + * a carried origin has none (r73). Equal positions under distinct ids can + * only be corruption — no order evidence either. + */ +function sameOriginOrder( + row: RefinementEvent, + other: RefinementEvent +): { rowAfter: boolean } | null { + const rowOrigin = refinementRowOrigin(row); + const otherOrigin = refinementRowOrigin(other); + if (rowOrigin === null || otherOrigin === null || rowOrigin.journal !== otherOrigin.journal) { + return null; + } + if (rowOrigin.seq === otherOrigin.seq) return null; + return { rowAfter: rowOrigin.seq > otherOrigin.seq }; +} + /** * Journal order for conflict detection. Rows copied from a removed sub-agent's * journal (sharedMemoryRowMigration.ts) were appended later than they * happened; their `sourceTs` restores the mutation's real position relative * to the owner's own rows. Same-instant ties fall back to append sequence. + * Rows of one origin order by that origin's sequence (sameOriginOrder). */ function isAfter(row: RefinementEvent, other: RefinementEvent): boolean { + const sameOrigin = sameOriginOrder(row, other); + if (sameOrigin !== null) return sameOrigin.rowAfter; const rowTs = isValidSourceClock(row.data.sourceTs) ? row.data.sourceTs : row.ts; const otherTs = isValidSourceClock(other.data.sourceTs) ? other.data.sourceTs : other.ts; return rowTs > otherTs || (rowTs === otherTs && row.seq > other.seq); @@ -555,13 +583,15 @@ function hasMalformedSourceClock(row: RefinementEvent): boolean { * the shared store's clock write failed (`orderUnknown`) has only * journal-local `ts`/`seq`, incomparable with other journals' rows — as does * a row whose clock value is malformed. Callers fail closed — such a pair - * conflicts in either direction (force overrides). + * conflicts in either direction (force overrides). Rows of one origin are + * always ordered (sameOriginOrder), whatever their clock values. */ function orderUnknown( row: RefinementEvent, target: RefinementEvent, targetRetargeted: boolean ): boolean { + if (sameOriginOrder(row, target) !== null) return false; if (row.data.orderUnknown === true || target.data.orderUnknown === true) return true; if (hasMalformedSourceClock(row) || hasMalformedSourceClock(target)) return true; // A retargeted target (see wasRetargeted) vs. a row of another journal. diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 5473bc79e85..3cbc85d87d9 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -1,6 +1,6 @@ import * as path from "node:path"; import assert from "@/common/utils/assert"; -import { isValidSourceClock } from "@/common/types/durableEvent"; +import { isValidSourceClock, refinementRowOrigin } from "@/common/types/durableEvent"; import { MemoryRefinementActionSchema, RefinementEvidenceSchema, @@ -251,6 +251,7 @@ export async function migrateSharedMemoryRefinementRows(args: { } const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); const postState = RefinementPostStateSchema.safeParse(row.data.postState); + const origin = refinementRowOrigin(row); // Throws: this is the only durable copy once the child's journal goes. const appended = await appendRefinementEventUnderBlobLock(ownerJournal, { sessionDir: args.ownerSessionDir, @@ -294,6 +295,12 @@ export async function migrateSharedMemoryRefinementRows(args: { ...(row.data.orderUnknown === true || !isValidSourceClock(row.data.sourceTs) || retargeted ? { orderUnknown: true as const } : {}), + // Order-unknown against the owner's rows, but not against each other: + // the source position (this child's journal, or the first origin of a + // copy-of-copy) lets the owner unwind the child's own overlapping + // history LIFO — the copies' owner-side `seq` is migration order, + // which a retried pass can permute (a re-copied row lands last). + ...(origin !== null ? { originJournal: origin.journal, originSeq: origin.seq } : {}), ...(row.data.runtime === "remote" ? { runtime: "remote" as const } : {}), }); publishedBlobs.push(...appended.publishedBlobs); From 5afeec36ac2713601cbbf0b9dabc8a92c3199fb2 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 11:43:27 +0000 Subject: [PATCH 80/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-f?= =?UTF-8?q?ourth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The legacy path remapper maps a created adoption record only while the owner target is still the recorded generation (`targetStamp`, the rule deletion reconciliation already applies): an unjournaled owner save with identical bytes makes the copy the owner's, so a child's pre-sharing create/edit row is refused (`replaced`) instead of deleting or rewriting it. A tombstoned record maps while its target is absent. - The rollback engine re-stamps the adopted copies its own retargeted applies rewrite, remove or compensate, so a child's remaining rows over the same note keep unwinding LIFO. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 75 +++++++++- src/node/services/memoryService.test.ts | 95 ++++++++++++- .../refinement/refinementRollback.test.ts | 133 ++++++++++++------ .../services/refinement/refinementRollback.ts | 27 ++++ 4 files changed, 284 insertions(+), 46 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 8d8e39a0644..96558b5e8c1 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -10,6 +10,7 @@ import type { Dirent } from "node:fs"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; +import writeFileAtomic from "write-file-atomic"; import type { RefinementInverse } from "@/common/types/refinement"; /** @@ -224,11 +225,13 @@ function setsEqual(a: ReadonlySet, b: ReadonlySet): boolean { /** Thrown for a legacy path the shared store does not represent (see below). */ export class LegacyPathNotAdoptedError extends Error { - constructor(legacyPath: string, reason: "not-adopted" | "owner-owned") { + constructor(legacyPath: string, reason: "not-adopted" | "owner-owned" | "replaced") { super( reason === "owner-owned" ? `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook; the shared workspace store holds an identical note the owner already had (adoption created nothing), so a rollback there would alter the owner's own note` - : `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` + : reason === "replaced" + ? `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook; its adopted copy in the shared workspace store was since replaced (rewritten, or deleted and recreated) outside this sub-agent's rows, so the file there is the owner's own and a rollback would alter or remove it` + : `'${legacyPath}' addresses this sub-agent's pre-sharing private notebook, and that note was not folded into the shared workspace store (never adopted, or unplaceable there): the shared notebook does not show it, so rolling it back there would change nothing visible` ); this.name = "LegacyPathNotAdoptedError"; } @@ -247,6 +250,16 @@ export class LegacyPathNotAdoptedError extends Error { * owner already had an identical note of its own): the child's rows never * touched that file, and applying their inverses there — a create row's * delete-files in particular — would alter or remove the owner's own note. + * Likewise a created record whose target is no longer THIS adoption's + * generation of the file (`targetStamp`, the same rule deletion + * reconciliation applies): an owner save is unjournaled and may keep the + * bytes, so neither peer rows nor post-state hashes would notice — the + * replacement is the owner's, and the mapping is refused (r74). The stamps + * are read when the remapper is created; callers create it under the store's + * mutation lock (the rollback engine re-derives it there), and the engine + * re-stamps the targets its own retargeted applies rewrite + * (refreshLegacyAdoptionTargetStamps) so the child's remaining rows for the + * same note stay mappable. * Only the manifest's `target` is trusted for the destination's relPath; * callers re-run their confinement checks on the mapped result. `strict` * (removal's row migration) throws on an unreadable manifest instead of @@ -271,6 +284,18 @@ export async function createLegacyPathRemapper(args: { // rename moves whatever is there, so an owner note added beside the // adopted copies (no refinement row of its own) would travel along // unnoticed. Precomputed for every directory prefix the manifest knows. + // Created records whose owner target is still this lineage's generation + // (see LegacyAdoptionRecord.targetStamp). A record reconciled as deleted + // is current while its target stays absent — or holds the generation a + // retargeted rollback recreated there (re-stamped below); anything else at + // that path is the owner's. + const currentGeneration = new Set(); + for (const [rel, record] of adopted) { + if (record.created !== true || record.pending === true) continue; + const stamp = await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/"))); + const stampCurrent = record.targetStamp !== undefined && stamp === record.targetStamp; + if (stampCurrent || (record.deleted === true && stamp === null)) currentGeneration.add(rel); + } const ownerSubtreeExact = new Map(); const directoryPrefixes = new Set(); for (const rel of adopted.keys()) { @@ -282,7 +307,7 @@ export async function createLegacyPathRemapper(args: { for (const dirRel of directoryPrefixes) { const descendants = [...adopted].filter(([rel]) => rel.startsWith(`${dirRel}/`)); const oneToOne = descendants.every( - ([rel, entry]) => entry.target === rel && entry.created === true && entry.pending !== true + ([rel, entry]) => entry.target === rel && currentGeneration.has(rel) ); const expected = new Set( descendants.filter(([, entry]) => entry.deleted !== true).map(([rel]) => rel) @@ -315,6 +340,7 @@ export async function createLegacyPathRemapper(args: { } if (record.pending === true) throw new LegacyPathNotAdoptedError(filePath, "not-adopted"); if (record.created !== true) throw new LegacyPathNotAdoptedError(filePath, "owner-owned"); + if (!currentGeneration.has(relPath)) throw new LegacyPathNotAdoptedError(filePath, "replaced"); return path.join(ownerRoot, ...record.target.split("/")); }; return { @@ -337,3 +363,46 @@ export async function createLegacyPathRemapper(args: { }, }; } + +/** + * Re-stamp adopted targets the rollback engine just rewrote or removed while + * applying a RETARGETED inverse (createLegacyPathRemapper mapped the child's + * legacy paths onto them): the write is the child's own lineage acting, so + * the new generation stays mappable for the child's remaining rows over the + * same note (create + edit unwind LIFO). Runs under the owner store's mutation + * lock the engine holds (the lock adoption passes take too). A target that is + * gone loses its stamp — nothing maps there until adoption places the note + * anew. Best-effort by contract: a failure here only leaves stale stamps, + * which refuse (never mutate) later. + */ +export async function refreshLegacyAdoptionTargetStamps(args: { + childSessionDir: string; + ownerSessionDir: string; + paths: readonly string[]; +}): Promise { + if (args.paths.length === 0) return; + const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const touched = new Set(); + for (const filePath of args.paths) { + const relative = path.relative(ownerRoot, path.resolve(filePath)); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) continue; + touched.add(relative.split(path.sep).join("/")); + } + if (touched.size === 0) return; + const manifestPath = legacyAdoptionManifestPath(path.resolve(args.childSessionDir)); + const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); + let dirty = false; + for (const record of adopted.values()) { + if (record.created !== true || record.pending === true) continue; + if (!touched.has(record.target)) continue; + const stamp = + (await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/")))) ?? undefined; + if (stamp === record.targetStamp) continue; + record.targetStamp = stamp; + dirty = true; + } + if (!dirty) return; + await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { + encoding: "utf-8", + }); +} diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 2a7bd28cd1b..2bffb8e89ae 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -19,7 +19,11 @@ import { type PinnedFileMutation, } from "./memoryService"; import { MemoryMetaService, memoryLogicalKey } from "./memoryMeta"; -import { legacyAdoptionManifestPath } from "./memoryLegacyAdoption"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, +} from "./memoryLegacyAdoption"; import { MemoryRefinementActionSchema, REFINEMENT_CAPTURE_MAX_FILES, @@ -4044,6 +4048,83 @@ describe("MemoryService", () => { expect(await pathExists(ownerCopy)).toBe(false); }); + it("re-stamps an adopted copy a retargeted rollback rewrites, refusing one the owner replaced", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "old.md"), "v2"); + const childJournal = sharedDurableEventJournal(childSessionDir); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "create", path: "/memories/workspace/old.md" }, + inverse: { op: "delete-files", paths: [path.join(legacyRoot, "old.md")] }, + postState: { + files: [{ path: path.join(legacyRoot, "old.md"), sha256: sha256Hex("v1") }], + }, + }, + }); + await childJournal.append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/old.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "old.md"), text: "v1" }], + }, + }, + }); + const [childCreate, childEdit] = await readRefinementEvents(childSessionDir); + await fixture.service.listIndexEntries({ ...fixture.ctx }); // adoption + const ownerCopy = path.join(ownerSessionDir, "memory", "old.md"); + const manifestPath = legacyAdoptionManifestPath(childSessionDir); + const recordStamp = async () => + (await readLegacyAdoptionManifest(manifestPath)).get("old.md")!.targetStamp; + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); + const rollback = (id: string) => + rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id, + evidence: { toolName: "test", actor: "user" }, + }); + // Rolling the edit back rewrites the copy: a new generation, but this + // lineage's own — the record follows it, so the create still maps. + expect((await rollback(childEdit.id)).success).toBe(true); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); + // An owner save meanwhile (Memory tab: unjournaled, bytes unchanged) + // makes the copy the owner's: the create's delete-files is refused, + // force or not, and the note stays. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fixture.service.saveFile( + { ...fixture.ctx, workspaceId: "ws-owner" }, + "/memories/workspace/old.md", + "v1", + sha256Hex("v1"), + "user" + ); + for (const force of [false, true]) { + const refused = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childCreate.id, + force, + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain("since replaced"); + } + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + }); + it("orders owner and child rows of the shared store by one store clock, advanced by rollbacks too", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -4356,13 +4437,21 @@ describe("MemoryService", () => { const legacyRoot = path.join(childSessionDir, "memory"); await fsPromises.mkdir(legacyRoot, { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, "legacy.md"), "v2"); + await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "legacy.md"), "v2"); await fsPromises.writeFile( legacyAdoptionManifestPath(path.dirname(legacyRoot)), JSON.stringify({ - "legacy.md": { content: "x", sidecar: "", target: "legacy.md", created: true }, + "legacy.md": { + content: "x", + sidecar: "", + target: "legacy.md", + created: true, + targetStamp: await adoptionTargetStamp( + path.join(ownerSessionDir, "memory", "legacy.md") + ), + }, }) ); - await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "legacy.md"), "v2"); await sharedDurableEventJournal(childSessionDir).append({ workspaceId: "ws-child", kind: "refinement", diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 10f62fd98b2..75dd5ff81d9 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -6,7 +6,12 @@ import * as path from "node:path"; import { REFINEMENT_INVERSE_BLOB_QUOTA_BYTES } from "@/common/types/refinement"; import { Config } from "@/node/config"; import { LocalRuntime } from "@/node/runtime/LocalRuntime"; -import { legacyAdoptionManifestPath } from "@/node/services/memoryLegacyAdoption"; +import { + adoptionTargetStamp, + legacyAdoptionManifestPath, + readLegacyAdoptionManifest, + type LegacyAdoptionRecord, +} from "@/node/services/memoryLegacyAdoption"; import { MemoryMetaService } from "@/node/services/memoryMeta"; import { MemoryService, type MemoryScopeContext } from "@/node/services/memoryService"; import { TestTempDir } from "@/node/services/tools/testHelpers"; @@ -879,6 +884,7 @@ describe("refinementRollback", () => { // Rows journaled while the workspace owned its store address // /memory (the legacy private notebook after an upgrade). await fixture.service.create(fixture.ctx, "/memories/workspace/note.md", "v1\n", "agent"); + const createRow = await lastRow(fixture.sessionDir); await fixture.service.strReplace( fixture.ctx, "/memories/workspace/note.md", @@ -892,14 +898,31 @@ describe("refinementRollback", () => { // The upgrade folded note.md into the task-tree owner's store (adoption // manifest beside the legacy files); orphan.md could not be placed. const ownerSessionDir = path.join(path.dirname(fixture.sessionDir), "ws-owner"); - await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "sub"), { recursive: true }); - await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "sub", "note.md"), "v2\n"); - await fsPromises.writeFile( - legacyAdoptionManifestPath(fixture.sessionDir), - JSON.stringify({ - "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, - }) - ); + const ownerRoot = path.join(ownerSessionDir, "memory"); + // Created records carry the generation of the copy the adoption wrote + // (LegacyAdoptionRecord.targetStamp); stamped from the file as it is now. + const writeManifest = async (records: Record) => { + for (const record of Object.values(records)) { + if ( + record.created === true && + record.deleted !== true && + record.targetStamp === undefined + ) { + record.targetStamp = + (await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/")))) ?? + undefined; + } + } + await fsPromises.writeFile( + legacyAdoptionManifestPath(fixture.sessionDir), + JSON.stringify(records) + ); + }; + await fsPromises.mkdir(path.join(ownerRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v2\n"); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + }); // The retargeted row was journaled against this session's private clock: // against an overlapping row of the OWNER's journal its order is unknown, // so the rollback is refused unless forced. @@ -943,6 +966,29 @@ describe("refinementRollback", () => { expect( await fsPromises.readFile(path.join(fixture.sessionDir, "memory", "note.md"), "utf-8") ).toBe("v2\n"); + // The rewrite is this lineage's own: the record is re-stamped to the new + // generation (r74), so the child's create row still maps onto the copy... + const restamped = ( + await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir)) + ).get("note.md")!; + expect(restamped.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "sub", "note.md"))) ?? undefined + ); + // ...but not after the owner replaced the copy outside the child's rows + // (a Memory-tab save is unjournaled and may keep the very same bytes): + // the file is the owner's now, and the create's delete-files is refused. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(ownerRoot, "sub", "note.md")); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v1\n"); + const replaced = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: createRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(replaced.success).toBe(false); + expect(replaced.success ? "" : replaced.error).toContain("since replaced"); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v1\n"); const refused = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: orphanRow.id, @@ -959,13 +1005,10 @@ describe("refinementRollback", () => { await fixture.service.create(fixture.ctx, "/memories/workspace/same.md", "same\n", "agent"); const sameRow = await lastRow(fixture.sessionDir); await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "same.md"), "same\n"); - await fsPromises.writeFile( - legacyAdoptionManifestPath(fixture.sessionDir), - JSON.stringify({ - "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, - "same.md": { content: "x", sidecar: "", target: "same.md" }, - }) - ); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + }); const ownerOwned = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: sameRow.id, @@ -979,16 +1022,14 @@ describe("refinementRollback", () => { // store keeps a tombstoned record: the child's pre-sharing delete row // still maps, and rolling it back restores the note in the shared store. await fixture.service.create(fixture.ctx, "/memories/workspace/gone.md", "g1\n", "agent"); + const goneCreateRow = await lastRow(fixture.sessionDir); await fixture.service.deletePath(fixture.ctx, "/memories/workspace/gone.md", "agent"); const deleteRow = await lastRow(fixture.sessionDir); - await fsPromises.writeFile( - legacyAdoptionManifestPath(fixture.sessionDir), - JSON.stringify({ - "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, - "same.md": { content: "x", sidecar: "", target: "same.md" }, - "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, - }) - ); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + }); const restoredDelete = await rollbackRefinement({ sessionDir: fixture.sessionDir, id: deleteRow.id, @@ -999,6 +1040,21 @@ describe("refinementRollback", () => { expect( await fsPromises.readFile(path.join(ownerSessionDir, "memory", "gone.md"), "utf-8") ).toBe("g1\n"); + // The recreated copy is stamped onto the tombstoned record, so the child's + // history over that note keeps unwinding: the create row now maps too. + expect( + (await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir))).get( + "gone.md" + )!.targetStamp + ).toBe((await adoptionTargetStamp(path.join(ownerRoot, "gone.md"))) ?? undefined); + const undoneGoneCreate = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: goneCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneGoneCreate.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "gone.md"))).toBe(false); // A pre-sharing DIRECTORY rename: the manifest records files only, so the // directory endpoints map through their adopted descendants (all landed // at their own relPath as this adoption's copies). @@ -1012,22 +1068,19 @@ describe("refinementRollback", () => { const dirRenameRow = await lastRow(fixture.sessionDir); await fsPromises.mkdir(path.join(ownerSessionDir, "memory", "newdir"), { recursive: true }); await fsPromises.writeFile(path.join(ownerSessionDir, "memory", "newdir", "a.md"), "a\n"); - await fsPromises.writeFile( - legacyAdoptionManifestPath(fixture.sessionDir), - JSON.stringify({ - "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, - "same.md": { content: "x", sidecar: "", target: "same.md" }, - "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, - "newdir/a.md": { content: "x", sidecar: "", target: "newdir/a.md", created: true }, - "olddir/a.md": { - content: "x", - sidecar: "", - target: "olddir/a.md", - created: true, - deleted: true, - }, - }) - ); + await writeManifest({ + "note.md": { content: "x", sidecar: "", target: "sub/note.md", created: true }, + "same.md": { content: "x", sidecar: "", target: "same.md" }, + "gone.md": { content: "x", sidecar: "", target: "gone.md", created: true, deleted: true }, + "newdir/a.md": { content: "x", sidecar: "", target: "newdir/a.md", created: true }, + "olddir/a.md": { + content: "x", + sidecar: "", + target: "olddir/a.md", + created: true, + deleted: true, + }, + }); // An owner note added beside the adopted copies (no refinement row of // its own) would travel with a structural rename: refused until the // subtree is exactly the adopted descendants again. diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 49420ec2139..21052f07d56 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -62,6 +62,7 @@ import { isWorkspaceRemovalTombstoned } from "@/node/services/workspaceRemoval"; import { createLegacyPathRemapper, LegacyPathNotAdoptedError, + refreshLegacyAdoptionTargetStamps, } from "@/node/services/memoryLegacyAdoption"; export type RefinementEvent = Extract; @@ -1263,6 +1264,29 @@ export async function rollbackRefinement( // partial rollback behind (no rollbackOf row, and a retry refuses on the // resulting divergence). const applied: RollbackApplied = { rollbackRowId: null, restored: [], deleted: [] }; + // Retargeted apply (r74): the adopted copies rewritten, removed or — + // after a failed apply — compensated are new generations of the files; + // re-stamp them so the child's remaining rows over the same notes still + // map (createLegacyPathRemapper refuses a copy that is no longer the + // recorded generation). A rename keeps inode and mtime, so it needs + // none. Still under the owner-store lock. + const restampAdoptedCopies = async (): Promise => { + if (remap === identityRemapper) return; + assert(opts.sharedWorkspaceMemorySessionDir !== undefined); + try { + await refreshLegacyAdoptionTargetStamps({ + childSessionDir: opts.sessionDir, + ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, + paths: [...applied.restored, ...applied.deleted], + }); + } catch (error) { + // Stale stamps only refuse later rollbacks (force overrides). + log.warn("[refinement] failed to re-stamp adopted legacy copies after a rollback", { + id: opts.id, + error, + }); + } + }; switch (inverse.op) { case "delete-files": try { @@ -1272,6 +1296,7 @@ export async function rollbackRefinement( } } catch (error) { await compensatePartialApply(applied.deleted, newInverse); + await restampAdoptedCopies(); throw error; } break; @@ -1305,6 +1330,7 @@ export async function rollbackRefinement( } } catch (error) { await compensatePartialApply([...applied.restored, ...applied.deleted], newInverse); + await restampAdoptedCopies(); throw error; } break; @@ -1316,6 +1342,7 @@ export async function rollbackRefinement( applied.renamed = { from: inverse.from, to: inverse.to }; break; } + await restampAdoptedCopies(); // Commit point: even if two processes double-entered the critical section // (theoretically possible — plain POSIX files cannot make the guard's From f096b6baee51b09e34f3169df5f9a4db8527e4d8 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 12:11:40 +0000 Subject: [PATCH 81/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-f?= =?UTF-8?q?ifth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A rename inverse whose destination (the name the child's pre-upgrade rename vacated) has no adoption record maps beside the adopted copies once its source side maps, so first-upgrade renames stay rollbackable. - Recovering a pending in-place replacement binds the generation now on disk instead of the overwritten one the pending record carried. - Retargeted rename rollbacks re-stamp both endpoints' records (by prefix): the vacated side loses its stamp, the tombstoned restored side takes the moved files' generation, and the directory proof counts such restored copies as present. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 55 +++++++++++++++---- src/node/services/memoryService.test.ts | 38 +++++++++++++ src/node/services/memoryService.ts | 13 +++-- .../refinement/refinementRollback.test.ts | 49 +++++++++++++++++ .../services/refinement/refinementRollback.ts | 11 +++- 5 files changed, 145 insertions(+), 21 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 96558b5e8c1..9d4dbd7e3b5 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -289,12 +289,17 @@ export async function createLegacyPathRemapper(args: { // is current while its target stays absent — or holds the generation a // retargeted rollback recreated there (re-stamped below); anything else at // that path is the owner's. - const currentGeneration = new Set(); + // Value: whether that generation is a file on disk (a tombstoned record + // whose copy a rollback restored counts as present). + const currentGeneration = new Map(); for (const [rel, record] of adopted) { if (record.created !== true || record.pending === true) continue; const stamp = await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/"))); - const stampCurrent = record.targetStamp !== undefined && stamp === record.targetStamp; - if (stampCurrent || (record.deleted === true && stamp === null)) currentGeneration.add(rel); + if (record.targetStamp !== undefined && stamp === record.targetStamp) { + currentGeneration.set(rel, "present"); + } else if (record.deleted === true && stamp === null) { + currentGeneration.set(rel, "absent"); + } } const ownerSubtreeExact = new Map(); const directoryPrefixes = new Set(); @@ -310,7 +315,7 @@ export async function createLegacyPathRemapper(args: { ([rel, entry]) => entry.target === rel && currentGeneration.has(rel) ); const expected = new Set( - descendants.filter(([, entry]) => entry.deleted !== true).map(([rel]) => rel) + descendants.filter(([rel]) => currentGeneration.get(rel) === "present").map(([rel]) => rel) ); ownerSubtreeExact.set( dirRel, @@ -321,10 +326,14 @@ export async function createLegacyPathRemapper(args: { ) ); } - const remapPath = (filePath: string): string => { + const legacyRelPath = (filePath: string): string | null => { const relative = path.relative(legacyRoot, path.resolve(filePath)); - if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return filePath; - const relPath = relative.split(path.sep).join("/"); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return null; + return relative.split(path.sep).join("/"); + }; + const remapPath = (filePath: string): string => { + const relPath = legacyRelPath(filePath); + if (relPath === null) return filePath; const record = adopted.get(relPath); if (record === undefined) { // A directory endpoint (a pre-sharing directory rename): the manifest @@ -343,14 +352,29 @@ export async function createLegacyPathRemapper(args: { if (!currentGeneration.has(relPath)) throw new LegacyPathNotAdoptedError(filePath, "replaced"); return path.join(ownerRoot, ...record.target.split("/")); }; + // The destination of a rename INVERSE is the name the child's rename + // vacated. A rename made before the first upgrade leaves no record for it + // (adoption saw only the post-rename names), yet the row is still + // rollbackable once its `from` side maps (r75): the vacated name lands + // beside the adopted copies. The engine requires it absent before moving, + // so an owner note there refuses like for any rename. + const remapRenameDestination = (filePath: string): string => { + const relPath = legacyRelPath(filePath); + if (relPath === null || adopted.has(relPath) || directoryPrefixes.has(relPath)) { + return remapPath(filePath); + } + return path.join(ownerRoot, ...relPath.split("/")); + }; return { path: remapPath, inverse: (inverse) => { switch (inverse.op) { case "delete-files": return { ...inverse, paths: inverse.paths.map(remapPath) }; - case "rename": - return { ...inverse, from: remapPath(inverse.from), to: remapPath(inverse.to) }; + case "rename": { + const from = remapPath(inverse.from); + return { ...inverse, from, to: remapRenameDestination(inverse.to) }; + } case "restore-files": return { ...inverse, @@ -369,8 +393,12 @@ export async function createLegacyPathRemapper(args: { * applying a RETARGETED inverse (createLegacyPathRemapper mapped the child's * legacy paths onto them): the write is the child's own lineage acting, so * the new generation stays mappable for the child's remaining rows over the - * same note (create + edit unwind LIFO). Runs under the owner store's mutation - * lock the engine holds (the lock adoption passes take too). A target that is + * same note (create + edit unwind LIFO). `paths` may be directories (a rename + * endpoint): every record whose target lies beneath is re-stamped, so after a + * retargeted rename the vacated side's records lose their stamp and the + * restored side's (tombstoned by the downgraded build's rename) take the + * moved files' generation (r75). Runs under the owner store's mutation lock + * the engine holds (the lock adoption passes take too). A target that is * gone loses its stamp — nothing maps there until adoption places the note * anew. Best-effort by contract: a failure here only leaves stale stamps, * which refuse (never mutate) later. @@ -394,7 +422,10 @@ export async function refreshLegacyAdoptionTargetStamps(args: { let dirty = false; for (const record of adopted.values()) { if (record.created !== true || record.pending === true) continue; - if (!touched.has(record.target)) continue; + const beneathTouched = [...touched].some( + (rel) => record.target === rel || record.target.startsWith(`${rel}/`) + ); + if (!beneathTouched) continue; const stamp = (await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/")))) ?? undefined; if (stamp === record.targetStamp) continue; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 2bffb8e89ae..5e44bbe2088 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2606,6 +2606,44 @@ describe("MemoryService", () => { }); expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe("v3"); expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // A crash after the replacement write but before the settled manifest: + // the pending record still carries the OVERWRITTEN generation's stamp. + // The retry binds the replacement on disk (r75) — settling the stale + // stamp would refuse the child's rollbacks as "replaced" and leave the + // copy behind when the source is deleted. + const settled2 = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { content: string; sidecar: string; target: string; targetStamp?: string } + > + )["note.md"]; + expect(settled2.targetStamp).toBeDefined(); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3-replaced"); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v3-replaced"); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { ...settled2, pending: true, replacementContent: sha256Hex("v3-replaced") }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ + ...fixture.ctx, + }); + const rebound = ( + JSON.parse(await fsPromises.readFile(manifestPath, "utf-8")) as Record< + string, + { pending?: boolean; targetStamp?: string } + > + )["note.md"]; + expect(rebound.pending).toBeUndefined(); + expect(rebound.targetStamp).not.toBe(settled2.targetStamp); + expect(rebound.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "note.md"))) ?? undefined + ); // The opposite crash window: the replacement bytes landed but the final // manifest write did not, and the downgraded build deletes the source // before the retry. The pending record names both hashes, so the copy diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index ac9183ac4d0..40e0fa7105f 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1429,14 +1429,15 @@ export class MemoryService extends EventEmitter { // A pending record is a copy this adoption wrote but could not // finish recording (crash or sidecar failure after the write): // the file holding exactly those bytes now is that write, so its - // current identity is the generation to bind. A settled record - // keeps the stamp it recorded — identical bytes in a different - // generation are the owner's (deletion then preserves). + // current identity is the generation to bind — also when the + // record carries a stamp, which is then the generation an + // interrupted in-place replacement OVERWROTE (r75). A settled + // record keeps the stamp it recorded — identical bytes in a + // different generation are the owner's (deletion then preserves). record.targetStamp = - previous.targetStamp ?? - (previous.pending === true + previous.pending === true ? ((await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined) - : undefined); + : previous.targetStamp; } else if ( previous.created === true && priorContent !== null && diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 75dd5ff81d9..4f55ea860ea 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1059,6 +1059,7 @@ describe("refinementRollback", () => { // directory endpoints map through their adopted descendants (all landed // at their own relPath as this adoption's copies). await fixture.service.create(fixture.ctx, "/memories/workspace/olddir/a.md", "a\n", "agent"); + const olddirCreateRow = await lastRow(fixture.sessionDir); await fixture.service.rename( fixture.ctx, "/memories/workspace/olddir", @@ -1128,6 +1129,54 @@ describe("refinementRollback", () => { await fsPromises.readFile(path.join(ownerSessionDir, "memory", "olddir", "a.md"), "utf-8") ).toBe("a\n"); expect(await pathExists(path.join(ownerSessionDir, "memory", "newdir"))).toBe(false); + // The generation moved with the file (r75): the vacated side's record + // loses its stamp, the tombstoned source record takes the moved file's — + // so the child's older rows at the restored name still map. + const afterRename = await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(fixture.sessionDir) + ); + expect(afterRename.get("newdir/a.md")!.targetStamp).toBeUndefined(); + expect(afterRename.get("olddir/a.md")!.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "olddir", "a.md"))) ?? undefined + ); + const undoneOlddirCreate = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: olddirCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneOlddirCreate.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "olddir", "a.md"))).toBe(false); + // A rename made BEFORE the first upgrade: adoption recorded only the + // post-rename names, so the inverse's destination (the vacated name) has + // no record — it still maps beside the adopted copies (r75). + await fixture.service.create(fixture.ctx, "/memories/workspace/dir2/b.md", "b\n", "agent"); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/dir2", + "/memories/workspace/dir3", + "agent" + ); + const firstUpgradeRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerRoot, "dir3"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "dir3", "b.md"), "b\n"); + await writeManifest({ + "dir3/b.md": { content: "x", sidecar: "", target: "dir3/b.md", created: true }, + }); + const undoneFirstUpgradeRename = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: firstUpgradeRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneFirstUpgradeRename.success).toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "dir2", "b.md"), "utf-8")).toBe("b\n"); + expect(await pathExists(path.join(ownerRoot, "dir3"))).toBe(false); + expect( + (await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir))).get( + "dir3/b.md" + )!.targetStamp + ).toBeUndefined(); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 21052f07d56..b4752f3645d 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -1268,8 +1268,9 @@ export async function rollbackRefinement( // after a failed apply — compensated are new generations of the files; // re-stamp them so the child's remaining rows over the same notes still // map (createLegacyPathRemapper refuses a copy that is no longer the - // recorded generation). A rename keeps inode and mtime, so it needs - // none. Still under the owner-store lock. + // recorded generation). A rename keeps inode and mtime but moves the + // generation to the other endpoint's records (r75). Still under the + // owner-store lock. const restampAdoptedCopies = async (): Promise => { if (remap === identityRemapper) return; assert(opts.sharedWorkspaceMemorySessionDir !== undefined); @@ -1277,7 +1278,11 @@ export async function rollbackRefinement( await refreshLegacyAdoptionTargetStamps({ childSessionDir: opts.sessionDir, ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, - paths: [...applied.restored, ...applied.deleted], + paths: [ + ...applied.restored, + ...applied.deleted, + ...(applied.renamed === undefined ? [] : [applied.renamed.from, applied.renamed.to]), + ], }); } catch (error) { // Stale stamps only refuse later rollbacks (force overrides). From b3c98e1f1a35ba582cbf639dcc7ddc2d6463d698 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 12:42:40 +0000 Subject: [PATCH 82/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-s?= =?UTF-8?q?ixth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A rename rollback whose restored side has no adoption record (the child renamed before its first upgrade) now writes tombstoned records for the moved copies, keyed by the legacy paths the child's older rows address and carrying the moved files' generation, so those rows keep rolling back and migrate on removal. The re-stamp runs for every sub-agent apply on the shared store — the child's own rollback rows (owner paths) included. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 71 ++++++++++++++----- .../refinement/refinementRollback.test.ts | 54 ++++++++++++-- .../services/refinement/refinementRollback.ts | 32 +++++---- 3 files changed, 123 insertions(+), 34 deletions(-) diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 9d4dbd7e3b5..5889d682107 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -390,48 +390,87 @@ export async function createLegacyPathRemapper(args: { /** * Re-stamp adopted targets the rollback engine just rewrote or removed while - * applying a RETARGETED inverse (createLegacyPathRemapper mapped the child's - * legacy paths onto them): the write is the child's own lineage acting, so + * applying an inverse on the child's behalf (createLegacyPathRemapper mapped + * the child's legacy paths onto them, or the row is the child's own rollback + * row over the shared store): the write is the child's own lineage acting, so * the new generation stays mappable for the child's remaining rows over the * same note (create + edit unwind LIFO). `paths` may be directories (a rename * endpoint): every record whose target lies beneath is re-stamped, so after a * retargeted rename the vacated side's records lose their stamp and the * restored side's (tombstoned by the downgraded build's rename) take the - * moved files' generation (r75). Runs under the owner store's mutation lock - * the engine holds (the lock adoption passes take too). A target that is - * gone loses its stamp — nothing maps there until adoption places the note - * anew. Best-effort by contract: a failure here only leaves stale stamps, + * moved files' generation (r75). A rename whose restored side has NO record + * (the child renamed before the first upgrade, so adoption only ever saw the + * new names) gets tombstoned records for the moved copies (r76): keyed by the + * legacy path the child's older rows address, carrying the moved file's + * generation — `deleted` because no legacy source exists there (reconciliation + * skips tombstones; a later source is a fresh note), yet mappable while the + * copy is that generation or absent again. Runs under the owner store's + * mutation lock the engine holds (the lock adoption passes take too). A target + * that is gone loses its stamp — nothing maps there until adoption places the + * note anew. Best-effort by contract: a failure here only leaves stale stamps, * which refuse (never mutate) later. */ export async function refreshLegacyAdoptionTargetStamps(args: { childSessionDir: string; ownerSessionDir: string; paths: readonly string[]; + renamed?: { from: string; to: string }; }): Promise { - if (args.paths.length === 0) return; const ownerRoot = path.join(path.resolve(args.ownerSessionDir), "memory"); + const ownerRel = (filePath: string): string | null => { + const relative = path.relative(ownerRoot, path.resolve(filePath)); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) return null; + return relative.split(path.sep).join("/"); + }; const touched = new Set(); for (const filePath of args.paths) { - const relative = path.relative(ownerRoot, path.resolve(filePath)); - if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) continue; - touched.add(relative.split(path.sep).join("/")); + const rel = ownerRel(filePath); + if (rel !== null) touched.add(rel); } + const renamed = + args.renamed === undefined + ? null + : { from: ownerRel(args.renamed.from), to: ownerRel(args.renamed.to) }; + if (renamed?.from != null) touched.add(renamed.from); + if (renamed?.to != null) touched.add(renamed.to); if (touched.size === 0) return; + const beneath = (target: string, rel: string): boolean => + target === rel || target.startsWith(`${rel}/`); const manifestPath = legacyAdoptionManifestPath(path.resolve(args.childSessionDir)); const adopted = await readLegacyAdoptionManifest(manifestPath, { strict: true }); let dirty = false; + const stampOf = async (target: string): Promise => + (await adoptionTargetStamp(path.join(ownerRoot, ...target.split("/")))) ?? undefined; for (const record of adopted.values()) { if (record.created !== true || record.pending === true) continue; - const beneathTouched = [...touched].some( - (rel) => record.target === rel || record.target.startsWith(`${rel}/`) - ); - if (!beneathTouched) continue; - const stamp = - (await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/")))) ?? undefined; + if (![...touched].some((rel) => beneath(record.target, rel))) continue; + const stamp = await stampOf(record.target); if (stamp === record.targetStamp) continue; record.targetStamp = stamp; dirty = true; } + if (renamed?.from != null && renamed.to != null) { + for (const [rel, record] of [...adopted]) { + // One-to-one copies only (the directory proof requires it; a lone file + // maps through its own record): the moved copy sits at the same + // relative position under the restored name. + if (record.created !== true || record.pending === true || record.target !== rel) continue; + if (!beneath(rel, renamed.from)) continue; + const movedRel = renamed.to + rel.slice(renamed.from.length); + if (adopted.has(movedRel)) continue; + const stamp = await stampOf(movedRel); + if (stamp === undefined) continue; // not moved after all: nothing to vouch for + adopted.set(movedRel, { + content: record.content, + sidecar: record.sidecar, + target: movedRel, + created: true, + deleted: true, + targetStamp: stamp, + }); + dirty = true; + } + } if (!dirty) return; await writeFileAtomic(manifestPath, JSON.stringify(Object.fromEntries(adopted)), { encoding: "utf-8", diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 4f55ea860ea..f596f2c8257 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1151,6 +1151,7 @@ describe("refinementRollback", () => { // post-rename names, so the inverse's destination (the vacated name) has // no record — it still maps beside the adopted copies (r75). await fixture.service.create(fixture.ctx, "/memories/workspace/dir2/b.md", "b\n", "agent"); + const dir2CreateRow = await lastRow(fixture.sessionDir); await fixture.service.rename( fixture.ctx, "/memories/workspace/dir2", @@ -1172,11 +1173,56 @@ describe("refinementRollback", () => { expect(undoneFirstUpgradeRename.success).toBe(true); expect(await fsPromises.readFile(path.join(ownerRoot, "dir2", "b.md"), "utf-8")).toBe("b\n"); expect(await pathExists(path.join(ownerRoot, "dir3"))).toBe(false); + // The restored side had no record: the moved copy gets a tombstoned one + // (r76) carrying its generation, so the child's older rows there map. + const manifest = () => + readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir)); + expect((await manifest()).get("dir3/b.md")!.targetStamp).toBeUndefined(); + const dir2Stamp = + (await adoptionTargetStamp(path.join(ownerRoot, "dir2", "b.md"))) ?? undefined; + expect((await manifest()).get("dir2/b.md")).toEqual({ + content: "x", + sidecar: "", + target: "dir2/b.md", + created: true, + deleted: true, + targetStamp: dir2Stamp, + }); + // Re-applying the rename through its rollback row (owner paths, no + // retargeting) moves the generation back; undoing that again restores it. + const renameRollbackRow = await lastRow(fixture.sessionDir); expect( - (await readLegacyAdoptionManifest(legacyAdoptionManifestPath(fixture.sessionDir))).get( - "dir3/b.md" - )!.targetStamp - ).toBeUndefined(); + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: renameRollbackRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect((await manifest()).get("dir3/b.md")!.targetStamp).toBe(dir2Stamp); + expect((await manifest()).get("dir2/b.md")!.targetStamp).toBeUndefined(); + const reapplyRow = await lastRow(fixture.sessionDir); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: reapplyRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect((await manifest()).get("dir2/b.md")!.targetStamp).toBe(dir2Stamp); + const undoneDir2Create = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: dir2CreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }); + expect(undoneDir2Create.success).toBe(true); + expect(await pathExists(path.join(ownerRoot, "dir2", "b.md"))).toBe(false); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index b4752f3645d..5fd1a364430 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -1264,25 +1264,29 @@ export async function rollbackRefinement( // partial rollback behind (no rollbackOf row, and a retry refuses on the // resulting divergence). const applied: RollbackApplied = { rollbackRowId: null, restored: [], deleted: [] }; - // Retargeted apply (r74): the adopted copies rewritten, removed or — - // after a failed apply — compensated are new generations of the files; - // re-stamp them so the child's remaining rows over the same notes still - // map (createLegacyPathRemapper refuses a copy that is no longer the - // recorded generation). A rename keeps inode and mtime but moves the - // generation to the other endpoint's records (r75). Still under the - // owner-store lock. + // A sub-agent's apply on the shared store (r74): the adopted copies + // rewritten, removed or — after a failed apply — compensated are new + // generations of the files; re-stamp them so the child's remaining rows + // over the same notes still map (createLegacyPathRemapper refuses a copy + // that is no longer the recorded generation). A rename keeps inode and + // mtime but moves the generation to the other endpoint's records (r75). + // Not only retargeted inverses: the child's own rollback rows carry + // owner paths already, and re-applying one is the same lineage acting. + // Still under the owner-store lock. const restampAdoptedCopies = async (): Promise => { - if (remap === identityRemapper) return; - assert(opts.sharedWorkspaceMemorySessionDir !== undefined); + if ( + kind !== "memory" || + opts.sharedWorkspaceMemorySessionDir === undefined || + path.resolve(opts.sharedWorkspaceMemorySessionDir) === path.resolve(opts.sessionDir) + ) { + return; + } try { await refreshLegacyAdoptionTargetStamps({ childSessionDir: opts.sessionDir, ownerSessionDir: opts.sharedWorkspaceMemorySessionDir, - paths: [ - ...applied.restored, - ...applied.deleted, - ...(applied.renamed === undefined ? [] : [applied.renamed.from, applied.renamed.to]), - ], + paths: [...applied.restored, ...applied.deleted], + ...(applied.renamed === undefined ? {} : { renamed: applied.renamed }), }); } catch (error) { // Stale stamps only refuse later rollbacks (force overrides). From ceba4a73b67e683e7124acbd321c9b087e670703 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 13:44:28 +0000 Subject: [PATCH 83/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-s?= =?UTF-8?q?eventh=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Rename rollbacks tombstone the restored side for conflict-imported copies too (matched by target, not key), and the engine re-stamps after the commit-point compensation as well. - Shared-memory row migration counts only a parseable rollback row as one: a corrupted rollback record no longer kills its intact target. - Removal pins each surviving child to the owner it resolves to now (pinDescendantWorkspaceMemoryOwners): stale pins are replaced, valid ones kept, and the read-back requires the exact value. - MemoryScopeContext.guardedWorkspaceId: a child's redirected consolidation run is refused by the child's removal tombstone at every read and commit. - Compaction boundaries no longer consume the closing epoch's policy record or deny marker (two backends can each close the same epoch); only the destructive boundary clears them, and carries copy instead of moving. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/agentSession.ts | 98 +++++++------------ ...Session.workspaceMemoryPolicyEpoch.test.ts | 48 ++++++--- .../services/memoryConsolidationService.ts | 7 ++ src/node/services/memoryLegacyAdoption.ts | 21 ++-- src/node/services/memoryService.test.ts | 84 ++++++++++++++++ src/node/services/memoryService.ts | 19 +++- .../services/memoryWorkspaceOwner.test.ts | 76 ++++++++++++++ src/node/services/memoryWorkspaceOwner.ts | 32 ++++++ .../refinement/refinementRollback.test.ts | 50 ++++++++++ .../services/refinement/refinementRollback.ts | 4 + .../refinement/sharedMemoryRowMigration.ts | 14 ++- .../services/workspaceMemoryDenyMarker.ts | 87 ++++++---------- .../services/workspaceMemoryPolicyEpochs.ts | 37 ++----- src/node/services/workspaceService.test.ts | 58 +++++------ src/node/services/workspaceService.ts | 25 ++--- 15 files changed, 443 insertions(+), 217 deletions(-) create mode 100644 src/node/services/memoryWorkspaceOwner.test.ts diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index 44cbcb1802a..e41207f9bb4 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -111,7 +111,6 @@ import { import { isWorkspaceArchived } from "@/common/utils/archive"; import { findWorkspaceEntry, resolveWorkspaceModelFallbackChain } from "@/node/services/taskUtils"; import { - deleteWorkspaceMemoryWritableForEpoch, setWorkspaceMemoryWritableForEpoch, workspaceMemoryWritableForEpoch, } from "@/node/services/workspaceMemoryPolicyEpochs"; @@ -1131,75 +1130,51 @@ export class AgentSession { } /** - * Start a fresh policy epoch: the in-memory mirror and the durable - * accumulator both forget the previous epoch's turns. Durable-or-throw like - * the other boundary invalidations: a stale persisted deny would refuse - * harvests of the new, possibly all-writable epoch forever (and a stale - * grant is never left behind by this path — grants are re-recorded per - * turn). Nothing to do when the field is already absent. + * Destructive boundary (/clear, context reset, history replace): the + * in-memory mirror and every durable epoch record forget the discarded + * transcript. Durable-or-throw like the other boundary invalidations: a + * stale persisted deny would refuse harvests of the new, possibly + * all-writable epoch forever (a destructive boundary reuses epoch -1, so a + * surviving `-1: false` would pin the new segment). Nothing to do when the + * field is already absent. + * + * A COMPACTION boundary clears nothing durable (r77): its closing epoch's + * record and deny marker stay until the next destructive boundary. Two + * backends can each commit a boundary closing the same epoch, and each + * completion observes that epoch's verdict — the first observer consuming + * the record would leave the second reading only its own stale mirror and + * harvesting a read-only turn. Readers key by epoch and epoch keys never + * recur before a destructive boundary, so retained records are inert; the + * cost is one boolean per compaction epoch in config.json until then. */ - private async resetWorkspaceMemoryWritable(options?: { closingEpoch: number }): Promise { - // Compaction (closing epoch given): the completion callback already - // forgot the mirror synchronously. Destructive boundary: the mirror is - // forgotten only once the durable clear below completed — a reset that - // reports success while the persisted `-1` deny survives would pin the - // whole new segment to the stored-false fast path. - if (options !== undefined) this.workspaceMemoryWritable = undefined; - // The session-dir deny marker (fallback for an unwritable config.json) - // belongs to the closing epoch too. Same fence idea as below: a deny - // recorded for the new epoch survives. + private async resetWorkspaceMemoryWritable(): Promise { + // The mirror is forgotten only once the durable clear below completed — + // a reset that reports success while the persisted `-1` deny survives + // would pin the whole new segment to the stored-false fast path. await clearWorkspaceMemoryDenyMarker( this.config.rootDir, - path.join(this.config.sessionsDir, this.workspaceId), - options + path.join(this.config.sessionsDir, this.workspaceId) ); // Strict load: an unreadable — or transiently ABSENT — config.json would // read as the empty default, in which this workspace has no records to // clear; the reset would report success and leave the stale records - // behind (a destructive boundary reuses epoch -1, so a surviving deny - // would pin the new segment). Throwing makes the boundary a retryable - // partial failure instead; a registered workspace always has a config. + // behind. Throwing makes the boundary a retryable partial failure + // instead; a registered workspace always has a config. const entry = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); if (entry?.workspace.workspaceMemoryWritableByEpoch === undefined) { this.workspaceMemoryWritable = undefined; return; } - if ( - options !== undefined && - workspaceMemoryWritableForEpoch(entry.workspace, options.closingEpoch) === undefined - ) { - return; - } await this.config.editConfig((cfg) => { const current = findWorkspaceEntry(cfg, this.workspaceId); if (current === null) return cfg; - // Fenced to the epoch being closed: another backend may already have - // recorded the first turn of the NEW epoch between the completion and - // this locked write; its record (a different epoch's) must survive. - // Records are per epoch and readers key by epoch anyway - // (WorkspaceService.recordWorkspaceMemoryWritable), so this delete is - // hygiene, not the correctness boundary. A destructive boundary - // (no closing epoch: /clear, reset, history replace) discards every - // epoch's transcript and so every record. - if (options !== undefined) { - deleteWorkspaceMemoryWritableForEpoch(current.workspace, options.closingEpoch); - } else { - delete current.workspace.workspaceMemoryWritableByEpoch; - } + delete current.workspace.workspaceMemoryWritableByEpoch; return cfg; }); // Verified read-back: Config.saveConfig swallows write failures, so the - // awaited edit alone does not prove the clear landed. A destructive - // boundary reuses epoch -1, and a surviving `-1: false` would pin the - // new segment to the stored-false fast path once the mirror is cleared - // below — so the mirror is cleared only after the durable state agrees. + // awaited edit alone does not prove the clear landed. const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); - const stale = - after !== null && - (options === undefined - ? after.workspace.workspaceMemoryWritableByEpoch !== undefined - : workspaceMemoryWritableForEpoch(after.workspace, options.closingEpoch) !== undefined); - if (stale) { + if (after?.workspace.workspaceMemoryWritableByEpoch !== undefined) { throw new Error( `Workspace memory policy reset did not persist for ${this.workspaceId} (config write swallowed?)` ); @@ -1212,8 +1187,9 @@ export class AgentSession { * closing epoch's policy, so its accumulator carries into the new epoch — * durably, by re-binding the config value and the deny marker recorded for * `closingEpoch` to `nextEpoch` (another backend's first turn of the new - * epoch reads by epoch and would otherwise see nothing). Fenced like the - * reset: a value already bound to another epoch is left alone. + * epoch reads by epoch and would otherwise see nothing). The closing + * epoch's value is copied, never consumed (see resetWorkspaceMemoryWritable): + * another backend's boundary closing the same epoch observes it too. */ private async carryWorkspaceMemoryWritable( closingEpoch: number, @@ -1249,10 +1225,9 @@ export class AgentSession { const next = workspaceMemoryWritableForEpoch(current.workspace, nextEpoch); carried = next === undefined ? closing : next && closing; setWorkspaceMemoryWritableForEpoch(current.workspace, nextEpoch, carried); - deleteWorkspaceMemoryWritableForEpoch(current.workspace, closingEpoch); return cfg; }); - // Consumed by another backend's boundary meanwhile: nothing to carry. + // Gone meanwhile (a destructive boundary): nothing to carry. if (carried === undefined) return; const after = findWorkspaceEntry(this.config.loadExistingConfigOrThrow(), this.workspaceId); if ( @@ -1455,14 +1430,15 @@ export class AgentSession { ); // New epoch. A preserved tail copies messages produced under this // epoch's policy into the next one, so the fail-closed accumulator - // carries over with them (re-bound to the new epoch key, durably); + // carries over with them (copied to the new epoch key, durably); // otherwise the next normal turn restarts it. The mirror forgets the - // closing epoch right here, synchronously; the durable reset runs + // closing epoch right here, synchronously; the durable carry runs // AFTER the completion observation settled (it reads the closing // epoch's marker/config) and is awaited by this session's next turn - // (settleWorkspaceMemoryPolicyEpoch). Other backends need no such - // wait: every durable value is bound to its epoch, so the closing - // epoch's value is invisible to their new-epoch turns regardless. + // (settleWorkspaceMemoryPolicyEpoch). The closing epoch's durable + // records are left in place (resetWorkspaceMemoryWritable explains + // why); every durable value is bound to its epoch, so they are + // invisible to the new epoch's turns on any backend. const closingEpoch = metadata.previousBoundaryHistorySequence ?? -1; const preservedTail = (metadata.preservedTailMessageCount ?? 0) > 0; if (!preservedTail) this.workspaceMemoryWritable = undefined; @@ -1471,7 +1447,7 @@ export class AgentSession { .then(() => preservedTail ? this.carryWorkspaceMemoryWritable(closingEpoch, metadata.summaryHistorySequence) - : this.resetWorkspaceMemoryWritable({ closingEpoch }) + : undefined ) .catch((error: unknown) => { log.warn("Failed to reset the workspace memory policy epoch", { diff --git a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts index 598b30a78fe..71c45d71b54 100644 --- a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts +++ b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts @@ -5,7 +5,11 @@ import type { Config } from "@/node/config"; import { SCRATCH_PROJECT_CONFIG_KEY } from "@/common/constants/scratch"; import { getErrorMessage } from "@/common/utils/errors"; import { findWorkspaceEntry } from "@/node/services/taskUtils"; -import { readWorkspaceMemoryDenyMarker } from "@/node/services/workspaceMemoryDenyMarker"; +import { + readWorkspaceMemoryDenyMarker, + workspaceMemoryDenyMarkerPath, + writeWorkspaceMemoryDenyMarker, +} from "@/node/services/workspaceMemoryDenyMarker"; import { AgentSession } from "./agentSession"; import { createStreamLifecycleMocks } from "./agentSession.testHarness"; import type { AIService } from "./aiService"; @@ -21,7 +25,7 @@ import { createTestHistoryService } from "./testHistoryService"; * falls back to the session-dir marker. */ interface SessionInternals { - resetWorkspaceMemoryWritable(options?: { closingEpoch: number }): Promise; + resetWorkspaceMemoryWritable(): Promise; carryWorkspaceMemoryWritable(closingEpoch: number, nextEpoch: number): Promise; } @@ -116,11 +120,15 @@ describe("AgentSession workspace memory policy epoch boundary", () => { test("carry re-binds the closing epoch's value and proves it, falling back to the deny marker", async () => { const { internals, sessionDir, records, setRecords, swallowNextWrite, config } = await createSession(); - // Proven carry: the closing deny moves to the new epoch key. + // Proven carry: the closing deny is copied to the new epoch key — and + // kept under its own, for another backend's boundary closing the same + // epoch (its completion observes the closing verdict too). await setRecords({ "-1": false }); await internals.carryWorkspaceMemoryWritable(-1, 7); - expect(records()).toEqual({ "7": false }); + expect(records()).toEqual({ "-1": false, "7": false }); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(false); + await internals.carryWorkspaceMemoryWritable(-1, 8); + expect(records()).toEqual({ "-1": false, "7": false, "8": false }); // Swallowed write: the deny never reached epoch 9 in config, so the // session-dir marker denies epoch 9 instead of nothing. @@ -191,16 +199,28 @@ describe("AgentSession workspace memory policy epoch boundary", () => { } expect(session.workspaceMemoryWritableMirror()).toBe(false); expect(records()).toEqual({ "-1": false }); + }); - // Compaction boundary, fenced to the closing epoch: the same proof. - await setRecords({ "3": false, "8": true }); - swallowNextWrite(); - expect( - await internals - .resetWorkspaceMemoryWritable({ closingEpoch: 3 }) - .then(() => null, getErrorMessage) - ).toMatch(/did not persist/); - await internals.resetWorkspaceMemoryWritable({ closingEpoch: 3 }); - expect(records()).toEqual({ "8": true }); + test("a compaction boundary consumes neither the closing epoch's marker nor its wildcard", async () => { + const { internals, sessionDir } = await createSession(); + // Two backends' boundaries close epoch 3: each carry finds the entry. + await writeWorkspaceMemoryDenyMarker(sessionDir, 3); + await internals.carryWorkspaceMemoryWritable(3, 8); + await internals.carryWorkspaceMemoryWritable(3, 10); + for (const epoch of [3, 8, 10]) { + expect(await readWorkspaceMemoryDenyMarker(sessionDir, epoch)).toBe(true); + } + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(false); + // A wildcard (inherited from a malformed marker) is a deny of unknown + // epoch: the carry copies it to the new epoch and keeps it as-is. + await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); + await writeWorkspaceMemoryDenyMarker(sessionDir, 12); + await internals.carryWorkspaceMemoryWritable(3, 14); + for (const epoch of [3, 14, 99]) { + expect(await readWorkspaceMemoryDenyMarker(sessionDir, epoch)).toBe(true); + } + // The destructive boundary is the only clear. + await internals.resetWorkspaceMemoryWritable(); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); }); }); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index d832e37701e..fc590a74a8d 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -1064,11 +1064,18 @@ export class MemoryConsolidationService extends EventEmitter { } const projectPath = resolveConsolidationProjectPath(workspace); + // A child's redirected run sweeps under the owner's identity; the + // child's own removal tombstone still refuses every read and commit of + // the run (MemoryScopeContext.guardedWorkspaceId) — a remover in another + // backend cannot abort this controller. const ctx: MemoryScopeContext = { runtime: null, checkoutCwd: "", workspaceId, projectPath, + ...(options.actingWorkspaceId !== undefined && options.actingWorkspaceId !== workspaceId + ? { guardedWorkspaceId: options.actingWorkspaceId } + : {}), }; const result = yield* Effect.promise(async () => diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 5889d682107..0c7f12b89ef 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -450,14 +450,19 @@ export async function refreshLegacyAdoptionTargetStamps(args: { dirty = true; } if (renamed?.from != null && renamed.to != null) { - for (const [rel, record] of [...adopted]) { - // One-to-one copies only (the directory proof requires it; a lone file - // maps through its own record): the moved copy sits at the same - // relative position under the restored name. - if (record.created !== true || record.pending === true || record.target !== rel) continue; - if (!beneath(rel, renamed.from)) continue; - const movedRel = renamed.to + rel.slice(renamed.from.length); - if (adopted.has(movedRel)) continue; + const targets = new Set([...adopted.values()].map((record) => record.target)); + for (const record of [...adopted.values()]) { + // Every copy beneath the vacated endpoint — at its own relPath (the + // directory proof requires one-to-one) or a lone file's conflict import + // under imported// (r77) — now sits at the same relative + // position under the restored name. The restored name IS the legacy + // path the child's older rows address: a restored side without a record + // mapped one-to-one (remapRenameDestination); one that had a record + // (any target) is re-stamped above and gets no second record. + if (record.created !== true || record.pending === true) continue; + if (!beneath(record.target, renamed.from)) continue; + const movedRel = renamed.to + record.target.slice(renamed.from.length); + if (adopted.has(movedRel) || targets.has(movedRel)) continue; const stamp = await stampOf(movedRel); if (stamp === undefined) continue; // not moved after all: nothing to vouch for adopted.set(movedRel, { diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 5e44bbe2088..7d7f4585fd3 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1420,6 +1420,42 @@ describe("MemoryService", () => { } }); + it("guards a context acting on a removed child's behalf like the child itself", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + // A child's consolidation run sweeps under the OWNER's identity; the + // child's removal by another backend (tombstone, no local signal) must + // still refuse that run's reads and commits in every scope. + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner", guardedWorkspaceId: "ws-child" }; + await fixture.service.create(ownerCtx, "/memories/workspace/n.md", "shared", "agent"); + await fixture.service.create(ownerCtx, "/memories/global/g.md", "global", "agent"); + const tombstonePath = workspaceRemovalTombstonePath(fixture.xumHome, "ws-child"); + await fsPromises.mkdir(path.dirname(tombstonePath), { recursive: true }); + await fsPromises.writeFile(tombstonePath, JSON.stringify({ workspaceId: "ws-child" })); + for (const attempt of [ + () => fixture.service.view(ownerCtx, "/memories/workspace/n.md"), + () => + fixture.service.strReplace(ownerCtx, "/memories/workspace/n.md", "shared", "x", "agent"), + () => fixture.service.strReplace(ownerCtx, "/memories/global/g.md", "global", "x", "agent"), + () => fixture.service.create(ownerCtx, "/memories/project/p.md", "p", "agent"), + ]) { + const result = await attempt(); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("ws-child was removed"); + } + expect( + await fsPromises.readFile( + path.join(fixture.config.sessionsDir, "ws-owner", "memory", "n.md"), + "utf-8" + ) + ).toBe("shared"); + // The owner's own contexts are unaffected. + const plainOwner = { ...fixture.ctx, workspaceId: "ws-owner" }; + expect((await fixture.service.view(plainOwner, "/memories/workspace/n.md")).success).toBe( + true + ); + }); + it("refuses a read whose workspace was tombstoned while the legacy adoption pass ran", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -3564,6 +3600,41 @@ describe("MemoryService", () => { expect(await migrate()).toBe(0); }); + it("migrates a row whose only rollback record is malformed instead of dropping both", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A rollback row that names the create but whose action and inverse + // are unusable: it cannot be copied, so counting it as a completed + // rollback would delete the child journal with neither inverse kept. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { op: "bogus" }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + ); + expect(copy).toBeDefined(); + expect(copy!.data.rollbackOf).toBeUndefined(); + }); + it("treats a malformed store clock as order-unknown instead of 'earlier than everything'", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -4133,6 +4204,19 @@ describe("MemoryService", () => { id, evidence: { toolName: "test", actor: "user" }, }); + // An apply that loses the rollback lock at the commit point is + // compensated; the compensated file is yet another generation, and the + // record follows that one too (r77) — otherwise the retry would refuse. + const lostLock = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id: childEdit.id, + evidence: { toolName: "test", actor: "user" }, + testOnlyBeforeCommit: () => Promise.reject(new Error("lost the rollback lock")), + }); + expect(lostLock.success).toBe(false); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v2"); + expect(await recordStamp()).toBe((await adoptionTargetStamp(ownerCopy)) ?? undefined); // Rolling the edit back rewrites the copy: a new generation, but this // lineage's own — the record follows it, so the create still maps. expect((await rollback(childEdit.id)).success).toBe(true); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 40e0fa7105f..2467853d6c1 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -99,6 +99,15 @@ export interface MemoryScopeContext { * and sidecar logical keys; empty when no project identity is available. */ projectPath: string; + /** + * A further workspace on whose behalf this context acts, guarded like the + * acting one: a sub-agent's consolidation run sweeps the OWNER's notebook + * under the owner's identity (`workspaceId`), and the child's removal — + * possibly by another backend, which cannot abort this run — must refuse + * every read and commit of that run at the tombstone check, not only its + * start (r77). + */ + guardedWorkspaceId?: string; } export type MemoryActor = "agent" | "user"; @@ -1826,7 +1835,15 @@ export class MemoryService extends EventEmitter { /** Acting workspace plus the store's owner: both must be alive to touch the store. */ private guardedWorkspaceIds(ctx: MemoryScopeContext, store: MemoryStore): string[] { const owner = this.storeOwnerWorkspaceId(store); - return [...new Set(owner === null ? [ctx.workspaceId] : [ctx.workspaceId, owner])]; + return [ + ...new Set([ + ctx.workspaceId, + ...(owner === null ? [] : [owner]), + ...(ctx.guardedWorkspaceId === undefined || ctx.guardedWorkspaceId === "" + ? [] + : [ctx.guardedWorkspaceId]), + ]), + ]; } /** diff --git a/src/node/services/memoryWorkspaceOwner.test.ts b/src/node/services/memoryWorkspaceOwner.test.ts new file mode 100644 index 00000000000..0ea3171d561 --- /dev/null +++ b/src/node/services/memoryWorkspaceOwner.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "bun:test"; +import type { Config } from "@/node/config"; +import { + pinDescendantWorkspaceMemoryOwners, + resolveWorkspaceMemoryOwnerId, +} from "./memoryWorkspaceOwner"; + +type ProjectsConfig = ReturnType; + +function topology( + workspaces: Array<{ id: string; parentWorkspaceId?: string; memoryOwnerWorkspaceId?: string }> +): ProjectsConfig { + return { + projects: new Map([ + ["/tmp/project", { workspaces: workspaces.map((ws) => ({ path: `/tmp/${ws.id}`, ...ws })) }], + ]), + } as unknown as ProjectsConfig; +} + +describe("pinDescendantWorkspaceMemoryOwners", () => { + it("pins each surviving child to the owner it resolves to now", () => { + const cfg = topology([ + { id: "ws-owner" }, + { id: "ws-other" }, + { id: "ws-mid", parentWorkspaceId: "ws-owner" }, + // No pin: the walk through ws-mid reaches ws-owner. + { id: "ws-plain", parentWorkspaceId: "ws-mid" }, + // Stale pin (its owner is gone): the resolver walks past it today, but + // once ws-mid is removed that walk would dangle — replaced. + { id: "ws-stale", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-gone" }, + // Valid pin to another live notebook: that is the notebook this child + // uses; kept (continuity), not redirected to ws-mid's owner. + { id: "ws-pinned", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-other" }, + // Not a child of the removed node: untouched. + { id: "ws-sibling", parentWorkspaceId: "ws-owner" }, + ]); + const before = Object.fromEntries( + ["ws-plain", "ws-stale", "ws-pinned"].map((id) => [ + id, + resolveWorkspaceMemoryOwnerId(cfg, id), + ]) + ); + expect(before).toEqual({ + "ws-plain": "ws-owner", + "ws-stale": "ws-owner", + "ws-pinned": "ws-other", + }); + + const pinned = pinDescendantWorkspaceMemoryOwners(cfg, "ws-mid"); + expect(Object.fromEntries(pinned)).toEqual(before); + const entries = [...cfg.projects.values()][0].workspaces; + const pinOf = (id: string) => entries.find((ws) => ws.id === id)!.memoryOwnerWorkspaceId; + expect(pinOf("ws-plain")).toBe("ws-owner"); + expect(pinOf("ws-stale")).toBe("ws-owner"); + expect(pinOf("ws-pinned")).toBe("ws-other"); + expect(pinOf("ws-sibling")).toBeUndefined(); + + // With ws-mid gone, every pinned child still resolves as before. + const after = topology( + entries + .filter((ws) => ws.id !== "ws-mid") + .map((ws) => ({ + id: ws.id!, + ...(ws.parentWorkspaceId === undefined + ? {} + : { parentWorkspaceId: ws.parentWorkspaceId }), + ...(ws.memoryOwnerWorkspaceId === undefined + ? {} + : { memoryOwnerWorkspaceId: ws.memoryOwnerWorkspaceId }), + })) + ); + for (const [id, owner] of Object.entries(before)) { + expect(resolveWorkspaceMemoryOwnerId(after, id)).toBe(owner); + } + }); +}); diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index b6688e2465c..90ed427ba50 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -90,6 +90,38 @@ export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: return resolver; } +/** + * Removal of `removedWorkspaceId`: pin each surviving direct child to the + * owner it resolves to NOW, so the notebook it uses stays the same once the + * chain through the removed node dangles. A child's own valid pin already + * decides its owner and is kept; one that is absent — or stale (its owner + * gone, which the resolver walks past today but could not once this node is + * gone: the child would drop to a private store) — is replaced by the walk + * through the removed node (r77). Mutates the entries in place; returns the + * pins written, for the caller's verified read-back. + */ +export function pinDescendantWorkspaceMemoryOwners( + cfg: ProjectsConfig, + removedWorkspaceId: string +): Map { + assert(removedWorkspaceId.length > 0, "pinDescendantWorkspaceMemoryOwners requires an id"); + const resolve = workspaceMemoryOwnerResolver(cfg); + const pinned = new Map(); + for (const project of cfg.projects.values()) { + for (const workspace of project.workspaces) { + if (workspace.parentWorkspaceId !== removedWorkspaceId || workspace.id === undefined) { + continue; + } + // Resolved before this loop mutates anything: every child's chain runs + // through the removed node, never through a sibling being pinned. + const owner = resolve(workspace.id); + workspace.memoryOwnerWorkspaceId = owner; + pinned.set(workspace.id, owner); + } + } + return pinned; +} + /** * Session dirs of the OTHER registered members of `workspaceId`'s task tree — * every workspace resolving to the same owner (the owner itself, siblings, diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index f596f2c8257..4ae64bac7ee 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1223,6 +1223,56 @@ describe("refinementRollback", () => { }); expect(undoneDir2Create.success).toBe(true); expect(await pathExists(path.join(ownerRoot, "dir2", "b.md"))).toBe(false); + // Same for a lone file whose adopted copy is a conflict import under + // imported// (the owner already had different content at the + // post-rename name): the moved copy gets its tombstoned record too (r77). + await fixture.service.create(fixture.ctx, "/memories/workspace/c.md", "c\n", "agent"); + const cCreateRow = await lastRow(fixture.sessionDir); + await fixture.service.rename( + fixture.ctx, + "/memories/workspace/c.md", + "/memories/workspace/d.md", + "agent" + ); + const fileRenameRow = await lastRow(fixture.sessionDir); + await fsPromises.mkdir(path.join(ownerRoot, "imported", "child"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "imported", "child", "d.md"), "c\n"); + await fsPromises.writeFile(path.join(ownerRoot, "d.md"), "owner's own d\n"); + await writeManifest({ + "d.md": { content: "x", sidecar: "", target: "imported/child/d.md", created: true }, + }); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: fileRenameRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect(await fsPromises.readFile(path.join(ownerRoot, "c.md"), "utf-8")).toBe("c\n"); + expect(await pathExists(path.join(ownerRoot, "imported", "child", "d.md"))).toBe(false); + expect(await fsPromises.readFile(path.join(ownerRoot, "d.md"), "utf-8")).toBe( + "owner's own d\n" + ); + expect((await manifest()).get("c.md")).toMatchObject({ + target: "c.md", + created: true, + deleted: true, + targetStamp: (await adoptionTargetStamp(path.join(ownerRoot, "c.md"))) ?? undefined, + }); + expect( + ( + await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: cCreateRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + }) + ).success + ).toBe(true); + expect(await pathExists(path.join(ownerRoot, "c.md"))).toBe(false); }); it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index 5fd1a364430..f7ab8398758 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -1365,6 +1365,10 @@ export async function rollbackRefinement( await fileLock.assertStillOwned(); } catch (error) { await compensateApplied(applied, newInverse); + // The stamps above described the applied state; the compensated + // files are new generations again (r77). A reversed rename re-stamps + // both endpoints (nothing is synthesized: the restored side is empty). + await restampAdoptedCopies(); throw error; } if (opts.testOnlyBeforeRollbackJournal !== undefined) { diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 3cbc85d87d9..45d676c6bc6 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -116,10 +116,20 @@ export async function migrateSharedMemoryRefinementRows(args: { }; // Liveness follows the whole rollback chain (rollback → rollback of the // rollback re-applies): an original row is live when it has been rolled - // back an even number of times. Rollback rows themselves are never copied. + // back an even number of times. Only a rollback row this migration could + // itself carry (parseable rollback action AND inverse) counts (r77): a + // corrupted one is skipped below, and letting it also kill its intact + // target would delete the child journal with neither inverse preserved. + // The copied target is then live on the owner side; its divergence checks + // refuse a re-apply that no longer matches the tree (force overrides). const rollbackByTarget = new Map( rows - .filter((row) => row.data.rollbackOf !== undefined) + .filter( + (row) => + row.data.rollbackOf !== undefined && + RollbackRefinementActionSchema.safeParse(row.data.action).success && + RefinementInverseSchema.safeParse(row.data.inverse).success + ) .map((row) => [row.data.rollbackOf!, row] as const) ); // Returns null on a corrupted (cyclic / absurdly long) lineage: such a row diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index a1b261832bc..28a5d9bd05c 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -10,16 +10,16 @@ * reads EVERY message of the epoch) would carry the refused turn's rows into * the shared notebook. This marker lives in the session dir — the same * durability domain as chat.jsonl — and is ANDed into the accumulator wherever - * it is consulted; it is cleared only at the epoch boundary that clears the - * config bit. + * it is consulted; it is cleared only at the destructive boundary that clears + * the config records. * * Like the config records, the marker is bound to its compaction epoch (the * opening boundary's history sequence, -1 before any boundary) and holds one - * entry per epoch (the newest few): a reader consulting it for another epoch - * ignores it, so a backend starting the new epoch before this one's boundary - * reset landed cannot inherit a stale deny — and a deny that backend records - * for the new epoch cannot displace the closing epoch's deny before the - * compacting backend observes it (see workspaceMemoryPolicyEpochs.ts). + * entry per epoch: a reader consulting it for another epoch ignores it, so a + * backend starting the new epoch cannot inherit a stale deny — and a deny that + * backend records for the new epoch cannot displace the closing epoch's deny + * before every compacting backend observes it (see + * workspaceMemoryPolicyEpochs.ts). * * Fail-closed by construction: a missing marker (or one without an entry for * this epoch) is "no deny"; a present entry for this epoch, a wildcard entry @@ -56,8 +56,8 @@ export async function writeWorkspaceMemoryDenyMarker( // possibly the only evidence of some other backend's read-only turn in // the closing epoch. This writer, recording a deny for its own epoch, // cannot know which epoch that was, so the well-formed file it leaves - // behind carries the deny forward as a wildcard until a boundary - // observation clears it (clearWorkspaceMemoryDenyMarker). + // behind carries the deny forward as a wildcard until a destructive + // boundary clears it (clearWorkspaceMemoryDenyMarker). const epochs = record === "absent" || record === null ? [] : record.epochs; const wildcard = record === null || (record !== "absent" && record.wildcard); await writeMarkerRecord(markerPath, [...epochs.filter((e) => e !== epoch), epoch], wildcard); @@ -72,8 +72,8 @@ async function writeMarkerRecord( epochs: readonly number[], wildcard: boolean ): Promise { - // Entries are removed only by the boundary observation that consumes them - // (clear/carry), never by count — see workspaceMemoryPolicyEpochs.ts. + // Entries are removed only by a destructive boundary, never by count or by + // the observation of a compaction boundary — see workspaceMemoryPolicyEpochs.ts. const retained = [...new Set(epochs)].sort((a, b) => b - a); await writeFileAtomic( markerPath, @@ -158,51 +158,25 @@ export async function readWorkspaceMemoryDenyMarkerForEpochs( } /** - * Epoch boundary: remove the closing epoch's deny, durable-or-throw (verified - * absent). `closingEpoch` fences the clear to that epoch's entry: a deny - * another backend recorded for the NEW epoch in the meantime must survive, - * so the file is removed only once no entry is left. Only a well-formed - * marker can hold other epochs' entries; a truncated or malformed one is - * stale state from some earlier epoch (every reader already treated it as a - * deny for as long as it existed) and is healed here — otherwise one corrupt - * file would force every later epoch's accumulator to false until a - * destructive history clear. Without `closingEpoch` (destructive boundary), - * every epoch's deny goes. + * Destructive boundary (/clear, context reset, history replace): every + * epoch's deny goes, durable-or-throw (verified absent). Compaction boundaries + * clear nothing: the closing epoch's entry may still be observed by another + * backend's boundary closing the same epoch (r77; see + * AgentSession.resetWorkspaceMemoryWritable), and a malformed marker stays a + * deny for every epoch until the transcript it may describe is discarded. + * Unreadable is not malformed: a marker that cannot be read is refused rather + * than deleted (the caller's reset fails and is retried). */ export async function clearWorkspaceMemoryDenyMarker( rootDir: string, - sessionDir: string, - options?: { closingEpoch: number } + sessionDir: string ): Promise { const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); - // Read-check-delete under the session-dir target lock the writer holds - // (WorkspaceService.recordWorkspaceMemoryWritable's fallback), so the fence - // cannot go stale between the read and the rm: a new-epoch deny written in - // that gap — possibly the only durable record of it — would otherwise be - // deleted right after its writer verified it. + // Under the session-dir target lock the writer holds + // (WorkspaceService.recordWorkspaceMemoryWritable's fallback). await withTargetMutationLock(rootDir, sessionDir, async () => { - if (options !== undefined) { - const record = await readMarkerRecord(markerPath); - if (record === "absent") return; - // Unreadable is not malformed: the file may hold a newer epoch's deny - // (possibly the only durable record of a read-only turn). Refuse; the - // caller's reset fails and is retried rather than deleting unknown state. - if (record === "unreadable") { - throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); - } - if (record !== null) { - // This boundary observed the closing epoch (deny included): the - // epoch-less wildcard, if any, is consumed with it. - const remaining = record.epochs.filter((epoch) => epoch !== options.closingEpoch); - if (remaining.length === record.epochs.length && !record.wildcard) return; - if (remaining.length > 0) { - await writeMarkerRecord(markerPath, remaining, false); - if (await readWorkspaceMemoryDenyMarker(sessionDir, options.closingEpoch)) { - throw new Error(`Workspace memory deny marker could not be cleared at ${markerPath}`); - } - return; - } - } + if ((await readMarkerRecord(markerPath)) === "unreadable") { + throw new Error(`Workspace memory deny marker is unreadable at ${markerPath}`); } await fsPromises.rm(markerPath, { force: true }); if (await readWorkspaceMemoryDenyMarker(sessionDir)) { @@ -235,14 +209,15 @@ export async function carryWorkspaceMemoryDenyMarker( } if (record === "absent" || record === null) return; if (!record.wildcard && !record.epochs.includes(closingEpoch)) return; - // The wildcard denied the closing epoch; carried as the new epoch's deny. + // The closing entry (or the wildcard, which denied the closing epoch too) + // is copied, not moved: another backend's boundary closing the same + // epoch must still find it (r77). The wildcard stays a deny of unknown + // epoch — narrowing it to this boundary's epochs would drop the deny + // for whichever epoch it actually recorded. await writeMarkerRecord( markerPath, - [ - ...record.epochs.filter((epoch) => epoch !== closingEpoch && epoch !== nextEpoch), - nextEpoch, - ], - false + [...record.epochs.filter((epoch) => epoch !== nextEpoch), nextEpoch], + record.wildcard ); if (!(await readWorkspaceMemoryDenyMarker(sessionDir, nextEpoch))) { throw new Error(`Workspace memory deny marker did not persist at ${markerPath}`); diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts index 7fabd3f9139..b142006c2a8 100644 --- a/src/node/services/workspaceMemoryPolicyEpochs.ts +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -10,13 +10,16 @@ * value. A single slot would be overwritten by the new epoch's grant, dropping * a deny recorded for the closing epoch — and the compacting backend's own * mirror (writable) would then grant the harvest of a read-only turn. A - * record is removed only by the observation that consumes it — the - * compacting session's boundary reset/carry (AgentSession) or a destructive - * boundary — never by count: a backend suspended between persisting its - * boundary and observing the closing policy must still find the record - * however many epochs other backends opened meanwhile. Records of a boundary - * whose observer never ran (crash in between) linger until the next - * destructive boundary; that residue is bounded by such crashes. + * record is removed only by a destructive boundary (AgentSession. + * resetWorkspaceMemoryWritable), never by a compaction boundary's observation + * and never by count: two backends can each commit a boundary closing the + * same epoch, and a backend suspended between persisting its boundary and + * observing the closing policy must still find the record however many + * epochs other backends opened meanwhile. Readers key strictly by epoch and + * epoch keys (boundary history sequences; -1 only before the first boundary + * of a segment) never recur before the destructive boundary that drops every + * record, so retained records are inert — one boolean per compaction epoch + * in config.json until then. */ import type { Workspace as WorkspaceConfigEntry } from "@/node/config"; import assert from "@/common/utils/assert"; @@ -74,23 +77,3 @@ export function setWorkspaceMemoryWritableForEpoch( [epochKey(epoch)]: writable, }; } - -/** Forget `epoch`'s record; removes the field once no record is left. */ -export function deleteWorkspaceMemoryWritableForEpoch( - entry: WorkspaceConfigEntry, - epoch: number -): void { - const current = policyRecords(entry); - if (current === undefined) return; - if (current === null) { - // Malformed container: nothing recoverable in it, heal by dropping it. - delete entry.workspaceMemoryWritableByEpoch; - return; - } - const key = epochKey(epoch); - if (!Object.hasOwn(current, key)) return; - const next = { ...entry.workspaceMemoryWritableByEpoch }; - delete next[key]; - if (Object.keys(next).length === 0) delete entry.workspaceMemoryWritableByEpoch; - else entry.workspaceMemoryWritableByEpoch = next; -} diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 64390e8ee02..1b43e5d3f43 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9432,25 +9432,23 @@ describe("WorkspaceService initialize", () => { true ); expect(persisted()).toBe(false); - // Malformed marker still denies; an epoch boundary — even the fenced - // compaction one, since a malformed file cannot claim to be a newer - // deny — heals it and the next epoch can become writable again. + // Malformed marker still denies; only a DESTRUCTIVE boundary heals it + // (a compaction boundary clears nothing: another backend's boundary + // closing the same epoch must still find every entry). await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); // Another backend's new-epoch deny write must not heal the malformed // marker away (it may be the only evidence of a read-only turn in the - // closing epoch): the deny is carried as a wildcard until the closing - // boundary observes and clears it. + // closing epoch): the deny is carried as a wildcard, denying every + // epoch, until the destructive boundary. await writeWorkspaceMemoryDenyMarker(sessionDir, 7); expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 3)).toBe(true); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); - expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 7 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "not json"); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); // A present but non-boolean wildcard is corruption, not "false": read as // malformed (deny for every epoch) rather than dropping the only @@ -9465,23 +9463,24 @@ describe("WorkspaceService initialize", () => { JSON.stringify({ epochs: [3] }) ); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(false); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 3 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); - // Unreadable is not malformed: a marker that cannot be read may hold a - // newer epoch's deny, so the fenced clear refuses instead of deleting it. + // Unreadable is not malformed: a marker that cannot be read may hold + // denies, so the clear refuses instead of deleting it. await writeWorkspaceMemoryDenyMarker(sessionDir, 9); const unreadableMarker = spyOn(fsPromises, "readFile").mockImplementationOnce((() => Promise.reject(Object.assign(new Error("EIO"), { code: "EIO" }))) as never); - const refusedClear = await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { - closingEpoch: -1, - }).then( + const refusedClear = await clearWorkspaceMemoryDenyMarker( + realConfig.rootDir, + sessionDir + ).then( () => null, (error: unknown) => (error instanceof Error ? error.message : String(error)) ); expect(refusedClear).toContain("unreadable"); unreadableMarker.mockRestore(); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 9)).toBe(true); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 9 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); await fsPromises.writeFile(workspaceMemoryDenyMarkerPath(sessionDir), "{}"); await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); @@ -9494,18 +9493,14 @@ describe("WorkspaceService initialize", () => { true ); expect(persisted()).toBe(true); - // The fenced clear (compaction boundary) keeps a deny recorded for the - // NEW epoch; readers of any other epoch ignore that deny. A new-epoch - // deny written before the clear does not displace the closing one. + // Entries of several epochs coexist; readers of any other epoch ignore + // each, and all of them go together at the destructive boundary. await writeWorkspaceMemoryDenyMarker(sessionDir, -1); await writeWorkspaceMemoryDenyMarker(sessionDir, 7); expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(true); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: -1 }); - expect(await readWorkspaceMemoryDenyMarker(sessionDir, 7)).toBe(true); - expect(await readWorkspaceMemoryDenyMarker(sessionDir, -1)).toBe(false); - expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(true); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 7 }); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(false); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); // The durable bit is bound to its epoch too: the closing epoch's @@ -9575,7 +9570,7 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persistedFor(16)).toBe(false); - await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { closingEpoch: 12 }); + await clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir); // A carried grant (or no carried record at all) changes nothing. await realConfig.editConfig((cfg) => { const entry = findWorkspaceEntry(cfg, "policy-scratch")!.workspace; @@ -9861,23 +9856,24 @@ describe("WorkspaceService initialize", () => { const sessionDir = path.join(realConfig.sessionsDir, "policy-lock"); await writeWorkspaceMemoryDenyMarker(sessionDir, -1); // Another backend's deny writer holds the session-dir lock while the - // boundary reset starts its read-check-delete: the reset must queue - // behind it and then see (and keep) the new epoch's marker. + // destructive boundary reset starts: the reset must queue behind it + // (a write landing between its rm and its verification would read as + // a failed removal) and then discard that deny with the rest of the + // discarded transcript's entries. let cleared = false; let clear: Promise | undefined; await withTargetMutationLock(realConfig.rootDir, sessionDir, async () => { - clear = clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir, { - closingEpoch: -1, - }).then(() => { + clear = clearWorkspaceMemoryDenyMarker(realConfig.rootDir, sessionDir).then(() => { cleared = true; }); await new Promise((resolve) => setTimeout(resolve, 20)); expect(cleared).toBe(false); await writeWorkspaceMemoryDenyMarker(sessionDir, 5); + expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); }); await clear; expect(cleared).toBe(true); - expect(await readWorkspaceMemoryDenyMarker(sessionDir, 5)).toBe(true); + expect(await readWorkspaceMemoryDenyMarker(sessionDir)).toBe(false); } finally { await cleanup(); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index c00028dbc55..67eaffb708c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -145,7 +145,10 @@ import { startRemovalTombstoneLease, TombstoneNotDurableError, } from "@/node/services/workspaceRemoval"; -import { resolveWorkspaceMemoryOwnerId } from "@/node/services/memoryWorkspaceOwner"; +import { + pinDescendantWorkspaceMemoryOwners, + resolveWorkspaceMemoryOwnerId, +} from "@/node/services/memoryWorkspaceOwner"; import { readWorkspaceMemoryDenyMarkerForEpochs, readWorkspaceMemoryDenyMarker, @@ -6357,27 +6360,15 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { verifiedSharedMemoryOwnerId = sharedMemoryOwnerId; if (sharedMemoryOwnerId !== workspaceId) { try { - const pinOwner = (cfg: ReturnType): string[] => { - const pinned: string[] = []; - for (const project of cfg.projects.values()) { - for (const workspace of project.workspaces) { - if (workspace.parentWorkspaceId === workspaceId) { - workspace.memoryOwnerWorkspaceId ??= sharedMemoryOwnerId; - if (workspace.id !== undefined) pinned.push(workspace.id); - } - } - } - return pinned; - }; - let pinnedIds: string[] = []; + let pinnedOwners = new Map(); await this.config.editConfig((cfg) => { - pinnedIds = pinOwner(cfg); + pinnedOwners = pinDescendantWorkspaceMemoryOwners(cfg, workspaceId); return cfg; }); const persisted = this.config.loadConfigOrDefault(); - for (const id of pinnedIds) { + for (const [id, owner] of pinnedOwners) { const entry = findWorkspaceEntry(persisted, id); - if (!entry?.workspace.memoryOwnerWorkspaceId) { + if (entry?.workspace.memoryOwnerWorkspaceId !== owner) { throw new Error(`memory owner pin for descendant ${id} did not persist`); } } From aa3aa30049f088c4e61cc214308b244accb9628d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 14:16:05 +0000 Subject: [PATCH 84/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-e?= =?UTF-8?q?ighth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One predicate (`isUsableRollbackRow`: rollbackOf + parseable rollback action + parseable inverse) now decides what counts as a rollback everywhere: the engine's rolled-back set, its chain walk and its already-rolled-back check, and migration's source liveness and owner rollback targets. A corrupt rollback row — in this journal, a peer's, or a corrupted owner-side copy — no longer hides a mutation that is still live on disk or blocks re-copying the intact source. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 109 ++++++++++++++++++ .../services/refinement/refinementRollback.ts | 30 ++++- .../refinement/sharedMemoryRowMigration.ts | 18 ++- 3 files changed, 145 insertions(+), 12 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 7d7f4585fd3..7b3ea3287e9 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -3635,6 +3635,115 @@ describe("MemoryService", () => { expect(copy!.data.rollbackOf).toBeUndefined(); }); + it("re-copies a rollback row whose earlier copy was corrupted instead of treating the target as rolled back", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + const rollback = async (id: string) => { + const result = await rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + id, + evidence: { toolName: "test", actor: "user" }, + }); + expect(result.success).toBe(true); + return (await readRefinementEvents(childSessionDir)).at(-1)!; + }; + // create → rolled back → re-applied: the create is live, both rollback + // rows are its lineage. + const undo = await rollback(createRow.id); + await rollback(undo.id); + const migrate = () => + migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await migrate()).toBe(3); + // The first ROLLBACK's copy is corrupted before the removal is retried. + // Its bare `rollbackOf` must not block re-copying the intact source + // rollback: the owner would keep an unusable rollback record over a + // target the engine then reads as live. + const journalPath = path.join(ownerSessionDir, "durable-events.jsonl"); + const rewritten = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${undo.id}"`)) return line; + const row = JSON.parse(line) as { data: { action: unknown } }; + row.data.action = { op: "bogus" }; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, rewritten.join("\n")); + expect(await migrate()).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + const createCopy = ownerRows.find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + )!; + const undoCopies = ownerRows.filter((row) => row.data.migratedFrom === `ws-child:${undo.id}`); + expect(undoCopies).toHaveLength(2); + expect( + undoCopies.filter((row) => (row.data.action as { op: string }).op === "rollback") + ).toHaveLength(1); + expect(undoCopies.every((row) => row.data.rollbackOf === createCopy.id)).toBe(true); + }); + + it("a peer's corrupt rollback row does not hide the peer's live edit from conflict detection", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerCtx = { ...fixture.ctx, workspaceId: "ws-owner" }; + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(ownerCtx, "/memories/workspace/dir/a.md", "o1", "agent"); + await fixture.service.rename( + ownerCtx, + "/memories/workspace/dir", + "/memories/workspace/moved", + "agent" + ); + const ownerRename = (await readRefinementEvents(ownerSessionDir)).at(-1)!; + // The child edits beneath the renamed destination (later by the store + // clock), then a rollback row naming that edit lands with a corrupt + // action: the edit is still on disk. + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/moved/a.md", + "o1", + "c2", + "agent" + ); + const childEdit = (await readRefinementEvents(childSessionDir)).at(-1)!; + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "moved", "a.md"), text: "c2" }], + }, + rollbackOf: childEdit.id, + }, + }); + const refused = await rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + `later refinement row ${childEdit.id}` + ); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }); + it("treats a malformed store clock as order-unknown instead of 'earlier than everything'", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index f7ab8398758..e5a756dc1ef 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -67,6 +67,23 @@ import { export type RefinementEvent = Extract; +/** + * A rollback row the engine (and shared-memory row migration) may trust for + * lineage: `rollbackOf` set AND a parseable rollback action AND a parseable + * inverse (r78). Persisted rows are raw JSON, so a row can name a target while + * its payload is corrupt; counting such a row as "target rolled back" would + * hide a mutation that is still live on disk (its rollback row cannot itself + * be rolled back or copied), so it is treated like no rollback at all — + * conflict detection then sees the original as live (fail closed). + */ +export function isUsableRollbackRow(row: RefinementEvent): boolean { + return ( + row.data.rollbackOf !== undefined && + RollbackRefinementActionSchema.safeParse(row.data.action).success && + RefinementInverseSchema.safeParse(row.data.inverse).success + ); +} + /** All refinement rows in the session journal (byId-deduped, seq order). */ export async function listRefinements(sessionDir: string): Promise { assert(sessionDir.length > 0, "listRefinements requires a session dir"); @@ -765,7 +782,7 @@ async function collectDivergence( // live rollback chain only conflicts when its net effect differs from the // state the target left behind (see liveRowConflictsWithTarget). const rolledBackIds = new Set( - rows.map((row) => row.data.rollbackOf).filter((id): id is string => id !== undefined) + rows.filter(isUsableRollbackRow).map((row) => row.data.rollbackOf!) ); for (const row of rows) { if (row.id === target.id) continue; @@ -902,6 +919,9 @@ function liveRowConflictsWithTarget( let current: RefinementEvent = row; const seen = new Set([row.id]); while (current.data.rollbackOf !== undefined) { + // A corrupt rollback row anywhere in the chain (isUsableRollbackRow): the + // chain's net effect cannot be established — assume conflict. + if (!isUsableRollbackRow(current)) return true; const original = rows.find((r) => r.id === current.data.rollbackOf); if (original === undefined || seen.has(original.id)) { return true; // Corrupt chain (missing root or cycle): assume conflict. @@ -1037,7 +1057,13 @@ export async function rollbackRefinement( `Row '${opts.id}' was produced by a remote (SSH/Docker) workspace runtime; its paths are not addressable on this host. Remote skill rollbacks are not supported.` ); } - const existingRollback = rows.find((row) => row.data.rollbackOf === opts.id); + // Same predicate as liveness (isUsableRollbackRow): a corrupt rollback + // row cannot be rolled back "instead", and lineage treats its target as + // still live — so the target itself stays rollbackable (the divergence + // checks below decide whether the tree still matches its inverse). + const existingRollback = rows.find( + (row) => row.data.rollbackOf === opts.id && isUsableRollbackRow(row) + ); if (existingRollback !== undefined) { throw new RollbackError( `Row '${opts.id}' was already rolled back by row '${existingRollback.id}'. Roll back that row instead to re-apply.` diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 45d676c6bc6..5282da02daf 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -21,7 +21,7 @@ import { type RefinementFileReference, type RefinementInverseDraft, } from "./refinementJournal"; -import { listRefinements } from "./refinementRollback"; +import { isUsableRollbackRow, listRefinements } from "./refinementRollback"; import { createLegacyPathRemapper, LegacyPathNotAdoptedError, @@ -123,14 +123,7 @@ export async function migrateSharedMemoryRefinementRows(args: { // The copied target is then live on the owner side; its divergence checks // refuse a re-apply that no longer matches the tree (force overrides). const rollbackByTarget = new Map( - rows - .filter( - (row) => - row.data.rollbackOf !== undefined && - RollbackRefinementActionSchema.safeParse(row.data.action).success && - RefinementInverseSchema.safeParse(row.data.inverse).success - ) - .map((row) => [row.data.rollbackOf!, row] as const) + rows.filter(isUsableRollbackRow).map((row) => [row.data.rollbackOf!, row] as const) ); // Returns null on a corrupted (cyclic / absurdly long) lineage: such a row // is treated as non-migratable instead of hanging removal. @@ -182,8 +175,13 @@ export async function migrateSharedMemoryRefinementRows(args: { } // Owner rows already rolled back (by anyone): a second rollback row for // the same target would corrupt the lineage the rollback engine walks. + // Only usable rollback rows count (r78): a corrupted rollback copy is + // excluded from ownerIdBySource above, so its intact source is copied + // again — and must not be blocked here by the bare `rollbackOf` of that + // very corruption, or the retry would leave the owner with an unusable + // rollback record over a target the engine then reads as live. const ownerRollbackTargets = new Set( - ownerRows.map((ownerRow) => ownerRow.data.rollbackOf).filter((id) => id !== undefined) + ownerRows.filter(isUsableRollbackRow).map((ownerRow) => ownerRow.data.rollbackOf!) ); for (const row of rows) { if (row.data.kind !== "memory") continue; From 36ee55a9d81fbf6a7eb3bf9f881fb5b159edcbbf Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Thu, 10 Sep 2026 15:15:49 +0000 Subject: [PATCH 85/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20seventy-n?= =?UTF-8?q?inth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Adoption reuses a settled record's target for a sidecar-only change only while the copy is still the recorded generation; otherwise the record stops claiming it and the child's pin toggle no longer folds in. - isUsableRollbackRow also requires the rollback action's `of` to agree with `rollbackOf`. - Harvest coverage accepts only a nonnegative safe-integer request bound. - A native row's origin binds to the journal it was read from (listRefinements tags rows), never to its persisted workspaceId. --- _Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/common/types/durableEvent.ts | 25 ++++-- .../memoryConsolidationService.test.ts | 15 +++- .../services/memoryConsolidationService.ts | 5 +- src/node/services/memoryService.test.ts | 76 ++++++++++++++++--- src/node/services/memoryService.ts | 27 +++++-- .../services/refinement/refinementRollback.ts | 26 +++++-- .../refinement/sharedMemoryRowMigration.ts | 2 +- 7 files changed, 142 insertions(+), 34 deletions(-) diff --git a/src/common/types/durableEvent.ts b/src/common/types/durableEvent.ts index 2771f0242bd..0af88055d6b 100644 --- a/src/common/types/durableEvent.ts +++ b/src/common/types/durableEvent.ts @@ -234,15 +234,24 @@ export interface RefinementRowOrigin { /** * The journal position that orders a refinement row against rows of the same * origin (RefinementDataSchema.originJournal). A native row is positioned by - * its own journal; a migrated copy only by a carried, well-formed origin — - * anything else about a copy is no order evidence (null). + * the journal it was READ from (`journalWorkspaceId`, the session's workspace + * id — the caller's knowledge, never the row's own `workspaceId`, which is + * persisted data a corrupt row could carry equal to another journal's; r79) + * and only while the row agrees with it. A migrated copy is positioned only + * by a carried, well-formed origin. Anything else is no order evidence (null). */ -export function refinementRowOrigin(row: { - workspaceId: string; - seq: number; - data: { migratedFrom?: string; originJournal?: unknown; originSeq?: unknown }; -}): RefinementRowOrigin | null { - if (row.data.migratedFrom === undefined) return { journal: row.workspaceId, seq: row.seq }; +export function refinementRowOrigin( + row: { + workspaceId: string; + seq: number; + data: { migratedFrom?: string; originJournal?: unknown; originSeq?: unknown }; + }, + journalWorkspaceId: string | undefined +): RefinementRowOrigin | null { + if (row.data.migratedFrom === undefined) { + if (journalWorkspaceId === undefined || row.workspaceId !== journalWorkspaceId) return null; + return { journal: journalWorkspaceId, seq: row.seq }; + } const { originJournal, originSeq } = row.data; if ( typeof originJournal !== "string" || diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index cace08b452e..e37fe09d68b 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1367,10 +1367,11 @@ describe("MemoryConsolidationService", () => { await fixture.addWorkspace("ws-clean"); await fixture.addWorkspace("ws-corrupt"); await fixture.addWorkspace("ws-unbounded"); + await fixture.addWorkspace("ws-fractional"); const seed = async ( workspaceId: string, foreignTurn: number | null | undefined, - options?: { withoutBound: boolean } + options?: { withoutBound?: boolean; fractionalBound?: boolean } ) => { const reset = createMuxMessage("reset-1", "assistant", "", { compactionBoundary: true, @@ -1399,7 +1400,10 @@ describe("MemoryConsolidationService", () => { await fixture.historyService.appendToHistory( workspaceId, createMuxMessage("reply-1", "assistant", "Noted.", { - requestHistorySequence: prompt.metadata?.historySequence, + requestHistorySequence: + options?.fractionalBound === true + ? (prompt.metadata?.historySequence ?? 0) + 0.5 + : prompt.metadata?.historySequence, workspaceMemoryPolicyEpoch: closingEpoch, }) ); @@ -1453,6 +1457,13 @@ describe("MemoryConsolidationService", () => { // or corrupt: its user row may be gone, so nothing else would surface it. const unbounded = await seed("ws-unbounded", -1, { withoutBound: true }); expect(unbounded.success).toBe(false); + // A bound outside the sequence domain (fractional) covers no user row: + // the turn stays uncovered and refuses (r79). + const fractional = await seed("ws-fractional", undefined, { fractionalBound: true }); + expect(fractional.success).toBe(false); + if (!fractional.success) expect(fractional.error).toContain("harvest refused"); + expect(fixture.modelCalls).toHaveLength(0); + expect(unbounded.success).toBe(false); if (!unbounded.success) expect(unbounded.error).toContain("another epoch"); expect(fixture.modelCalls).toHaveLength(0); diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index fc590a74a8d..f23c3c0e816 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -393,7 +393,10 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb return "the compacted epoch holds a turn whose memory policy was recorded for another epoch; harvest refused (fail closed)"; } const bound = message.metadata?.requestHistorySequence; - if (typeof bound !== "number") continue; + // Persisted rows are raw JSON: only a history sequence in the clock's + // domain covers anything (r79); a fractional or negative value leaves the + // turn uncovered and the refusal below fails closed. + if (typeof bound !== "number" || !Number.isSafeInteger(bound) || bound < 0) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 7b3ea3287e9..4086fde26fb 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2262,6 +2262,25 @@ describe("MemoryService", () => { await fixture.metaService.setPinned(childKey, true); await fixture.service.listIndexEntries({ ...fixture.ctx }); expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + // ...but not once the copy is the OWNER's generation (deleted and + // recreated with identical bytes): a later child toggle no longer folds + // in (r79), and the record stops claiming the copy — the same rule + // deletion reconciliation and the rollback remapper apply. + const ownerCopy = path.join(fixture.config.sessionsDir, "ws-owner", "memory", "note.md"); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fixture.metaService.setPinned(childKey, false); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(true); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); }); it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { @@ -3729,19 +3748,54 @@ describe("MemoryService", () => { rollbackOf: childEdit.id, }, }); - const refused = await rollbackRefinement({ - sessionDir: ownerSessionDir, - id: ownerRename.id, - listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], - evidence: { toolName: "test", actor: "user" }, + const attempt = () => + rollbackRefinement({ + sessionDir: ownerSessionDir, + id: ownerRename.id, + listSharedWorkspaceMemoryPeerSessionDirs: () => [childSessionDir], + evidence: { toolName: "test", actor: "user" }, + }); + const expectRefused = async () => { + const refused = await attempt(); + expect(refused.success).toBe(false); + expect(refused.success ? "" : refused.error).toContain( + `later refinement row ${childEdit.id}` + ); + expect( + await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") + ).toBe("c2"); + }; + await expectRefused(); + // Lineage fields that disagree (a well-formed rollback action naming + // another row) are corrupt too (r79). + const childJournalPath = path.join(childSessionDir, "durable-events.jsonl"); + const rewriteChildRows = async (edit: (row: Record) => void) => { + const rewritten = (await fsPromises.readFile(childJournalPath, "utf-8")) + .split("\n") + .map((line) => { + if (line.trim() === "") return line; + const row = JSON.parse(line) as Record; + edit(row); + return JSON.stringify(row); + }); + await fsPromises.writeFile(childJournalPath, rewritten.join("\n")); + }; + await rewriteChildRows((row) => { + const data = row.data as { rollbackOf?: string; action: unknown }; + if (data.rollbackOf === childEdit.id) data.action = { op: "rollback", of: "other-row" }; + }); + await expectRefused(); + // A peer row whose persisted workspaceId was corrupted to the OWNER's + // must not be ordered by the owner journal's sequence (r79): the origin + // binds to the journal the row was read from, so the row falls back to + // the store clock and still reads as the later mutation. + await rewriteChildRows((row) => { + row.workspaceId = "ws-owner"; }); - expect(refused.success).toBe(false); - expect(refused.success ? "" : refused.error).toContain( - `later refinement row ${childEdit.id}` - ); expect( - await fsPromises.readFile(path.join(ownerSessionDir, "memory", "moved", "a.md"), "utf-8") - ).toBe("c2"); + (await readRefinementEvents(childSessionDir)).every((row) => row.workspaceId === "ws-owner") + ).toBe(true); + await expectRefused(); }); it("treats a malformed store clock as order-unknown instead of 'earlier than everything'", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 2467853d6c1..9c1f8fd6dac 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1408,6 +1408,10 @@ export class MemoryService extends EventEmitter { continue; // folded in earlier, nothing changed since } let target: { relPath: string; write: boolean; replaces?: boolean } | null = null; + // A child's pin toggle folds into the copy only while the copy is + // this adoption's generation (see below); an owner-owned file keeps + // the owner's pin. + let foldChildPin = true; if (previous !== undefined) { // The recorded target is reused only while it still holds bytes // this adoption put there — the owner may have edited, replaced or @@ -1434,7 +1438,6 @@ export class MemoryService extends EventEmitter { } if (priorContent === content) { target = { relPath: previous.target, write: false }; - record.created = previous.created === true; // A pending record is a copy this adoption wrote but could not // finish recording (crash or sidecar failure after the write): // the file holding exactly those bytes now is that write, so its @@ -1442,11 +1445,23 @@ export class MemoryService extends EventEmitter { // record carries a stamp, which is then the generation an // interrupted in-place replacement OVERWROTE (r75). A settled // record keeps the stamp it recorded — identical bytes in a - // different generation are the owner's (deletion then preserves). - record.targetStamp = - previous.pending === true - ? ((await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined) + // different generation are the owner's (r79: the same rule + // deletion reconciliation and the rollback remapper apply), so + // the record stops claiming the copy and the child's sidecar + // changes no longer reach it. + const currentStamp = + (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; + const ours = + previous.created === true && + (previous.pending === true || + (previous.targetStamp !== undefined && previous.targetStamp === currentStamp)); + record.created = ours; + record.targetStamp = !ours + ? undefined + : previous.pending === true + ? currentStamp : previous.targetStamp; + foldChildPin = previous.created !== true || ours; } else if ( previous.created === true && priorContent !== null && @@ -1537,7 +1552,7 @@ export class MemoryService extends EventEmitter { projectPath: ctx.projectPath, workspaceId: owner, }), - { pinned: childPinChanged ? "source" : "target" } + { pinned: childPinChanged && foldChildPin ? "source" : "target" } ); } catch (error) { log.warn( diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index e5a756dc1ef..f3a2717e120 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -77,9 +77,14 @@ export type RefinementEvent = Extract; * conflict detection then sees the original as live (fail closed). */ export function isUsableRollbackRow(row: RefinementEvent): boolean { + if (row.data.rollbackOf === undefined) return false; + const action = RollbackRefinementActionSchema.safeParse(row.data.action); + // The two lineage fields must agree (r79): a row whose action names one + // target while `rollbackOf` names another is corrupt, and trusting either + // side would hide a live mutation behind the other. return ( - row.data.rollbackOf !== undefined && - RollbackRefinementActionSchema.safeParse(row.data.action).success && + action.success && + action.data.of === row.data.rollbackOf && RefinementInverseSchema.safeParse(row.data.inverse).success ); } @@ -88,9 +93,20 @@ export function isUsableRollbackRow(row: RefinementEvent): boolean { export async function listRefinements(sessionDir: string): Promise { assert(sessionDir.length > 0, "listRefinements requires a session dir"); const events = await sharedDurableEventJournal(sessionDir).read(); - return events.filter((event): event is RefinementEvent => event.kind === "refinement"); + const rows = events.filter((event): event is RefinementEvent => event.kind === "refinement"); + const journalWorkspaceId = path.basename(path.resolve(sessionDir)); + for (const row of rows) journalOfRow.set(row, journalWorkspaceId); + return rows; } +/** + * The journal (session workspace id) each row object was read from, recorded + * by listRefinements: origin comparisons (sameOriginOrder) must not trust a + * row's persisted `workspaceId` for that (r79). Rows obtained any other way + * have no known journal and compare as order-unknown. + */ +const journalOfRow = new WeakMap(); + export interface RollbackRefinementOptions { sessionDir: string; /** Envelope `id` of the refinement row to roll back. */ @@ -562,8 +578,8 @@ function sameOriginOrder( row: RefinementEvent, other: RefinementEvent ): { rowAfter: boolean } | null { - const rowOrigin = refinementRowOrigin(row); - const otherOrigin = refinementRowOrigin(other); + const rowOrigin = refinementRowOrigin(row, journalOfRow.get(row)); + const otherOrigin = refinementRowOrigin(other, journalOfRow.get(other)); if (rowOrigin === null || otherOrigin === null || rowOrigin.journal !== otherOrigin.journal) { return null; } diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 5282da02daf..80e191172e0 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -259,7 +259,7 @@ export async function migrateSharedMemoryRefinementRows(args: { } const evidence = RefinementEvidenceSchema.safeParse(row.data.evidence); const postState = RefinementPostStateSchema.safeParse(row.data.postState); - const origin = refinementRowOrigin(row); + const origin = refinementRowOrigin(row, args.childWorkspaceId); // Throws: this is the only durable copy once the child's journal goes. const appended = await appendRefinementEventUnderBlobLock(ownerJournal, { sessionDir: args.ownerSessionDir, From 5bd3061c766903273bd66be6e613e66779399b4f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 00:39:24 +0000 Subject: [PATCH 86/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eightieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Policy epoch identities are never reused across destructive clears: a full clear opens a new history segment (history-segment.json) above every cleared sequence, appended rows carry the segment stamp, the boundary-less epoch is -(segmentStart + 1) (workspaceMemoryPolicyEpochOf) and compaction completion reports it as closingPolicyEpoch. Also: replaced adoption copies stop folding child pins (explicit `replaced` state), the revision token's notebook scan is memoized per owner, migration judges rollback copies with isUsableRollbackRow, tail copies require an integer request bound, and AIService removes the assistant placeholder on stream startup failure. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high` • Cost: `$844.45`_ --- src/common/orpc/schemas/memory.ts | 1 + src/common/types/compaction.ts | 8 ++ src/common/types/message.ts | 12 ++ .../utils/messages/compactionBoundary.test.ts | 69 +++++++++ .../utils/messages/compactionBoundary.ts | 47 ++++++ src/node/services/agentSession.ts | 20 ++- src/node/services/aiService.test.ts | 40 ++++++ src/node/services/aiService.ts | 7 + src/node/services/compactionHandler.test.ts | 10 ++ src/node/services/compactionHandler.ts | 17 ++- src/node/services/historyService.test.ts | 81 +++++++++-- src/node/services/historyService.ts | 134 ++++++++++++++++-- .../memoryConsolidationService.test.ts | 61 ++++++++ .../services/memoryConsolidationService.ts | 3 +- src/node/services/memoryLegacyAdoption.ts | 15 +- src/node/services/memoryService.test.ts | 73 ++++++++++ src/node/services/memoryService.ts | 57 +++++++- .../refinement/sharedMemoryRowMigration.ts | 13 +- src/node/services/turnRequestBuilder.ts | 26 ++-- .../services/workspaceMemoryDenyMarker.ts | 8 +- src/node/services/workspaceService.test.ts | 7 +- src/node/services/workspaceService.ts | 3 +- 22 files changed, 643 insertions(+), 69 deletions(-) diff --git a/src/common/orpc/schemas/memory.ts b/src/common/orpc/schemas/memory.ts index d857e31b781..09e2f571fa0 100644 --- a/src/common/orpc/schemas/memory.ts +++ b/src/common/orpc/schemas/memory.ts @@ -101,6 +101,7 @@ export const CompactionCompletionMetadataSchema = z.object({ summaryHistorySequence: z.number(), compactionEpoch: z.number(), previousBoundaryHistorySequence: z.number().optional(), + closingPolicyEpoch: z.number().optional(), compactionRequestMessageId: z.string(), // RLM keep-recent floor: preserved-tail copies appended after the boundary. preservedTailMessageCount: z.number().optional(), diff --git a/src/common/types/compaction.ts b/src/common/types/compaction.ts index 02e3b35146b..ce9d73ebfde 100644 --- a/src/common/types/compaction.ts +++ b/src/common/types/compaction.ts @@ -12,6 +12,14 @@ export interface CompactionCompletionMetadata { summaryHistorySequence: number; compactionEpoch: number; previousBoundaryHistorySequence?: number; + /** + * The workspace-memory policy epoch the compacted rows belonged to + * (workspaceMemoryPolicyEpochOf over the compacted history): the key their + * turns recorded under and the one the harvest and the policy carry read. + * Absent on records persisted by builds before the history segment stamp + * (compactionClosingPolicyEpoch derives their legacy identity). + */ + closingPolicyEpoch?: number; compactionRequestMessageId: string; /** * RLM keep-recent floor: number of preserved-tail copies appended after the diff --git a/src/common/types/message.ts b/src/common/types/message.ts index 46e216fca56..85a77ceebc3 100644 --- a/src/common/types/message.ts +++ b/src/common/types/message.ts @@ -975,6 +975,18 @@ export interface MuxMetadata { */ workspaceMemoryPolicyEpoch?: number; historySequence?: number; // Assigned by backend for global message ordering (required when writing to history) + /** + * Start sequence of the history segment this row was appended in + * (HistoryService, `history-segment.json`). A full clear opens a new + * segment whose sequences continue above every sequence the cleared + * history ever used, so a boundary's history sequence — and the + * boundary-less epoch identity `-(segmentStart + 1)` derived from this + * stamp (workspaceMemoryPolicyEpochOf) — never recurs across destructive + * clears: a turn started before a clear cannot be mistaken for one of the + * segment that replaced it. Omitted in the first segment (start 0), which + * legacy rows without the stamp belong to as well. + */ + historySegment?: number; /** Provider step boundaries in parts, persisted so continuous compaction can keep complete steps. */ stepStartPartIndices?: number[]; duration?: number; diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index e56effb1104..da1794a8818 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -3,14 +3,83 @@ import { describe, expect, it } from "bun:test"; import { createMuxMessage } from "@/common/types/message"; import { + compactionClosingPolicyEpoch, epochHasPriorTurnRows, findLatestCompactionBoundaryIndex, findLatestContextBoundaryIndex, hasProviderEligibleMessages, sliceMessagesForProviderFromLatestContextBoundary, sliceMessagesFromLatestCompactionBoundary, + workspaceMemoryPolicyEpochOf, } from "./compactionBoundary"; +describe("workspaceMemoryPolicyEpochOf", () => { + const boundary = (id: string, historySequence: number) => + createMuxMessage(id, "assistant", "summary", { + compacted: "user", + compactionBoundary: true, + compactionEpoch: 1, + historySequence, + }); + + it("is -1 before any boundary for legacy and first-segment rows", () => { + expect(workspaceMemoryPolicyEpochOf([])).toBe(-1); + expect( + workspaceMemoryPolicyEpochOf([ + createMuxMessage("u1", "user", "a", { historySequence: 0 }), + createMuxMessage("u2", "user", "b", { historySequence: 1, historySegment: 0 }), + ]) + ).toBe(-1); + }); + + it("derives a segment-unique boundary-less identity from the segment stamp", () => { + // Rows of a later segment (after a full clear) never share the cleared + // segment's -1; any subset of the segment yields the same identity. + const rows = [ + createMuxMessage("u1", "user", "a", { historySequence: 7, historySegment: 7 }), + createMuxMessage("a1", "assistant", "b", { historySequence: 8, historySegment: 7 }), + ]; + expect(workspaceMemoryPolicyEpochOf(rows)).toBe(-8); + expect(workspaceMemoryPolicyEpochOf(rows.slice(1))).toBe(-8); + // A row that lost its stamp (rewritten in place by an older build) + // cannot pull the segment back to the legacy identity. + expect( + workspaceMemoryPolicyEpochOf([ + ...rows, + createMuxMessage("a2", "assistant", "c", { historySequence: 9 }), + ]) + ).toBe(-8); + }); + + it("ignores malformed stamps and prefers the latest boundary's sequence", () => { + expect( + workspaceMemoryPolicyEpochOf([ + createMuxMessage("u1", "user", "a", { + historySequence: 3, + historySegment: -4 as unknown as number, + }), + createMuxMessage("u2", "user", "b", { historySequence: 4, historySegment: 2.5 }), + ]) + ).toBe(-1); + expect( + workspaceMemoryPolicyEpochOf([ + boundary("s1", 5), + createMuxMessage("u1", "user", "a", { historySequence: 6, historySegment: 5 }), + ]) + ).toBe(5); + }); +}); + +describe("compactionClosingPolicyEpoch", () => { + it("prefers the recorded closing epoch and falls back to the legacy identity", () => { + expect( + compactionClosingPolicyEpoch({ closingPolicyEpoch: -8, previousBoundaryHistorySequence: 3 }) + ).toBe(-8); + expect(compactionClosingPolicyEpoch({ previousBoundaryHistorySequence: 3 })).toBe(3); + expect(compactionClosingPolicyEpoch({})).toBe(-1); + }); +}); + describe("findLatestCompactionBoundaryIndex", () => { it("returns the newest compaction boundary via reverse scan", () => { const messages = [ diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index aacf01a6350..3610bfe196e 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -92,6 +92,53 @@ export function latestContextBoundaryHistorySequence( return latest; } +/** + * Start sequence of the history segment `messages` belong to (the largest + * `historySegment` stamp among them; 0 for legacy rows and the first + * segment). Every row of a segment carries the same stamp, so any non-empty + * subset of the segment yields the same value; a full clear opens a segment + * with a strictly larger start (HistoryService). + */ +export function historySegmentStart(messages: readonly MuxMessage[]): number { + let start = 0; + for (const message of messages) { + const segment = message.metadata?.historySegment; + if (typeof segment !== "number" || !Number.isSafeInteger(segment) || segment < 0) continue; + if (segment > start) start = segment; + } + return start; +} + +/** + * The compaction epoch `messages` (an active-context read) belong to, as the + * workspace-memory write policy keys it: the latest durable boundary's + * history sequence, or `-(segmentStart + 1)` before any boundary of the + * segment (-1 in the first segment, as before the stamp existed). Sequences + * never recur across full clears (each clear opens a segment above every + * sequence used so far), so neither identity is ever reused for a different + * conversation: a turn that recorded its policy under the pre-clear identity + * cannot pass as one of the post-clear epoch, however the clear and its + * appends interleave across backends. Compaction completion reports the same + * value as `closingPolicyEpoch`, so the completion-side observation and every + * backend's turn records agree on which epoch a value belongs to. + */ +export function workspaceMemoryPolicyEpochOf(messages: readonly MuxMessage[]): number { + return latestContextBoundaryHistorySequence(messages) ?? -(historySegmentStart(messages) + 1); +} + +/** + * The policy epoch a compaction closed: `closingPolicyEpoch` when the + * completion recorded it, else the identity older builds used (the previous + * boundary's sequence, -1 before any) so persisted legacy records still key + * their turns' stamps. + */ +export function compactionClosingPolicyEpoch(metadata: { + closingPolicyEpoch?: number; + previousBoundaryHistorySequence?: number; +}): number { + return metadata.closingPolicyEpoch ?? metadata.previousBoundaryHistorySequence ?? -1; +} + /** * Locate the latest durable context boundary in reverse chronological order. * diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index e41207f9bb4..92773a6e217 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -8,7 +8,10 @@ import { estimateFreshRequestTokensForModel } from "./contextBudgetCounting"; import type { RequestAssemblySnapshot } from "./events/eventSpine"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import { createContextBudgetRejectedMessage } from "@/common/utils/messages/contextBudgetRejection"; -import { sliceMessagesForProviderFromLatestContextBoundary } from "@/common/utils/messages/compactionBoundary"; +import { + compactionClosingPolicyEpoch, + sliceMessagesForProviderFromLatestContextBoundary, +} from "@/common/utils/messages/compactionBoundary"; import { randomUUID } from "crypto"; import { sandboxHostService } from "./sandbox/sandboxHostService"; import { applyToolPolicyToNames, isSessionHistoryDisabled } from "@/common/utils/tools/toolPolicy"; @@ -1132,11 +1135,14 @@ export class AgentSession { /** * Destructive boundary (/clear, context reset, history replace): the * in-memory mirror and every durable epoch record forget the discarded - * transcript. Durable-or-throw like the other boundary invalidations: a - * stale persisted deny would refuse harvests of the new, possibly - * all-writable epoch forever (a destructive boundary reuses epoch -1, so a - * surviving `-1: false` would pin the new segment). Nothing to do when the - * field is already absent. + * transcript. The new segment's epoch identities never recur (a full clear + * continues history sequences above the cleared segment, so neither a + * boundary's sequence nor the boundary-less `-(segmentStart + 1)` is + * reused; workspaceMemoryPolicyEpochOf), so the discarded records are + * inert to the new segment's readers — dropping them is hygiene, kept + * durable-or-throw like the other boundary invalidations so a reported + * success never leaves stale state behind. Nothing to do when the field is + * already absent. * * A COMPACTION boundary clears nothing durable (r77): its closing epoch's * record and deny marker stay until the next destructive boundary. Two @@ -1439,7 +1445,7 @@ export class AgentSession { // records are left in place (resetWorkspaceMemoryWritable explains // why); every durable value is bound to its epoch, so they are // invisible to the new epoch's turns on any backend. - const closingEpoch = metadata.previousBoundaryHistorySequence ?? -1; + const closingEpoch = compactionClosingPolicyEpoch(metadata); const preservedTail = (metadata.preservedTailMessageCount ?? 0) > 0; if (!preservedTail) this.workspaceMemoryWritable = undefined; const reset = observed diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index 6897bc32523..a6a325b3be6 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -8,6 +8,7 @@ import * as path from "node:path"; import { describe, it, expect, beforeEach, afterEach, mock, spyOn } from "bun:test"; +import { Err } from "@/common/types/result"; import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries"; import { AIService, resolveMuxProjectRootForHostFs } from "./aiService"; import { discoverAvailableSubagentsForToolContext } from "./turnContextAssembler"; @@ -2503,6 +2504,45 @@ describe("AIService.streamMessage compaction boundary slicing", () => { expect(initialMetadata.routeProvider).toBe("openrouter"); }); + it("removes the assistant placeholder when stream startup fails", async () => { + using xumHome = new DisposableTempDir("ai-service-startup-failure-placeholder"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-startup-failure-placeholder"; + const metadata = createLocalWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(xumHome.path, metadata); + const internals = harness.service as unknown as { + historyService: HistoryService; + streamManager: StreamManager; + }; + const deleted: string[] = []; + spyOn(internals.historyService, "deleteMessage").mockImplementation((_workspaceId, id) => { + deleted.push(id); + return Promise.resolve({ success: true, data: undefined }); + }); + spyOn(internals.streamManager, "startStream").mockResolvedValue( + Err({ type: "unknown", raw: "temp dir creation failed" }) + ); + + const result = await harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "continue")], + workspaceId, + modelString: "openai:gpt-5.2", + thinkingLevel: "medium", + }); + + expect(result.success).toBe(false); + // The placeholder carried the request bound and policy epoch stamp of a + // turn that never ran; left behind it would vouch for the user batch at + // the harvest gate (epochHarvestRefusal). + const appended = (internals.historyService.appendToHistory as ReturnType).mock + .calls as unknown as Array<[string, { id: string; role: string }]>; + const placeholder = appended.find(([, message]) => message.role === "assistant")?.[1]; + if (!placeholder) throw new Error("Expected an appended assistant placeholder"); + expect(deleted).toEqual([placeholder.id]); + }); + it("passes muxMetadata into initial stream metadata", async () => { using xumHome = new DisposableTempDir("ai-service-mux-metadata"); const projectPath = path.join(xumHome.path, "project"); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index a004638b864..04cc3971ec2 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -1026,6 +1026,13 @@ export class AIService extends EventEmitter { startupState.pendingRunMetadataId = null; } buildOutcome.logStartOutcome("stream_start_failed", streamResult.error.type); + // No stream ran for this turn. Its placeholder already carries the + // request bound and the policy epoch stamp (they must be on the row + // StreamManager finalizes), which would present the user batch to the + // harvest gate as covered by a turn — remove it like an aborted + // startup's, so the never-started turn stays excluded until a retry + // actually runs it. + await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); return Err(streamResult.error); } diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 3de6d3846d2..92fb46d0c89 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1945,6 +1945,14 @@ describe("CompactionHandler", () => { createMuxMessage("a9", "assistant", "old-build turn with a corrupted bound", { requestHistorySequence: null as unknown as number, }), + // A stamped turn whose bound is outside the sequence domain covers + // nothing (the harvest gate applies the same rule, r79/r80): its + // user row stays unstamped, the row itself keeps its stamp. + createMuxMessage("u10", "user", "question behind a fractional bound"), + createMuxMessage("a10", "assistant", "fractional bound", { + requestHistorySequence: boundarySequence + 17.5, + workspaceMemoryPolicyEpoch: boundarySequence, + }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1968,6 +1976,8 @@ describe("CompactionHandler", () => { undefined, // a7 (malformed stamp) boundarySequence, // a8 (no stamp, no bound: synthetic payload row) undefined, // a9 (no stamp, malformed bound: not a synthetic row) + undefined, // u10 (its turn's bound is fractional: never covered) + boundarySequence, // a10 (recorded stamp kept) ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 93e0109fc0c..4bf222ab2b4 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -43,6 +43,7 @@ import { isDurableContextBoundaryMarker, latestContextBoundaryHistorySequence, sliceMessagesFromLatestCompactionBoundary, + workspaceMemoryPolicyEpochOf, } from "@/common/utils/messages/compactionBoundary"; import { extractReadFilePaths, mergeReadFilePaths } from "@/common/utils/messages/extractReadFiles"; import { @@ -1270,7 +1271,7 @@ export class CompactionHandler { ); const idMap = new Map(params.tail.map((row) => [row.id, createPreservedTailCopyMessageId()])); // Same closing epoch the completion metadata reports below. - const closingPolicyEpoch = latestContextBoundaryHistorySequence(params.messages) ?? -1; + const closingPolicyEpoch = workspaceMemoryPolicyEpochOf(params.messages); const copies = this.buildCoveredTailCopies(params.tail, idMap, closingPolicyEpoch).map( (copy) => { // Continuous compaction prunes the just-finished answer too. Keep recent pages @@ -1339,6 +1340,7 @@ export class CompactionHandler { summaryHistorySequence: sequence, compactionEpoch: epoch, previousBoundaryHistorySequence: latestContextBoundaryHistorySequence(params.messages), + closingPolicyEpoch: workspaceMemoryPolicyEpochOf(params.messages), compactionRequestMessageId: boundary.id, preservedTailMessageCount: copies.length, }); @@ -1402,6 +1404,9 @@ export class CompactionHandler { assert(Number.isInteger(nextCompactionEpoch), "next compaction epoch must be an integer"); const previousBoundaryHistorySequence = latestContextBoundaryHistorySequence(messages); + // The policy epoch the compacted rows belong to: the key their turns + // recorded under (TurnRequestBuilder) and the one the harvest reads. + const closingPolicyEpoch = workspaceMemoryPolicyEpochOf(messages); const maxExistingHistorySequence = this.getMaxExistingHistorySequence(messages); // For idle compaction, preserve the original recency timestamp so the workspace @@ -1509,7 +1514,7 @@ export class CompactionHandler { messages, compactionRequestMessageId, summaryMessage.id, - previousBoundaryHistorySequence ?? -1 + closingPolicyEpoch ); const persistenceResult = @@ -1578,6 +1583,7 @@ export class CompactionHandler { summaryHistorySequence: persistedSequence, compactionEpoch: nextCompactionEpoch, previousBoundaryHistorySequence, + closingPolicyEpoch, compactionRequestMessageId, preservedTailMessageCount: preservedTailCopies.length, }); @@ -1680,7 +1686,12 @@ export class CompactionHandler { if (message.role !== "assistant") continue; const bound = message.metadata?.requestHistorySequence; const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; - if (typeof bound !== "number" || typeof policyEpoch !== "number") continue; + // Same domain check as the harvest gate (epochHarvestRefusal, r79): a + // fractional or negative bound covers nothing there, so it must not + // stamp a batch here either — the copies would then carry an epoch + // without ever having been covered, and the association would grant + // in a later epoch what the gate refused in this one. + if (typeof policyEpoch !== "number" || !isNonNegativeInteger(bound)) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; for (const id of [ diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 4d4ef681bf3..a5fe524a62a 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -1409,13 +1409,15 @@ describe("HistoryService", () => { const cachedNext = counters.sequenceCounters.get(nextWorkspace); const next = row("next"); expect((await restarted.appendToHistory(nextWorkspace, next)).success).toBe(true); - expect(next.metadata?.historySequence).toBe(method === "clear" ? 0 : 101); + // A clear opens a new history segment above every cleared sequence + // (history-segment.json) rather than restarting at 0. + expect(next.metadata?.historySequence).toBe(101); const reloaded = new HistoryService(config); const later = row("later"); expect((await reloaded.appendToHistory(nextWorkspace, later)).success).toBe(true); - expect(later.metadata?.historySequence).toBe(method === "clear" ? 1 : 102); + expect(later.metadata?.historySequence).toBe(102); if (method !== "archive delete") { - expect(cachedNext).toBe(method === "clear" ? 0 : 101); + expect(cachedNext).toBe(101); } } ); @@ -2548,18 +2550,49 @@ describe("HistoryService", () => { expect(exists).toBe(false); }); - it("should reset sequence counter", async () => { + it("continues sequences above the cleared history in a new segment", async () => { const workspaceId = "workspace1"; - const msg1 = createMuxMessage("msg1", "user", "Hello"); - - await service.appendToHistory(workspaceId, msg1); + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.appendToHistory(workspaceId, createMuxMessage("msg2", "user", "Second")); await service.clearHistory(workspaceId); - const msg2 = createMuxMessage("msg2", "user", "New message"); - await service.appendToHistory(workspaceId, msg2); + // A sequence — or a policy epoch identity derived from one — must never + // name rows of two different conversations, so the cleared segment's + // sequences are retired for good and the new rows carry their segment. + const msg3 = createMuxMessage("msg3", "user", "New message"); + await service.appendToHistory(workspaceId, msg3); + expect(msg3.metadata?.historySequence).toBe(2); + expect(msg3.metadata?.historySegment).toBe(2); - const messages = await collectFullHistory(service, workspaceId); - expect(messages[0].metadata?.historySequence).toBe(0); + // Durable across a restart: a fresh service scanning the (short) new + // history must not fall back to the cleared sequences. + const restarted = new HistoryService(config); + const msg4 = createMuxMessage("msg4", "user", "After restart"); + await restarted.appendToHistory(workspaceId, msg4); + expect(msg4.metadata?.historySequence).toBe(3); + expect(msg4.metadata?.historySegment).toBe(2); + + // A second clear opens a segment strictly above the previous one even + // though only the new segment's rows were cleared. + await restarted.clearHistory(workspaceId); + const msg5 = createMuxMessage("msg5", "user", "Third conversation"); + await restarted.appendToHistory(workspaceId, msg5); + expect(msg5.metadata?.historySequence).toBe(4); + expect(msg5.metadata?.historySegment).toBe(4); + }); + + it("stamps rows appended by a foreign backend after a clear with the new segment", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + // A second backend with a stale cached counter (multi-instance). + const foreign = new HistoryService(config); + await foreign.appendToHistory(workspaceId, createMuxMessage("msg2", "user", "Foreign")); + await service.clearHistory(workspaceId); + + const late = createMuxMessage("msg3", "assistant", "in flight before the clear"); + expect((await foreign.appendToHistory(workspaceId, late)).success).toBe(true); + expect(late.metadata?.historySequence).toBe(2); + expect(late.metadata?.historySegment).toBe(2); }); it("should succeed when clearing non-existent history", async () => { @@ -2570,16 +2603,38 @@ describe("HistoryService", () => { expect(result.success).toBe(true); }); - it("should reset sequence counter even when file doesn't exist", async () => { + it("opens a new segment even when clearing an empty history", async () => { const workspaceId = "workspace-no-history"; + // A turn admitted before the clear may still persist its policy under + // the boundary-less identity of the segment being cleared; the new + // segment must not share it, so the segment moves on regardless. await service.clearHistory(workspaceId); const msg = createMuxMessage("msg1", "user", "First"); await service.appendToHistory(workspaceId, msg); const messages = await collectFullHistory(service, workspaceId); - expect(messages[0].metadata?.historySequence).toBe(0); + expect(messages[0].metadata?.historySequence).toBe(1); + expect(messages[0].metadata?.historySegment).toBe(1); + }); + + it("refuses to append when the segment file is unreadable", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory(workspaceId); + await fs.writeFile( + path.join(config.sessionsDir, workspaceId, "history-segment.json"), + "not json" + ); + + // Reading a corrupt segment as 0 would restart at the retired sequences. + const restarted = new HistoryService(config); + const result = await restarted.appendToHistory( + workspaceId, + createMuxMessage("msg2", "user", "Second") + ); + expect(result.success).toBe(false); }); }); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index e23f88c8e49..20ebb21fc89 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -195,6 +195,12 @@ function stripContextUsage(message: MuxMessage): MuxMessage { }; } +/** The persisted row's segment stamp, kept across in-place replacement (see MuxMetadata.historySegment). */ +function preservedHistorySegment(existing: MuxMessage): { historySegment?: number } { + const segment = existing.metadata?.historySegment; + return segment === undefined ? {} : { historySegment: segment }; +} + function getCompactionMetadataToPreserve( workspaceId: string, existingMessage: MuxMessage, @@ -414,8 +420,22 @@ export class HistoryService { private readonly CHAT_FILE = CHAT_FILE_NAME; private readonly CHAT_ARCHIVE_FILE = CHAT_ARCHIVE_FILE_NAME; private readonly PARTIAL_FILE = "partial.json"; + /** + * `{ start }`: the first history sequence of the current history segment. + * A full clear does not restart sequences at 0 — it opens a new segment + * above every sequence the cleared history used (advanceHistorySegment), + * so no sequence, and no epoch identity derived from one (a boundary's + * sequence, or `-(start + 1)` before any boundary; see + * workspaceMemoryPolicyEpochOf), is ever reused for a different + * conversation. Rows appended in a later segment carry its start as + * `metadata.historySegment`. Absent = first segment (start 0). + */ + private readonly HISTORY_SEGMENT_FILE = "history-segment.json"; // Track next sequence number per workspace in memory private sequenceCounters = new Map(); + // Current segment start per workspace, loaded with the counter + // (getMaxHistorySequence) and advanced by full clears under the history lock. + private historySegmentStarts = new Map(); // Workspaces whose chat.jsonl was already checked for a sealed (pre-boundary) // prefix this process. Guards the lazy one-time migration of legacy files; // new boundaries rotate eagerly at write time. @@ -1808,8 +1828,81 @@ export class HistoryService { return newest; } + private getHistorySegmentPath(workspaceId: string): string { + return path.join(this.getSessionDir(workspaceId), this.HISTORY_SEGMENT_FILE); + } + + /** + * Current segment start from `history-segment.json` (0 when absent), + * refreshing the cache. Unreadable or malformed content throws rather + * than reading as 0: a cleared workspace whose file cannot be read would + * otherwise restart at 0 and reuse the sequences the clear retired. + */ + private async readHistorySegmentStart(workspaceId: string): Promise { + let raw: string; + try { + raw = await fs.readFile(this.getHistorySegmentPath(workspaceId), "utf-8"); + } catch (error) { + if (!isErrnoWithCode(error, "ENOENT")) throw error; + this.historySegmentStarts.set(workspaceId, 0); + return 0; + } + const parsed: unknown = JSON.parse(raw); + const start: unknown = + typeof parsed === "object" && parsed !== null + ? (parsed as { start?: unknown }).start + : undefined; + if (!isNonNegativeInteger(start)) { + throw new Error(`Malformed history segment file for ${workspaceId}: ${raw}`); + } + this.historySegmentStarts.set(workspaceId, start); + return start; + } + + /** + * Open the next history segment (full clear, under the history write + * lock, BEFORE the files are rewritten): its start is above every sequence + * the history being cleared used, the cached counter, AND the previous + * start — an already-empty history still moves on, because a turn admitted + * before the clear can otherwise persist its policy under the boundary-less + * identity the cleared segment and the new one would share. Persisted + * before the rows are removed so a crash in between leaves the history + * intact rather than an empty history whose counter restarts at 0. + */ + private async advanceHistorySegmentUnderHistoryLock( + workspaceId: string, + clearedSequences: readonly number[] + ): Promise { + // Loads the segment cache as a side effect and floors at previousStart - 1. + const persistedMax = await this.getMaxHistorySequence(workspaceId); + const previousStart = this.historySegmentStarts.get(workspaceId) ?? 0; + const start = + clearedSequences.reduce( + (max, sequence) => (sequence > max ? sequence : max), + Math.max(persistedMax, (this.sequenceCounters.get(workspaceId) ?? 0) - 1, previousStart) + ) + 1; + assert(isNonNegativeInteger(start), "history segment start must be a non-negative integer"); + await ensurePrivateDir(this.getSessionDir(workspaceId)); + await writeFileAtomic(this.getHistorySegmentPath(workspaceId), JSON.stringify({ start })); + this.historySegmentStarts.set(workspaceId, start); + this.sequenceCounters.set(workspaceId, start); + } + + /** + * Stamp a freshly sequenced row with the segment it is appended in (first + * segment rows stay unstamped, byte-identical to legacy rows). The cache is + * loaded by the in-lock counter refresh every append path runs first. + */ + private stampHistorySegment(workspaceId: string, metadata: MuxMetadata): MuxMetadata { + const start = this.historySegmentStarts.get(workspaceId); + assert(start !== undefined, "history segment start must be loaded before rows are stamped"); + return start === 0 ? metadata : { ...metadata, historySegment: start }; + } + private async getMaxHistorySequence(workspaceId: string): Promise { - let maxSequence = -1; + // Floor: a cleared history has no rows, but its sequences continue above + // the retired segment (see HISTORY_SEGMENT_FILE). + let maxSequence = (await this.readHistorySegmentStart(workspaceId)) - 1; // Full scan of the active file (cheap post-rotation; see getNextHistorySequence // for why we don't trust the tail alone). @@ -2643,9 +2736,9 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { + message.metadata = this.stampHistorySegment(workspaceId, { historySequence: nextSeqNum, - }; + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } else { // Message already has metadata, but may need historySequence assigned @@ -2670,6 +2763,9 @@ export class HistoryService { ); } this.sequenceCounters.set(workspaceId, existingSeqNum + 1); + // A pre-sequenced row (recovered partial) is appended in the + // current segment like any other. + message.metadata = this.stampHistorySegment(workspaceId, message.metadata); } else { // Has metadata but no historySequence, assign one const nextSeqNum = await this.getNextHistorySequence(workspaceId); @@ -2677,10 +2773,10 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { + message.metadata = this.stampHistorySegment(workspaceId, { ...message.metadata, historySequence: nextSeqNum, - }; + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } } @@ -3019,7 +3115,10 @@ export class HistoryService { isNonNegativeInteger(nextSeqNum), "getNextHistorySequence must return a non-negative integer" ); - message.metadata = { ...message.metadata, historySequence: nextSeqNum }; + message.metadata = this.stampHistorySegment(workspaceId, { + ...message.metadata, + historySequence: nextSeqNum, + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); } // Atomic all-or-nothing commit (r48): fs.appendFile is not @@ -3331,12 +3430,15 @@ export class HistoryService { message ); - // Preserve the historySequence, update everything else. + // Preserve the historySequence (and the segment the row was appended + // in: the replacement was built from an in-memory copy that may + // predate the stamp), update everything else. messages[i] = { ...message, metadata: { ...message.metadata, ...(preservedCompactionMetadata ?? {}), + ...preservedHistorySegment(existingMessage), historySequence: targetSequence, }, }; @@ -3457,6 +3559,7 @@ export class HistoryService { metadata: { ...summaryMessage.metadata, ...(preservedCompactionMetadata ?? {}), + ...preservedHistorySegment(messages[i]), historySequence: targetSequence, }, }; @@ -3475,10 +3578,10 @@ export class HistoryService { "persistBoundaryWithTailCopies append expects an unsequenced summary" ); const nextSeqNum = await this.getNextHistorySequence(workspaceId); - summaryMessage.metadata = { + summaryMessage.metadata = this.stampHistorySegment(workspaceId, { ...summaryMessage.metadata, historySequence: nextSeqNum, - }; + }); this.sequenceCounters.set(workspaceId, nextSeqNum + 1); persistedSummary = summaryMessage; messages.push(summaryMessage); @@ -3490,7 +3593,10 @@ export class HistoryService { "persistBoundaryWithTailCopies expects unsequenced tail copies" ); const seq = await this.getNextHistorySequence(workspaceId); - copy.metadata = { ...copy.metadata, historySequence: seq }; + copy.metadata = this.stampHistorySegment(workspaceId, { + ...copy.metadata, + historySequence: seq, + }); this.sequenceCounters.set(workspaceId, seq + 1); messages.push(copy); } @@ -4038,8 +4144,8 @@ export class HistoryService { workspaceId ).advanceGenerationUnderHistoryLock(); } + await this.advanceHistorySegmentUnderHistoryLock(workspaceId, allSequences); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); - this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); } @@ -4097,8 +4203,8 @@ export class HistoryService { await this.getContinuousCompactionJournal( workspaceId ).advanceGenerationUnderHistoryLock(); + await this.advanceHistorySegmentUnderHistoryLock(workspaceId, allSequences); await this.rewriteHistoryFilesUnlocked(workspaceId, null, null); - this.sequenceCounters.set(workspaceId, 0); return Ok(allSequences); } @@ -4260,6 +4366,7 @@ export class HistoryService { const archiveFloor = (await this.getArchiveTailMaxSequence(newWorkspaceId)) + 1; this.sequenceCounters.set(newWorkspaceId, Math.max(oldCounter, archiveFloor)); this.sequenceCounters.delete(oldWorkspaceId); + this.historySegmentStarts.delete(oldWorkspaceId); return Ok(undefined); } @@ -4273,6 +4380,9 @@ export class HistoryService { // Transfer sequence counter to new workspace ID this.sequenceCounters.set(newWorkspaceId, oldCounter); this.sequenceCounters.delete(oldWorkspaceId); + // The segment file moved with the session directory; the new id's + // cache is loaded on its next append. + this.historySegmentStarts.delete(oldWorkspaceId); log.debug( `Migrated ${messages.length} messages from ${oldWorkspaceId} to ${newWorkspaceId}` diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index e37fe09d68b..ce45d1911f6 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1242,6 +1242,67 @@ describe("MemoryConsolidationService", () => { expect(harvested?.status).toBe("completed"); }); + it("keys the harvest by the recorded closing epoch, refusing a turn stamped before a clear", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + await fixture.addWorkspace("ws-stale"); + // A full clear opens a new history segment: the boundary-less identity + // becomes -(segmentStart + 1) (workspaceMemoryPolicyEpochOf) and a turn + // admitted before the clear, stamped -1, can no longer pass as one of the + // new segment even though its row landed after the clear. + const seed = async (workspaceId: string, staleStamp: boolean) => { + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("old-1", "user", "before the clear") + ); + await fixture.historyService.clearHistory(workspaceId); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory(workspaceId, prompt); + const segmentStart = prompt.metadata?.historySegment; + if (segmentStart === undefined || segmentStart <= 0) + throw new Error("expected a new segment"); + const closingPolicyEpoch = -(segmentStart + 1); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: staleStamp ? -1 : closingPolicyEpoch, + }) + ); + await fixture.historyService.appendToHistory( + workspaceId, + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory(workspaceId, summary); + return fixture.service.maybeHarvestThenSweep({ + workspaceId, + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + closingPolicyEpoch, + }); + }; + const stale = await seed("ws-stale", true); + expect(stale.success).toBe(false); + if (!stale.success) expect(stale.error).toContain("another epoch"); + expect(fixture.modelCalls).toHaveLength(0); + const current = await seed("ws-dream", false); + expect(current.success).toBe(true); + expect(fixture.modelCalls.length).toBeGreaterThan(0); + }); + it("refuses to harvest an epoch holding user rows no turn's request snapshot covers", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); // Backend A snapshots its request through pref-1; backend B appends a diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index f23c3c0e816..b935d96cdf6 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -69,6 +69,7 @@ import { log } from "@/node/services/log"; import type { HistoryService } from "@/node/services/historyService"; import { isTokenBudgetInternalMessage, type MuxMessage } from "@/common/types/message"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; +import { compactionClosingPolicyEpoch } from "@/common/utils/messages/compactionBoundary"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; @@ -1358,7 +1359,7 @@ export class MemoryConsolidationService extends EventEmitter { // was recorded for THIS epoch (see epochHarvestRefusal). Uncovered rows // have an unknown policy the grant evaluated at completion could not // have accounted for. Terminal refusal: a retry would replay that grant. - const refusal = epochHarvestRefusal(messages, metadata.previousBoundaryHistorySequence ?? -1); + const refusal = epochHarvestRefusal(messages, compactionClosingPolicyEpoch(metadata)); if (refusal !== null) { yield* Effect.promise(() => self.recordRefusedHarvest(metadata, refusal)); return yield* Effect.fail(new HarvestRefusedError(refusal)); diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index 0c7f12b89ef..bf02a6e1306 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -55,6 +55,15 @@ export interface LegacyAdoptionRecord { * stamp could not be taken): never unchanged — the copy is preserved. */ targetStamp?: string; + /** + * The copy this adoption created was since replaced outside it (rewritten, + * or deleted and recreated to identical bytes: `targetStamp` no longer + * matches), so the file is the owner's own. Kept apart from a note the + * owner already had when it was first adopted (`created` never set): that + * one still folds the child's pin toggles, a replaced copy never does — + * `created` alone cannot tell the two apart once provenance is lost. + */ + replaced?: boolean; /** * Hash of the bytes an in-place replacement is about to write (set on the * pending prior record, cleared once the pass completes). With `content` @@ -84,8 +93,9 @@ export interface LegacyAdoptionRecord { * Parse one manifest record. Lifecycle flags are raw JSON: a value that is * neither absent nor boolean fails CLOSED — `pending`/`pendingDeletion` read * as set (the pass is redone), `created`/`deleted` as unset (no destructive - * provenance; the source is reconciled as a plain unlisted note) — so a - * corrupted flag can never make an interrupted pass look settled. + * provenance; the source is reconciled as a plain unlisted note), `replaced` + * as set (the child's pin no longer reaches the file) — so a corrupted flag + * can never make an interrupted pass look settled. */ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null { if (typeof value !== "object" || value === null) return null; @@ -117,6 +127,7 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null pending: flag(record.pending, true), pendingDeletion: flag(record.pendingDeletion, true), deleted: flag(record.deleted, false), + replaced: flag(record.replaced, true), replacementContent: record.replacementContent, targetStamp: record.targetStamp, }; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 4086fde26fb..177c35aa6af 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1232,6 +1232,47 @@ describe("MemoryService", () => { expect(fixture.service.resolveWorkspaceMemoryOwnerId("ws-child")).toBe("ws-child"); }); + it("rescans the owner notebook for the revision token only on root changes or per interval", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + await fixture.service.create(fixture.ctx, "/memories/workspace/dir/nested.md", "v1", "agent"); + const foreign = new MemoryService(fixture.config, new MemoryMetaService(fixture.xumHome)); + const nestedStats = () => + lstatSpy.mock.calls.filter(([target]) => String(target).startsWith(ownerRoot)).length; + const lstatSpy = spyOn(fsPromises, "lstat"); + try { + const first = await foreign.workspaceMemoryRevision("ws-owner"); + expect(nestedStats()).toBeGreaterThan(0); + // A cached-context probe on every turn (and every Memory tab interval) + // must not walk the notebook again: the clock and sidecar segments + // carry this build's writes, the scan only exists for downgraded ones. + lstatSpy.mockClear(); + expect(await foreign.workspaceMemoryRevision("ws-owner")).toBe(first); + expect(nestedStats()).toBe(0); + // A note added at the root moves the root mtime (one stat): rescanned + // at once, token changed — a downgraded build's new note shows up. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(ownerRoot, "old-build.md"), "downgraded write"); + const afterRootWrite = await foreign.workspaceMemoryRevision("ws-owner"); + expect(afterRootWrite).not.toBe(first); + expect(nestedStats()).toBeGreaterThan(0); + // This build's own writes need no rescan to be seen: the clock moved. + lstatSpy.mockClear(); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/dir/nested.md", + "v1", + "v2", + "agent" + ); + expect(await foreign.workspaceMemoryRevision("ws-owner")).not.toBe(afterRootWrite); + expect(nestedStats()).toBe(0); + } finally { + lstatSpy.mockRestore(); + } + }); + it("advances the owner store's revision token on shared writes, visible to another backend", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); @@ -2280,7 +2321,19 @@ describe("MemoryService", () => { ).get("note.md")!; expect(record.created).toBe(false); expect(record.targetStamp).toBeUndefined(); + expect(record.replaced).toBe(true); expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The record now persists without `created`, like a note the owner had + // all along — but that one folds child toggles, this one must not: the + // next toggle (a fresh process, so nothing is remembered in memory) + // leaves the owner-owned replacement alone too (r80). + await fixture.metaService.setPinned(childKey, true); + await fixture.metaService.setPinned(ownerKey, false); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); }); it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { @@ -3708,6 +3761,26 @@ describe("MemoryService", () => { undoCopies.filter((row) => (row.data.action as { op: string }).op === "rollback") ).toHaveLength(1); expect(undoCopies.every((row) => row.data.rollbackOf === createCopy.id)).toBe(true); + // A copy whose action still PARSES but names another target is corrupt + // by the engine's own rule (isUsableRollbackRow, r79): the migration + // must judge copies by that same predicate (r80), or a retried removal + // skips the intact source and the lineage is lost. + const disagreeing = (await fsPromises.readFile(journalPath, "utf-8")) + .split("\n") + .map((line) => { + if (!line.includes(`"migratedFrom":"ws-child:${undo.id}"`)) return line; + const row = JSON.parse(line) as { data: { action: { op: string; of?: string } } }; + if (row.data.action.op === "rollback") row.data.action.of = "someone-else"; + return JSON.stringify(row); + }); + await fsPromises.writeFile(journalPath, disagreeing.join("\n")); + expect(await migrate()).toBe(1); + const recopied = (await readRefinementEvents(ownerSessionDir)).filter( + (row) => + row.data.migratedFrom === `ws-child:${undo.id}` && + (row.data.action as { op: string; of?: string }).of === createCopy.id + ); + expect(recopied).toHaveLength(1); }); it("a peer's corrupt rollback row does not hide the peer's live edit from conflict detection", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 9c1f8fd6dac..80a565bb0e3 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -480,6 +480,16 @@ async function legacyStoreStamp(childSessionDir: string, legacyRoot: string): Pr return `${revision ?? "none"}:${rootMtime}:${fileStamps.join("\u0001")}`; } +/** + * Minimum spacing of the owner-store file scan a workspaceMemoryRevision + * token carries (legacyStoreStamp over the shared notebook). The scan exists + * only for a DOWNGRADED build's in-place edits, which move no clock; this + * build's writers — local or another backend — advance the store clock, + * which every token reads fresh. Unthrottled, each cached-context probe (every + * turn) and every Memory tab interval would list and lstat the whole notebook. + */ +const OWNER_STORE_SCAN_INTERVAL_MS = 60_000; + /** A stat failure that proves the path is absent (vs. one that says nothing about it). */ function isMissingPathError(error: unknown): boolean { const code = (error as NodeJS.ErrnoException | null)?.code; @@ -871,6 +881,12 @@ export class MemoryService extends EventEmitter { */ private readonly legacyStoreCheckedAgainst = new Map(); + /** Last owner-store file scan per owner (see ownerStoreStamp). */ + private readonly ownerStoreStampMemo = new Map< + string, + { rootMtime: string; stamp: string; scannedAt: number } + >(); + /** * Owner of the workspace scope for this context ("" when there is no * workspace). Public so callers that key sidecar metadata for the same @@ -1409,8 +1425,10 @@ export class MemoryService extends EventEmitter { } let target: { relPath: string; write: boolean; replaces?: boolean } | null = null; // A child's pin toggle folds into the copy only while the copy is - // this adoption's generation (see below); an owner-owned file keeps - // the owner's pin. + // this adoption's generation (see below) or the owner's identical + // note it was folded into at first adoption; a copy the owner + // replaced since keeps the owner's pin — recorded as `replaced`, so + // the next pass still knows (its `created` is gone either way). let foldChildPin = true; if (previous !== undefined) { // The recorded target is reused only while it still holds bytes @@ -1461,7 +1479,9 @@ export class MemoryService extends EventEmitter { : previous.pending === true ? currentStamp : previous.targetStamp; - foldChildPin = previous.created !== true || ours; + const replaced = previous.replaced === true || (previous.created === true && !ours); + if (replaced) record.replaced = true; + foldChildPin = !replaced; } else if ( previous.created === true && priorContent !== null && @@ -2235,9 +2255,9 @@ export class MemoryService extends EventEmitter { .filter(([key]) => key.startsWith(ownerKeyPrefix)) .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) ); - const token = `${revision === null ? "missing" : String(revision)}\u0000${await legacyStoreStamp( - ownerSessionDir, - workspaceMemoryStorePath(this.config.sessionsDir, owner) + const token = `${revision === null ? "missing" : String(revision)}\u0000${await this.ownerStoreStamp( + owner, + ownerSessionDir )}\u0000${ownerSidecar}`; // A redirected sub-agent's token also tracks its legacy private notebook // (see legacyAdoptionCheckKey): its next store access adopts the change, @@ -2246,6 +2266,31 @@ export class MemoryService extends EventEmitter { return `${token}\u0000${(await this.legacyAdoptionCheckKey(workspaceId, owner)).checkKey}`; } + /** + * The token's downgrade-compatibility segment. The per-file scan reruns + * when the notebook root's mtime moved (one stat: a note created, deleted + * or renamed at the top level by any build) and otherwise at most once per + * OWNER_STORE_SCAN_INTERVAL_MS per owner, shared by every session and tab + * probing that store. Only a downgraded build's in-place edit of an + * existing note waits for the interval; this build's writes advance the + * clock segment. + */ + private async ownerStoreStamp(owner: string, ownerSessionDir: string): Promise { + const root = workspaceMemoryStorePath(this.config.sessionsDir, owner); + const rootMtime = await fsPromises + .stat(root) + .then((stat) => String(stat.mtimeMs)) + .catch(() => "missing"); + const memo = this.ownerStoreStampMemo.get(owner); + const now = Date.now(); + if (memo?.rootMtime === rootMtime && now - memo.scannedAt < OWNER_STORE_SCAN_INTERVAL_MS) { + return memo.stamp; + } + const stamp = await legacyStoreStamp(ownerSessionDir, root); + this.ownerStoreStampMemo.set(owner, { rootMtime, stamp, scannedAt: now }); + return stamp; + } + /** The store clock segment of a workspaceMemoryRevision token (tests, diagnostics). */ static revisionClockOf(token: string): string { return token.split("\u0000", 1)[0] ?? token; diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 80e191172e0..807c8a7c5ba 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -163,14 +163,19 @@ export async function migrateSharedMemoryRefinementRows(args: { // is corrupted would otherwise make a retried removal skip its intact // source, delete the child session, and leave the owner with nothing // but an unusable rollback record. Such a source is copied again (the - // corrupted row stays behind as an audit record). + // corrupted row stays behind as an audit record). A rollback copy is + // judged by the engine's own predicate (isUsableRollbackRow): its action + // must also name the `rollbackOf` target, or the engine rejects the copy + // while this pass would have counted it — and a retried removal would + // then skip the intact child source and delete its journal. const ownerIdBySource = new Map(); for (const ownerRow of ownerRows) { if (ownerRow.data.migratedFrom === undefined || ownerRow.data.kind !== "memory") continue; const usable = - RefinementInverseSchema.safeParse(ownerRow.data.inverse).success && - (MemoryRefinementActionSchema.safeParse(ownerRow.data.action).success || - RollbackRefinementActionSchema.safeParse(ownerRow.data.action).success); + ownerRow.data.rollbackOf === undefined + ? RefinementInverseSchema.safeParse(ownerRow.data.inverse).success && + MemoryRefinementActionSchema.safeParse(ownerRow.data.action).success + : isUsableRollbackRow(ownerRow); if (usable) ownerIdBySource.set(ownerRow.data.migratedFrom, ownerRow.id); } // Owner rows already rolled back (by anyone): a second rollback row for diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 40d08fd3b56..6e01de4e26c 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -39,7 +39,7 @@ import type { ModelMessage, MuxMessage, MuxMessageMetadata } from "@/common/type import { createMuxMessage } from "@/common/types/message"; import { epochHasPriorTurnRows, - latestContextBoundaryHistorySequence, + workspaceMemoryPolicyEpochOf, } from "@/common/utils/messages/compactionBoundary"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; import type { MuxProviderOptions } from "@/common/types/providerOptions"; @@ -1534,11 +1534,13 @@ export class TurnRequestBuilder { return epochHasPriorTurnRows(activeContextMessages, currentBatch); })(); // The compaction epoch this turn's policy accumulates over: the latest - // durable boundary's history sequence (any kind), -1 before any boundary - // — the same identity compaction completion reports as - // previousBoundaryHistorySequence, so the completion-side observation - // and every backend's turn records agree on which epoch a value belongs to. - const policyEpoch = latestContextBoundaryHistorySequence(messages) ?? -1; + // durable boundary's history sequence (any kind), else the history + // segment's boundary-less identity — never reused across full clears + // (see workspaceMemoryPolicyEpochOf) — the same identity compaction + // completion reports as closingPolicyEpoch, so the completion-side + // observation and every backend's turn records agree on which epoch a + // value belongs to. + const policyEpoch = workspaceMemoryPolicyEpochOf(messages); // A preserved-tail boundary (RLM keep-recent copies follow it) re-appends // rows produced under EARLIER epochs' policies: those accumulators are // part of this epoch. The compacting session re-binds the closing one to @@ -1554,9 +1556,10 @@ export class TurnRequestBuilder { const tailCopies = activeContextMessages.filter( (message) => message.metadata?.rlmPreservedTailCopy === true ); - // A usable source epoch is an integer in [-1, policyEpoch) — -1 before - // any boundary, else a boundary's history sequence (persisted history is - // unvalidated). A copy without one — persisted by a build + // A usable source epoch is an integer below policyEpoch — a segment's + // boundary-less identity (negative), else a boundary's history sequence, + // both earlier than this epoch's (persisted history is unvalidated). A + // copy without one — persisted by a build // before the field, a re-copy of such a copy, or a malformed value — // carries a policy nobody can look up: it is excluded from the prior-turn // check like every copy, so without this the epoch would grant on the @@ -1565,10 +1568,7 @@ export class TurnRequestBuilder { // such copy in the active context. const usableSourceEpoch = (message: MuxMessage): number | undefined => { const epoch = message.metadata?.rlmPreservedTailSourcePolicyEpoch; - return typeof epoch === "number" && - Number.isInteger(epoch) && - epoch >= -1 && - epoch < policyEpoch + return typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch < policyEpoch ? epoch : undefined; }; diff --git a/src/node/services/workspaceMemoryDenyMarker.ts b/src/node/services/workspaceMemoryDenyMarker.ts index 28a5d9bd05c..1fa6b4b19e4 100644 --- a/src/node/services/workspaceMemoryDenyMarker.ts +++ b/src/node/services/workspaceMemoryDenyMarker.ts @@ -101,13 +101,13 @@ async function readMarkerRecord( const parsed: unknown = JSON.parse(raw); if (typeof parsed !== "object" || parsed === null) return null; const { epochs, wildcard } = parsed as { epochs?: unknown; wildcard?: unknown }; - // Epochs are -1 (before any boundary) or a boundary's history sequence: - // anything outside that domain is corruption, read as malformed (deny). + // Epochs are a segment's boundary-less identity (negative) or a + // boundary's history sequence (workspaceMemoryPolicyEpochOf): anything + // outside the integer domain is corruption, read as malformed (deny). if ( !Array.isArray(epochs) || !epochs.every( - (epoch): epoch is number => - typeof epoch === "number" && Number.isSafeInteger(epoch) && epoch >= -1 + (epoch): epoch is number => typeof epoch === "number" && Number.isSafeInteger(epoch) ) ) { return null; diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 1b43e5d3f43..f291b517476 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9552,13 +9552,14 @@ describe("WorkspaceService initialize", () => { // 12 → 16 between two separate reads could hide the entry from both). expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1, 12])).toBe(true); expect(await readWorkspaceMemoryDenyMarkerForEpochs(sessionDir, [16, -1])).toBe(false); - // Epochs outside the domain (-1 or a history sequence) are corruption: - // the marker reads as malformed, a deny for every epoch. + // Epochs outside the integer domain (a segment's negative boundary-less + // identity or a history sequence) are corruption: the marker reads as + // malformed, a deny for every epoch. const markerPath = workspaceMemoryDenyMarkerPath(sessionDir); const savedMarker = await fsPromises.readFile(markerPath, "utf-8"); await fsPromises.writeFile( markerPath, - JSON.stringify({ deniedAt: Date.now(), epochs: [-2], wildcard: false }) + JSON.stringify({ deniedAt: Date.now(), epochs: [2.5], wildcard: false }) ); expect(await readWorkspaceMemoryDenyMarker(sessionDir, 16)).toBe(true); await fsPromises.writeFile(markerPath, savedMarker); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 67eaffb708c..63acae8127c 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -120,6 +120,7 @@ import { extractEditedFilePaths } from "@/common/utils/messages/extractEditedFil import { buildCompactionMessageText } from "@/common/utils/compaction/compactionPrompt"; import { CONTEXT_BOUNDARY_KINDS, + compactionClosingPolicyEpoch, hasProviderEligibleMessages, isDurableCompactedMarker, sliceMessagesForProviderFromLatestContextBoundary, @@ -4640,7 +4641,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // silently drop another backend's persisted deny, letting this // session's own writable mirror grant the harvest — skip it instead // (fail closed; nothing is recorded, the epoch is simply not harvested). - const closingEpoch = metadata.previousBoundaryHistorySequence ?? -1; + const closingEpoch = compactionClosingPolicyEpoch(metadata); let persistedWritable: boolean | undefined; try { const entry = findWorkspaceEntry( From 8383b8570798b86899d87f6d3a558e202edd9117 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 02:16:28 +0000 Subject: [PATCH 87/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-fi?= =?UTF-8?q?rst=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Placeholder discard denies the epoch when the row cannot be removed; fork snapshots carry history-segment.json; segment starts must be safe integers and a malformed segment file is quarantined and reseeded instead of failing every append; source rollback rows are gated by isUsableRollbackRow and a live row with a malformed action aborts the handover; duplicated user row ids fail every id-keyed policy accounting closed. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 17 ++++ .../utils/messages/compactionBoundary.ts | 25 ++++- src/node/services/aiService.ts | 10 +- src/node/services/compactionHandler.ts | 9 +- src/node/services/historyService.test.ts | 71 ++++++++++++-- src/node/services/historyService.ts | 94 +++++++++++++++---- .../memoryConsolidationService.test.ts | 46 +++++++++ .../services/memoryConsolidationService.ts | 10 +- src/node/services/memoryService.test.ts | 60 ++++++++++++ .../refinement/sharedMemoryRowMigration.ts | 20 +++- src/node/services/turnRequestBuilder.ts | 19 +++- 11 files changed, 341 insertions(+), 40 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index da1794a8818..844dcadcb6c 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -4,6 +4,7 @@ import { createMuxMessage } from "@/common/types/message"; import { compactionClosingPolicyEpoch, + duplicateUserMessageIds, epochHasPriorTurnRows, findLatestCompactionBoundaryIndex, findLatestContextBoundaryIndex, @@ -70,6 +71,22 @@ describe("workspaceMemoryPolicyEpochOf", () => { }); }); +describe("duplicateUserMessageIds", () => { + it("names ids shared by several user rows and makes them prior turns", () => { + const rows = [ + createMuxMessage("u1", "user", "first", { historySequence: 0 }), + createMuxMessage("a1", "assistant", "reply", { historySequence: 1 }), + createMuxMessage("u1", "user", "same id again", { historySequence: 2 }), + ]; + expect([...duplicateUserMessageIds(rows)]).toEqual(["u1"]); + expect(duplicateUserMessageIds(rows.slice(0, 2)).size).toBe(0); + // The current batch is matched by id: the duplicated id would hide the + // earlier row from the prior-turn check, so it counts as one. + expect(epochHasPriorTurnRows(rows, new Set(["u1"]))).toBe(true); + expect(epochHasPriorTurnRows(rows.slice(0, 2), new Set(["u1"]))).toBe(false); + }); +}); + describe("compactionClosingPolicyEpoch", () => { it("prefers the recorded closing epoch and falls back to the legacy identity", () => { expect( diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 3610bfe196e..efcf5a13b04 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -263,12 +263,35 @@ export function epochHasPriorTurnRows( activeContextMessages: readonly MuxMessage[], currentBatch: ReadonlySet ): boolean { + // Batches are matched by row id: two user rows sharing one id (persisted + // history is raw JSON) would both read as the current batch, hiding the + // earlier one from this check and from the harvest gate's coverage — so a + // duplicated id is itself an unaccounted prior turn (fail closed). + const duplicated = duplicateUserMessageIds(activeContextMessages); return activeContextMessages.some( (message) => message.role === "user" && - !currentBatch.has(message.id) && + (duplicated.has(message.id) || !currentBatch.has(message.id)) && message.metadata?.muxMetadata?.type !== "compaction-request" && message.metadata?.rlmPreservedTailCopy !== true && !isTokenBudgetInternalMessage(message) ); } + +/** + * Ids carried by more than one user row of `messages`. The policy checks + * account for user rows by id (a turn's batch is its user row plus the + * prelude ids that row lists), so an id shared by two rows would let the + * accounting of one vouch for the other; callers treat such ids as + * unaccounted for. + */ +export function duplicateUserMessageIds(messages: readonly MuxMessage[]): ReadonlySet { + const seen = new Set(); + const duplicated = new Set(); + for (const message of messages) { + if (message.role !== "user") continue; + if (seen.has(message.id)) duplicated.add(message.id); + else seen.add(message.id); + } + return duplicated; +} diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 04cc3971ec2..233dd3871fe 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -1026,12 +1026,10 @@ export class AIService extends EventEmitter { startupState.pendingRunMetadataId = null; } buildOutcome.logStartOutcome("stream_start_failed", streamResult.error.type); - // No stream ran for this turn. Its placeholder already carries the - // request bound and the policy epoch stamp (they must be on the row - // StreamManager finalizes), which would present the user batch to the - // harvest gate as covered by a turn — remove it like an aborted - // startup's, so the never-started turn stays excluded until a retry - // actually runs it. + // No stream ran for this turn: discard its placeholder like an + // aborted startup's (TurnRequestBuilder denies the epoch when the row + // cannot be removed), so the never-started turn stays excluded from + // the harvest until a retry actually runs it. await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); return Err(streamResult.error); } diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 4bf222ab2b4..f5f57f58acd 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -42,6 +42,7 @@ import { isDurableCompactedMarker, isDurableContextBoundaryMarker, latestContextBoundaryHistorySequence, + duplicateUserMessageIds, sliceMessagesFromLatestCompactionBoundary, workspaceMemoryPolicyEpochOf, } from "@/common/utils/messages/compactionBoundary"; @@ -1682,6 +1683,10 @@ export class CompactionHandler { } } const coveredEpochById = new Map(); + // An id shared by two user rows names no batch: neither copy is stamped + // (the harvest gate refuses such an epoch; the copies must not present + // either row as covered to the next one). + const duplicated = duplicateUserMessageIds(tailRows); for (const message of tailRows) { if (message.role !== "assistant") continue; const bound = message.metadata?.requestHistorySequence; @@ -1698,7 +1703,9 @@ export class CompactionHandler { anchor.id, ...getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds), ]) { - if (!coveredEpochById.has(id)) coveredEpochById.set(id, policyEpoch); + if (!coveredEpochById.has(id) && !duplicated.has(id)) { + coveredEpochById.set(id, policyEpoch); + } } } return tailRows.map((row) => { diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index a5fe524a62a..8107bb3155b 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2619,22 +2619,75 @@ describe("HistoryService", () => { expect(messages[0].metadata?.historySegment).toBe(1); }); - it("refuses to append when the segment file is unreadable", async () => { + it("quarantines a malformed segment file and reseeds above the visible rows", async () => { const workspaceId = "workspace1"; await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); await service.clearHistory(workspaceId); - await fs.writeFile( - path.join(config.sessionsDir, workspaceId, "history-segment.json"), - "not json" + const msg2 = createMuxMessage("msg2", "user", "Second"); + await service.appendToHistory(workspaceId, msg2); + expect(msg2.metadata?.historySegment).toBe(1); + const segmentPath = path.join(config.sessionsDir, workspaceId, "history-segment.json"); + await fs.writeFile(segmentPath, "not json"); + + // Self-healing rather than refusing every append until the file is + // deleted by hand: the corrupt file is quarantined and the segment + // reseeded above every sequence still visible, so the row lands and + // carries a fresh stamp (the epoch identity moves on — fail closed for + // the policy recorded under the old one, never a reused sequence). + const restarted = new HistoryService(config); + const msg3 = createMuxMessage("msg3", "user", "Third"); + expect((await restarted.appendToHistory(workspaceId, msg3)).success).toBe(true); + expect(msg3.metadata?.historySequence).toBe(2); + expect(msg3.metadata?.historySegment).toBe(2); + expect(JSON.parse(await fs.readFile(segmentPath, "utf-8"))).toEqual({ start: 2 }); + const quarantined = (await fs.readdir(path.dirname(segmentPath))).filter((name) => + name.startsWith("history-segment.json.corrupt-") ); + expect(quarantined).toHaveLength(1); + // A clear after the repair opens the next segment as usual. + await restarted.clearHistory(workspaceId); + const msg4 = createMuxMessage("msg4", "user", "Fourth"); + await restarted.appendToHistory(workspaceId, msg4); + expect(msg4.metadata?.historySegment).toBe(3); + }); - // Reading a corrupt segment as 0 would restart at the retired sequences. - const restarted = new HistoryService(config); - const result = await restarted.appendToHistory( - workspaceId, - createMuxMessage("msg2", "user", "Second") + it("refuses to open a segment above an unsafe persisted sequence", async () => { + const workspaceId = "workspace1"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + // `start + 1` cannot move past 2^53: a clear that pretended to would + // hand the new segment the old one's identity. + await fs.writeFile( + path.join(workspaceDir, "chat.jsonl"), + JSON.stringify({ + ...createMuxMessage("huge", "user", "absurd sequence", { + historySequence: Number.MAX_SAFE_INTEGER + 1, + }), + workspaceId, + }) + "\n" ); + const result = await service.clearHistory(workspaceId); expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("safe integer"); + expect(await fs.readFile(path.join(workspaceDir, "chat.jsonl"), "utf-8")).toContain("huge"); + }); + + it("forks the current segment along with the history snapshot", async () => { + await service.appendToHistory("source", createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory("source"); + const kept = createMuxMessage("msg2", "user", "Kept"); + await service.appendToHistory("source", kept); + expect(kept.metadata?.historySegment).toBe(1); + + // The copied rows carry the source's stamp: the fork continues that + // segment instead of appending unstamped rows (segment 0) beside them. + expect((await service.copyHistorySnapshotToNewWorkspace("source", "fork")).success).toBe( + true + ); + const forked = createMuxMessage("msg3", "user", "In the fork"); + await service.appendToHistory("fork", forked); + expect(forked.metadata?.historySequence).toBe(2); + expect(forked.metadata?.historySegment).toBe(1); }); }); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index c6ec757c9c8..cfbf5f18472 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1802,6 +1802,10 @@ export class HistoryService { Ok({ archive: await this.readExistingFile(this.getChatArchivePath(sourceWorkspaceId)), chat: await this.readExistingFile(this.getChatHistoryPath(sourceWorkspaceId)), + // The copied rows carry the source's segment stamp; the target must + // continue that segment (same floor, same stamp) or its own appends + // would sit unstamped beside stamped rows. + segment: await this.readExistingFile(this.getHistorySegmentPath(sourceWorkspaceId)), }) ); if (!snapshot.success) { @@ -1816,6 +1820,7 @@ export class HistoryService { for (const [targetPath, contents] of [ [this.getChatArchivePath(targetWorkspaceId), snapshot.data.archive], [this.getChatHistoryPath(targetWorkspaceId), snapshot.data.chat], + [this.getHistorySegmentPath(targetWorkspaceId), snapshot.data.segment], ] as const) { if (contents === null) { await fs.rm(targetPath, { force: true }); @@ -1823,6 +1828,7 @@ export class HistoryService { await writeFileAtomic(targetPath, contents); } } + this.historySegmentStarts.delete(targetWorkspaceId); return Ok(undefined); } ); @@ -1894,30 +1900,78 @@ export class HistoryService { } /** - * Current segment start from `history-segment.json` (0 when absent), - * refreshing the cache. Unreadable or malformed content throws rather - * than reading as 0: a cleared workspace whose file cannot be read would - * otherwise restart at 0 and reuse the sequences the clear retired. + * Current segment start from `history-segment.json` (0 when absent), or + * null when the file is present but malformed. Unreadable (EACCES, EIO) + * throws: a cleared workspace whose file cannot be read would otherwise + * restart at 0 and reuse the sequences the clear retired. Only a safe + * integer is a usable start — above 2^53, `start + 1` can equal `start` + * and a clear would fail to open a strictly newer segment. */ - private async readHistorySegmentStart(workspaceId: string): Promise { + private async readHistorySegmentStart(workspaceId: string): Promise { let raw: string; try { raw = await fs.readFile(this.getHistorySegmentPath(workspaceId), "utf-8"); } catch (error) { if (!isErrnoWithCode(error, "ENOENT")) throw error; - this.historySegmentStarts.set(workspaceId, 0); return 0; } - const parsed: unknown = JSON.parse(raw); + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } const start: unknown = typeof parsed === "object" && parsed !== null ? (parsed as { start?: unknown }).start : undefined; - if (!isNonNegativeInteger(start)) { - throw new Error(`Malformed history segment file for ${workspaceId}: ${raw}`); + return isNonNegativeInteger(start) && Number.isSafeInteger(start) ? start : null; + } + + /** + * Self-healing for a malformed `history-segment.json` (under the history + * write lock): the file is quarantined next to itself and reseeded with a + * start above every sequence still visible (the rows, the cached counter + * and the cached start), so appends and clears keep working instead of + * failing on every attempt until someone deletes the file by hand. The + * retired range a corrupt file may have named is not recoverable; rows + * appended after the reseed carry the new stamp, so the segment's + * boundary-less epoch identity changes and any policy recorded under the + * old one is simply never consulted again (harvests of that epoch fail + * closed rather than reading a stale record). + */ + private async reseedHistorySegmentUnderHistoryLock( + workspaceId: string, + visibleMaxSequence: number + ): Promise { + const segmentPath = this.getHistorySegmentPath(workspaceId); + const quarantinePath = `${segmentPath}.corrupt-${Date.now()}`; + log.warn("Quarantining malformed history segment file and reseeding the segment", { + workspaceId, + quarantinePath, + }); + await fs.rename(segmentPath, quarantinePath); + const start = + Math.max( + visibleMaxSequence, + (this.sequenceCounters.get(workspaceId) ?? 0) - 1, + this.historySegmentStarts.get(workspaceId) ?? 0 + ) + 1; + await this.writeHistorySegmentStart(workspaceId, start); + return start; + } + + private async writeHistorySegmentStart(workspaceId: string, start: number): Promise { + // Reachable from persisted data (an absurd sequence in a hand-edited row): + // refuse rather than persist a start that `+ 1` cannot move past. + if (!isNonNegativeInteger(start) || !Number.isSafeInteger(start)) { + throw new Error( + `Cannot open a new history segment for ${workspaceId}: start ${start} is not a safe integer` + ); } + await ensurePrivateDir(this.getSessionDir(workspaceId)); + await writeFileAtomic(this.getHistorySegmentPath(workspaceId), JSON.stringify({ start })); this.historySegmentStarts.set(workspaceId, start); - return start; } /** @@ -1942,10 +1996,7 @@ export class HistoryService { (max, sequence) => (sequence > max ? sequence : max), Math.max(persistedMax, (this.sequenceCounters.get(workspaceId) ?? 0) - 1, previousStart) ) + 1; - assert(isNonNegativeInteger(start), "history segment start must be a non-negative integer"); - await ensurePrivateDir(this.getSessionDir(workspaceId)); - await writeFileAtomic(this.getHistorySegmentPath(workspaceId), JSON.stringify({ start })); - this.historySegmentStarts.set(workspaceId, start); + await this.writeHistorySegmentStart(workspaceId, start); this.sequenceCounters.set(workspaceId, start); } @@ -1960,10 +2011,16 @@ export class HistoryService { return start === 0 ? metadata : { ...metadata, historySegment: start }; } + /** + * Called under the history write lock only (every caller assigns + * sequences): a malformed segment file is repaired in place here. + */ private async getMaxHistorySequence(workspaceId: string): Promise { // Floor: a cleared history has no rows, but its sequences continue above // the retired segment (see HISTORY_SEGMENT_FILE). - let maxSequence = (await this.readHistorySegmentStart(workspaceId)) - 1; + const segmentStart = await this.readHistorySegmentStart(workspaceId); + if (segmentStart !== null) this.historySegmentStarts.set(workspaceId, segmentStart); + let maxSequence = (segmentStart ?? this.historySegmentStarts.get(workspaceId) ?? 0) - 1; // Full scan of the active file (cheap post-rotation; see getNextHistorySequence // for why we don't trust the tail alone). @@ -1977,8 +2034,11 @@ export class HistoryService { // The archive holds strictly-older sequences than chat.jsonl, so it only // decides the counter when chat.jsonl is missing/hand-edited. const archiveMax = await this.getArchiveTailMaxSequence(workspaceId); - - return Math.max(maxSequence, archiveMax); + const max = Math.max(maxSequence, archiveMax); + if (segmentStart === null) { + return (await this.reseedHistorySegmentUnderHistoryLock(workspaceId, max)) - 1; + } + return max; } /** diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index ce45d1911f6..8e406ef5732 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1303,6 +1303,52 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls.length).toBeGreaterThan(0); }); + it("refuses to harvest an epoch whose user rows share an id", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // Coverage is keyed by id: covering the later row would mark the earlier + // (never-covered) row covered too, so a duplicated id refuses outright. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("pref-1", "user", "Read-only agent's prompt, never answered") + ); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests." + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("sharing one id"); + expect(fixture.modelCalls).toHaveLength(0); + }); + it("refuses to harvest an epoch holding user rows no turn's request snapshot covers", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); // Backend A snapshots its request through pref-1; backend B appends a diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index b935d96cdf6..67c6ea6312d 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -69,7 +69,10 @@ import { log } from "@/node/services/log"; import type { HistoryService } from "@/node/services/historyService"; import { isTokenBudgetInternalMessage, type MuxMessage } from "@/common/types/message"; import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrelude"; -import { compactionClosingPolicyEpoch } from "@/common/utils/messages/compactionBoundary"; +import { + compactionClosingPolicyEpoch, + duplicateUserMessageIds, +} from "@/common/utils/messages/compactionBoundary"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; import type { MemoryScopeContext, MemoryService } from "@/node/services/memoryService"; @@ -363,6 +366,11 @@ class HarvestRefusedError extends Error { * carrying neither agent nor repository content. */ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: number): string | null { + // Coverage is keyed by row id: a duplicated user id (raw-JSON history) + // would let one row's turn vouch for the other, so it refuses outright. + if (duplicateUserMessageIds(messages).size > 0) { + return "the compacted epoch holds user rows sharing one id; harvest refused (fail closed)"; + } const userRows: Array<{ message: MuxMessage; sequence: number }> = []; for (const message of messages) { const sequence = message.metadata?.historySequence; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 177c35aa6af..6b5b218a97b 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -3783,6 +3783,66 @@ describe("MemoryService", () => { expect(recopied).toHaveLength(1); }); + it("never turns a corrupt source rollback row into a usable owner rollback", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A rollback row whose parseable action names ANOTHER target than + // `rollbackOf`: the engine rejects it (isUsableRollbackRow); remapping + // `of` to the create's copy would make the owner believe the create was + // rolled back while its bytes are still on disk. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "rollback", of: "someone-else" }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const ownerRows = await readRefinementEvents(ownerSessionDir); + expect(ownerRows.filter((row) => row.data.rollbackOf !== undefined)).toHaveLength(0); + }); + + it("refuses to hand over a live row whose action is malformed instead of dropping it", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + // A live edit with a usable inverse but a corrupt action: its inverse + // paths are the evidence conflict detection needs, and the owner journal + // cannot carry it without an action — removal must not delete the only + // copy (a forced removal accepts the loss explicitly). + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "bogus" }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + }, + }); + const attempt = migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }); + expect(await attempt.then(() => null, getErrorMessage)).toContain("action is malformed"); + }); + it("a peer's corrupt rollback row does not hide the peer's live edit from conflict detection", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 807c8a7c5ba..65c33aade22 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -197,11 +197,29 @@ export async function migrateSharedMemoryRefinementRows(args: { if (row.data.rollbackOf === undefined) { if (isLive(row.id) !== true) continue; const parsed = MemoryRefinementActionSchema.safeParse(row.data.action); - if (!parsed.success) continue; + if (!parsed.success) { + // A live edit whose action is corrupt but whose inverse still + // parses is evidence conflict detection needs (its inverse paths + // mark the child's later mutation over the same files); the owner + // journal cannot carry it without an action, so removal must not + // delete the only copy — throw, like a row that cannot be + // persisted. A forced removal accepts the loss explicitly. + if (RefinementInverseSchema.safeParse(row.data.inverse).success) { + throw new Error( + `refinement row ${row.id} of ${args.childWorkspaceId} is live but its action is malformed; it cannot be handed over to the owner journal` + ); + } + continue; + } action = parsed.data; } else { // Child journal order puts a rollback row after its target, so the // target's copy (from an earlier pass or this loop) is known here. + // The engine's own usability rule applies to the SOURCE row too: a + // row whose parseable action names another target than `rollbackOf` + // is corrupt and must not be turned into a usable owner rollback by + // rewriting `of` (that would suppress a possibly live mutation). + if (!isUsableRollbackRow(row)) continue; rollbackOf = ownerIdBySource.get(`${args.childWorkspaceId}:${row.data.rollbackOf}`); if (rollbackOf === undefined || ownerRollbackTargets.has(rollbackOf)) continue; const parsed = RollbackRefinementActionSchema.safeParse(row.data.action); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 42418067dfc..c0058656de6 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -935,7 +935,7 @@ export class TurnRequestBuilder { const recordStartupPhaseTiming = context.recordStartupPhaseTiming; let pendingRunMetadataId: string | null = context.startupState.pendingRunMetadataId; - const deleteAbortedPlaceholder = async (messageId: string): Promise => { + const deleteAbortedPlaceholder = async (messageId: string): Promise => { const deleteResult = await this.dependencies.historyService.deleteMessage( workspaceId, messageId @@ -948,6 +948,7 @@ export class TurnRequestBuilder { deleteResult.error ); } + return deleteResult.success; }; // Mode (plan|exec|compact) is derived from the selected agent definition. const effectiveMuxProviderOptions: MuxProviderOptions = muxProviderOptions ?? {}; @@ -1624,6 +1625,16 @@ export class TurnRequestBuilder { }); return true; }; + // Placeholder of a turn that never ran (startup aborted or failed): it + // carries the request bound and the policy stamp the finalized row needs, + // so left behind it would present the user batch to the harvest gate as + // covered by a turn. When its removal fails (I/O, a concurrent rewrite) + // the epoch is denied instead — fail closed rather than trust a row + // nobody can confirm is gone. + const discardPlaceholder = async (messageId: string): Promise => { + if (await deleteAbortedPlaceholder(messageId)) return; + await persistWorkspaceMemoryWritable(false); + }; const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic // repo hook execution (tool_env/tool_pre/tool_post) while keeping @@ -3171,7 +3182,7 @@ export class TurnRequestBuilder { } if (combinedAbortSignal.aborted) { - await deleteAbortedPlaceholder(assistantMessageId); + await discardPlaceholder(assistantMessageId); return { type: "finished", result: Ok( @@ -3393,7 +3404,7 @@ export class TurnRequestBuilder { } catch (error) { if (error instanceof ContextBudgetExceededError) { runLanguageModelCleanup(modelResult.data.model); - await deleteAbortedPlaceholder(assistantMessageId); + await discardPlaceholder(assistantMessageId); return { type: "finished", result: Err(error.details) }; } throw error; @@ -3492,7 +3503,7 @@ export class TurnRequestBuilder { type: "ready", turnExecutionOptions, assistantMessageId, - deleteAbortedPlaceholder, + deleteAbortedPlaceholder: discardPlaceholder, logStartOutcome, ...(finalWorkspaceMemoryWritable ? { From a22b8fe7362c644d0c71ce7f5f8072219be276b0 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 03:23:33 +0000 Subject: [PATCH 88/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-se?= =?UTF-8?q?cond=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A placeholder that can be neither removed nor denied surfaces the policy persist error instead of the turn's own outcome; a corrupt segment file in an emptied workspace reseeds above the wall clock so no earlier identity recurs; migration liveness counts only memory-kind rollback rows. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/aiService.test.ts | 58 +++++++++++++++++++ src/node/services/aiService.ts | 18 +++++- src/node/services/historyService.test.ts | 31 ++++++++-- src/node/services/historyService.ts | 9 ++- src/node/services/memoryService.test.ts | 34 +++++++++++ .../refinement/sharedMemoryRowMigration.ts | 14 +++-- src/node/services/turnRequestBuilder.ts | 33 ++++++++--- 7 files changed, 178 insertions(+), 19 deletions(-) diff --git a/src/node/services/aiService.test.ts b/src/node/services/aiService.test.ts index a6a325b3be6..bbe50245152 100644 --- a/src/node/services/aiService.test.ts +++ b/src/node/services/aiService.test.ts @@ -11,6 +11,7 @@ import { describe, it, expect, beforeEach, afterEach, mock, spyOn } from "bun:te import { Err } from "@/common/types/result"; import { resolveModelForMetadata } from "@/common/utils/providers/modelEntries"; import { AIService, resolveMuxProjectRootForHostFs } from "./aiService"; +import { WORKSPACE_MEMORY_POLICY_PERSIST_ERROR } from "./turnRequestBuilder"; import { discoverAvailableSubagentsForToolContext } from "./turnContextAssembler"; import { normalizeAnthropicBaseURL, @@ -2543,6 +2544,63 @@ describe("AIService.streamMessage compaction boundary slicing", () => { expect(deleted).toEqual([placeholder.id]); }); + it("denies the epoch when a failed turn's placeholder cannot be removed, and surfaces a lost deny", async () => { + using xumHome = new DisposableTempDir("ai-service-startup-failure-undeletable"); + const projectPath = path.join(xumHome.path, "project"); + await fs.mkdir(projectPath, { recursive: true }); + + const workspaceId = "workspace-startup-failure-undeletable"; + const metadata = createLocalWorkspaceMetadata(workspaceId, projectPath); + const harness = createHarness(xumHome.path, metadata); + const internals = harness.service as unknown as { + historyService: HistoryService; + streamManager: StreamManager; + }; + // The harness runs without a memory tool, so start() already records a + // deny for the turn; the discard adds a second one. `outcomes` scripts + // the sink's durability per call. + const recorded: boolean[] = []; + let outcomes: boolean[] = []; + harness.service.turnRequestBuilderBindings.workspaceMemoryPolicySink = { + recordWorkspaceMemoryWritable: (_workspaceId, writable) => { + recorded.push(writable); + return Promise.resolve(outcomes.shift() ?? true); + }, + }; + spyOn(internals.historyService, "deleteMessage").mockResolvedValue(Err("concurrent rewrite")); + spyOn(internals.streamManager, "startStream").mockResolvedValue( + Err({ type: "unknown", raw: "temp dir creation failed" }) + ); + const stream = () => + harness.service.streamMessage({ + messages: [createMuxMessage("latest-user", "user", "continue")], + workspaceId, + modelString: "openai:gpt-5.2", + thinkingLevel: "medium", + }); + + // The stamped placeholder stays behind: the epoch is denied instead, and + // the startup error is still the result. + const denied = await stream(); + expect(denied.success).toBe(false); + if (!denied.success) + expect(denied.error).toEqual({ type: "unknown", raw: "temp dir creation failed" }); + expect(recorded).toEqual([false, false]); + + // Neither removal nor deny durable: that failure is the result, not the + // startup error — the row is vouching for a batch no model saw. + outcomes = [true, false]; + const lost = await stream(); + expect(recorded).toEqual([false, false, false, false]); + expect(lost.success).toBe(false); + if (!lost.success) { + expect(lost.error.type).toBe("unknown"); + expect(lost.error.type === "unknown" ? lost.error.raw : "").toContain( + WORKSPACE_MEMORY_POLICY_PERSIST_ERROR + ); + } + }); + it("passes muxMetadata into initial stream metadata", async () => { using xumHome = new DisposableTempDir("ai-service-mux-metadata"); const projectPath = path.join(xumHome.path, "project"); diff --git a/src/node/services/aiService.ts b/src/node/services/aiService.ts index 233dd3871fe..99056d95075 100644 --- a/src/node/services/aiService.ts +++ b/src/node/services/aiService.ts @@ -23,6 +23,7 @@ import { type PreparedStreamMessage, type PreparedTurnRequest, type TurnRequestBuildContext, + WORKSPACE_MEMORY_POLICY_PERSIST_ERROR, } from "./turnRequestBuilder"; export { replaceOrAppendMessageById } from "./turnRequestBuilder"; export type { StreamMessageOptions } from "./turnRequestBuilder"; @@ -1029,8 +1030,17 @@ export class AIService extends EventEmitter { // No stream ran for this turn: discard its placeholder like an // aborted startup's (TurnRequestBuilder denies the epoch when the row // cannot be removed), so the never-started turn stays excluded from - // the harvest until a retry actually runs it. - await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); + // the harvest until a retry actually runs it. When neither could be + // made durable, that failure — not the startup error — is the result: + // the stamped row is still there, vouching for a batch no model saw. + if (!(await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId))) { + return Err({ + type: "unknown", + raw: `${WORKSPACE_MEMORY_POLICY_PERSIST_ERROR} (stream startup failed first: ${ + streamResult.error.type + })`, + }); + } return Err(streamResult.error); } @@ -1039,7 +1049,9 @@ export class AIService extends EventEmitter { this.clearTrackedPendingDevToolsRunMetadata(buildOutcome.assistantMessageId); startupState.pendingRunMetadataId = null; } - await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId); + if (!(await buildOutcome.deleteAbortedPlaceholder(buildOutcome.assistantMessageId))) { + return Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }); + } } else { // The stream is live: durable effects gated on "actually started" // (memory harvest grant) may land now. diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 8107bb3155b..9ec856de434 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2634,12 +2634,19 @@ describe("HistoryService", () => { // reseeded above every sequence still visible, so the row lands and // carries a fresh stamp (the epoch identity moves on — fail closed for // the policy recorded under the old one, never a reused sequence). + // The reseeded start is floored by the wall clock: an empty history + // after a restart has no row, counter or cached start to prove a + // segment identity was never used before, and a small reseed could + // repeat one a late backend still names. + const before = Date.now(); const restarted = new HistoryService(config); const msg3 = createMuxMessage("msg3", "user", "Third"); expect((await restarted.appendToHistory(workspaceId, msg3)).success).toBe(true); - expect(msg3.metadata?.historySequence).toBe(2); - expect(msg3.metadata?.historySegment).toBe(2); - expect(JSON.parse(await fs.readFile(segmentPath, "utf-8"))).toEqual({ start: 2 }); + const reseeded = msg3.metadata?.historySegment; + if (reseeded === undefined) throw new Error("expected a reseeded stamp"); + expect(reseeded).toBeGreaterThan(before); + expect(msg3.metadata?.historySequence).toBe(reseeded); + expect(JSON.parse(await fs.readFile(segmentPath, "utf-8"))).toEqual({ start: reseeded }); const quarantined = (await fs.readdir(path.dirname(segmentPath))).filter((name) => name.startsWith("history-segment.json.corrupt-") ); @@ -2648,7 +2655,23 @@ describe("HistoryService", () => { await restarted.clearHistory(workspaceId); const msg4 = createMuxMessage("msg4", "user", "Fourth"); await restarted.appendToHistory(workspaceId, msg4); - expect(msg4.metadata?.historySegment).toBe(3); + expect(msg4.metadata?.historySegment).toBe(reseeded + 1); + }); + + it("reseeds an empty workspace's corrupt segment above any identity used before", async () => { + const workspaceId = "workspace1"; + await service.appendToHistory(workspaceId, createMuxMessage("msg1", "user", "Hello")); + await service.clearHistory(workspaceId); + // Segment 1 (identity -2) was used; the file is corrupted with nothing + // else surviving: no rows, and a fresh process has no cached state. + await fs.writeFile( + path.join(config.sessionsDir, workspaceId, "history-segment.json"), + "{broken" + ); + const restarted = new HistoryService(config); + const msg = createMuxMessage("msg2", "user", "After repair"); + expect((await restarted.appendToHistory(workspaceId, msg)).success).toBe(true); + expect(msg.metadata?.historySegment).toBeGreaterThan(1); }); it("refuses to open a segment above an unsafe persisted sequence", async () => { diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index cfbf5f18472..f0993aa9b29 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1951,11 +1951,18 @@ export class HistoryService { quarantinePath, }); await fs.rename(segmentPath, quarantinePath); + // An empty history after a restart leaves no row, counter or cached start + // to floor at, and a small reseed (1) could repeat the identity of an + // earlier segment that a late backend still names in its policy records. + // The wall clock is the one monotone source that survives all of that: + // sequences only need to compare, so the jump is harmless, and every + // later clear continues above it. const start = Math.max( visibleMaxSequence, (this.sequenceCounters.get(workspaceId) ?? 0) - 1, - this.historySegmentStarts.get(workspaceId) ?? 0 + this.historySegmentStarts.get(workspaceId) ?? 0, + Date.now() ) + 1; await this.writeHistorySegmentStart(workspaceId, start); return start; diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 6b5b218a97b..c0d895c442a 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -3816,6 +3816,40 @@ describe("MemoryService", () => { expect(ownerRows.filter((row) => row.data.rollbackOf !== undefined)).toHaveLength(0); }); + it("does not let a non-memory rollback row kill a memory row's liveness during migration", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerSessionDir = path.join(fixture.config.sessionsDir, "ws-owner"); + await fixture.service.create(fixture.ctx, "/memories/workspace/n.md", "v1", "agent"); + const [createRow] = await readRefinementEvents(childSessionDir); + // A well-formed rollback row whose `kind` is not "memory": the copy loop + // skips it, so it must not count as a completed rollback either — or + // removal would delete the journal without copying the live create. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "skill", + action: { op: "rollback", of: createRow.id }, + inverse: { op: "delete-files", paths: [path.join(ownerSessionDir, "memory", "n.md")] }, + rollbackOf: createRow.id, + }, + }); + expect( + await migrateSharedMemoryRefinementRows({ + childSessionDir, + childWorkspaceId: "ws-child", + ownerSessionDir, + ownerWorkspaceId: "ws-owner", + }) + ).toBe(1); + const copy = (await readRefinementEvents(ownerSessionDir)).find( + (row) => row.data.migratedFrom === `ws-child:${createRow.id}` + ); + expect(copy).toBeDefined(); + }); + it("refuses to hand over a live row whose action is malformed instead of dropping it", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/refinement/sharedMemoryRowMigration.ts b/src/node/services/refinement/sharedMemoryRowMigration.ts index 65c33aade22..733c3814b54 100644 --- a/src/node/services/refinement/sharedMemoryRowMigration.ts +++ b/src/node/services/refinement/sharedMemoryRowMigration.ts @@ -21,7 +21,7 @@ import { type RefinementFileReference, type RefinementInverseDraft, } from "./refinementJournal"; -import { isUsableRollbackRow, listRefinements } from "./refinementRollback"; +import { isUsableRollbackRow, listRefinements, type RefinementEvent } from "./refinementRollback"; import { createLegacyPathRemapper, LegacyPathNotAdoptedError, @@ -122,8 +122,14 @@ export async function migrateSharedMemoryRefinementRows(args: { // target would delete the child journal with neither inverse preserved. // The copied target is then live on the owner side; its divergence checks // refuse a re-apply that no longer matches the tree (force overrides). + // Only rollback rows this migration can carry count: a rollback row whose + // `kind` is not "memory" (corrupted, or another taxonomy's) is skipped by + // the copy loop below, so counting it here would mark its memory target + // dead and let removal delete the journal without copying either. + const isMigratableRollbackRow = (row: RefinementEvent): boolean => + row.data.kind === "memory" && isUsableRollbackRow(row); const rollbackByTarget = new Map( - rows.filter(isUsableRollbackRow).map((row) => [row.data.rollbackOf!, row] as const) + rows.filter(isMigratableRollbackRow).map((row) => [row.data.rollbackOf!, row] as const) ); // Returns null on a corrupted (cyclic / absurdly long) lineage: such a row // is treated as non-migratable instead of hanging removal. @@ -186,7 +192,7 @@ export async function migrateSharedMemoryRefinementRows(args: { // very corruption, or the retry would leave the owner with an unusable // rollback record over a target the engine then reads as live. const ownerRollbackTargets = new Set( - ownerRows.filter(isUsableRollbackRow).map((ownerRow) => ownerRow.data.rollbackOf!) + ownerRows.filter(isMigratableRollbackRow).map((ownerRow) => ownerRow.data.rollbackOf!) ); for (const row of rows) { if (row.data.kind !== "memory") continue; @@ -219,7 +225,7 @@ export async function migrateSharedMemoryRefinementRows(args: { // row whose parseable action names another target than `rollbackOf` // is corrupt and must not be turned into a usable owner rollback by // rewriting `of` (that would suppress a possibly live mutation). - if (!isUsableRollbackRow(row)) continue; + if (!isMigratableRollbackRow(row)) continue; rollbackOf = ownerIdBySource.get(`${args.childWorkspaceId}:${row.data.rollbackOf}`); if (rollbackOf === undefined || ownerRollbackTargets.has(rollbackOf)) continue; const parsed = RollbackRefinementActionSchema.safeParse(row.data.action); diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index c0058656de6..7485afb4d15 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -552,7 +552,13 @@ type TurnRequestBuildOutcome = type: "ready"; turnExecutionOptions: TurnExecutionOptions; assistantMessageId: string; - deleteAbortedPlaceholder: (messageId: string) => Promise; + /** + * Remove the never-run turn's stamped placeholder, or deny its epoch + * when that fails; false when neither could be made durable — the + * caller must report WORKSPACE_MEMORY_POLICY_PERSIST_ERROR instead of + * its own outcome. + */ + deleteAbortedPlaceholder: (messageId: string) => Promise; logStartOutcome: (outcome: "started" | "stream_start_failed", errorType?: string) => void; /** * Durable side effects that must only land once the stream has actually @@ -572,7 +578,7 @@ export interface PreparedTurnRequest extends AsyncDisposable { } /** Turn refused because a memory-policy DENY could not be made durable (see persistWorkspaceMemoryWritable). */ -const WORKSPACE_MEMORY_POLICY_PERSIST_ERROR = +export const WORKSPACE_MEMORY_POLICY_PERSIST_ERROR = "Could not persist this workspace's read-only memory policy; refusing to start the turn so a restart cannot fall back to a stale write permission. Retry once the config directory is writable."; type PreparedTurnRequestOutcome = @@ -1631,9 +1637,12 @@ export class TurnRequestBuilder { // covered by a turn. When its removal fails (I/O, a concurrent rewrite) // the epoch is denied instead — fail closed rather than trust a row // nobody can confirm is gone. - const discardPlaceholder = async (messageId: string): Promise => { - if (await deleteAbortedPlaceholder(messageId)) return; - await persistWorkspaceMemoryWritable(false); + // Returns false only when neither the deletion nor the deny could be made + // durable: the caller must then surface that instead of its own outcome, + // since a stamped placeholder nobody could remove or deny stays behind. + const discardPlaceholder = async (messageId: string): Promise => { + if (await deleteAbortedPlaceholder(messageId)) return true; + return persistWorkspaceMemoryWritable(false); }; const projectTrusted = isWorkspaceProjectTrusted(this.dependencies.config, metadata); // projectAutomationDisabled: benchmark harnesses opt out of automatic @@ -3182,7 +3191,12 @@ export class TurnRequestBuilder { } if (combinedAbortSignal.aborted) { - await discardPlaceholder(assistantMessageId); + if (!(await discardPlaceholder(assistantMessageId))) { + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } return { type: "finished", result: Ok( @@ -3404,7 +3418,12 @@ export class TurnRequestBuilder { } catch (error) { if (error instanceof ContextBudgetExceededError) { runLanguageModelCleanup(modelResult.data.model); - await discardPlaceholder(assistantMessageId); + if (!(await discardPlaceholder(assistantMessageId))) { + return { + type: "finished", + result: Err({ type: "unknown", raw: WORKSPACE_MEMORY_POLICY_PERSIST_ERROR }), + }; + } return { type: "finished", result: Err(error.details) }; } throw error; From e5cb003a762245bd7d0a9a949e07c226a26d6e79 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 03:47:48 +0000 Subject: [PATCH 89/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-th?= =?UTF-8?q?ird=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Next history sequences must be safe integers everywhere they are derived (a segment start at the boundary is malformed and reseeds; a persisted row at the boundary refuses the append), and a corrupt policy-record container is healed by the next write instead of pinning every epoch to its blanket deny. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/historyService.test.ts | 33 +++++++++++++++++++ src/node/services/historyService.ts | 29 ++++++++++++---- .../services/workspaceMemoryPolicyEpochs.ts | 11 +++++++ src/node/services/workspaceService.test.ts | 13 ++++++++ src/node/services/workspaceService.ts | 10 ++++-- 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 9ec856de434..34d7789793b 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2674,6 +2674,39 @@ describe("HistoryService", () => { expect(msg.metadata?.historySegment).toBeGreaterThan(1); }); + it("treats a segment start at the safe-integer boundary as malformed and refuses unsafe next sequences", async () => { + const workspaceId = "workspace1"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + // `start` is safe but `start + 1` is not: the file cannot seed a + // counter that moves, so it is quarantined and reseeded like any other + // malformed file. + const segmentPath = path.join(workspaceDir, "history-segment.json"); + await fs.writeFile(segmentPath, JSON.stringify({ start: Number.MAX_SAFE_INTEGER })); + const msg = createMuxMessage("msg1", "user", "Hello"); + expect((await service.appendToHistory(workspaceId, msg)).success).toBe(true); + const reseeded = msg.metadata?.historySegment; + if (reseeded === undefined) throw new Error("expected a reseeded stamp"); + expect(Number.isSafeInteger(reseeded + 1)).toBe(true); + expect(reseeded).toBeLessThan(Number.MAX_SAFE_INTEGER); + // A persisted row at the boundary leaves no safe next sequence: the + // append refuses rather than assign a value `+ 1` cannot move past. + const other = "workspace2"; + await fs.mkdir(path.join(config.sessionsDir, other), { recursive: true }); + await fs.writeFile( + path.join(config.sessionsDir, other, "chat.jsonl"), + JSON.stringify({ + ...createMuxMessage("edge", "user", "at the edge", { + historySequence: Number.MAX_SAFE_INTEGER, + }), + workspaceId: other, + }) + "\n" + ); + const refused = await service.appendToHistory(other, createMuxMessage("m", "user", "x")); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("safe integer"); + }); + it("refuses to open a segment above an unsafe persisted sequence", async () => { const workspaceId = "workspace1"; const workspaceDir = path.join(config.sessionsDir, workspaceId); diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index f0993aa9b29..5c73c786dd1 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -1925,7 +1925,10 @@ export class HistoryService { typeof parsed === "object" && parsed !== null ? (parsed as { start?: unknown }).start : undefined; - return isNonNegativeInteger(start) && Number.isSafeInteger(start) ? start : null; + // The start itself AND the sequences assigned from it must stay safe + // (`start + 1` past 2^53 no longer moves): a file at the boundary is + // malformed too and reseeds (which then refuses an unsafe reseed). + return isNonNegativeInteger(start) && Number.isSafeInteger(start + 1) ? start : null; } /** @@ -2871,15 +2874,27 @@ export class HistoryService { // User rationale: a stale partial or hand-edited chat.jsonl can leave an old // historySequence at the tail. Initializing from the tail would make the next // live message look like an edit/truncation to the renderer, so scan for max. - const nextSeqNum = (await this.getMaxHistorySequence(workspaceId)) + 1; - assert( - isNonNegativeInteger(nextSeqNum), - "next history sequence counter must be a non-negative integer" - ); + const nextSeqNum = await this.getNextPersistedHistorySequence(workspaceId); this.sequenceCounters.set(workspaceId, nextSeqNum); return nextSeqNum; } + /** + * `max persisted sequence + 1`, refused when that is not a safe integer. + * Reachable from persisted data (a hand-edited row at 2^53 - 1): an unsafe + * next sequence would stop moving under `+ 1` and let a later finalization + * replace an unrelated row, so appends fail instead of assigning it. + */ + private async getNextPersistedHistorySequence(workspaceId: string): Promise { + const nextSeqNum = (await this.getMaxHistorySequence(workspaceId)) + 1; + if (!isNonNegativeInteger(nextSeqNum) || !Number.isSafeInteger(nextSeqNum)) { + throw new Error( + `History sequences of ${workspaceId} are exhausted: next sequence ${nextSeqNum} is not a safe integer` + ); + } + return nextSeqNum; + } + /** * Internal helper for appending to history without acquiring lock. */ @@ -3196,7 +3211,7 @@ export class HistoryService { * precedes every operation (active file is bounded by rotation). */ private async refreshSequenceCounterUnderWriteLock(workspaceId: string): Promise { - const persistedNext = (await this.getMaxHistorySequence(workspaceId)) + 1; + const persistedNext = await this.getNextPersistedHistorySequence(workspaceId); const cached = this.sequenceCounters.get(workspaceId); if (cached === undefined || persistedNext > cached) { this.sequenceCounters.set(workspaceId, persistedNext); diff --git a/src/node/services/workspaceMemoryPolicyEpochs.ts b/src/node/services/workspaceMemoryPolicyEpochs.ts index b142006c2a8..fe7c8519407 100644 --- a/src/node/services/workspaceMemoryPolicyEpochs.ts +++ b/src/node/services/workspaceMemoryPolicyEpochs.ts @@ -52,6 +52,17 @@ export function workspaceMemoryWritableForEpoch( return typeof value === "boolean" ? value : false; } +/** + * Whether the persisted container is corrupt (present but not a plain + * object). Every epoch then reads as a deny; the next write replaces the + * container (setWorkspaceMemoryWritableForEpoch), so writers must not take + * a no-write fast path on the strength of that deny or the corruption — and + * the blanket deny — would outlive every epoch until a destructive clear. + */ +export function hasMalformedWorkspaceMemoryPolicyRecords(entry: WorkspaceConfigEntry): boolean { + return policyRecords(entry) === null; +} + /** * The persisted container: `undefined` when absent, `null` when present but * not a plain object (raw JSON, no schema validation upstream). diff --git a/src/node/services/workspaceService.test.ts b/src/node/services/workspaceService.test.ts index 17763e3eb7d..82e7fb65e36 100644 --- a/src/node/services/workspaceService.test.ts +++ b/src/node/services/workspaceService.test.ts @@ -9705,6 +9705,19 @@ describe("WorkspaceService initialize", () => { }) ).toBe(true); expect(persistedFor(60)).not.toBe(true); + // Healed on that write (no fast path on a corrupt container's blanket + // deny, r83): the container is a plain object again holding this + // epoch's deny, and the next epoch's first turn grants normally. + const healed = findWorkspaceEntry(realConfig.loadConfigOrDefault(), "policy-scratch")! + .workspace.workspaceMemoryWritableByEpoch; + expect(healed).toEqual({ "60": false }); + expect( + await service.recordWorkspaceMemoryWritable("policy-scratch", true, { + epochHasPriorTurns: false, + policyEpoch: 62, + }) + ).toBe(true); + expect(persistedFor(62)).toBe(true); setWorkspaceMemoryWritableForEpoch(corrupted, 61, true); expect(corrupted.workspaceMemoryWritableByEpoch).toEqual({ "61": true }); } diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index 4374663930d..fdc82f76865 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -367,6 +367,7 @@ import { import { findWorkspaceEntry } from "@/node/services/taskUtils"; import { setWorkspaceMemoryWritableForEpoch, + hasMalformedWorkspaceMemoryPolicyRecords, workspaceMemoryWritableForEpoch, } from "@/node/services/workspaceMemoryPolicyEpochs"; import type { WorktreeArchiveSnapshotService } from "@/node/services/worktreeArchiveSnapshotService"; @@ -4462,10 +4463,13 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { (mirror ?? true) && writable; // Fast path (no write): the outcome cannot differ from the stored value — - // it is already false, or already true and this turn grants. + // it is already false, or already true and this turn grants. Not when + // the stored "false" is a corrupt container's blanket deny: the write + // below replaces the container with a healed record (this epoch stays + // denied; later epochs read their own records again). if ( - stored === false || - (stored === true && conjunction(stored, carriedFor(before.workspace))) + !hasMalformedWorkspaceMemoryPolicyRecords(before.workspace) && + (stored === false || (stored === true && conjunction(stored, carriedFor(before.workspace)))) ) { session?.recordWorkspaceMemoryWritable(stored); return true; From 9aa1bdef1b2d1309162711631d16832d8b14fd2c Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 04:27:28 +0000 Subject: [PATCH 90/98] =?UTF-8?q?=F0=9F=A4=96=20tests:=20drop=20the=20work?= =?UTF-8?q?flow=20card's=20placeholder=20sequence=20in=20the=20auto-resume?= =?UTF-8?q?=20fixture?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The builder's MAX_SAFE_INTEGER placeholder is replaced in production before the card is appended; HistoryService now refuses a counter it could not advance past that value, so the fixture assigns sequences the way WorkspaceService does. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/taskService.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/node/services/taskService.test.ts b/src/node/services/taskService.test.ts index 4044f662853..f75e9af3a87 100644 --- a/src/node/services/taskService.test.ts +++ b/src/node/services/taskService.test.ts @@ -9337,6 +9337,10 @@ describe("TaskService", () => { ); slashCard.metadata = { ...slashCard.metadata, + // Like WorkspaceService's slash-command append, drop the builder's + // placeholder sequence: HistoryService assigns the real one (and refuses + // a counter it could not advance past). + historySequence: undefined, muxMetadata: { type: WORKFLOW_RUN_CARD_DISPLAY_METADATA_TYPE, runId: workflowRunId }, }; const appendCard = await historyService.appendToHistory(rootWorkspaceId, slashCard); From 48735ddca752ba75105c165fad1bc0ee4eaa8983 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 05:10:09 +0000 Subject: [PATCH 91/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-fo?= =?UTF-8?q?urth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A memory owner pin is honored only once the recorded parent is gone (a live parent chain takes precedence and removal re-pins from it); the probe token fingerprints a redirected child's legacy notebook through the same throttled scan as the owner's; the workspace memory clock refuses to advance past the safe-integer range; and a tombstoned adoption record counts as absent only when the stat proves it. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryLegacyAdoption.ts | 44 ++++++++--- src/node/services/memoryService.test.ts | 79 +++++++++++++++++++ src/node/services/memoryService.ts | 52 +++++++++--- .../services/memoryWorkspaceOwner.test.ts | 34 +++++++- src/node/services/memoryWorkspaceOwner.ts | 33 +++++--- .../workspaceMemoryRevision.test.ts | 41 ++++++++++ .../refinement/workspaceMemoryRevision.ts | 9 +++ 7 files changed, 254 insertions(+), 38 deletions(-) create mode 100644 src/node/services/refinement/workspaceMemoryRevision.test.ts diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index bf02a6e1306..a77eb7bf25a 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -188,18 +188,32 @@ export async function readLegacyAdoptionManifest( } /** - * The file identity a LegacyAdoptionRecord.targetStamp records; null when the - * file cannot be stat'ed (the record then carries no stamp: preserved). + * The file identity a LegacyAdoptionRecord.targetStamp records, or why there + * is none: "absent" only when the stat PROVES the path is gone (ENOENT / + * ENOTDIR); any other failure (EACCES, EIO) is "unreadable" — it says + * nothing about the path, so callers deciding on absence must refuse. */ -export async function adoptionTargetStamp(absPath: string): Promise { +export async function adoptionTargetPresence( + absPath: string +): Promise<{ stamp: string } | "absent" | "unreadable"> { try { const stat = await fsPromises.lstat(absPath, { bigint: true }); - return `${stat.ino}:${stat.size}:${stat.mtimeNs}`; - } catch { - return null; + return { stamp: `${stat.ino}:${stat.size}:${stat.mtimeNs}` }; + } catch (error) { + const code = (error as NodeJS.ErrnoException | null)?.code; + return code === "ENOENT" || code === "ENOTDIR" ? "absent" : "unreadable"; } } +/** + * The file identity a LegacyAdoptionRecord.targetStamp records; null when the + * file cannot be stat'ed (the record then carries no stamp: preserved). + */ +export async function adoptionTargetStamp(absPath: string): Promise { + const presence = await adoptionTargetPresence(absPath); + return typeof presence === "string" ? null : presence.stamp; +} + /** * Every non-directory entry (files, symlinks, anything) under `absDir`, * recursively, as relPaths prefixed with `dirRel`; empty when the directory @@ -301,14 +315,24 @@ export async function createLegacyPathRemapper(args: { // retargeted rollback recreated there (re-stamped below); anything else at // that path is the owner's. // Value: whether that generation is a file on disk (a tombstoned record - // whose copy a rollback restored counts as present). + // whose copy a rollback restored counts as present). Absence must be + // PROVEN (ENOENT/ENOTDIR): a target that merely cannot be stat'ed right + // now (EACCES, EIO) may well hold an owner-created replacement, and + // calling it absent would let a restore or rename land on it — such a + // record is simply not current (the rollback is refused as owner-owned). const currentGeneration = new Map(); for (const [rel, record] of adopted) { if (record.created !== true || record.pending === true) continue; - const stamp = await adoptionTargetStamp(path.join(ownerRoot, ...record.target.split("/"))); - if (record.targetStamp !== undefined && stamp === record.targetStamp) { + const presence = await adoptionTargetPresence( + path.join(ownerRoot, ...record.target.split("/")) + ); + if ( + record.targetStamp !== undefined && + typeof presence !== "string" && + presence.stamp === record.targetStamp + ) { currentGeneration.set(rel, "present"); - } else if (record.deleted === true && stamp === null) { + } else if (record.deleted === true && presence === "absent") { currentGeneration.set(rel, "absent"); } } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index c0d895c442a..fb3015d0df1 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1268,6 +1268,25 @@ describe("MemoryService", () => { ); expect(await foreign.workspaceMemoryRevision("ws-owner")).not.toBe(afterRootWrite); expect(nestedStats()).toBe(0); + // A redirected child's token also fingerprints its legacy notebook: + // that scan is memoized the same way (r84) — the first probe walks + // it, a repeated probe does not, a new legacy note is seen at once. + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(path.join(legacyRoot, "dir"), { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "dir", "old.md"), "legacy"); + const legacyStats = () => + lstatSpy.mock.calls.filter(([target]) => String(target).startsWith(legacyRoot + path.sep)) + .length; + lstatSpy.mockClear(); + const childToken = await foreign.workspaceMemoryRevision("ws-child"); + expect(legacyStats()).toBeGreaterThan(0); + lstatSpy.mockClear(); + expect(await foreign.workspaceMemoryRevision("ws-child")).toBe(childToken); + expect(legacyStats()).toBe(0); + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "new.md"), "downgraded write"); + expect(await foreign.workspaceMemoryRevision("ws-child")).not.toBe(childToken); + expect(legacyStats()).toBeGreaterThan(0); } finally { lstatSpy.mockRestore(); } @@ -2572,6 +2591,66 @@ describe("MemoryService", () => { expect(readopted["note.md"].deleted).toBeUndefined(); }); + it("refuses to restore a deleted legacy note while its owner target cannot be inspected", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const childSessionDir = path.join(fixture.config.sessionsDir, "ws-child"); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(childSessionDir, "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + // The pre-sharing delete: its inverse restores the legacy path. + await sharedDurableEventJournal(childSessionDir).append({ + workspaceId: "ws-child", + kind: "refinement", + data: { + kind: "memory", + action: { op: "delete", path: "/memories/workspace/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(legacyRoot, "note.md"), text: "v1" }], + }, + }, + }); + const deleteRow = (await readRefinementEvents(childSessionDir)).at(-1)!; + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(path.join(ownerRoot, "note.md"))).toBe(false); + const target = path.join(ownerRoot, "note.md"); + const rollback = () => + rollbackRefinement({ + sessionDir: childSessionDir, + sharedWorkspaceMemorySessionDir: path.dirname(ownerRoot), + id: deleteRow.id, + evidence: { toolName: "test", actor: "user" }, + }); + // The tombstoned record is current only while the target is PROVEN + // absent: a stat failure that proves nothing (EACCES) may hide an + // owner-created replacement, so the restore is refused rather than + // landing on it. + const realLstat = fsPromises.lstat.bind(fsPromises); + const unreadable = spyOn(fsPromises, "lstat").mockImplementation((( + p: Parameters[0], + ...rest: unknown[] + ) => + String(p) === target + ? Promise.reject(Object.assign(new Error("EACCES"), { code: "EACCES" })) + : (realLstat as (...args: unknown[]) => unknown)(p, ...rest)) as never); + try { + const refused = await rollback(); + expect(refused.success).toBe(false); + expect(await pathExists(target)).toBe(false); + } finally { + unreadable.mockRestore(); + } + // Proven absent: the restore lands in the shared store. + const restored = await rollback(); + expect(restored.success).toBe(true); + expect(await fsPromises.readFile(target, "utf-8")).toBe("v1"); + }); + it("ignores a manifest a downgraded child wrote into its model-writable legacy root", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 80a565bb0e3..50c81d59cc7 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -881,7 +881,7 @@ export class MemoryService extends EventEmitter { */ private readonly legacyStoreCheckedAgainst = new Map(); - /** Last owner-store file scan per owner (see ownerStoreStamp). */ + /** Last file scan per store (owner notebooks and redirected children's legacy notebooks; see throttledStoreStamp). */ private readonly ownerStoreStampMemo = new Map< string, { rootMtime: string; stamp: string; scannedAt: number } @@ -1260,7 +1260,15 @@ export class MemoryService extends EventEmitter { */ private async legacyAdoptionCheckKey( childId: string, - owner: string + owner: string, + options?: { + /** + * Probe-token use: the legacy file scan is memoized like the owner + * store's (throttledStoreStamp). The adoption pass itself must stay + * exact (it decides whether to re-run) and never passes this. + */ + throttled: boolean; + } ): Promise<{ legacyRootKind: Awaited>; checkKey: string }> { const childSessionDir = path.join(this.config.sessionsDir, childId); const legacyRoot = path.join(childSessionDir, "memory"); @@ -1278,9 +1286,13 @@ export class MemoryService extends EventEmitter { .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) ) : ""; - const checkKey = `${owner}\u0000${legacyRootKind}\u0000${ - legacyRootKind === "dir" ? await legacyStoreStamp(childSessionDir, legacyRoot) : "" - }\u0000${childSidecarFingerprint}`; + const legacyStamp = + legacyRootKind !== "dir" + ? "" + : options?.throttled === true + ? await this.throttledStoreStamp(`legacy\u0000${childId}`, childSessionDir, legacyRoot) + : await legacyStoreStamp(childSessionDir, legacyRoot); + const checkKey = `${owner}\u0000${legacyRootKind}\u0000${legacyStamp}\u0000${childSidecarFingerprint}`; return { legacyRootKind, checkKey }; } @@ -2263,7 +2275,9 @@ export class MemoryService extends EventEmitter { // (see legacyAdoptionCheckKey): its next store access adopts the change, // so the cached context must miss as soon as the legacy state moves. if (owner === workspaceId) return token; - return `${token}\u0000${(await this.legacyAdoptionCheckKey(workspaceId, owner)).checkKey}`; + return `${token}\u0000${ + (await this.legacyAdoptionCheckKey(workspaceId, owner, { throttled: true })).checkKey + }`; } /** @@ -2275,19 +2289,35 @@ export class MemoryService extends EventEmitter { * existing note waits for the interval; this build's writes advance the * clock segment. */ - private async ownerStoreStamp(owner: string, ownerSessionDir: string): Promise { - const root = workspaceMemoryStorePath(this.config.sessionsDir, owner); + private ownerStoreStamp(owner: string, ownerSessionDir: string): Promise { + return this.throttledStoreStamp( + `owner\u0000${owner}`, + ownerSessionDir, + workspaceMemoryStorePath(this.config.sessionsDir, owner) + ); + } + + /** + * legacyStoreStamp with the per-store memo described at ownerStoreStamp; + * also used for a redirected child's legacy notebook in its probe token, + * which otherwise walks that directory on every probe as well. + */ + private async throttledStoreStamp( + memoKey: string, + sessionDir: string, + root: string + ): Promise { const rootMtime = await fsPromises .stat(root) .then((stat) => String(stat.mtimeMs)) .catch(() => "missing"); - const memo = this.ownerStoreStampMemo.get(owner); + const memo = this.ownerStoreStampMemo.get(memoKey); const now = Date.now(); if (memo?.rootMtime === rootMtime && now - memo.scannedAt < OWNER_STORE_SCAN_INTERVAL_MS) { return memo.stamp; } - const stamp = await legacyStoreStamp(ownerSessionDir, root); - this.ownerStoreStampMemo.set(owner, { rootMtime, stamp, scannedAt: now }); + const stamp = await legacyStoreStamp(sessionDir, root); + this.ownerStoreStampMemo.set(memoKey, { rootMtime, stamp, scannedAt: now }); return stamp; } diff --git a/src/node/services/memoryWorkspaceOwner.test.ts b/src/node/services/memoryWorkspaceOwner.test.ts index 0ea3171d561..af26d03554d 100644 --- a/src/node/services/memoryWorkspaceOwner.test.ts +++ b/src/node/services/memoryWorkspaceOwner.test.ts @@ -28,8 +28,11 @@ describe("pinDescendantWorkspaceMemoryOwners", () => { // Stale pin (its owner is gone): the resolver walks past it today, but // once ws-mid is removed that walk would dangle — replaced. { id: "ws-stale", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-gone" }, - // Valid pin to another live notebook: that is the notebook this child - // uses; kept (continuity), not redirected to ws-mid's owner. + // Pin to another live notebook while the parent is still registered: + // a state this code never writes (pins are recorded as an ancestor is + // removed). The live chain wins — the child has been using ws-owner's + // notebook — and the removal re-pins it to that (r84), rather than + // letting corrupt raw config redirect it across task trees. { id: "ws-pinned", parentWorkspaceId: "ws-mid", memoryOwnerWorkspaceId: "ws-other" }, // Not a child of the removed node: untouched. { id: "ws-sibling", parentWorkspaceId: "ws-owner" }, @@ -43,7 +46,7 @@ describe("pinDescendantWorkspaceMemoryOwners", () => { expect(before).toEqual({ "ws-plain": "ws-owner", "ws-stale": "ws-owner", - "ws-pinned": "ws-other", + "ws-pinned": "ws-owner", }); const pinned = pinDescendantWorkspaceMemoryOwners(cfg, "ws-mid"); @@ -52,7 +55,7 @@ describe("pinDescendantWorkspaceMemoryOwners", () => { const pinOf = (id: string) => entries.find((ws) => ws.id === id)!.memoryOwnerWorkspaceId; expect(pinOf("ws-plain")).toBe("ws-owner"); expect(pinOf("ws-stale")).toBe("ws-owner"); - expect(pinOf("ws-pinned")).toBe("ws-other"); + expect(pinOf("ws-pinned")).toBe("ws-owner"); expect(pinOf("ws-sibling")).toBeUndefined(); // With ws-mid gone, every pinned child still resolves as before. @@ -73,4 +76,27 @@ describe("pinDescendantWorkspaceMemoryOwners", () => { expect(resolveWorkspaceMemoryOwnerId(after, id)).toBe(owner); } }); + + it("honors a pin only once the recorded parent is gone", () => { + const live = topology([ + { id: "ws-owner" }, + { id: "ws-other" }, + { id: "ws-child", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-other" }, + { id: "ws-grand", parentWorkspaceId: "ws-child" }, + ]); + // Parent registered: the chain decides, for the child and everything below it. + expect(resolveWorkspaceMemoryOwnerId(live, "ws-child")).toBe("ws-owner"); + expect(resolveWorkspaceMemoryOwnerId(live, "ws-grand")).toBe("ws-owner"); + // Parent gone: the (live) pin decides; a pin whose owner is gone too + // leaves the child on its own store. + const dangling = topology([ + { id: "ws-other" }, + { id: "ws-child", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-other" }, + { id: "ws-grand", parentWorkspaceId: "ws-child" }, + { id: "ws-orphan", parentWorkspaceId: "ws-owner", memoryOwnerWorkspaceId: "ws-gone" }, + ]); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-child")).toBe("ws-other"); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-grand")).toBe("ws-other"); + expect(resolveWorkspaceMemoryOwnerId(dangling, "ws-orphan")).toBe("ws-orphan"); + }); }); diff --git a/src/node/services/memoryWorkspaceOwner.ts b/src/node/services/memoryWorkspaceOwner.ts index 90ed427ba50..5e23f9289d4 100644 --- a/src/node/services/memoryWorkspaceOwner.ts +++ b/src/node/services/memoryWorkspaceOwner.ts @@ -70,14 +70,21 @@ export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: } return workspaceId; } - // A pinned owner (recorded when an intermediate ancestor was removed) - // short-circuits the walk; if that owner is itself gone, fall through to - // the parent chain, which then dangles and resolves to self. - const pinned = entry.memoryOwnerWorkspaceId; - if (pinned !== undefined && pinned !== "" && byId.has(pinned)) { - return pinned; - } + // A pinned owner is recorded when an intermediate ancestor is removed + // (pinDescendantWorkspaceMemoryOwners), so it only speaks for a chain + // that DANGLES: while the recorded parent is still registered the walk + // follows it, and a pin that disagrees with a live parent (raw config, + // never produced by this code) heals on the next removal instead of + // redirecting the child into an unrelated tree's notebook. With the + // parent gone, a live pin decides; a pin whose owner is gone too leaves + // the child on its own store. const parentWorkspaceId = entry.parentWorkspaceId; + const parentLive = + parentWorkspaceId !== undefined && parentWorkspaceId !== "" && byId.has(parentWorkspaceId); + if (!parentLive) { + const pinned = entry.memoryOwnerWorkspaceId; + if (pinned !== undefined && pinned !== "" && byId.has(pinned)) return pinned; + } if (parentWorkspaceId === undefined || parentWorkspaceId === "") return current; current = parentWorkspaceId; } @@ -93,12 +100,12 @@ export function workspaceMemoryOwnerResolver(cfg: ProjectsConfig): (workspaceId: /** * Removal of `removedWorkspaceId`: pin each surviving direct child to the * owner it resolves to NOW, so the notebook it uses stays the same once the - * chain through the removed node dangles. A child's own valid pin already - * decides its owner and is kept; one that is absent — or stale (its owner - * gone, which the resolver walks past today but could not once this node is - * gone: the child would drop to a private store) — is replaced by the walk - * through the removed node (r77). Mutates the entries in place; returns the - * pins written, for the caller's verified read-back. + * chain through the removed node dangles. The pin is whatever the walk + * resolves to while the node is still registered — an existing pin is + * overwritten by it (a live parent takes precedence over a pin in the + * resolver, so that IS the notebook the child has been using), and a stale + * one (its owner gone) is replaced likewise. Mutates the entries in place; + * returns the pins written, for the caller's verified read-back. */ export function pinDescendantWorkspaceMemoryOwners( cfg: ProjectsConfig, diff --git a/src/node/services/refinement/workspaceMemoryRevision.test.ts b/src/node/services/refinement/workspaceMemoryRevision.test.ts new file mode 100644 index 00000000000..02374ade83c --- /dev/null +++ b/src/node/services/refinement/workspaceMemoryRevision.test.ts @@ -0,0 +1,41 @@ +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import * as fsPromises from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import { + advanceWorkspaceMemoryRevision, + readWorkspaceMemoryRevision, + workspaceMemoryRevisionPath, +} from "./workspaceMemoryRevision"; + +describe("advanceWorkspaceMemoryRevision", () => { + let sessionDir: string; + beforeEach(async () => { + sessionDir = await fsPromises.mkdtemp(path.join(os.tmpdir(), "xum-memory-revision-")); + }); + afterEach(async () => { + await fsPromises.rm(sessionDir, { recursive: true, force: true }); + }); + + it("advances monotonically past a clock that runs ahead of wall time", async () => { + const ahead = Date.now() + 60_000; + await fsPromises.writeFile(workspaceMemoryRevisionPath(sessionDir), String(ahead)); + expect(await advanceWorkspaceMemoryRevision(sessionDir)).toBe(ahead + 1); + expect(await readWorkspaceMemoryRevision(sessionDir)).toBe(ahead + 1); + }); + + it("refuses to advance an exhausted clock and leaves the file as it is", async () => { + // 2^53 - 1 reads as valid, but `+ 1` is no longer a safe integer: writing + // it would persist a value the strict reader rejects forever. + const revisionPath = workspaceMemoryRevisionPath(sessionDir); + await fsPromises.writeFile(revisionPath, String(Number.MAX_SAFE_INTEGER)); + const attempt = advanceWorkspaceMemoryRevision(sessionDir); + expect( + await attempt.then( + () => null, + (error: unknown) => String(error) + ) + ).toContain("exhausted"); + expect(await fsPromises.readFile(revisionPath, "utf-8")).toBe(String(Number.MAX_SAFE_INTEGER)); + }); +}); diff --git a/src/node/services/refinement/workspaceMemoryRevision.ts b/src/node/services/refinement/workspaceMemoryRevision.ts index a2933db9c18..acbb41f4810 100644 --- a/src/node/services/refinement/workspaceMemoryRevision.ts +++ b/src/node/services/refinement/workspaceMemoryRevision.ts @@ -76,6 +76,15 @@ async function readWorkspaceMemoryRevisionStrict(ownerSessionDir: string): Promi export async function advanceWorkspaceMemoryRevision(ownerSessionDir: string): Promise { const previous = (await readWorkspaceMemoryRevisionStrict(ownerSessionDir)) ?? 0; const next = Math.max(Date.now(), previous + 1); + // A persisted clock at 2^53 - 1 reads as valid but cannot advance: writing + // `previous + 1` would persist a value the strict reader rejects forever + // and hand out a malformed `sourceTs`. Refuse instead (callers treat the + // throw as "order unknown"), leaving the file as it is. + if (!Number.isSafeInteger(next)) { + throw new Error( + `workspace memory revision at ${workspaceMemoryRevisionPath(ownerSessionDir)} is exhausted: ${previous}` + ); + } // Atomic: a crash mid-write must not leave a truncated value the strict // reader would reject forever (every later row order-unknown). await writeFileAtomic(workspaceMemoryRevisionPath(ownerSessionDir), String(next)); From 1f2453fafa3613bbc64975e500270bc4a7940f26 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 05:35:13 +0000 Subject: [PATCH 92/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-fi?= =?UTF-8?q?fth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A requestPreludeMessageIds entry accounts for a row only when that row has prelude shape (synthetic), in the harvest gate, the prior-turn check and the tail-copy stamps, so a listing naming an ordinary user turn cannot make it read as covered. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 17 +++++-- .../utils/messages/compactionBoundary.ts | 25 ++++++++-- src/node/services/compactionHandler.test.ts | 16 +++--- src/node/services/compactionHandler.ts | 21 +++++--- .../memoryConsolidationService.test.ts | 50 +++++++++++++++++++ .../services/memoryConsolidationService.ts | 13 +++-- src/node/services/turnRequestBuilder.ts | 17 +++---- 7 files changed, 125 insertions(+), 34 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index 844dcadcb6c..2afba8f6d75 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -82,8 +82,9 @@ describe("duplicateUserMessageIds", () => { expect(duplicateUserMessageIds(rows.slice(0, 2)).size).toBe(0); // The current batch is matched by id: the duplicated id would hide the // earlier row from the prior-turn check, so it counts as one. - expect(epochHasPriorTurnRows(rows, new Set(["u1"]))).toBe(true); - expect(epochHasPriorTurnRows(rows.slice(0, 2), new Set(["u1"]))).toBe(false); + const current = { userMessageId: "u1", preludeMessageIds: new Set() }; + expect(epochHasPriorTurnRows(rows, current)).toBe(true); + expect(epochHasPriorTurnRows(rows.slice(0, 2), current)).toBe(false); }); }); @@ -467,7 +468,7 @@ describe("sliceMessagesFromLatestCompactionBoundary", () => { }); describe("epochHasPriorTurnRows", () => { - const current = new Set(["u-now", "p-now"]); + const current = { userMessageId: "u-now", preludeMessageIds: new Set(["p-now"]) }; const withRows = (...rows: Array>) => epochHasPriorTurnRows( rows.map((args) => createMuxMessage(...args)), @@ -475,11 +476,19 @@ describe("epochHasPriorTurnRows", () => { ); it("counts an earlier user turn but not the batch being started", () => { - expect(withRows(["u-now", "user", "now"], ["p-now", "user", "prelude"])).toBe(false); + expect( + withRows(["u-now", "user", "now"], ["p-now", "user", "prelude", { synthetic: true }]) + ).toBe(false); expect(withRows(["u-old", "user", "earlier"], ["u-now", "user", "now"])).toBe(true); expect(withRows(["a-old", "assistant", "answer"], ["u-now", "user", "now"])).toBe(false); }); + it("exempts a listed prelude id only for a row of prelude shape", () => { + // A prelude listing naming an ordinary user turn (raw history) must not + // hide that turn from the unknown-history rule. + expect(withRows(["u-now", "user", "now"], ["p-now", "user", "a real earlier turn"])).toBe(true); + }); + it("ignores rows that are no turn of the epoch: compaction requests, tail copies, token-budget internals", () => { expect( withRows( diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index efcf5a13b04..fecbeaae4f7 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -261,23 +261,42 @@ export function sliceMessagesForProviderFromLatestContextBoundary( */ export function epochHasPriorTurnRows( activeContextMessages: readonly MuxMessage[], - currentBatch: ReadonlySet + currentTurn: { userMessageId: string | undefined; preludeMessageIds: ReadonlySet } ): boolean { // Batches are matched by row id: two user rows sharing one id (persisted // history is raw JSON) would both read as the current batch, hiding the // earlier one from this check and from the harvest gate's coverage — so a - // duplicated id is itself an unaccounted prior turn (fail closed). + // duplicated id is itself an unaccounted prior turn (fail closed). A + // prelude listing exempts only rows of prelude shape (isRequestPreludeRow): + // an ordinary user turn named there stays a prior turn. const duplicated = duplicateUserMessageIds(activeContextMessages); return activeContextMessages.some( (message) => message.role === "user" && - (duplicated.has(message.id) || !currentBatch.has(message.id)) && + (duplicated.has(message.id) || + !( + message.id === currentTurn.userMessageId || + (currentTurn.preludeMessageIds.has(message.id) && isRequestPreludeRow(message)) + )) && message.metadata?.muxMetadata?.type !== "compaction-request" && message.metadata?.rlmPreservedTailCopy !== true && !isTokenBudgetInternalMessage(message) ); } +/** + * Whether a row has the shape of a request prelude row — one the backend + * appends with a turn and lists in the user row's `requestPreludeMessageIds` + * (@mention/skill/MCP snapshots, family payloads): always `synthetic`. The + * id-keyed policy accounting (prior-turn check, harvest coverage, tail-copy + * stamps) honors a prelude listing only for such rows, so a listing that + * names an ordinary user turn (persisted history is raw JSON) cannot make + * that turn read as accounted for. + */ +export function isRequestPreludeRow(message: MuxMessage): boolean { + return message.metadata?.synthetic === true; +} + /** * Ids carried by more than one user row of `messages`. The policy checks * account for user rows by id (a turn's batch is its user row plus the diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 92fb46d0c89..0cb17654c6b 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1912,23 +1912,26 @@ describe("CompactionHandler", () => { // Sequences: the boundary sits at boundarySequence, the two first-epoch // copies at +1/+2, so the rows seeded here start at +3. await seedHistory( - createMuxMessage("p2", "user", "prelude snapshot"), + createMuxMessage("p2", "user", "prelude snapshot", { synthetic: true }), + // Listed as a prelude row but of ordinary shape: never stamped through + // the listing (r85). + createMuxMessage("px", "user", "a real turn's question listed as prelude"), createMuxMessage("u2", "user", "second question", { - requestPreludeMessageIds: ["p2"], + requestPreludeMessageIds: ["p2", "px"], }), createMuxMessage("a2", "assistant", "second answer", { - requestHistorySequence: boundarySequence + 4, // anchors on u2 + requestHistorySequence: boundarySequence + 5, // anchors on u2 workspaceMemoryPolicyEpoch: -1, }), createMuxMessage("u3", "user", "third question"), createMuxMessage("ux", "user", "foreign backend's batch"), createMuxMessage("a3", "assistant", "third answer", { - requestHistorySequence: boundarySequence + 6, // anchors on u3, not ux + requestHistorySequence: boundarySequence + 7, // anchors on u3, not ux workspaceMemoryPolicyEpoch: boundarySequence, }), createMuxMessage("u4", "user", "old-build question"), createMuxMessage("a4", "assistant", "old-build answer", { - requestHistorySequence: boundarySequence + 9, // anchors on u4 + requestHistorySequence: boundarySequence + 10, // anchors on u4 }), createMuxMessage("u5", "user", "unanswered question"), // A row that recorded its policy under a foreign epoch but lost its @@ -1950,7 +1953,7 @@ describe("CompactionHandler", () => { // user row stays unstamped, the row itself keeps its stamp. createMuxMessage("u10", "user", "question behind a fractional bound"), createMuxMessage("a10", "assistant", "fractional bound", { - requestHistorySequence: boundarySequence + 17.5, + requestHistorySequence: boundarySequence + 18.5, workspaceMemoryPolicyEpoch: boundarySequence, }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) @@ -1964,6 +1967,7 @@ describe("CompactionHandler", () => { -1, // copy(u1) -1, // copy(a1) -1, // p2 (prelude of u2) + undefined, // px (listed, but no prelude shape) -1, // u2 -1, // a2 boundarySequence, // u3 diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index f5f57f58acd..7d9d5514117 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -43,6 +43,7 @@ import { isDurableContextBoundaryMarker, latestContextBoundaryHistorySequence, duplicateUserMessageIds, + isRequestPreludeRow, sliceMessagesFromLatestCompactionBoundary, workspaceMemoryPolicyEpochOf, } from "@/common/utils/messages/compactionBoundary"; @@ -1676,11 +1677,12 @@ export class CompactionHandler { closingPolicyEpoch: number ): MuxMessage[] { const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + const userRowById = new Map(); for (const message of tailRows) { const sequence = message.metadata?.historySequence; - if (message.role === "user" && typeof sequence === "number") { - userRows.push({ message, sequence }); - } + if (message.role !== "user") continue; + userRowById.set(message.id, message); + if (typeof sequence === "number") userRows.push({ message, sequence }); } const coveredEpochById = new Map(); // An id shared by two user rows names no batch: neither copy is stamped @@ -1699,10 +1701,15 @@ export class CompactionHandler { if (typeof policyEpoch !== "number" || !isNonNegativeInteger(bound)) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; - for (const id of [ - anchor.id, - ...getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds), - ]) { + // Same prelude rule as the harvest gate: a listed id stamps a user row + // only when that row has prelude shape. + const prelude = getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds).filter( + (id) => { + const listed = userRowById.get(id); + return listed === undefined || isRequestPreludeRow(listed); + } + ); + for (const id of [anchor.id, ...prelude]) { if (!coveredEpochById.has(id) && !duplicated.has(id)) { coveredEpochById.set(id, policyEpoch); } diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 8e406ef5732..6e64eccf323 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1303,6 +1303,56 @@ describe("MemoryConsolidationService", () => { expect(fixture.modelCalls.length).toBeGreaterThan(0); }); + it("refuses to harvest when a prelude listing names an ordinary user turn", async () => { + using fixture = await createFixture({ modelFactory: harvestCandidateModel }); + // A read-only backend's prompt was never answered; the next turn's user + // row lists it as a "prelude" (raw history). The listed row is not a + // synthetic prelude row, so it stays uncovered and the harvest refuses. + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("late-1", "user", "Read-only agent's prompt, turn not yet started") + ); + const prompt = createMuxMessage( + "pref-1", + "user", + "Please remember that I prefer concise tests.", + { + requestPreludeMessageIds: ["late-1"], + } + ); + await fixture.historyService.appendToHistory("ws-dream", prompt); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("reply-1", "assistant", "Noted.", { + requestHistorySequence: prompt.metadata?.historySequence, + workspaceMemoryPolicyEpoch: -1, + }) + ); + await fixture.historyService.appendToHistory( + "ws-dream", + createMuxMessage("compact-request", "user", "Please compact", { + muxMetadata: { type: "compaction-request", rawCommand: "/compact", parsed: {} }, + }) + ); + const summary = createMuxMessage("summary-1", "assistant", "Summary.", { + compactionBoundary: true, + compacted: "user", + compactionEpoch: 1, + }); + await fixture.historyService.appendToHistory("ws-dream", summary); + const result = await fixture.service.maybeHarvestThenSweep({ + workspaceId: "ws-dream", + workspaceMemoryWritable: true, + summaryMessageId: "summary-1", + summaryHistorySequence: summary.metadata?.historySequence ?? -1, + compactionEpoch: 1, + compactionRequestMessageId: "compact-request", + }); + expect(result.success).toBe(false); + if (!result.success) expect(result.error).toContain("never recorded"); + expect(fixture.modelCalls).toHaveLength(0); + }); + it("refuses to harvest an epoch whose user rows share an id", async () => { using fixture = await createFixture({ modelFactory: harvestCandidateModel }); // Coverage is keyed by id: covering the later row would mark the earlier diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index 67c6ea6312d..d67ffc07a43 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -72,6 +72,7 @@ import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrel import { compactionClosingPolicyEpoch, duplicateUserMessageIds, + isRequestPreludeRow, } from "@/common/utils/messages/compactionBoundary"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; import { runMemoryConsolidation } from "@/node/services/memoryConsolidation"; @@ -372,10 +373,12 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb return "the compacted epoch holds user rows sharing one id; harvest refused (fail closed)"; } const userRows: Array<{ message: MuxMessage; sequence: number }> = []; + const userRowById = new Map(); for (const message of messages) { const sequence = message.metadata?.historySequence; - if (message.role === "user" && typeof sequence === "number") - userRows.push({ message, sequence }); + if (message.role !== "user") continue; + userRowById.set(message.id, message); + if (typeof sequence === "number") userRows.push({ message, sequence }); } const covered = new Set(); for (const message of messages) { @@ -409,8 +412,12 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); + // A listed id covers a row only when that row has prelude shape: a + // listing naming an ordinary user turn (raw history) would otherwise + // let this turn vouch for content it never consumed. for (const id of getRequestPreludeMessageIds(anchor.metadata?.requestPreludeMessageIds)) { - covered.add(id); + const listed = userRowById.get(id); + if (listed === undefined || isRequestPreludeRow(listed)) covered.add(id); } } const uncovered = messages.some( diff --git a/src/node/services/turnRequestBuilder.ts b/src/node/services/turnRequestBuilder.ts index 7485afb4d15..4cca2a7e293 100644 --- a/src/node/services/turnRequestBuilder.ts +++ b/src/node/services/turnRequestBuilder.ts @@ -1551,17 +1551,12 @@ export class TurnRequestBuilder { // WorkspaceService.recordWorkspaceMemoryWritable). The turn being started // is its last user row plus that row's prelude snapshots; compaction // request rows open an epoch rather than belong to one. - const epochHasPriorTurns = ((): boolean => { - const currentBatch = new Set( - latestUserMessage === undefined - ? [] - : [ - latestUserMessage.id, - ...getRequestPreludeMessageIds(latestUserMessage.metadata?.requestPreludeMessageIds), - ] - ); - return epochHasPriorTurnRows(activeContextMessages, currentBatch); - })(); + const epochHasPriorTurns = epochHasPriorTurnRows(activeContextMessages, { + userMessageId: latestUserMessage?.id, + preludeMessageIds: new Set( + getRequestPreludeMessageIds(latestUserMessage?.metadata?.requestPreludeMessageIds) + ), + }); // The compaction epoch this turn's policy accumulates over: the latest // durable boundary's history sequence (any kind), else the history // segment's boundary-less identity — never reused across full clears From 0099ff0561d508ad00024a4a8d21225aea6821f3 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 06:13:08 +0000 Subject: [PATCH 93/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-si?= =?UTF-8?q?xth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prelude shape now means a synthetic assistant row or a synthetic user row carrying an @mention/skill/MCP snapshot, so synthetic user turns (auto-resume, goal continuations) stay turns of their own. History sequences without a safe successor are skipped like fractions, so appends continue and /clear removes them instead of every append and clear refusing. The session's workspace-memory writable mirror is keyed by policy epoch and re-binds to the summary sequence on a preserved tail, so another backend's epoch change cannot carry a stale value. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 27 +++++++- .../utils/messages/compactionBoundary.ts | 26 +++++-- src/node/services/agentSession.ts | 40 ++++++++--- ...Session.workspaceMemoryPolicyEpoch.test.ts | 23 +++++-- src/node/services/compactionHandler.test.ts | 5 +- src/node/services/historyService.test.ts | 68 ++++++++++--------- src/node/services/historyService.ts | 16 ++++- .../memoryConsolidationService.test.ts | 5 +- src/node/services/workspaceService.ts | 14 ++-- 9 files changed, 159 insertions(+), 65 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index 2afba8f6d75..1f571216766 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -477,7 +477,10 @@ describe("epochHasPriorTurnRows", () => { it("counts an earlier user turn but not the batch being started", () => { expect( - withRows(["u-now", "user", "now"], ["p-now", "user", "prelude", { synthetic: true }]) + withRows( + ["u-now", "user", "now"], + ["p-now", "user", "prelude", { synthetic: true, fileAtMentionSnapshot: ["@a.md"] }] + ) ).toBe(false); expect(withRows(["u-old", "user", "earlier"], ["u-now", "user", "now"])).toBe(true); expect(withRows(["a-old", "assistant", "answer"], ["u-now", "user", "now"])).toBe(false); @@ -487,6 +490,28 @@ describe("epochHasPriorTurnRows", () => { // A prelude listing naming an ordinary user turn (raw history) must not // hide that turn from the unknown-history rule. expect(withRows(["u-now", "user", "now"], ["p-now", "user", "a real earlier turn"])).toBe(true); + // Nor a synthetic user TURN (auto-resume, CLI goal continuation): no snapshot. + expect( + withRows(["u-now", "user", "now"], ["p-now", "user", "continue", { synthetic: true }]) + ).toBe(true); + for (const marker of [ + { agentSkillSnapshot: { skillName: "s", scope: "project" as const, sha256: "x" } }, + { + mcpPromptSnapshot: { + serverName: "srv", + promptName: "p", + commandKey: "srv:p", + invokingMessageId: "u-now", + }, + }, + ]) { + expect( + withRows( + ["u-now", "user", "now"], + ["p-now", "user", "snapshot", { synthetic: true, ...marker }] + ) + ).toBe(false); + } }); it("ignores rows that are no turn of the epoch: compaction requests, tail copies, token-budget internals", () => { diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index fecbeaae4f7..d7331156122 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -286,15 +286,27 @@ export function epochHasPriorTurnRows( /** * Whether a row has the shape of a request prelude row — one the backend - * appends with a turn and lists in the user row's `requestPreludeMessageIds` - * (@mention/skill/MCP snapshots, family payloads): always `synthetic`. The - * id-keyed policy accounting (prior-turn check, harvest coverage, tail-copy - * stamps) honors a prelude listing only for such rows, so a listing that - * names an ordinary user turn (persisted history is raw JSON) cannot make - * that turn read as accounted for. + * appends with a turn and lists in the user row's `requestPreludeMessageIds`: + * a synthetic USER row carrying the snapshot it materializes (@mention file + * snapshot, agent skill snapshot, MCP prompt snapshot), or a synthetic + * ASSISTANT row (family payloads). `synthetic` alone is not enough: the + * backend also persists synthetic user TURNS (auto-resume, CLI goal + * continuations), which are turns of their own. The id-keyed policy + * accounting (prior-turn check, harvest coverage, tail-copy stamps) honors a + * prelude listing only for prelude-shaped rows, so a listing that names any + * other user row (persisted history is raw JSON) cannot make that turn read + * as accounted for. */ export function isRequestPreludeRow(message: MuxMessage): boolean { - return message.metadata?.synthetic === true; + const metadata = message.metadata; + if (metadata?.synthetic !== true) return false; + if (message.role === "assistant") return true; + return ( + message.role === "user" && + (metadata.fileAtMentionSnapshot !== undefined || + metadata.agentSkillSnapshot !== undefined || + metadata.mcpPromptSnapshot !== undefined) + ); } /** diff --git a/src/node/services/agentSession.ts b/src/node/services/agentSession.ts index f0ee4139b6c..2e6dc115870 100644 --- a/src/node/services/agentSession.ts +++ b/src/node/services/agentSession.ts @@ -1110,15 +1110,31 @@ export class AgentSession { * context boundary (compaction without a preserved tail, /clear, context * reset, destructive history replace) via resetWorkspaceMemoryWritable. */ - private workspaceMemoryWritable: boolean | undefined; + private workspaceMemoryWritable: { epoch: number; writable: boolean } | undefined; - recordWorkspaceMemoryWritable(effective: boolean): void { - this.workspaceMemoryWritable = effective; + /** + * Record the mirror for `epoch` (the policy epoch the turn was recorded + * under). Keyed like the durable records: with several backends over one + * chat.jsonl, another backend's no-tail compaction or destructive clear + * opens a new epoch without any callback here, so an unkeyed mirror would + * carry the previous epoch's value into the new one — a stale deny would + * pin its first writable turn, a stale grant would keep the unknown-history + * rule from failing closed. + */ + recordWorkspaceMemoryWritable(effective: boolean, epoch: number): void { + assert(Number.isInteger(epoch), "workspace memory policy mirror epoch must be an integer"); + this.workspaceMemoryWritable = { epoch, writable: effective }; } - /** The mirror, for WorkspaceService's conjunction (undefined until a turn recorded). */ - workspaceMemoryWritableMirror(): boolean | undefined { - return this.workspaceMemoryWritable; + /** + * The mirror for `epoch`, for WorkspaceService's conjunction and the + * completion observation: undefined until a turn of THAT epoch recorded + * (a value recorded for another epoch says nothing about this one). + */ + workspaceMemoryWritableMirror(epoch: number): boolean | undefined { + return this.workspaceMemoryWritable?.epoch === epoch + ? this.workspaceMemoryWritable.writable + : undefined; } /** In-flight durable epoch reset started by a no-tail compaction (see the completion callback). */ @@ -1428,7 +1444,8 @@ export class AgentSession { this.coordinator.recordCompactionSummary( (metadata.preservedTailMessageCount ?? 0) > 0 ? metadata.summaryMessageId : null ); - const closing = this.workspaceMemoryWritable; + const closingEpoch = compactionClosingPolicyEpoch(metadata); + const closing = this.workspaceMemoryWritableMirror(closingEpoch); const observed = Promise.resolve( onCompactionComplete?.({ ...metadata, @@ -1446,9 +1463,14 @@ export class AgentSession { // records are left in place (resetWorkspaceMemoryWritable explains // why); every durable value is bound to its epoch, so they are // invisible to the new epoch's turns on any backend. - const closingEpoch = compactionClosingPolicyEpoch(metadata); const preservedTail = (metadata.preservedTailMessageCount ?? 0) > 0; - if (!preservedTail) this.workspaceMemoryWritable = undefined; + // The mirror follows the durable carry: a preserved tail re-binds the + // closing epoch's value to the new epoch (the copies were produced + // under it); otherwise the new epoch starts without one. + this.workspaceMemoryWritable = + preservedTail && closing !== undefined + ? { epoch: metadata.summaryHistorySequence, writable: closing } + : undefined; const reset = observed .catch(() => undefined) .then(() => diff --git a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts index 71c45d71b54..35841442c7c 100644 --- a/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts +++ b/src/node/services/agentSession.workspaceMemoryPolicyEpoch.test.ts @@ -160,32 +160,43 @@ describe("AgentSession workspace memory policy epoch boundary", () => { expect(await readWorkspaceMemoryDenyMarker(sessionDir, 13)).toBe(false); }); + test("the mirror answers only for the epoch it was recorded under", async () => { + const { session } = await createSession(); + session.recordWorkspaceMemoryWritable(false, 7); + expect(session.workspaceMemoryWritableMirror(7)).toBe(false); + // Another backend's boundary opened epoch 12 without any callback here: + // the stale value must neither deny the new epoch nor mask its unknown + // history. + expect(session.workspaceMemoryWritableMirror(12)).toBeUndefined(); + expect(session.workspaceMemoryWritableMirror(-1)).toBeUndefined(); + }); + test("reset clears the mirror only once the durable clear is proven", async () => { const { session, internals, records, setRecords, swallowNextWrite, config } = await createSession(); // Destructive boundary: every record goes, then the mirror. - session.recordWorkspaceMemoryWritable(false); + session.recordWorkspaceMemoryWritable(false, -1); await setRecords({ "-1": false, "5": true }); await internals.resetWorkspaceMemoryWritable(); expect(records()).toBeUndefined(); - expect(session.workspaceMemoryWritableMirror()).toBeUndefined(); + expect(session.workspaceMemoryWritableMirror(-1)).toBeUndefined(); // Swallowed write: a surviving `-1: false` would pin the new segment to // the stored-false fast path — the reset must fail (retryable) and keep // the mirror rather than report success. - session.recordWorkspaceMemoryWritable(false); + session.recordWorkspaceMemoryWritable(false, -1); await setRecords({ "-1": false }); swallowNextWrite(); expect( await internals.resetWorkspaceMemoryWritable().then(() => null, getErrorMessage) ).toMatch(/did not persist/); expect(records()).toEqual({ "-1": false }); - expect(session.workspaceMemoryWritableMirror()).toBe(false); + expect(session.workspaceMemoryWritableMirror(-1)).toBe(false); // An ABSENT config.json is not the empty default: a registered workspace // always has one, so its absence is transient — the reset must fail // (retryable) rather than clear the mirror over records it never saw. - session.recordWorkspaceMemoryWritable(false); + session.recordWorkspaceMemoryWritable(false, -1); await setRecords({ "-1": false }); const configPath = path.join(config.rootDir, "config.json"); const savedConfig = await fsPromises.readFile(configPath); @@ -197,7 +208,7 @@ describe("AgentSession workspace memory policy epoch boundary", () => { } finally { await fsPromises.writeFile(configPath, savedConfig); } - expect(session.workspaceMemoryWritableMirror()).toBe(false); + expect(session.workspaceMemoryWritableMirror(-1)).toBe(false); expect(records()).toEqual({ "-1": false }); }); diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 0cb17654c6b..14c6f76e036 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1912,7 +1912,10 @@ describe("CompactionHandler", () => { // Sequences: the boundary sits at boundarySequence, the two first-epoch // copies at +1/+2, so the rows seeded here start at +3. await seedHistory( - createMuxMessage("p2", "user", "prelude snapshot", { synthetic: true }), + createMuxMessage("p2", "user", "prelude snapshot", { + synthetic: true, + fileAtMentionSnapshot: ["@notes.md"], + }), // Listed as a prelude row but of ordinary shape: never stamped through // the listing (r85). createMuxMessage("px", "user", "a real turn's question listed as prelude"), diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 34d7789793b..6ca25e51f73 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2674,7 +2674,7 @@ describe("HistoryService", () => { expect(msg.metadata?.historySegment).toBeGreaterThan(1); }); - it("treats a segment start at the safe-integer boundary as malformed and refuses unsafe next sequences", async () => { + it("treats a segment start at the safe-integer boundary as malformed", async () => { const workspaceId = "workspace1"; const workspaceDir = path.join(config.sessionsDir, workspaceId); await fs.mkdir(workspaceDir, { recursive: true }); @@ -2689,43 +2689,49 @@ describe("HistoryService", () => { if (reseeded === undefined) throw new Error("expected a reseeded stamp"); expect(Number.isSafeInteger(reseeded + 1)).toBe(true); expect(reseeded).toBeLessThan(Number.MAX_SAFE_INTEGER); - // A persisted row at the boundary leaves no safe next sequence: the - // append refuses rather than assign a value `+ 1` cannot move past. - const other = "workspace2"; - await fs.mkdir(path.join(config.sessionsDir, other), { recursive: true }); - await fs.writeFile( - path.join(config.sessionsDir, other, "chat.jsonl"), - JSON.stringify({ - ...createMuxMessage("edge", "user", "at the edge", { - historySequence: Number.MAX_SAFE_INTEGER, - }), - workspaceId: other, - }) + "\n" - ); - const refused = await service.appendToHistory(other, createMuxMessage("m", "user", "x")); - expect(refused.success).toBe(false); - if (!refused.success) expect(refused.error).toContain("safe integer"); }); - it("refuses to open a segment above an unsafe persisted sequence", async () => { - const workspaceId = "workspace1"; + it("ignores persisted sequences without a safe successor for the append and clear floors", async () => { + const workspaceId = "workspace2"; const workspaceDir = path.join(config.sessionsDir, workspaceId); await fs.mkdir(workspaceDir, { recursive: true }); - // `start + 1` cannot move past 2^53: a clear that pretended to would - // hand the new segment the old one's identity. + // Hand-edited rows at and past 2^53 - 1 cannot floor a counter that + // has to move; they are skipped like fractional sequences (never + // refused), so the user can still send and /clear removes them. await fs.writeFile( path.join(workspaceDir, "chat.jsonl"), - JSON.stringify({ - ...createMuxMessage("huge", "user", "absurd sequence", { - historySequence: Number.MAX_SAFE_INTEGER + 1, - }), - workspaceId, - }) + "\n" + [ + { ...createMuxMessage("sane", "user", "sane", { historySequence: 4 }), workspaceId }, + { + ...createMuxMessage("edge", "user", "at the edge", { + historySequence: Number.MAX_SAFE_INTEGER, + }), + workspaceId, + }, + { + ...createMuxMessage("huge", "user", "past the edge", { + historySequence: Number.MAX_SAFE_INTEGER + 1, + }), + workspaceId, + }, + ] + .map((row) => JSON.stringify(row)) + .join("\n") + "\n" ); - const result = await service.clearHistory(workspaceId); - expect(result.success).toBe(false); - if (!result.success) expect(result.error).toContain("safe integer"); - expect(await fs.readFile(path.join(workspaceDir, "chat.jsonl"), "utf-8")).toContain("huge"); + const appended = createMuxMessage("m", "user", "x"); + expect((await service.appendToHistory(workspaceId, appended)).success).toBe(true); + expect(appended.metadata?.historySequence).toBe(5); + expect((await service.clearHistory(workspaceId)).success).toBe(true); + expect( + await fs.access(path.join(workspaceDir, "chat.jsonl")).then( + () => true, + () => false + ) + ).toBe(false); + const next = createMuxMessage("n", "user", "y"); + expect((await service.appendToHistory(workspaceId, next)).success).toBe(true); + expect(next.metadata?.historySequence).toBe(6); + expect(next.metadata?.historySegment).toBe(6); }); it("forks the current segment along with the history snapshot", async () => { diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index 5c73c786dd1..dab0441caab 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -197,6 +197,14 @@ function stripContextUsage(message: MuxMessage): MuxMessage { }; } +/** + * A persisted history sequence the counter can be floored at: a nonnegative + * integer whose successor is still a safe integer (see getNewestHistorySequence). + */ +function isUsableHistorySequence(value: unknown): value is number { + return isNonNegativeInteger(value) && Number.isSafeInteger(value + 1); +} + /** The persisted row's segment stamp, kept across in-place replacement (see MuxMetadata.historySegment). */ function preservedHistorySegment(existing: MuxMessage): { historySegment?: number } { const segment = existing.metadata?.historySegment; @@ -1883,7 +1891,11 @@ export class HistoryService { for (const message of messages) { const sequence = message.metadata?.historySequence; - if (!isNonNegativeInteger(sequence)) { + // A sequence without a safe successor (>= 2^53 - 1; a hand-edited row) + // is malformed like a fraction: it cannot floor a counter that has to + // move, so it is skipped — appends continue from the sane rows and a + // clear removes it — rather than refusing every append and clear. + if (!isUsableHistorySequence(sequence)) { continue; } @@ -2003,7 +2015,7 @@ export class HistoryService { const previousStart = this.historySegmentStarts.get(workspaceId) ?? 0; const start = clearedSequences.reduce( - (max, sequence) => (sequence > max ? sequence : max), + (max, sequence) => (isUsableHistorySequence(sequence) && sequence > max ? sequence : max), Math.max(persistedMax, (this.sequenceCounters.get(workspaceId) ?? 0) - 1, previousStart) ) + 1; await this.writeHistorySegmentStart(workspaceId, start); diff --git a/src/node/services/memoryConsolidationService.test.ts b/src/node/services/memoryConsolidationService.test.ts index 6e64eccf323..db92fc5eb74 100644 --- a/src/node/services/memoryConsolidationService.test.ts +++ b/src/node/services/memoryConsolidationService.test.ts @@ -1732,7 +1732,10 @@ describe("MemoryConsolidationService", () => { ); } const snapshots = [`${ids.summary}-snap-a`, `${ids.summary}-snap-b`].map((id) => - createMuxMessage(id, "user", "Snapshot content", { synthetic: true }) + createMuxMessage(id, "user", "Snapshot content", { + synthetic: true, + fileAtMentionSnapshot: ["@notes.md"], + }) ); for (const snapshot of snapshots) { await fixture.historyService.appendToHistory("ws-dream", snapshot); diff --git a/src/node/services/workspaceService.ts b/src/node/services/workspaceService.ts index fdc82f76865..a67a65a8b92 100644 --- a/src/node/services/workspaceService.ts +++ b/src/node/services/workspaceService.ts @@ -4317,7 +4317,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // A no-tail compaction's durable epoch reset may still be in flight: read // nothing of the closing epoch (accumulator, marker) before it settled. await session?.settleWorkspaceMemoryPolicyEpoch(); - const mirror = session?.workspaceMemoryWritableMirror(); + const mirror = session?.workspaceMemoryWritableMirror(policyEpoch); const sessionDir = path.join(this.config.sessionsDir, workspaceId); // A deny that config.json cannot hold falls back to the session dir: the // turn's user row is already durable in chat.jsonl there, so the deny @@ -4343,7 +4343,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); if (!written) { log.debug("Skipping workspace memory deny marker for removed workspace", { workspaceId }); - session?.recordWorkspaceMemoryWritable(false); + session?.recordWorkspaceMemoryWritable(false, policyEpoch); return true; } } catch (markerError: unknown) { @@ -4354,10 +4354,10 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); // Process-local floor: this session at least keeps refusing until // the next successful persist writes the false it now mirrors. - session?.recordWorkspaceMemoryWritable(false); + session?.recordWorkspaceMemoryWritable(false, policyEpoch); return false; } - session?.recordWorkspaceMemoryWritable(false); + session?.recordWorkspaceMemoryWritable(false, policyEpoch); return true; }; // Strict load of an EXISTING file: a config.json that is missing (strict @@ -4384,7 +4384,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { // Unregistered workspace: nothing durable to update and no stale // permission to invalidate (harvests fail closed on the missing value). if (before === null) { - session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable); + session?.recordWorkspaceMemoryWritable((mirror ?? true) && writable, policyEpoch); return true; } // The persisted bit is the epoch accumulator, fail-closed: the harvest @@ -4471,7 +4471,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { !hasMalformedWorkspaceMemoryPolicyRecords(before.workspace) && (stored === false || (stored === true && conjunction(stored, carriedFor(before.workspace)))) ) { - session?.recordWorkspaceMemoryWritable(stored); + session?.recordWorkspaceMemoryWritable(stored, policyEpoch); return true; } effective = conjunction(stored, carriedFor(before.workspace)); @@ -4495,7 +4495,7 @@ export class WorkspaceService extends EventEmitter implements WorkspaceHost { }); return denyDurableFallback(getErrorMessage(error)); } - session?.recordWorkspaceMemoryWritable(effective); + session?.recordWorkspaceMemoryWritable(effective, policyEpoch); const persistedEntry = findWorkspaceEntry(this.config.loadConfigOrDefault(), workspaceId); const persisted = persistedEntry === null ? undefined : storedFor(persistedEntry.workspace); if (persisted !== effective) { From 42e3e3695fa7efa04f1908863517f9e2ee6f4c3e Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 06:48:03 +0000 Subject: [PATCH 94/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-se?= =?UTF-8?q?venth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One predicate decides which history segment starts are persisted and read back, so a clear can no longer write a start the next read rejects. A persisted row floors the counter only when the segment that retires it is such a start, appends assign only sequences that remain usable floors, and a reseed that cannot be persisted refuses before quarantining the malformed file instead of leaving the segment to restart at 0. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/historyService.test.ts | 44 ++++++++++++- src/node/services/historyService.ts | 82 +++++++++++++++--------- 2 files changed, 94 insertions(+), 32 deletions(-) diff --git a/src/node/services/historyService.test.ts b/src/node/services/historyService.test.ts index 6ca25e51f73..8ccf9902a52 100644 --- a/src/node/services/historyService.test.ts +++ b/src/node/services/historyService.test.ts @@ -2691,17 +2691,55 @@ describe("HistoryService", () => { expect(reseeded).toBeLessThan(Number.MAX_SAFE_INTEGER); }); + it("refuses a reseed it cannot persist without quarantining the malformed file", async () => { + const workspaceId = "workspace1"; + const workspaceDir = path.join(config.sessionsDir, workspaceId); + await fs.mkdir(workspaceDir, { recursive: true }); + const segmentPath = path.join(workspaceDir, "history-segment.json"); + // A readable start at the ceiling: the first append of that segment is + // refused (the sequence could not be retired), but the start is cached. + await fs.writeFile(segmentPath, JSON.stringify({ start: Number.MAX_SAFE_INTEGER - 1 })); + expect( + (await service.appendToHistory(workspaceId, createMuxMessage("m1", "user", "x"))).success + ).toBe(false); + // The file then turns malformed: the reseed floors at the cached start + // and would land past the ceiling. It refuses BEFORE renaming, so the + // malformed file stays in place and the next attempt does not read an + // absent file as segment 0 and reuse the retired identity. + await fs.writeFile(segmentPath, "garbage"); + for (let attempt = 0; attempt < 2; attempt++) { + const refused = await service.appendToHistory( + workspaceId, + createMuxMessage(`m${attempt + 2}`, "user", "x") + ); + expect(refused.success).toBe(false); + if (!refused.success) expect(refused.error).toContain("safe successor"); + expect(await fs.readFile(segmentPath, "utf-8")).toBe("garbage"); + } + expect((await fs.readdir(workspaceDir)).filter((name) => name.includes(".corrupt-"))).toEqual( + [] + ); + }); + it("ignores persisted sequences without a safe successor for the append and clear floors", async () => { const workspaceId = "workspace2"; const workspaceDir = path.join(config.sessionsDir, workspaceId); await fs.mkdir(workspaceDir, { recursive: true }); - // Hand-edited rows at and past 2^53 - 1 cannot floor a counter that - // has to move; they are skipped like fractional sequences (never - // refused), so the user can still send and /clear removes them. + // Hand-edited rows at and past 2^53 - 2 cannot floor a counter that + // has to move, nor open the segment that retires them (2^53 - 2 would + // put the next start at 2^53 - 1, which the reader rejects); they are + // skipped like fractional sequences (never refused), so the user can + // still send and /clear removes them. await fs.writeFile( path.join(workspaceDir, "chat.jsonl"), [ { ...createMuxMessage("sane", "user", "sane", { historySequence: 4 }), workspaceId }, + { + ...createMuxMessage("below", "user", "below the edge", { + historySequence: Number.MAX_SAFE_INTEGER - 1, + }), + workspaceId, + }, { ...createMuxMessage("edge", "user", "at the edge", { historySequence: Number.MAX_SAFE_INTEGER, diff --git a/src/node/services/historyService.ts b/src/node/services/historyService.ts index dab0441caab..ef5cc24a6f1 100644 --- a/src/node/services/historyService.ts +++ b/src/node/services/historyService.ts @@ -198,13 +198,24 @@ function stripContextUsage(message: MuxMessage): MuxMessage { } /** - * A persisted history sequence the counter can be floored at: a nonnegative - * integer whose successor is still a safe integer (see getNewestHistorySequence). + * A history segment start that can be persisted AND read back: a nonnegative + * integer whose successor is still a safe integer (`start + 1` past 2^53 no + * longer moves). One predicate for the writer and the reader, so the service + * never persists a start it would treat as malformed on the next read. */ -function isUsableHistorySequence(value: unknown): value is number { +function isUsableHistorySegmentStart(value: unknown): value is number { return isNonNegativeInteger(value) && Number.isSafeInteger(value + 1); } +/** + * A history sequence the counter can be floored at and that a full clear can + * retire (see getNewestHistorySequence): a nonnegative integer whose successor + * is a usable segment start, so every floor derived from it stays writable. + */ +function isUsableHistorySequence(value: unknown): value is number { + return isNonNegativeInteger(value) && isUsableHistorySegmentStart(value + 1); +} + /** The persisted row's segment stamp, kept across in-place replacement (see MuxMetadata.historySegment). */ function preservedHistorySegment(existing: MuxMessage): { historySegment?: number } { const segment = existing.metadata?.historySegment; @@ -1891,10 +1902,11 @@ export class HistoryService { for (const message of messages) { const sequence = message.metadata?.historySequence; - // A sequence without a safe successor (>= 2^53 - 1; a hand-edited row) - // is malformed like a fraction: it cannot floor a counter that has to - // move, so it is skipped — appends continue from the sane rows and a - // clear removes it — rather than refusing every append and clear. + // A sequence whose successor is not a usable segment start (>= 2^53 - 2; + // a hand-edited row) is malformed like a fraction: it cannot floor a + // counter that has to move nor be retired by a clear, so it is skipped — + // appends continue from the sane rows and a clear removes it — rather + // than refusing every append and clear. if (!isUsableHistorySequence(sequence)) { continue; } @@ -1937,10 +1949,9 @@ export class HistoryService { typeof parsed === "object" && parsed !== null ? (parsed as { start?: unknown }).start : undefined; - // The start itself AND the sequences assigned from it must stay safe - // (`start + 1` past 2^53 no longer moves): a file at the boundary is - // malformed too and reseeds (which then refuses an unsafe reseed). - return isNonNegativeInteger(start) && Number.isSafeInteger(start + 1) ? start : null; + // Same predicate as the writer: a file at the boundary is malformed too + // and reseeds (which refuses an unusable reseed before touching the file). + return isUsableHistorySegmentStart(start) ? start : null; } /** @@ -1959,13 +1970,6 @@ export class HistoryService { workspaceId: string, visibleMaxSequence: number ): Promise { - const segmentPath = this.getHistorySegmentPath(workspaceId); - const quarantinePath = `${segmentPath}.corrupt-${Date.now()}`; - log.warn("Quarantining malformed history segment file and reseeding the segment", { - workspaceId, - quarantinePath, - }); - await fs.rename(segmentPath, quarantinePath); // An empty history after a restart leaves no row, counter or cached start // to floor at, and a small reseed (1) could repeat the identity of an // earlier segment that a late backend still names in its policy records. @@ -1979,18 +1983,36 @@ export class HistoryService { this.historySegmentStarts.get(workspaceId) ?? 0, Date.now() ) + 1; + // Refuse BEFORE quarantining: a rename followed by a refused write would + // leave no segment file, and the next read would restart the segment at + // 0 — the retired range's identity reused, which the file exists to prevent. + this.requireUsableHistorySegmentStart(workspaceId, start); + const segmentPath = this.getHistorySegmentPath(workspaceId); + const quarantinePath = `${segmentPath}.corrupt-${Date.now()}`; + log.warn("Quarantining malformed history segment file and reseeding the segment", { + workspaceId, + quarantinePath, + }); + await fs.rename(segmentPath, quarantinePath); await this.writeHistorySegmentStart(workspaceId, start); return start; } - private async writeHistorySegmentStart(workspaceId: string, start: number): Promise { - // Reachable from persisted data (an absurd sequence in a hand-edited row): - // refuse rather than persist a start that `+ 1` cannot move past. - if (!isNonNegativeInteger(start) || !Number.isSafeInteger(start)) { + /** + * Reachable from persisted data (the cached counter or start of a + * hand-edited history at the 2^53 boundary): refuse rather than persist a + * start the reader would reject. + */ + private requireUsableHistorySegmentStart(workspaceId: string, start: number): void { + if (!isUsableHistorySegmentStart(start)) { throw new Error( - `Cannot open a new history segment for ${workspaceId}: start ${start} is not a safe integer` + `Cannot open a new history segment for ${workspaceId}: start ${String(start)} has no safe successor` ); } + } + + private async writeHistorySegmentStart(workspaceId: string, start: number): Promise { + this.requireUsableHistorySegmentStart(workspaceId, start); await ensurePrivateDir(this.getSessionDir(workspaceId)); await writeFileAtomic(this.getHistorySegmentPath(workspaceId), JSON.stringify({ start })); this.historySegmentStarts.set(workspaceId, start); @@ -2892,16 +2914,18 @@ export class HistoryService { } /** - * `max persisted sequence + 1`, refused when that is not a safe integer. - * Reachable from persisted data (a hand-edited row at 2^53 - 1): an unsafe - * next sequence would stop moving under `+ 1` and let a later finalization - * replace an unrelated row, so appends fail instead of assigning it. + * `max persisted sequence + 1`, refused when that is not itself a usable + * sequence. Reachable from persisted data (a hand-edited segment start at + * the 2^53 boundary): a row assigned there could not floor the counter on + * the next load (its duplicate would let a later finalization replace an + * unrelated row) nor be retired by a clear, so appends fail instead of + * writing it. */ private async getNextPersistedHistorySequence(workspaceId: string): Promise { const nextSeqNum = (await this.getMaxHistorySequence(workspaceId)) + 1; - if (!isNonNegativeInteger(nextSeqNum) || !Number.isSafeInteger(nextSeqNum)) { + if (!isUsableHistorySequence(nextSeqNum)) { throw new Error( - `History sequences of ${workspaceId} are exhausted: next sequence ${nextSeqNum} is not a safe integer` + `History sequences of ${workspaceId} are exhausted: next sequence ${String(nextSeqNum)} has no safe successor` ); } return nextSeqNum; From 1061c458480b987a36af5b876c7bb3406760f3f7 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 07:14:50 +0000 Subject: [PATCH 95/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-ei?= =?UTF-8?q?ghth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A downgraded child's legacy edit replaces its adopted copy in place only while that copy is still this adoption's generation (settled records by stamp, a pending fresh adoption by its pending write); a copy the owner deleted and recreated with the same bytes is the owner's, so the edit is imported beside it instead. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 33 +++++++++++++++++++++++++ src/node/services/memoryService.ts | 17 ++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index fb3015d0df1..3ce1cc127e4 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2355,6 +2355,39 @@ describe("MemoryService", () => { expect((await fixture.metaService.getPinnedKeys()).has(ownerKey)).toBe(false); }); + it("imports a legacy edit beside an adopted copy the owner recreated with the same bytes", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v1"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const ownerCopy = path.join(ownerRoot, "note.md"); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + // The owner deletes the copy and recreates it with the adopted bytes — + // the owner's generation now — and, before any pass re-inspects it, + // the downgraded build edits the legacy source. The bytes still hash + // to the record's, but the stamp no longer matches: the edit must not + // replace the owner's note in place; it lands in the import directory. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.rm(ownerCopy); + await fsPromises.writeFile(ownerCopy, "v1"); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v2"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(ownerCopy, "utf-8")).toBe("v1"); + expect( + await fsPromises.readFile(path.join(ownerRoot, "imported", "ws-child", "note.md"), "utf-8") + ).toBe("v2"); + const record = ( + await readLegacyAdoptionManifest( + legacyAdoptionManifestPath(path.join(fixture.config.sessionsDir, "ws-child")) + ) + ).get("note.md")!; + expect(record.target).toBe("imported/ws-child/note.md"); + expect(record.created).toBe(true); + }); + it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 50c81d59cc7..9fc242ea1cd 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1499,7 +1499,22 @@ export class MemoryService extends EventEmitter { priorContent !== null && [previous.content, previous.replacementContent].includes(sha256Hex(priorContent)) ) { - target = { relPath: previous.target, write: true, replaces: true }; + // The bytes alone do not prove the copy is still this adoption's: + // the owner may have deleted the copy and recreated it with the + // very same bytes (r87) — then it is the owner's generation (the + // rule above), and the child's edit must not replace it in place + // but land elsewhere. A settled record proves its generation by + // stamp; a pending record without one is a fresh adoption whose + // write the retry finds (the identical-bytes rule), while a + // pending replacement keeps the overwritten generation's stamp + // and is held to it — the old bytes must still be that file. + const currentStamp = + (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; + const ours = + previous.targetStamp !== undefined + ? previous.targetStamp === currentStamp + : previous.pending === true; + if (ours) target = { relPath: previous.target, write: true, replaces: true }; } } if (target === null) { From 5ba3627f287dd2a0de2752523b21f4fd79bb52aa Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 08:01:29 +0000 Subject: [PATCH 96/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20eighty-ni?= =?UTF-8?q?nth=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A prelude listing exempts a synthetic user row only when its snapshot has the shape the backend writes. A retargeted pre-sharing row is order-unknown against every row not proven to share its origin, whatever the persisted workspace ids say. Legacy adoption stages a copy's bytes in a hidden owner store entry, takes their identity there and records it in the pending manifest before installing them by rename, so a retry recognizes the copy by stamp on either side of an interrupted install and a byte match alone — another backend's identical note at the planned target — never counts as provenance. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.test.ts | 22 ++ .../utils/messages/compactionBoundary.ts | 36 +++- src/node/services/memoryLegacyAdoption.ts | 11 + src/node/services/memoryService.test.ts | 84 +++++++- src/node/services/memoryService.ts | 202 ++++++++++++------ .../refinement/refinementRollback.test.ts | 57 +++++ .../services/refinement/refinementRollback.ts | 8 +- 7 files changed, 342 insertions(+), 78 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.test.ts b/src/common/utils/messages/compactionBoundary.test.ts index 1f571216766..bf24d88bda1 100644 --- a/src/common/utils/messages/compactionBoundary.test.ts +++ b/src/common/utils/messages/compactionBoundary.test.ts @@ -512,6 +512,28 @@ describe("epochHasPriorTurnRows", () => { ) ).toBe(false); } + // Raw history: a snapshot field holding `null` or a value without the + // backend's shape is no snapshot — the row stays a turn of its own. + for (const malformed of [ + { fileAtMentionSnapshot: null }, + { fileAtMentionSnapshot: "@a.md" }, + { fileAtMentionSnapshot: [1] }, + { agentSkillSnapshot: null }, + { agentSkillSnapshot: { skillName: "s" } }, + { mcpPromptSnapshot: {} }, + { mcpPromptSnapshot: { serverName: "srv", promptName: "p" } }, + ]) { + const row = createMuxMessage("p-now", "user", "looks like a prelude", { synthetic: true }); + expect( + epochHasPriorTurnRows( + [ + createMuxMessage("u-now", "user", "now"), + { ...row, metadata: { ...row.metadata, ...(malformed as object) } }, + ], + current + ) + ).toBe(true); + } }); it("ignores rows that are no turn of the epoch: compaction requests, tail copies, token-budget internals", () => { diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index d7331156122..6936c4db0bb 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -6,7 +6,7 @@ import { import { isPositiveInteger } from "@/common/utils/numbers"; import { hasProviderReplayableContent } from "@/common/utils/messages/providerEligibility"; -import type { MuxMessage } from "@/common/types/message"; +import type { MuxMessage, MuxMetadata } from "@/common/types/message"; import { isTokenBudgetInternalMessage } from "@/common/types/message"; export { CONTEXT_BOUNDARY_KINDS }; @@ -295,7 +295,9 @@ export function epochHasPriorTurnRows( * accounting (prior-turn check, harvest coverage, tail-copy stamps) honors a * prelude listing only for prelude-shaped rows, so a listing that names any * other user row (persisted history is raw JSON) cannot make that turn read - * as accounted for. + * as accounted for. The snapshot must have the shape the backend writes + * (r89): a `null` or arbitrary value under the field — corruption, or a + * turn hand-edited to look like a prelude — grants nothing. */ export function isRequestPreludeRow(message: MuxMessage): boolean { const metadata = message.metadata; @@ -303,9 +305,33 @@ export function isRequestPreludeRow(message: MuxMessage): boolean { if (message.role === "assistant") return true; return ( message.role === "user" && - (metadata.fileAtMentionSnapshot !== undefined || - metadata.agentSkillSnapshot !== undefined || - metadata.mcpPromptSnapshot !== undefined) + (isFileAtMentionSnapshotShape(metadata.fileAtMentionSnapshot) || + isAgentSkillSnapshotShape(metadata.agentSkillSnapshot) || + isMcpPromptSnapshotShape(metadata.mcpPromptSnapshot)) + ); +} + +function isFileAtMentionSnapshotShape(value: unknown): boolean { + return Array.isArray(value) && value.every((token) => typeof token === "string"); +} + +function isAgentSkillSnapshotShape(value: unknown): boolean { + if (value === null || typeof value !== "object") return false; + const snapshot = value as Partial>; + return ( + typeof snapshot.skillName === "string" && + typeof snapshot.scope === "string" && + typeof snapshot.sha256 === "string" + ); +} + +function isMcpPromptSnapshotShape(value: unknown): boolean { + if (value === null || typeof value !== "object") return false; + const snapshot = value as Partial>; + return ( + typeof snapshot.serverName === "string" && + typeof snapshot.promptName === "string" && + typeof snapshot.commandKey === "string" ); } diff --git a/src/node/services/memoryLegacyAdoption.ts b/src/node/services/memoryLegacyAdoption.ts index a77eb7bf25a..857bfd4ecea 100644 --- a/src/node/services/memoryLegacyAdoption.ts +++ b/src/node/services/memoryLegacyAdoption.ts @@ -71,6 +71,13 @@ export interface LegacyAdoptionRecord { * adoption's on either side of an interrupted write. */ replacementContent?: string; + /** + * Identity (`ino:size:mtimeNs`) of the staged bytes an in-place replacement + * is about to install, taken on the staging entry before the install (a + * rename keeps it) and set together with `replacementContent`. A retry + * finds the installed copy by this stamp; a byte match alone never counts. + */ + replacementStamp?: string; /** * Reconciliation of a deleted source is under way: the copy is about to be * (or was just) removed. Set before the removal so a crash between the @@ -117,6 +124,9 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null return null; } if (record.targetStamp !== undefined && typeof record.targetStamp !== "string") return null; + if (record.replacementStamp !== undefined && typeof record.replacementStamp !== "string") { + return null; + } const flag = (raw: unknown, malformed: boolean): boolean | undefined => raw === undefined ? undefined : typeof raw === "boolean" ? raw : malformed; return { @@ -129,6 +139,7 @@ function parseLegacyAdoptionRecord(value: unknown): LegacyAdoptionRecord | null deleted: flag(record.deleted, false), replaced: flag(record.replaced, true), replacementContent: record.replacementContent, + replacementStamp: record.replacementStamp, targetStamp: record.targetStamp, }; } diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 3ce1cc127e4..669bbed7a00 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2386,6 +2386,74 @@ describe("MemoryService", () => { ).get("note.md")!; expect(record.target).toBe("imported/ws-child/note.md"); expect(record.created).toBe(true); + expect(record.targetStamp).toBe( + (await adoptionTargetStamp(path.join(ownerRoot, "imported", "ws-child", "note.md"))) ?? + undefined + ); + // The staged bytes were installed by rename; nothing lingers. + expect(await pathExists(path.join(ownerRoot, ".adoption-staging"))).toBe(false); + }); + + it("never claims a copy by byte match alone: a pending record without its receipt's generation", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "same bytes"); + // A fresh adoption crashed after recording its pending manifest but + // BEFORE installing the copy — the receipt names staged bytes that + // never reached the target. Another backend (or the owner) then created + // an owner note with the very same bytes at the planned target. + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + await fsPromises.mkdir(path.dirname(manifestPath), { recursive: true }); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + targetStamp: "1:10:1", + }, + }) + ); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "same bytes"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const record = (await readLegacyAdoptionManifest(manifestPath)).get("note.md")!; + expect(record.pending).toBeUndefined(); + expect(record.created).toBe(false); + expect(record.targetStamp).toBeUndefined(); + // The same for an older build's stamp-less pending record: ambiguous, + // so it claims nothing. + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "note.md": { + content: sha256Hex("same bytes"), + sidecar: "", + target: "note.md", + created: true, + pending: true, + }, + }) + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + expect((await readLegacyAdoptionManifest(manifestPath)).get("note.md")!.created).toBe(false); + // Deleting the legacy source therefore leaves the owner's note alone. + await fsPromises.rm(path.join(legacyRoot, "note.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( + "same bytes" + ); + // No staged bytes linger in the owner store. + expect(await pathExists(path.join(ownerRoot, ".adoption-staging"))).toBe(false); }); it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { @@ -2841,10 +2909,17 @@ describe("MemoryService", () => { await new Promise((resolve) => setTimeout(resolve, 5)); await fsPromises.writeFile(path.join(legacyRoot, "note.md"), "v3-replaced"); await fsPromises.writeFile(path.join(ownerRoot, "note.md"), "v3-replaced"); + // The receipt the pass took on the staged bytes (a rename keeps it). + const receipt = async () => (await adoptionTargetStamp(path.join(ownerRoot, "note.md")))!; await fsPromises.writeFile( manifestPath, JSON.stringify({ - "note.md": { ...settled2, pending: true, replacementContent: sha256Hex("v3-replaced") }, + "note.md": { + ...settled2, + pending: true, + replacementContent: sha256Hex("v3-replaced"), + replacementStamp: await receipt(), + }, }) ); await new MemoryService( @@ -2878,7 +2953,12 @@ describe("MemoryService", () => { await fsPromises.writeFile( manifestPath, JSON.stringify({ - "note.md": { ...settled3, pending: true, replacementContent: sha256Hex("v4") }, + "note.md": { + ...settled3, + pending: true, + replacementContent: sha256Hex("v4"), + replacementStamp: await receipt(), + }, }) ); await new Promise((resolve) => setTimeout(resolve, 5)); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 9fc242ea1cd..59f7d7204e9 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -17,7 +17,7 @@ * documented limitation. */ import { EventEmitter } from "events"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import * as fsPromises from "node:fs/promises"; import * as path from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -431,6 +431,12 @@ interface MemoryStore { * relPath the owner already uses with different content (adoptLegacyPrivateStore). */ const LEGACY_IMPORT_DIR = "imported"; +/** + * Hidden (dot-entry: never listed, never indexed) owner-store directory the + * adoption pass stages a copy's bytes in before installing them by rename + * (adoptLegacyPrivateStoreOrThrow); emptied at the start of every pass. + */ +const LEGACY_ADOPTION_STAGING_DIR = ".adoption-staging"; /** * Pin bit of a manifest record's child sidecar fingerprint. No child entry at @@ -1386,6 +1392,9 @@ export class MemoryService extends EventEmitter { // the cap and hide an adopted note's only copy once readable again. let remainingCapacity = MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles({ strict: true })).length; + // Staged bytes a crashed pass never installed: their records claim + // nothing (no file carries the receipt), so they are simply dropped. + await store.remove(LEGACY_ADOPTION_STAGING_DIR); let capacityExhausted = false; let manifestDirty = false; let imported = 0; @@ -1435,7 +1444,14 @@ export class MemoryService extends EventEmitter { ) { continue; // folded in earlier, nothing changed since } - let target: { relPath: string; write: boolean; replaces?: boolean } | null = null; + // `generation`: the stamp of the copy an in-place replacement installs + // over, re-checked right before the install. + let target: { + relPath: string; + write: boolean; + replaces?: boolean; + generation?: string; + } | null = null; // A child's pin toggle folds into the copy only while the copy is // this adoption's generation (see below) or the owner's identical // note it was folded into at first adoption; a copy the owner @@ -1466,55 +1482,44 @@ export class MemoryService extends EventEmitter { skipped++; continue; } + // Ours only while the copy is a generation this adoption installed + // (LegacyAdoptionRecord.targetStamp / replacementStamp — receipts + // taken on the staged bytes BEFORE they appear at the target, so + // even a pass interrupted between manifest and install left one). + // Identical bytes in another generation are the owner's (r79: the + // same rule deletion reconciliation and the rollback remapper + // apply): the owner may have deleted and recreated the note with + // the very same bytes, or — a pass interrupted before its install + // — another backend may have created it at the planned target + // (r89). `pending` alone is never provenance: a stamp-less pending + // record (an older build's) is ambiguous and claims nothing. + const currentStamp = + (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; + const ours = + previous.created === true && + currentStamp !== undefined && + (currentStamp === previous.targetStamp || currentStamp === previous.replacementStamp); if (priorContent === content) { target = { relPath: previous.target, write: false }; - // A pending record is a copy this adoption wrote but could not - // finish recording (crash or sidecar failure after the write): - // the file holding exactly those bytes now is that write, so its - // current identity is the generation to bind — also when the - // record carries a stamp, which is then the generation an - // interrupted in-place replacement OVERWROTE (r75). A settled - // record keeps the stamp it recorded — identical bytes in a - // different generation are the owner's (r79: the same rule - // deletion reconciliation and the rollback remapper apply), so - // the record stops claiming the copy and the child's sidecar - // changes no longer reach it. - const currentStamp = - (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; - const ours = - previous.created === true && - (previous.pending === true || - (previous.targetStamp !== undefined && previous.targetStamp === currentStamp)); record.created = ours; - record.targetStamp = !ours - ? undefined - : previous.pending === true - ? currentStamp - : previous.targetStamp; + record.targetStamp = ours ? currentStamp : undefined; const replaced = previous.replaced === true || (previous.created === true && !ours); if (replaced) record.replaced = true; foldChildPin = !replaced; } else if ( - previous.created === true && + ours && priorContent !== null && [previous.content, previous.replacementContent].includes(sha256Hex(priorContent)) ) { - // The bytes alone do not prove the copy is still this adoption's: - // the owner may have deleted the copy and recreated it with the - // very same bytes (r87) — then it is the owner's generation (the - // rule above), and the child's edit must not replace it in place - // but land elsewhere. A settled record proves its generation by - // stamp; a pending record without one is a fresh adoption whose - // write the retry finds (the identical-bytes rule), while a - // pending replacement keeps the overwritten generation's stamp - // and is held to it — the old bytes must still be that file. - const currentStamp = - (await adoptionTargetStamp(store.physicalPath(previous.target))) ?? undefined; - const ours = - previous.targetStamp !== undefined - ? previous.targetStamp === currentStamp - : previous.pending === true; - if (ours) target = { relPath: previous.target, write: true, replaces: true }; + // Legacy bytes edited on the downgraded build while the copy is + // still this adoption's (either side of an interrupted + // replacement): replaced in place. + target = { + relPath: previous.target, + write: true, + replaces: true, + generation: currentStamp, + }; } } if (target === null) { @@ -1546,27 +1551,88 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - // Provenance BEFORE the copy: interrupted here (crash, or the - // sidecar fold below failing), the retry finds the owner file - // already identical and takes the no-write path — without this - // record it would read as the owner's own note, and a legacy - // deletion could then never follow it out of the shared store. - // A replacement keeps the PRIOR record (old hash, same target) - // while pending: interrupted before the write, the retry must still - // recognize the surviving old bytes as this adoption's copy and - // replace them, not import the new bytes elsewhere. + // Staged install: the bytes are written to a hidden staging entry + // of the owner store first and their identity taken there (a + // rename keeps ino, size and mtime), so the manifest can record the + // receipt of the copy BEFORE the copy appears at the target. A pass + // interrupted at any point then leaves a record that either names + // a file not yet there (nothing claimed) or names the installed + // generation by stamp; a plain byte match never has to stand in + // for provenance. Without the record, an installed copy would read + // as the owner's own note, and a legacy deletion could then never + // follow it out of the shared store. A replacement keeps the PRIOR + // record (old hash and stamp, same target) while pending: on either + // side of the install the retry recognizes the file by its stamp. + const stagingRelPath = `${LEGACY_ADOPTION_STAGING_DIR}/${randomUUID()}`; + try { + await store.assertContained(stagingRelPath); + await store.writeFile(stagingRelPath, content); + } catch (error) { + log.warn("[MemoryService] cannot stage a legacy note for adoption; retrying later", { + childId, + relPath, + error, + }); + skipped++; + continue; + } + const stagedStamp = await adoptionTargetStamp(store.physicalPath(stagingRelPath)); + if (stagedStamp === null) { + await store.remove(stagingRelPath); + skipped++; + continue; + } adopted.set( relPath, target.replaces === true && previous !== undefined - ? { ...previous, pending: true, replacementContent: record.content } - : { ...record, target: target.relPath, created: true, pending: true } + ? { + ...previous, + pending: true, + replacementContent: record.content, + replacementStamp: stagedStamp, + } + : { + ...record, + target: target.relPath, + created: true, + pending: true, + targetStamp: stagedStamp, + } ); await writeManifest(); - await store.writeFile(target.relPath, content); + // The destination as decided above, re-checked under the lock right + // before the install: a fresh placement must still be free, a + // replacement must still be the generation it was decided against. + // Anything else is owner state the rename must not clobber — the + // staged bytes are dropped, the record restored, and the note is + // placed on the next pass. + const installable = + target.replaces === true + ? (await adoptionTargetStamp(store.physicalPath(target.relPath))) === + target.generation + : (await store.kind(target.relPath, { strict: true })) === null; + if (!installable) { + await store.remove(stagingRelPath); + if (previous === undefined) adopted.delete(relPath); + else adopted.set(relPath, previous); + await writeManifest(); + log.warn( + "[MemoryService] adoption destination changed before install; retrying later", + { + childId, + relPath, + target: target.relPath, + } + ); + skipped++; + continue; + } + await store.rename(stagingRelPath, target.relPath); if (target.replaces !== true) remainingCapacity--; imported++; record.created = true; - // The generation of the file just written (see targetStamp). + // The generation of the file just installed (see targetStamp): the + // staged receipt, unless the filesystem re-stamped the rename. record.targetStamp = (await adoptionTargetStamp(store.physicalPath(target.relPath))) ?? undefined; } @@ -1694,24 +1760,21 @@ export class MemoryService extends EventEmitter { } } } - // Ours only while it is THIS generation of the file (targetStamp, - // taken right after this adoption's write): identical bytes in a - // file the owner deleted and recreated, or edited and restored, are - // the owner's, and a record without a stamp preserves. The one - // exception is the far side of an interrupted in-place replacement: - // the pending record names the bytes about to be written, and a - // file holding exactly those (never before in the store) is that - // write, whose stamp the crash kept from being recorded. + // Ours only while it is a generation this adoption installed + // (targetStamp; replacementStamp on the far side of an interrupted + // in-place replacement — both receipts taken on the staged bytes, + // so a crash cannot have kept them from being recorded): identical + // bytes in a file the owner deleted and recreated, or edited and + // restored, are the owner's, and a record without a stamp preserves. const currentHash = current === null ? null : sha256Hex(current); + const stamp = await adoptionTargetStamp(store.physicalPath(previous.target)); const unchanged = currentHash !== null && - ((previous.pending === true && - previous.replacementContent !== undefined && - currentHash === previous.replacementContent) || - (currentHash === previous.content && - previous.targetStamp !== undefined && - (await adoptionTargetStamp(store.physicalPath(previous.target))) === - previous.targetStamp)); + stamp !== null && + ((currentHash === previous.content && stamp === previous.targetStamp) || + (previous.pending === true && + currentHash === previous.replacementContent && + stamp === previous.replacementStamp)); // A listed note may now point at this very target (the downgraded // build renamed `a.md` to the path its conflict copy was adopted // under, and the new record reused the identical file): the target @@ -1780,6 +1843,7 @@ export class MemoryService extends EventEmitter { manifestDirty = true; } if (manifestDirty) await writeManifest(); + await store.remove(LEGACY_ADOPTION_STAGING_DIR); if (capacityExhausted) { log.warn( "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up", diff --git a/src/node/services/refinement/refinementRollback.test.ts b/src/node/services/refinement/refinementRollback.test.ts index 4ae64bac7ee..b10fe891150 100644 --- a/src/node/services/refinement/refinementRollback.test.ts +++ b/src/node/services/refinement/refinementRollback.test.ts @@ -1275,6 +1275,63 @@ describe("refinementRollback", () => { expect(await pathExists(path.join(ownerRoot, "c.md"))).toBe(false); }); + it("keeps a retargeted row order-unknown against a peer row whose workspaceId is corrupted to its own", async () => { + using fixture = await createFixture(); + await fixture.service.create(fixture.ctx, "/memories/workspace/note.md", "v1\n", "agent"); + await fixture.service.strReplace( + fixture.ctx, + "/memories/workspace/note.md", + "v1", + "v2", + "agent" + ); + const editRow = await lastRow(fixture.sessionDir); + const ownerSessionDir = path.join(path.dirname(fixture.sessionDir), "ws-owner"); + const ownerRoot = path.join(ownerSessionDir, "memory"); + await fsPromises.mkdir(path.join(ownerRoot, "sub"), { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "sub", "note.md"), "v2\n"); + const record: LegacyAdoptionRecord = { + content: "x", + sidecar: "", + target: "sub/note.md", + created: true, + targetStamp: (await adoptionTargetStamp(path.join(ownerRoot, "sub", "note.md"))) ?? undefined, + }; + await fsPromises.writeFile( + legacyAdoptionManifestPath(fixture.sessionDir), + JSON.stringify({ "note.md": record }) + ); + // The owner's row persisted with THIS workspace's id (corruption): it is + // still another journal's row, with the shared store's clock — a + // comparison of that clock against the retargeted row's private one + // would order the peer edit "earlier" and let the rollback overwrite it. + await sharedDurableEventJournal(ownerSessionDir).append({ + workspaceId: path.basename(fixture.sessionDir), + kind: "refinement", + data: { + kind: "memory", + action: { op: "str_replace", path: "/memories/workspace/sub/note.md" }, + inverse: { + op: "restore-files", + files: [{ path: path.join(ownerSessionDir, "memory", "sub", "note.md"), text: "v2\n" }], + }, + sourceTs: 1, + }, + }); + const unordered = await rollbackRefinement({ + sessionDir: fixture.sessionDir, + id: editRow.id, + evidence: EVIDENCE, + sharedWorkspaceMemorySessionDir: ownerSessionDir, + listSharedWorkspaceMemoryPeerSessionDirs: () => [ownerSessionDir], + }); + expect(unordered.success).toBe(false); + expect(unordered.success ? "" : unordered.error).toContain( + "order relative to this row is unknown" + ); + expect(await fsPromises.readFile(path.join(ownerRoot, "sub", "note.md"), "utf-8")).toBe("v2\n"); + }); + it("journals the rollback row before releasing the target locks (no durable-order inversion)", async () => { using fixture = await createFixture(); const virtualPath = "/memories/global/order.md"; diff --git a/src/node/services/refinement/refinementRollback.ts b/src/node/services/refinement/refinementRollback.ts index f3a2717e120..76a07f80c57 100644 --- a/src/node/services/refinement/refinementRollback.ts +++ b/src/node/services/refinement/refinementRollback.ts @@ -628,8 +628,12 @@ function orderUnknown( if (sameOriginOrder(row, target) !== null) return false; if (row.data.orderUnknown === true || target.data.orderUnknown === true) return true; if (hasMalformedSourceClock(row) || hasMalformedSourceClock(target)) return true; - // A retargeted target (see wasRetargeted) vs. a row of another journal. - return targetRetargeted && row.workspaceId !== target.workspaceId; + // A retargeted target (see wasRetargeted) vs. any row not proven to share + // its origin: the target's clock is a private store's, so no other clock + // orders it. Not decided by persisted `workspaceId`s (r79): a peer row's + // corrupted to the target's would otherwise be ordered by clock and a + // later peer mutation overwritten by the rollback. + return targetRetargeted; } /** From ba9ebc70b3d671ee909ce63f1a77cc6c98a1422f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 08:24:32 +0000 Subject: [PATCH 97/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20ninetieth?= =?UTF-8?q?=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a renamed legacy note lands on its own conflict copy, the successor record inherits the generation observed on disk — the receipt that proved the copy unchanged — rather than the predecessor's targetStamp, which on the far side of an interrupted in-place replacement names the overwritten generation and would make the copy read as replaced by the owner at once. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- src/node/services/memoryService.test.ts | 58 +++++++++++++++++++++++++ src/node/services/memoryService.ts | 9 +++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 669bbed7a00..060b8edeaf9 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -2063,6 +2063,64 @@ describe("MemoryService", () => { expect(await pathExists(importedCopy)).toBe(false); }); + it("transfers the installed generation to the successor across an interrupted replacement", async () => { + using fixture = await createFixture("ws-child"); + await registerTaskTree(fixture); + const ownerRoot = path.join(fixture.config.sessionsDir, "ws-owner", "memory"); + const legacyRoot = path.join(fixture.config.sessionsDir, "ws-child", "memory"); + await fsPromises.mkdir(ownerRoot, { recursive: true }); + await fsPromises.mkdir(legacyRoot, { recursive: true }); + await fsPromises.writeFile(path.join(ownerRoot, "a.md"), "owner's a"); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a"); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + const importedCopy = path.join(ownerRoot, "imported", "ws-child", "a.md"); + const manifestPath = legacyAdoptionManifestPath(path.dirname(legacyRoot)); + const settled = (await readLegacyAdoptionManifest(manifestPath)).get("a.md")!; + // The downgraded build edits a.md; the replacement pass installed the + // new bytes (a new generation) but crashed before settling: the record + // is pending with the OVERWRITTEN generation's targetStamp and the + // installed one's replacementStamp. + await new Promise((resolve) => setTimeout(resolve, 5)); + await fsPromises.writeFile(path.join(legacyRoot, "a.md"), "child's a2"); + await fsPromises.rm(importedCopy); + await fsPromises.writeFile(importedCopy, "child's a2"); + const installed = (await adoptionTargetStamp(importedCopy))!; + expect(installed).not.toBe(settled.targetStamp); + await fsPromises.writeFile( + manifestPath, + JSON.stringify({ + "a.md": { + ...settled, + pending: true, + replacementContent: sha256Hex("child's a2"), + replacementStamp: installed, + }, + }) + ); + // Before the retry, the source is renamed onto the conflict-copy path: + // the successor record reuses the installed file, and must inherit the + // generation actually on disk — not the overwritten one, which would + // make the copy read as replaced by the owner at once. + await fsPromises.mkdir(path.join(legacyRoot, "imported", "ws-child"), { recursive: true }); + await fsPromises.rename( + path.join(legacyRoot, "a.md"), + path.join(legacyRoot, "imported", "ws-child", "a.md") + ); + await new MemoryService( + fixture.config, + new MemoryMetaService(fixture.xumHome) + ).listIndexEntries({ ...fixture.ctx }); + const successor = (await readLegacyAdoptionManifest(manifestPath)).get( + "imported/ws-child/a.md" + )!; + expect(successor).toMatchObject({ target: "imported/ws-child/a.md", created: true }); + expect(successor.targetStamp).toBe(installed); + // With the right generation, deleting the renamed source removes the copy. + await fsPromises.rm(path.join(legacyRoot, "imported", "ws-child", "a.md")); + await fixture.service.listIndexEntries({ ...fixture.ctx }); + expect(await pathExists(importedCopy)).toBe(false); + }); + it("keeps an owner-edited conflict copy the owner's when a renamed legacy note lands on it", async () => { using fixture = await createFixture("ws-child"); await registerTaskTree(fixture); diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 59f7d7204e9..8939e8b043b 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -1795,9 +1795,14 @@ export class MemoryService extends EventEmitter { // Only a copy still holding the adopted bytes is ours to hand // over; one the owner edited since is the owner's, and the // successor keeps its own (non-created) provenance. - if (unchanged && successor[1].created !== true) { + if (unchanged && stamp !== null && successor[1].created !== true) { successor[1].created = true; - successor[1].targetStamp = previous.targetStamp; + // The generation observed on disk — the receipt `unchanged` + // matched (r90: on the far side of an interrupted replacement + // that is `replacementStamp`, not the overwritten generation's + // `targetStamp`, which would make the successor read as + // replaced by the owner at once). + successor[1].targetStamp = stamp; manifestDirty = true; } } else if (unchanged) { From 15ea436ca06ac65da46dc33f4132a881b82ff3d9 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Fri, 11 Sep 2026 08:53:07 +0000 Subject: [PATCH 98/98] =?UTF-8?q?=F0=9F=A4=96=20fix:=20address=20ninety-fi?= =?UTF-8?q?rst=20Codex=20round=20on=20shared=20sub-agent=20memory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harvest gate, the tail-copy stamps and the segment start share one predicate for a persisted history sequence (a nonnegative safe integer), so an unsafe request bound stamps nothing the gate would refuse. Adoption staging moves out of the memory namespace to a directory beside the owner's memory root: a legacy note under any in-namespace path, however named, is adopted and kept, and the install rename is fenced so a failure fails the note rather than the pass. --- _Generated with `xum` • Model: `anthropic:claude-fable-5-1` • Thinking: `high`_ --- .../utils/messages/compactionBoundary.ts | 14 ++++- src/node/services/compactionHandler.test.ts | 9 ++++ src/node/services/compactionHandler.ts | 8 +-- .../services/memoryConsolidationService.ts | 3 +- src/node/services/memoryService.test.ts | 26 +++++++-- src/node/services/memoryService.ts | 53 ++++++++++++++----- 6 files changed, 91 insertions(+), 22 deletions(-) diff --git a/src/common/utils/messages/compactionBoundary.ts b/src/common/utils/messages/compactionBoundary.ts index 6936c4db0bb..c83a816a2fb 100644 --- a/src/common/utils/messages/compactionBoundary.ts +++ b/src/common/utils/messages/compactionBoundary.ts @@ -92,6 +92,18 @@ export function latestContextBoundaryHistorySequence( return latest; } +/** + * A persisted history sequence (stamp, request bound, segment start) in the + * clock's domain: a nonnegative safe integer. History rows are raw JSON, so + * every policy check reading one — the harvest gate, the tail-copy stamps, + * the segment start — must apply this same predicate (r90): a fractional, + * negative or unsafe value covers nothing anywhere, or one check would grant + * what another refused. + */ +export function isPersistedHistorySequence(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + /** * Start sequence of the history segment `messages` belong to (the largest * `historySegment` stamp among them; 0 for legacy rows and the first @@ -103,7 +115,7 @@ export function historySegmentStart(messages: readonly MuxMessage[]): number { let start = 0; for (const message of messages) { const segment = message.metadata?.historySegment; - if (typeof segment !== "number" || !Number.isSafeInteger(segment) || segment < 0) continue; + if (!isPersistedHistorySequence(segment)) continue; if (segment > start) start = segment; } return start; diff --git a/src/node/services/compactionHandler.test.ts b/src/node/services/compactionHandler.test.ts index 14c6f76e036..9a1b8b6e2c3 100644 --- a/src/node/services/compactionHandler.test.ts +++ b/src/node/services/compactionHandler.test.ts @@ -1959,6 +1959,13 @@ describe("CompactionHandler", () => { requestHistorySequence: boundarySequence + 18.5, workspaceMemoryPolicyEpoch: boundarySequence, }), + // An unsafe integer is outside the domain too (r90): the gate refuses + // it, so the copy must not read as covered either. + createMuxMessage("u11", "user", "question behind an unsafe bound"), + createMuxMessage("a11", "assistant", "unsafe bound", { + requestHistorySequence: Number.MAX_SAFE_INTEGER + 1, + workspaceMemoryPolicyEpoch: boundarySequence, + }), createStampedCompactionRequest("compact-req-2", boundarySequence + 1) ); expect(await handler.handleCompletion(createStreamEndEvent("Summary 2"))).toBe(true); @@ -1985,6 +1992,8 @@ describe("CompactionHandler", () => { undefined, // a9 (no stamp, malformed bound: not a synthetic row) undefined, // u10 (its turn's bound is fractional: never covered) boundarySequence, // a10 (recorded stamp kept) + undefined, // u11 (its turn's bound is an unsafe integer: never covered) + boundarySequence, // a11 (recorded stamp kept) ]); }); diff --git a/src/node/services/compactionHandler.ts b/src/node/services/compactionHandler.ts index 7d9d5514117..94d1bc5fdd7 100644 --- a/src/node/services/compactionHandler.ts +++ b/src/node/services/compactionHandler.ts @@ -43,6 +43,7 @@ import { isDurableContextBoundaryMarker, latestContextBoundaryHistorySequence, duplicateUserMessageIds, + isPersistedHistorySequence, isRequestPreludeRow, sliceMessagesFromLatestCompactionBoundary, workspaceMemoryPolicyEpochOf, @@ -1693,12 +1694,13 @@ export class CompactionHandler { if (message.role !== "assistant") continue; const bound = message.metadata?.requestHistorySequence; const policyEpoch = message.metadata?.workspaceMemoryPolicyEpoch; - // Same domain check as the harvest gate (epochHarvestRefusal, r79): a - // fractional or negative bound covers nothing there, so it must not + // Same domain check as the harvest gate (epochHarvestRefusal, r79 — + // the shared predicate, so an unsafe integer is refused alike, r90): a + // bound outside the clock's domain covers nothing there, so it must not // stamp a batch here either — the copies would then carry an epoch // without ever having been covered, and the association would grant // in a later epoch what the gate refused in this one. - if (typeof policyEpoch !== "number" || !isNonNegativeInteger(bound)) continue; + if (typeof policyEpoch !== "number" || !isPersistedHistorySequence(bound)) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; // Same prelude rule as the harvest gate: a listed id stamps a user row diff --git a/src/node/services/memoryConsolidationService.ts b/src/node/services/memoryConsolidationService.ts index d67ffc07a43..888c9ae9237 100644 --- a/src/node/services/memoryConsolidationService.ts +++ b/src/node/services/memoryConsolidationService.ts @@ -72,6 +72,7 @@ import { getRequestPreludeMessageIds } from "@/common/utils/messages/requestPrel import { compactionClosingPolicyEpoch, duplicateUserMessageIds, + isPersistedHistorySequence, isRequestPreludeRow, } from "@/common/utils/messages/compactionBoundary"; import { runMemoryHarvest } from "@/node/services/memoryHarvest"; @@ -408,7 +409,7 @@ function epochHarvestRefusal(messages: readonly MuxMessage[], closingEpoch: numb // Persisted rows are raw JSON: only a history sequence in the clock's // domain covers anything (r79); a fractional or negative value leaves the // turn uncovered and the refusal below fails closed. - if (typeof bound !== "number" || !Number.isSafeInteger(bound) || bound < 0) continue; + if (!isPersistedHistorySequence(bound)) continue; const anchor = userRows.findLast((row) => row.sequence <= bound)?.message; if (anchor === undefined) continue; covered.add(anchor.id); diff --git a/src/node/services/memoryService.test.ts b/src/node/services/memoryService.test.ts index 060b8edeaf9..c6f04c3e022 100644 --- a/src/node/services/memoryService.test.ts +++ b/src/node/services/memoryService.test.ts @@ -1911,6 +1911,16 @@ describe("MemoryService", () => { await fsPromises.mkdir(path.join(legacyRoot, ".hidden"), { recursive: true }); await fsPromises.writeFile(path.join(legacyRoot, ".note"), "dot note"); await fsPromises.writeFile(path.join(legacyRoot, ".hidden", "n.md"), "nested dot note"); + // Every in-namespace path is a note's, including one named like the + // pass's own staging area (which lives OUTSIDE the memory root, r91): + // adoption must never mistake it for staged bytes and remove it. + await fsPromises.mkdir(path.join(legacyRoot, "memory-adoption-staging"), { + recursive: true, + }); + await fsPromises.writeFile( + path.join(legacyRoot, "memory-adoption-staging", "n.md"), + "staging-named note" + ); await fsPromises.writeFile( path.join(legacyRoot, ".DS_Store"), Buffer.from([0, 0, 1, 255, 254]) @@ -1926,6 +1936,9 @@ describe("MemoryService", () => { "nested dot note" ); expect(await pathExists(path.join(ownerRoot, ".DS_Store"))).toBe(false); + expect( + await fsPromises.readFile(path.join(ownerRoot, "memory-adoption-staging", "n.md"), "utf-8") + ).toBe("staging-named note"); // Removing the stray entry lets a retried (non-forced) handover complete. await fsPromises.rm(path.join(legacyRoot, ".DS_Store")); await fixture.service.adoptLegacyPrivateStoreForRemoval("ws-child", "ws-owner"); @@ -2448,8 +2461,11 @@ describe("MemoryService", () => { (await adoptionTargetStamp(path.join(ownerRoot, "imported", "ws-child", "note.md"))) ?? undefined ); - // The staged bytes were installed by rename; nothing lingers. - expect(await pathExists(path.join(ownerRoot, ".adoption-staging"))).toBe(false); + // The staged bytes were installed by rename; nothing lingers (the + // staging dir sits beside the memory root, outside the namespace). + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); }); it("never claims a copy by byte match alone: a pending record without its receipt's generation", async () => { @@ -2510,8 +2526,10 @@ describe("MemoryService", () => { expect(await fsPromises.readFile(path.join(ownerRoot, "note.md"), "utf-8")).toBe( "same bytes" ); - // No staged bytes linger in the owner store. - expect(await pathExists(path.join(ownerRoot, ".adoption-staging"))).toBe(false); + // No staged bytes linger beside the owner store. + expect(await pathExists(path.join(path.dirname(ownerRoot), "memory-adoption-staging"))).toBe( + false + ); }); it("keeps adoption provenance when the pass is interrupted between copy and manifest", async () => { diff --git a/src/node/services/memoryService.ts b/src/node/services/memoryService.ts index 8939e8b043b..30e6e55b099 100644 --- a/src/node/services/memoryService.ts +++ b/src/node/services/memoryService.ts @@ -432,11 +432,17 @@ interface MemoryStore { */ const LEGACY_IMPORT_DIR = "imported"; /** - * Hidden (dot-entry: never listed, never indexed) owner-store directory the - * adoption pass stages a copy's bytes in before installing them by rename + * Directory beside the owner's memory root (in its session dir, OUTSIDE the + * model-writable memory namespace — a legacy note may legitimately live under + * any in-namespace path, dot-entries included) where the adoption pass stages + * a copy's bytes before installing them by rename * (adoptLegacyPrivateStoreOrThrow); emptied at the start of every pass. */ -const LEGACY_ADOPTION_STAGING_DIR = ".adoption-staging"; +const LEGACY_ADOPTION_STAGING_DIR_NAME = "memory-adoption-staging"; + +function legacyAdoptionStagingDir(store: MemoryStore): string { + return path.join(path.dirname(store.physicalRoot), LEGACY_ADOPTION_STAGING_DIR_NAME); +} /** * Pin bit of a manifest record's child sidecar fingerprint. No child entry at @@ -1394,7 +1400,8 @@ export class MemoryService extends EventEmitter { MEMORY_MAX_FILES_PER_SCOPE - (await store.listFiles({ strict: true })).length; // Staged bytes a crashed pass never installed: their records claim // nothing (no file carries the receipt), so they are simply dropped. - await store.remove(LEGACY_ADOPTION_STAGING_DIR); + const stagingDir = legacyAdoptionStagingDir(store); + await fsPromises.rm(stagingDir, { recursive: true, force: true }); let capacityExhausted = false; let manifestDirty = false; let imported = 0; @@ -1563,10 +1570,10 @@ export class MemoryService extends EventEmitter { // follow it out of the shared store. A replacement keeps the PRIOR // record (old hash and stamp, same target) while pending: on either // side of the install the retry recognizes the file by its stamp. - const stagingRelPath = `${LEGACY_ADOPTION_STAGING_DIR}/${randomUUID()}`; + const stagingPath = path.join(stagingDir, randomUUID()); try { - await store.assertContained(stagingRelPath); - await store.writeFile(stagingRelPath, content); + await fsPromises.mkdir(stagingDir, { recursive: true }); + await writeFileAtomic(stagingPath, content, { encoding: "utf-8" }); } catch (error) { log.warn("[MemoryService] cannot stage a legacy note for adoption; retrying later", { childId, @@ -1576,9 +1583,9 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - const stagedStamp = await adoptionTargetStamp(store.physicalPath(stagingRelPath)); + const stagedStamp = await adoptionTargetStamp(stagingPath); if (stagedStamp === null) { - await store.remove(stagingRelPath); + await fsPromises.rm(stagingPath, { force: true }); skipped++; continue; } @@ -1611,11 +1618,14 @@ export class MemoryService extends EventEmitter { ? (await adoptionTargetStamp(store.physicalPath(target.relPath))) === target.generation : (await store.kind(target.relPath, { strict: true })) === null; - if (!installable) { - await store.remove(stagingRelPath); + const restoreRecord = async () => { + await fsPromises.rm(stagingPath, { force: true }); if (previous === undefined) adopted.delete(relPath); else adopted.set(relPath, previous); await writeManifest(); + }; + if (!installable) { + await restoreRecord(); log.warn( "[MemoryService] adoption destination changed before install; retrying later", { @@ -1627,7 +1637,24 @@ export class MemoryService extends EventEmitter { skipped++; continue; } - await store.rename(stagingRelPath, target.relPath); + // The install: same session dir, so a plain rename (an EXDEV — the + // memory root mounted apart from its session dir — fails this note, + // not the pass). + try { + const destination = store.physicalPath(target.relPath); + await fsPromises.mkdir(path.dirname(destination), { recursive: true }); + await fsPromises.rename(stagingPath, destination); + } catch (error) { + await restoreRecord(); + log.warn("[MemoryService] cannot install a staged legacy note; retrying later", { + childId, + relPath, + target: target.relPath, + error, + }); + skipped++; + continue; + } if (target.replaces !== true) remainingCapacity--; imported++; record.created = true; @@ -1848,7 +1875,7 @@ export class MemoryService extends EventEmitter { manifestDirty = true; } if (manifestDirty) await writeManifest(); - await store.remove(LEGACY_ADOPTION_STAGING_DIR); + await fsPromises.rm(stagingDir, { recursive: true, force: true }); if (capacityExhausted) { log.warn( "[MemoryService] shared workspace notebook is full; legacy notes left in the sub-agent's private directory until space frees up",