diff --git a/apps/desktop/src/main/__tests__/main-process-diagnostics.test.ts b/apps/desktop/src/main/__tests__/main-process-diagnostics.test.ts index 434bd94710..dc75925683 100644 --- a/apps/desktop/src/main/__tests__/main-process-diagnostics.test.ts +++ b/apps/desktop/src/main/__tests__/main-process-diagnostics.test.ts @@ -270,6 +270,26 @@ test('copies Desktop diagnostics while Runtime Host is unavailable', async () => runtimeHostProcessLogs: () => [ '[2026-08-20T00:00:00.000Z] ERROR [runtime-host] local Host child exited: pid=42 code=23 signal=none', ], + runtimeHostConnections: () => [{ + epoch: 'guest-target', + target: { + profile: { + id: 'offline-guest', name: 'Shared Session', kind: 'remote', + access: 'session_guest', rootId: 'a'.repeat(64), + transport: { kind: 'tls', url: 'wss://example.com' }, + }, + credential: 'private-guest-credential', + }, + readiness: 'reconnecting', + reconnect: { + failures: 27, + firstFailureAt: Date.parse('2026-09-09T00:00:00Z'), + lastFailureAt: Date.parse('2026-09-09T01:00:00Z'), + }, + error: Object.assign(new Error('route unavailable api_key=sk-secretvalue123'), { + code: 'peer_reachability_needs_repair', + }), + }], resolveActiveRuntimeHost: () => undefined, resolveRuntimeHost: () => ({ getDiagnostics: async () => { @@ -296,6 +316,12 @@ test('copies Desktop diagnostics while Runtime Host is unavailable', async () => /Recent local Runtime Host process exits \(1\)[\s\S]*pid=42 code=23 signal=none/, ); assert.match(clipboard, /Diagnostics unavailable: Runtime Host disconnected/); + assert.match(clipboard, /Runtime Host connections \(1\)\n"offline-guest": reconnecting/); + assert.match(clipboard, /Failed attempts: 27/); + assert.match(clipboard, /First failure: 2026-09-09T00:00:00.000Z/); + assert.match(clipboard, /Last failure: 2026-09-09T01:00:00.000Z/); + assert.match(clipboard, /Latest error \[peer_reachability_needs_repair\]: route unavailable/); + assert.doesNotMatch(clipboard, /private-guest-credential|sk-secretvalue123/); }); test('acknowledges one previous-run notice while keeping its diagnostics copyable', async () => { diff --git a/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts b/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts index 9fb28dac17..ce5038da77 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-desktop-manager.test.ts @@ -23,6 +23,7 @@ import type { BotIncomingMessage } from '@maka/runtime/bots'; import { RuntimeHostOperationError, RuntimeHostPeerError, + RuntimeHostPeerReachabilityUnavailableError, RuntimeHostPermanentReconnectError, RuntimeHostRequestInterruptedError, type RuntimeHostSpawnedProcess, @@ -1245,6 +1246,88 @@ test('keeps an initially unavailable Direct target live and wakes it on new rout await manager.close(); }); +test('repeated offline Guest failures preserve Local readiness without rebroadcasting each retry', async (t) => { + const local = candidateHarness(); + const remote = candidateHarness({ hostId: 'a'.repeat(64), ownership: 'external' }); + const warn = t.mock.method(console, 'warn', () => {}); + const info = t.mock.method(console, 'info', () => {}); + const logCount = () => warn.mock.callCount() + info.mock.callCount(); + const guestErrors: string[] = []; + let attempts = 0; + let recovered = false; + let changedFailure = false; + const manager = await startRuntimeHostDesktopManager( + {} as DesktopRuntimeHostCandidateStartInput, + { + startCandidate: async (input) => { + if (!input.profileTarget) return ready(local.candidate); + attempts++; + if (recovered) return ready(remote.candidate); + throw changedFailure + ? new RuntimeHostPeerError('coordination_unavailable', 'relay unavailable') + : new RuntimeHostPeerReachabilityUnavailableError('12D3KooWpeer'); + }, + onTargetStateChanged: (state) => { + if (state.target.profile.id === 'offline-guest' && state.readiness !== 'ready' && state.error) { + guestErrors.push(state.error.message); + } + }, + reconnectBackoff: { minMs: 60_000, maxMs: 60_000 }, + }, + ); + t.after(() => manager.close()); + await manager.mountGuest(peerTarget('offline-guest', 'session_guest'), () => {}); + const initialPublications = guestErrors.length; + const initialWarnings = logCount(); + assert.equal(initialWarnings, 1); + for (let retry = 0; retry < 3; retry++) { + manager.wakePeerRecovery('offline-guest'); + await new Promise((resolve) => setImmediate(resolve)); + } + assert.ok(attempts >= 4, 'retries remain live'); + assert.equal(guestErrors.length, initialPublications, 'identical errors are not Host transitions'); + assert.equal(logCount(), initialWarnings, 'an offline error is logged once'); + assert.equal(manager.defaultProfileId(), 'local'); + assert.equal(manager.current('local')?.candidate, local.candidate); + + changedFailure = true; + manager.wakePeerRecovery('offline-guest'); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(guestErrors.at(-1), 'relay unavailable', 'a different failure updates diagnostics'); + for (let retry = 0; retry < 20; retry++) { + changedFailure = !changedFailure; + manager.wakePeerRecovery('offline-guest'); + await new Promise((resolve) => setImmediate(resolve)); + } + assert.equal(logCount(), initialWarnings, 'changing dial errors do not append logs during one outage'); + const diagnostic = manager.entries().find((state) => state.target.profile.id === 'offline-guest'); + assert.equal(diagnostic?.reconnect?.failures, attempts, 'diagnostics retain every failed attempt'); + assert.ok(diagnostic?.reconnect); + assert.ok(diagnostic.reconnect.lastFailureAt >= diagnostic.reconnect.firstFailureAt); + recovered = true; + manager.wakePeerRecovery('offline-guest'); + await manager.waitUntilReady('offline-guest'); + assert.equal(manager.current('offline-guest')?.candidate, remote.candidate); + assert.equal(manager.current('local')?.candidate, local.candidate); + assert.equal(logCount(), initialWarnings + 1, 'recovery logs one summary'); + assert.equal(info.mock.calls.at(-1)?.arguments[1]?.failedAttempts, attempts - 1); + assert.equal( + manager.entries().find((state) => state.target.profile.id === 'offline-guest')?.reconnect, + undefined, + 'a recovered target no longer has pending failures', + ); + recovered = false; + remote.disconnect(); + await new Promise((resolve) => setImmediate(resolve)); + manager.wakePeerRecovery('offline-guest'); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(logCount(), initialWarnings + 2, 'a later outage is reported again'); + assert.equal( + manager.entries().find((state) => state.target.profile.id === 'offline-guest')?.reconnect?.failures, + 1, + ); +}); + test('marks a retrying Direct target unavailable on permanent failure', async () => { const local = candidateHarness(); const permanent = new RuntimeHostPermanentReconnectError('credential rejected'); diff --git a/apps/desktop/src/main/__tests__/runtime-host-managed-services.test.ts b/apps/desktop/src/main/__tests__/runtime-host-managed-services.test.ts index 91003ebb3d..4a7be3d104 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-managed-services.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-managed-services.test.ts @@ -18,11 +18,14 @@ */ import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, test } from "node:test"; import { createClientRuntimeHostProfileCatalog } from "@maka/runtime-host/client"; +import { resolveDesktopRuntimeHostStartup } from "../runtime-host-profile-service.js"; import { createDesktopRuntimeHostManagedServiceStore, findDesktopRuntimeHostManagedServiceBinding, @@ -251,3 +254,44 @@ test("persists a WSL deployment through its environment control route", async () ); await assert.rejects(store.save(profile, deployedService), /already bound/u); }); + + +test("waits for a live deployment writer and recovers after it is killed", async (t) => { + const root = await mkdtemp(join(tmpdir(), "maka-managed-deployment-lock-")); + roots.push(root); + const store = createDesktopRuntimeHostManagedServiceStore(root); + await store.save(profile, deployedService); + const before = await store.read(); + const child = spawn(process.execPath, [ + "--input-type=module", + "--eval", + [ + `import { withProcessLifetimeFileUpdateLock } from ${JSON.stringify(import.meta.resolve("@maka/storage/process-lifetime-file-update-lock"))};`, + "await withProcessLifetimeFileUpdateLock(process.argv[1], async () => {", + " process.send('locked');", + " await new Promise(() => setInterval(() => {}, 1000));", + "});", + ].join("\n"), + join(root, "runtime-host-deployments.json"), + ], { stdio: ["ignore", "ignore", "inherit", "ipc"] }); + const exited = once(child, "exit"); + t.after(async () => { + if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await exited; + }); + await Promise.race([ + once(child, "message"), + exited.then(() => { throw new Error("Deployment writer exited before acquiring its lock"); }), + ]); + + // Startup may reclaim old directory markers, but cannot remove a current + // writer's marker or release its OS lease. + await resolveDesktopRuntimeHostStartup(root); + let settled = false; + const pending = store.read().finally(() => { settled = true; }); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(settled, false); + child.kill("SIGKILL"); + await exited; + assert.deepEqual(await pending, before); +}); diff --git a/apps/desktop/src/main/__tests__/runtime-host-new-task-preload.test.ts b/apps/desktop/src/main/__tests__/runtime-host-new-task-preload.test.ts new file mode 100644 index 0000000000..a6b2289b60 --- /dev/null +++ b/apps/desktop/src/main/__tests__/runtime-host-new-task-preload.test.ts @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +import assert from 'node:assert/strict'; +import { createRequire } from 'node:module'; +import { fileURLToPath } from 'node:url'; +import { runInNewContext } from 'node:vm'; +import test from 'node:test'; +import { build } from 'esbuild'; +import type { MakaBridge } from '../../preload/bridge-contract.js'; + +async function loadBridge(changeDuringRead: 'guest' | 'owner') { + const owner = { + hostId: 'local-host', targetEpoch: 'local-epoch', profileId: 'local', + profileName: 'Local', profileKind: 'local', profileAccess: 'owner', readiness: 'ready', + }; + const guest = { + hostId: 'shared-host', targetEpoch: 'shared-epoch', profileId: 'shared', + profileName: 'Shared', profileKind: 'remote', profileAccess: 'session_guest', + readiness: 'reconnecting', + }; + const listeners = new Map void>>(); + const emit = (channel: string, value: unknown) => { + for (const listener of listeners.get(channel) ?? []) listener({}, value); + }; + let catalogReads = 0; + let projectReads = 0; + const scopedCalls: string[] = []; + const ipcRenderer = { + on(channel: string, listener: (...args: unknown[]) => void) { + const handlers = listeners.get(channel) ?? new Set(); + handlers.add(listener); + listeners.set(channel, handlers); + }, + off(channel: string, listener: (...args: unknown[]) => void) { + listeners.get(channel)?.delete(listener); + }, + send() {}, + async invoke(channel: string, scope?: { hostId?: string }) { + if (scope?.hostId) { + scopedCalls.push(scope.hostId); + assert.equal(scope.hostId, owner.hostId, 'Guest reconnects must not redirect Owner reads'); + } + switch (channel) { + case 'runtime-host:activeIdentity': return { ...owner }; + case 'runtime-host:identities': return [{ ...owner }, { ...guest }]; + case 'runtime-host-profiles:getSnapshot': + catalogReads++; + return { + defaultProfileId: 'local', + entries: [ + { profile: { id: 'local', name: 'Local', kind: 'local' }, + hostId: owner.hostId, enabled: true, readiness: 'ready' }, + { profile: { id: 'shared', name: 'Shared', kind: 'remote', access: 'session_guest' }, + hostId: guest.hostId, enabled: true, readiness: 'reconnecting' }, + ], + }; + case 'projects:getSnapshot': + projectReads++; + if (changeDuringRead === 'guest') { + for (let index = 0; index < 20; index++) { + emit('runtime-host-profiles:changed', { + ...guest, epoch: guest.targetEpoch, isDefault: false, + }); + } + } else if (projectReads === 1) { + owner.targetEpoch = 'replacement-epoch'; + emit('runtime-host-profiles:changed', { + ...owner, epoch: owner.targetEpoch, isDefault: true, + }); + } + return { + projects: [], + capabilities: { chooseClientDirectory: true, selectNoProject: true }, + }; + case 'app:info': return { projectId: null, projectGit: {} }; + case 'settings:get': return { projects: {}, chatDefaults: {} }; + case 'onboarding:getSnapshot': return { + state: { kind: 'ready_empty' }, milestones: [], sessions: [], connections: [], + defaultSlug: null, chatModelChoices: [], sessionSendOutcomes: {}, + }; + case 'session-local:catalog': return [{ scope: owner, sessions: [], authoritative: true }]; + case 'session-collaboration:mount:list': + case 'sessions:list': return []; + default: throw new Error('Unexpected channel: ' + channel); + } + }, + }; + let bridge: MakaBridge | undefined; + const bundle = await build({ + entryPoints: [fileURLToPath(new URL('../../../src/preload/preload.ts', import.meta.url))], + bundle: true, write: false, platform: 'node', format: 'cjs', external: ['electron'], + }); + const require = createRequire(import.meta.url); + runInNewContext(bundle.outputFiles[0]!.text, { + require: (id: string) => id === 'electron' ? { + ipcRenderer, + contextBridge: { exposeInMainWorld: (name: string, value: MakaBridge) => { + if (name === 'maka') bridge = value; + } }, + } : require(id), + process: { env: {} }, Buffer, console, setTimeout, clearTimeout, TextEncoder, TextDecoder, + crypto: globalThis.crypto, + }); + assert.ok(bridge); + return { bridge, catalogReads: () => catalogReads, scopedCalls }; +} + +test('offline Guest notifications cannot starve the Local new-task catalog or redirect onboarding', async () => { + const { bridge, catalogReads, scopedCalls } = await loadBridge('guest'); + let invalidations = 0; + const unsubscribe = bridge.newTasks.subscribeChanges(() => invalidations++); + try { + const catalog = await bridge.newTasks.getCatalog(); + assert.equal(catalogReads(), 1, 'Guest state changes cannot invalidate an Owner catalog read'); + assert.equal(invalidations, 0); + assert.equal(catalog.defaultProfileId, 'local'); + assert.equal(catalog.hosts.length, 1); + assert.equal(catalog.hosts[0]?.profile.id, 'local'); + assert.equal(catalog.hosts[0]?.readiness, 'ready'); + const snapshot = await bridge.onboarding.getSnapshot(); + assert.equal(snapshot.state.kind, 'ready_empty'); + assert.ok(scopedCalls.length > 0); + assert.ok(scopedCalls.every(hostId => hostId === 'local-host')); + } finally { + unsubscribe(); + } +}); + +test('an Owner replacement still invalidates the new-task catalog', async () => { + const { bridge, catalogReads } = await loadBridge('owner'); + const catalog = await bridge.newTasks.getCatalog(); + assert.equal(catalogReads(), 2); + assert.equal(catalog.hosts[0]?.readiness, 'ready'); +}); diff --git a/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts b/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts index 14aed85c9f..3667dbc56b 100644 --- a/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts +++ b/apps/desktop/src/main/__tests__/runtime-host-profile-service.test.ts @@ -106,6 +106,48 @@ afterEach(async () => { ); }); +for (const hasDeployment of [false, true]) { + test(`recovers an abandoned deployment lock before loading Host choices (saved=${hasDeployment})`, async () => { + const root = await clientRoot(); + const catalog = createClientRuntimeHostProfileCatalog(root); + const managedServices = createDesktopRuntimeHostManagedServiceStore(root); + if (hasDeployment) { + await catalog.create(MANAGED_PROFILE, "token"); + await managedServices.save(MANAGED_PROFILE, MANAGED_SERVICE); + } + const before = await managedServices.read(); + await mkdir(join(root, "runtime-host-deployments.json.lock")); + + const startup = await resolveDesktopRuntimeHostStartup(root, { catalog }); + const service = createDesktopRuntimeHostProfileService({ + clientDataRoot: root, + startup, + catalog, + managedServices, + states: () => [ready({ profile: LOCAL_RUNTIME_HOST_PROFILE })], + enable: async () => undefined, + disable: async () => undefined, + setDefault: () => undefined, + finalizePairing: async () => undefined, + }); + const snapshot = await service.getSnapshot(); + assert.equal(snapshot.defaultProfileId, LOCAL_RUNTIME_HOST_PROFILE.id); + assert.equal(snapshot.entries[0]?.readiness, "ready"); + assert.equal(snapshot.entries.length, hasDeployment ? 2 : 1); + assert.deepEqual(await managedServices.read(), before); + if (hasDeployment) assert.equal(snapshot.entries[1]?.managedService, true); + }); +} + +test("does not discard unexpected contents in an abandoned deployment lock", async () => { + const root = await clientRoot(); + const lock = join(root, "runtime-host-deployments.json.lock"); + await mkdir(lock); + await writeFile(join(lock, "unexpected"), "retain me"); + await assert.rejects(resolveDesktopRuntimeHostStartup(root), { code: "ENOTEMPTY" }); + assert.equal(await readFile(join(lock, "unexpected"), "utf8"), "retain me"); +}); + test("migrates the former selected Host into enabled and default preferences", async () => { const root = await clientRoot(); await createClientRuntimeHostProfileCatalog(root).create(PROFILE, "token"); diff --git a/apps/desktop/src/main/main-process-diagnostics.ts b/apps/desktop/src/main/main-process-diagnostics.ts index a5b39dccdd..7adc9c9463 100644 --- a/apps/desktop/src/main/main-process-diagnostics.ts +++ b/apps/desktop/src/main/main-process-diagnostics.ts @@ -32,6 +32,7 @@ import { type DesktopTargetScope, } from '../shared/runtime-host-identity.js'; import type { MainProcessRecoveryEvidence } from './main-process-recovery-journal.js'; +import type { RuntimeHostDesktopTargetState } from './runtime-host-desktop-manager.js'; const INPUT_LIMITS = { title: 512, @@ -130,6 +131,7 @@ export interface DesktopDiagnosticsDeps { readonly environment: () => DesktopDiagnosticEnvironment; readonly mainLogs: () => readonly string[]; readonly runtimeHostProcessLogs?: () => readonly string[]; + readonly runtimeHostConnections?: () => readonly RuntimeHostDesktopTargetState[]; readonly resolveActiveRuntimeHost: () => RuntimeHostDiagnosticsClient | undefined; readonly resolveRuntimeHost: (scope: DesktopTargetScope) => RuntimeHostDiagnosticsClient | undefined; readonly writeClipboard: (value: string) => void; @@ -389,6 +391,7 @@ export async function copyDesktopDiagnosticReport( runtimeExecution, undefined, deps.runtimeHostProcessLogs?.() ?? [], + deps.runtimeHostConnections?.() ?? [], ), ); } @@ -401,6 +404,7 @@ export function formatDesktopDiagnosticReport( runtimeExecution: RuntimeHostExecutionDiagnosticRead | undefined = undefined, capturedAt = new Date(), runtimeHostProcessLogs: readonly string[] = [], + runtimeHostConnections: readonly RuntimeHostDesktopTargetState[] = [], ): string { const lines = ['Maka Desktop diagnostic report', `Captured at: ${capturedAt.toISOString()}`]; const rendererContext = @@ -462,6 +466,25 @@ export function formatDesktopDiagnosticReport( ); } + if (input.surface !== 'previous_main_process_interruption' && runtimeHostConnections.length > 0) { + lines.push('', `Runtime Host connections (${runtimeHostConnections.length})`); + for (const state of runtimeHostConnections) { + lines.push(`${JSON.stringify(state.target.profile.id)}: ${state.readiness}`); + if (state.reconnect) { + lines.push( + ` Failed attempts: ${state.reconnect.failures}`, + ` First failure: ${new Date(state.reconnect.firstFailureAt).toISOString()}`, + ` Last failure: ${new Date(state.reconnect.lastFailureAt).toISOString()}`, + ); + } + if (state.readiness !== 'ready' && state.error) { + const code = 'code' in state.error && typeof state.error.code === 'string' + ? ` [${state.error.code}]` : ''; + lines.push(` Latest error${code}: ${boundedDiagnosticError(state.error)}`); + } + } + } + lines.push('', 'Runtime Host'); if (runtimeHost.ok) { const host = runtimeHost.value; diff --git a/apps/desktop/src/main/runtime-host-boot.ts b/apps/desktop/src/main/runtime-host-boot.ts index 7760beef8d..85db56710b 100644 --- a/apps/desktop/src/main/runtime-host-boot.ts +++ b/apps/desktop/src/main/runtime-host-boot.ts @@ -370,6 +370,7 @@ const desktopDiagnostics: DesktopDiagnosticsDeps = { }), mainLogs: () => mainProcessLogBuffer.snapshot(), runtimeHostProcessLogs: () => runtimeHostProcessLogBuffer.snapshot(), + runtimeHostConnections: () => runtimeHostManager?.entries() ?? [], resolveActiveRuntimeHost: () => { const scope = activeRuntimeHostRef(); return scope ? resolveRuntimeHostDiagnostics(scope) : undefined; diff --git a/apps/desktop/src/main/runtime-host-desktop-manager.ts b/apps/desktop/src/main/runtime-host-desktop-manager.ts index db9a709858..5e69fec06e 100644 --- a/apps/desktop/src/main/runtime-host-desktop-manager.ts +++ b/apps/desktop/src/main/runtime-host-desktop-manager.ts @@ -33,6 +33,7 @@ import { RuntimeHostPermanentReconnectError, RuntimeHostRemoteCompatibilityError, RuntimeHostPeerError, + RuntimeHostPeerReachabilityUnavailableError, RuntimeHostRequestInterruptedError, runtimeHostStartupError, LOCAL_RUNTIME_HOST_PROFILE, @@ -120,7 +121,7 @@ export interface RuntimeHostDesktopTargetSnapshot { readonly candidate?: DesktopRuntimeHostCandidate; } -export type RuntimeHostDesktopTargetState = +export type RuntimeHostDesktopTargetState = ( | { readonly epoch: string; readonly target: ResolvedRuntimeHostProfile; @@ -140,7 +141,14 @@ export type RuntimeHostDesktopTargetState = readonly readiness: 'unavailable'; readonly hostId?: string; readonly error: Error; - }; + } +) & { + readonly reconnect?: { + readonly failures: number; + readonly firstFailureAt: number; + readonly lastFailureAt: number; + }; +}; export type DesktopLocalHostRetirement = | { readonly kind: 'active_tasks' } @@ -989,10 +997,37 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { : false, ...(initialSignal ? { initialSignal } : {}), onReconnectError: (error) => { - console.warn('[runtime-host] reconnect attempt failed:', error); - if (target.valid && target.state.readiness !== 'ready') { - this.#publishState(target, { ...target.state, error }); + if (!target.valid || target.state.readiness === 'ready') return; + const previous = target.state.error; + const last = target.state.reconnect; + const now = Date.now(); + // Keep one live diagnostic per outage, even when successive dials + // fail differently. Routine retries must not evict unrelated logs. + target.state = { + ...target.state, + error, + reconnect: { + failures: (last?.failures ?? 0) + 1, + firstFailureAt: last?.firstFailureAt ?? now, + lastFailureAt: now, + }, + }; + if (!last) { + if (error instanceof RuntimeHostPeerReachabilityUnavailableError || + error instanceof RuntimeHostPeerError) { + console.info('[runtime-host] reconnecting:', { + profileId: target.target.profile.id, + code: error.code, + message: error.message, + }); + } else { + console.warn('[runtime-host] reconnecting:', target.target.profile.id, error); + } } + if (previous?.name === error.name && previous.message === error.message && + ('code' in previous ? previous.code : undefined) === + ('code' in error ? error.code : undefined)) return; + this.#publishState(target, target.state); }, onFatalError: (error) => { if (starting) { @@ -1431,8 +1466,19 @@ class RuntimeHostDesktopManagerImpl implements RuntimeHostDesktopManager { ): void { // A retry starting is not evidence of recovery. Keep its last failure until // a connection succeeds (or a newer failure replaces it). - if (state.readiness === 'reconnecting' && !state.error && target.state.readiness !== 'ready') { - state = { ...state, ...(target.state.error ? { error: target.state.error } : {}) }; + if (state.readiness !== 'ready' && target.state.readiness !== 'ready') { + state = { + ...state, + ...(!state.error && target.state.error ? { error: target.state.error } : {}), + ...(target.state.reconnect ? { reconnect: target.state.reconnect } : {}), + }; + } + if (state.readiness === 'ready' && target.state.reconnect) { + console.info('[runtime-host] connection restored:', { + profileId: target.target.profile.id, + failedAttempts: target.state.reconnect.failures, + durationMs: Math.max(0, Date.now() - target.state.reconnect.firstFailureAt), + }); } target.state = state; try { diff --git a/apps/desktop/src/main/runtime-host-managed-services.ts b/apps/desktop/src/main/runtime-host-managed-services.ts index 07ab66773c..1c3547d058 100644 --- a/apps/desktop/src/main/runtime-host-managed-services.ts +++ b/apps/desktop/src/main/runtime-host-managed-services.ts @@ -36,7 +36,7 @@ import { decodeRuntimeHostOperatorCommand, type RuntimeHostOperatorCommand, } from "@maka/runtime-host/operator"; -import { withFileUpdateLock } from "@maka/storage/file-update-lock"; +import { withProcessLifetimeFileUpdateLock } from "@maka/storage/process-lifetime-file-update-lock"; import { syncDirectory } from "@maka/storage/stable-storage"; const SCHEMA_VERSION = 2; @@ -375,7 +375,7 @@ class FileDesktopRuntimeHostManagedServiceStore implements DesktopRuntimeHostMan async #exclusive(operation: () => Promise): Promise { await mkdir(dirname(this.#path), { recursive: true, mode: 0o700 }); - return withFileUpdateLock(this.#path, operation); + return withProcessLifetimeFileUpdateLock(this.#path, operation); } } diff --git a/apps/desktop/src/main/runtime-host-profile-service.ts b/apps/desktop/src/main/runtime-host-profile-service.ts index fdc3c01407..3ae654d2f1 100644 --- a/apps/desktop/src/main/runtime-host-profile-service.ts +++ b/apps/desktop/src/main/runtime-host-profile-service.ts @@ -169,6 +169,12 @@ export async function resolveDesktopRuntimeHostStartup( readPreferences?: () => Promise; } = {}, ): Promise { + // The Desktop single-instance lock is held before startup opens any stores. + // Reclaim only legacy directory markers here, before concurrent readers can + // start; current deployment writers use a process-lifetime OS lease. + await recoverAbandonedDesktopFileUpdateLock( + join(clientDataRoot, "runtime-host-deployments.json"), + ); const preferencesPath = join(clientDataRoot, PREFERENCES_FILE); let preferences: DesktopRuntimeHostPreferences; let preferencesReadFailure: Error | undefined; @@ -226,7 +232,7 @@ export async function resolveDesktopRuntimeHostStartup( ), ); if (obsoleteProfileIds.size > 0) { - await recoverAbandonedProfileLock(join(clientDataRoot, PROFILE_FILE)); + await recoverAbandonedDesktopFileUpdateLock(join(clientDataRoot, PROFILE_FILE)); } for (const profileId of obsoleteProfileIds) await catalog.remove(profileId); if (obsoleteProfileIds.size > 0) document = await catalog.read(); @@ -341,7 +347,7 @@ export function createDesktopRuntimeHostProfileService(input: { const mutateProfiles = (operation: () => Promise): Promise => mutate(async () => { - await recoverAbandonedProfileLock(profilePath); + await recoverAbandonedDesktopFileUpdateLock(profilePath); assertPreferencesWritable(); if (pairingReadFailure) { throw new Error( @@ -1470,8 +1476,8 @@ function assertRootIsNotEnabled( } } -async function recoverAbandonedProfileLock(profilePath: string): Promise { - const lockPath = `${profilePath}.lock`; +async function recoverAbandonedDesktopFileUpdateLock(targetPath: string): Promise { + const lockPath = `${targetPath}.lock`; const lock = await lstat(lockPath).catch((error: unknown) => { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; throw error; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 4da7c882cd..f3c89333c6 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -263,6 +263,7 @@ import { projectDesktopSharedSessionSummary } from '../shared/shared-session-cat let activeRuntimeHost: DesktopTargetScope | undefined; let activeRuntimeHostGeneration = 0; +let newTaskCatalogGeneration = 0; type RuntimeHostScopeKey = string; const runtimeHostScopes = new Map(); const runtimeHostProfiles = new Map(); @@ -353,7 +354,12 @@ ipcRenderer.on( ) { activeRuntimeHostGeneration += 1; } - for (const listener of newTaskChangeListeners) listener(); + // Guest mounts can only participate in their shared Sessions. Their + // reconnects cannot change the Hosts/projects available for a new task. + if (change.profileAccess === 'owner') { + newTaskCatalogGeneration += 1; + for (const listener of newTaskChangeListeners) listener(); + } }, ); @@ -575,7 +581,7 @@ async function selectedRuntimeHostScope( async function loadNewTaskCatalog(): Promise { for (let attempt = 0; attempt < 2; attempt += 1) { - const generation = activeRuntimeHostGeneration; + const generation = newTaskCatalogGeneration; const profiles = await ipcRenderer.invoke( 'runtime-host-profiles:getSnapshot', ) as DesktopRuntimeHostProfileSnapshot; @@ -635,7 +641,7 @@ async function loadNewTaskCatalog(): Promise { } }), ); - if (generation !== activeRuntimeHostGeneration) continue; + if (generation !== newTaskCatalogGeneration) continue; return { defaultProfileId: profiles.defaultProfileId, hosts }; } throw new Error('Runtime Host targets changed while the new-task catalog was loading'); diff --git a/packages/runtime-host/src/__tests__/peer-native.test.ts b/packages/runtime-host/src/__tests__/peer-native.test.ts index 8c97c972f6..8f6ce7675f 100644 --- a/packages/runtime-host/src/__tests__/peer-native.test.ts +++ b/packages/runtime-host/src/__tests__/peer-native.test.ts @@ -27,6 +27,7 @@ import { test } from 'node:test'; import { createRuntimeHostPeerClient, RuntimeHostPeerReachabilityUnavailableError, + type RuntimeHostPeerRouteResolution, } from '../client/peer-client.js'; import { PEER_REACHABILITY_MAX_CLOCK_SKEW_MS } from '../peer-reachability/index.js'; import { @@ -315,6 +316,46 @@ module.exports = { ); assert.equal(native.default.stats.requests.length, requestCount); + let resolution: RuntimeHostPeerRouteResolution = { + state: 'exhausted', + routeHints: [], + coordinationRelays: [], + transitRelayPeerIds: [], + }; + let notifyResolution = () => {}; + const detachExhausted = client.attachRouteResolver({ + resolveRoutes: () => resolution, + subscribeRoutes: (_peerId, listener) => { + notifyResolution = listener; + return () => {}; + }, + prepareRoutes: async () => { + resolution = { ...resolution, state: 'recovering' }; + notifyResolution(); + resolution = { ...resolution, state: 'exhausted' }; + notifyResolution(); + }, + }); + let recoveryWakeups = 0; + const unsubscribeRecovery = client.subscribeRoutes('offline', () => recoveryWakeups++); + for (let attempt = 0; attempt < 3; attempt++) { + await assert.rejects( + client.connect({ ...peerConnectInput('offline'), routeHints: [] }), + RuntimeHostPeerReachabilityUnavailableError, + ); + } + assert.equal(recoveryWakeups, 0, 'empty recovery sweeps must not wake their own retries'); + resolution = { ...resolution, state: 'available', coordinationRelays: ['/memory/new-relay'] }; + notifyResolution(); + assert.equal(recoveryWakeups, 1, 'a new candidate wakes the offline connection'); + notifyResolution(); + assert.equal(recoveryWakeups, 1, 'unchanged candidates do not bypass backoff'); + resolution = { ...resolution, state: 'exhausted', coordinationRelays: [] }; + notifyResolution(); + assert.equal(recoveryWakeups, 1, 'losing the final candidate does not wake a retry'); + unsubscribeRecovery(); + detachExhausted(); + let connectivityWakeups = 0; const unsubscribeConnectivity = client.subscribeRoutes('restored', () => { connectivityWakeups += 1; diff --git a/packages/runtime-host/src/client/peer-client.ts b/packages/runtime-host/src/client/peer-client.ts index 3b9df63d94..707cf29bfc 100644 --- a/packages/runtime-host/src/client/peer-client.ts +++ b/packages/runtime-host/src/client/peer-client.ts @@ -99,6 +99,7 @@ export interface RuntimeHostPeerClient { readonly relayCandidates: readonly RuntimeHostPeerTransitRelayCandidate[]; }): Promise; attachRouteResolver(resolver: RuntimeHostPeerRouteResolver): () => void; + /** Notify reconnect owners when candidates change or a peer becomes connected. */ subscribeRoutes(peerId: string, listener: () => void): () => void; observeAuthenticatedReachability(input: { readonly expectedPeerId: string; @@ -289,6 +290,29 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { } subscribeRoutes(peerId: string, listener: () => void): () => void { + const snapshot = () => + this.#connectionResolution( + { peerId, routeHints: [], directDeadlineMs: 0 }, + 'application', + false, + ); + let previous = snapshot(); + let connected = this.isConnected(peerId); + return this.#subscribeRouteResolution(peerId, () => { + const next = snapshot(); + const nextConnected = this.isConnected(peerId); + const available = + next.state === 'available' && + (!sameCandidates(previous, next) || (nextConnected && !connected)); + previous = next; + connected = nextConnected; + // A dial's own recovery sweep toggles recovering/exhausted. Waking its + // reconnect owner for those transitions would bypass every backoff. + if (available) listener(); + }); + } + + #subscribeRouteResolution(peerId: string, listener: () => void): () => void { const listeners = this.#routeListeners.get(peerId) ?? new Set<() => void>(); const first = listeners.size === 0; listeners.add(listener); @@ -540,7 +564,7 @@ class RuntimeHostPeerClientImpl implements RuntimeHostPeerClient { void updateTail.catch(() => undefined); }; const unsubscribe = - kind === 'application' ? this.subscribeRoutes(input.peerId, update) : undefined; + kind === 'application' ? this.#subscribeRouteResolution(input.peerId, update) : undefined; let connection: Promise; try { connection = endpoint[kind === 'application' ? 'connect' : 'connectMeshControl']({