From bc509b754bf9d2aa229c02c466e6e0fbcf83f80d Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Sat, 5 Sep 2026 07:13:27 +0800 Subject: [PATCH 1/6] test(runtime-host): bound owned-Host lifecycle tests to kernel contracts Fixes the intermittent failures reported in #4776. The test-side analysis and fix shape are @UncertaintyDeterminesYou4ndMe's (proposed in #4784 and donated in the issue after that PR was withdrawn); this commit implements it so the findings are not lost. - The launch-owner exit test gated its assertion on the Client's own `connection.closed`, which the Client aborts after a 2 s unanswered liveness probe, so a merely busy Host resolved it while still running. The assertion now waits on the process itself with a 20 s bound derived from the kernel's `shutdownGraceMs` contract, below the launcher's new 60 s idle grace. - The owned launch fixture's `idleGraceMs` (10 s) could expire mid-test and let an idle exit masquerade as an owner-loss exit; it now sits at 60 s with an explicit `initialConnectionTimeoutMs`. - "Exits promptly" now asserts shutdown start (the kernel's published `draining` registration) separately from shutdown completion, and the settle bound (15 s) sits above the kernel's own 10 s grace instead of inside it, so a starved-but-clean Host no longer reports an unclean exit. Fixes #4776 Generated-by: GLM-5.3-Flash (ZCode) --- .../fixtures/owned-authority-launcher.ts | 8 ++- .../src/__tests__/host-kernel.test.ts | 18 ++++++- .../src/__tests__/owned-candidate.test.ts | 52 +++++++++++++++++-- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts index 6e67dcd363..04799e3c96 100644 --- a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts +++ b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts @@ -32,7 +32,13 @@ const attempt = await launchOwnedRuntimeHostCandidate({ rootPath, expectedRootId, entrypoint: new URL('../../execution-candidate-main.js', import.meta.url), - idleGraceMs: 10_000, + // The idle grace only has to outlast the test, and it has to stay clear of + // any bound a test puts on an owner-loss exit: a Candidate that exits + // because it went idle must never be mistaken for one that exited because + // its launch owner died. The first-connection deadline stays short so a + // Candidate no Client ever reaches still exits on its own. + idleGraceMs: 60_000, + initialConnectionTimeoutMs: 10_000, inheritableAuthorityLeaseFd: leaseFd, launchOwnerClientInstanceId: clientInstanceId, }).spawned; diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 2e330f305a..dace5c2fbb 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -1916,12 +1916,26 @@ describe('non-serving Runtime Host kernel', () => { launcher.kill('SIGKILL'); await waitForExit(launcher); + // The process is the only thing that reports the claim. A Client's + // `connection.closed` does not: it is that Client's own transport, and + // the Client aborts it after its liveness probe goes unanswered for two + // seconds. A Host that is merely busy therefore resolves it while still + // running, and gating the exit assertion on it starts the exit budget at + // a moment that has nothing to do with the Host's shutdown. + // + // The bound comes from the kernel's contract rather than from an + // interval this test could predict. Owner loss cannot close a + // composition before its startup settles, and the shutdown that follows + // is bounded by `shutdownGraceMs` (10 s), after which the kernel + // force-terminates. Twenty seconds therefore sits above every + // legitimate exit and below the launcher's 60 s idle grace, so it cannot + // be satisfied by a Candidate that merely went idle. + await waitForProcessExit(launchedPid, 20_000); await withTimeout( connected.connection.closed, 5_000, - 'authority-supervised Candidate survived its launch owner', + 'authority-supervised Candidate exited without closing its Client connection', ); - await waitForProcessExit(launchedPid); paths.resources.forgetPid(launchedPid); }); }); diff --git a/packages/runtime-host/src/__tests__/owned-candidate.test.ts b/packages/runtime-host/src/__tests__/owned-candidate.test.ts index 3c264f145d..e260349ccb 100644 --- a/packages/runtime-host/src/__tests__/owned-candidate.test.ts +++ b/packages/runtime-host/src/__tests__/owned-candidate.test.ts @@ -37,6 +37,11 @@ import { type CandidateExitDetails, type OwnedCandidateAttempt, } from '../client/launcher.js'; +import { + resolveExistingStorageRoot, + resolveExistingStorageRootControlDirectory, +} from '@maka/storage/root-authority'; +import { readHostRegistration } from '../control/registration.js'; test('owned connection keeps a fresh Host alive for its full election window', async () => { const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-')); @@ -257,12 +262,20 @@ test('owned Host exits promptly after its first connection closes', async () => assert.equal(result.kind, 'connected', connectFailure(result)); if (result.kind !== 'connected') return; + const controlDirectory = await resolveHostControlDirectory(rootPath, result.connection.rootId); await result.connection.close(); - // Prompt means the owned launch's idleGraceMs of 0, as opposed to the 30 s - // default grace, so the bound only has to sit well below that. Shutdown takes - // about 30 ms on an idle machine and stretches past 500 ms under a full CI - // suite while still exiting cleanly: the Host is starved, not stuck. - assert.equal(await result.host.settle(5_000), true); + // Promptness is when the Host starts shutting down, not how long shutting + // down takes: the owned launch's idleGraceMs is 0 against a 30 s default. + // The kernel publishes its draining registration as the first step of + // shutdown, so the registration reports the idle grace directly. Reading it + // from `settle` alone could not separate the two, which is why a loaded + // machine that only made the shutdown itself slow failed this assertion. + await waitForHostShutdownStart(controlDirectory, 10_000); + // The exit is a second claim with a bound of its own, and the kernel sets + // it: shutdown gets `shutdownGraceMs` (10 s) to close every resource before + // the kernel force-terminates the process. Anything below that fails a Host + // that is starved rather than stuck. + assert.equal(await result.host.settle(15_000), true); }); test('an exited owned Candidate permits one real successor in the same election', { @@ -436,6 +449,35 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () => assert.deepEqual(await readdir(rootPath), []); }); +async function resolveHostControlDirectory(rootPath: string, rootId: string): Promise { + const capability = await resolveExistingStorageRoot({ + path: rootPath, + kind: 'interactive', + expectedRootId: rootId, + }); + const { controlDirectory } = await resolveExistingStorageRootControlDirectory(capability); + return controlDirectory; +} + +/** + * Resolves once the Host has begun shutting down. `draining` is the state the + * kernel publishes before it does any shutdown work, and the registration is + * removed near the end of that work, so either observation proves shutdown + * started; the Host was serving this Client, so its registration existed. + */ +async function waitForHostShutdownStart( + controlDirectory: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const registration = await readHostRegistration(controlDirectory).catch(() => undefined); + if (!registration || registration.state === 'draining') return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error('owned Host did not begin shutting down after its first connection closed'); +} + function connectFailure( result: | Awaited> From a60a2dfbd380a4fab8f598e2e23b9cceb77a3d4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=B3=E5=A4=A9=E8=B1=AA?= Date: Mon, 7 Sep 2026 22:01:13 +0800 Subject: [PATCH 2/6] test(runtime-host): pin the owner-loss recovery window and surface registration read errors Follow-up to the P2 review on #4814. Two changes: - The owner-loss exit test kills the launcher before the launch-owner guard binds, so the remaining recovery time was part of its exit bound with no kernel deadline. A test-only entry now delays composition creation by a fixed 5 s, the kill lands inside that pinned window, and the deadline becomes delay + shutdownGraceMs + margin. The comment derives the bound instead of asserting it. - waitForHostShutdownStart swallowed every readHostRegistration error, masquerading I/O or decode failures as "shutdown started"; readHostRegistration already maps a missing file to undefined, so the unguarded await surfaces real errors. Generated-by: GLM-5.3-Flash (ZCode) --- .../fixtures/owned-authority-launcher.ts | 9 ++-- .../src/__tests__/host-kernel.test.ts | 34 +++++++++++---- .../src/__tests__/owned-candidate.test.ts | 5 ++- .../owned-candidate-delayed-recovery-main.ts | 43 +++++++++++++++++++ .../owned-candidate-recovery-delay.ts | 29 +++++++++++++ 5 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts create mode 100644 packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts diff --git a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts index 04799e3c96..a490a9ca5d 100644 --- a/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts +++ b/packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts @@ -20,10 +20,11 @@ import { openSync } from 'node:fs'; import { launchOwnedRuntimeHostCandidate } from '../../client/launcher.js'; -const [rootPath, expectedRootId, leasePath, clientInstanceId] = process.argv.slice(2); +const [rootPath, expectedRootId, leasePath, clientInstanceId, entrypointOverride] = + process.argv.slice(2); if (!rootPath || !expectedRootId || !leasePath || !clientInstanceId) { throw new Error( - 'usage: owned-authority-launcher ', + 'usage: owned-authority-launcher [entrypoint]', ); } @@ -31,7 +32,9 @@ const leaseFd = openSync(leasePath, 'a+'); const attempt = await launchOwnedRuntimeHostCandidate({ rootPath, expectedRootId, - entrypoint: new URL('../../execution-candidate-main.js', import.meta.url), + // Tests that bound an owner-loss exit pass a test-only entry whose + // startup window they control; the production entry stays the default. + entrypoint: new URL(entrypointOverride ?? '../../execution-candidate-main.js', import.meta.url), // The idle grace only has to outlast the test, and it has to stay clear of // any bound a test puts on an owner-loss exit: a Candidate that exits // because it went idle must never be mistaken for one that exited because diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 6ca88eda8c..407a78356b 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -88,6 +88,7 @@ import type { RuntimeHostCompositionSource } from '../server/host-composition.js import { createUnavailableDomainOperationHandlers } from '../server/operation-dispatcher.js'; import { HostChangeFeed } from '../server/host-change-feed.js'; import { FramedTransport, RuntimeHostTransportError } from '../transport/framed-transport.js'; +import { OWNED_CANDIDATE_RECOVERY_DELAY_MS } from '../test-only/owned-candidate-recovery-delay.js'; import { prepareStorageRootControlDirectory, resolveRootControlNamespace, @@ -2204,6 +2205,11 @@ describe('non-serving Runtime Host kernel', () => { capability.rootId, join(paths.base, 'authority-lease-probe'), launchOwnerClientInstanceId, + // The owner-loss exit bound below covers the recovery window, so + // this run pins startup behind the delayed-recovery entry instead + // of leaving the bound at the mercy of however long an unassisted + // recovery takes under load. + '../../test-only/owned-candidate-delayed-recovery-main.js', ], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, ), @@ -2233,14 +2239,26 @@ describe('non-serving Runtime Host kernel', () => { // running, and gating the exit assertion on it starts the exit budget at // a moment that has nothing to do with the Host's shutdown. // - // The bound comes from the kernel's contract rather than from an - // interval this test could predict. Owner loss cannot close a - // composition before its startup settles, and the shutdown that follows - // is bounded by `shutdownGraceMs` (10 s), after which the kernel - // force-terminates. Twenty seconds therefore sits above every - // legitimate exit and below the launcher's 60 s idle grace, so it cannot - // be satisfied by a Candidate that merely went idle. - await waitForProcessExit(launchedPid, 20_000); + // The bound is a sum of contracts the test controls rather than an + // interval this test could predict. A launch-owner Client is admitted + // while the Host is still recovering, and the guard that closes the + // Host on owner loss binds only after startup returns, so the + // remaining recovery time is part of the bound and has no kernel + // deadline. The delayed-recovery entry pins that window to + // OWNED_CANDIDATE_RECOVERY_DELAY_MS (5 s), and the shutdown that + // follows is bounded by `shutdownGraceMs` (10 s), after which the + // kernel force-terminates. The deadline is that delay plus the + // shutdown grace and margin — twenty seconds — which sits above every + // legitimate exit and below the launcher's 60 s idle grace, so it + // cannot be satisfied by a Candidate that merely went idle. The kill + // lands inside the pinned window, so this exercises owner loss before + // the guard binds; Windows locally terminates such grandchildren + // abruptly without a JS exit event, so only CI verdicts count as + // cross-platform evidence for this assertion. + await waitForProcessExit( + launchedPid, + OWNED_CANDIDATE_RECOVERY_DELAY_MS + 15_000, + ); await withTimeout( connected.connection.closed, 5_000, diff --git a/packages/runtime-host/src/__tests__/owned-candidate.test.ts b/packages/runtime-host/src/__tests__/owned-candidate.test.ts index e260349ccb..bc25e6a05e 100644 --- a/packages/runtime-host/src/__tests__/owned-candidate.test.ts +++ b/packages/runtime-host/src/__tests__/owned-candidate.test.ts @@ -471,7 +471,10 @@ async function waitForHostShutdownStart( ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - const registration = await readHostRegistration(controlDirectory).catch(() => undefined); + // readHostRegistration already maps a missing file to `undefined`; the + // unguarded await lets real I/O or decode errors fail this wait loudly + // instead of masquerading as "shutdown started". + const registration = await readHostRegistration(controlDirectory); if (!registration || registration.state === 'draining') return; await new Promise((resolve) => setTimeout(resolve, 10)); } diff --git a/packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts b/packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts new file mode 100644 index 0000000000..d1f03f8a23 --- /dev/null +++ b/packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts @@ -0,0 +1,43 @@ +#!/usr/bin/env node +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Test-only Candidate entry for the owner-loss lifecycle tests. A + * launch-owner Client is admitted while the Host is still recovering, and + * the guard that closes the Host on owner loss binds only after startup + * returns, so the remaining recovery time is part of any owner-loss exit + * bound and has no kernel deadline. Delaying composition creation by a fixed + * interval turns that window into a number the test controls (see + * `OWNED_CANDIDATE_RECOVERY_DELAY_MS`) instead of however long an unassisted + * recovery happens to take under load. The run still goes through the real + * Runtime Host composition — only its start is deferred. + */ +import { runExecutionCandidateEntry } from '../candidate-entry.js'; +import { createExecutionRuntimeHostComposition } from '../server/execution-composition.js'; +import { OWNED_CANDIDATE_RECOVERY_DELAY_MS } from './owned-candidate-recovery-delay.js'; + +await runExecutionCandidateEntry(process.argv.slice(2), import.meta.url, { + dependencies: { + createComposition: async (context, compositionOptions) => { + await new Promise((resolve) => setTimeout(resolve, OWNED_CANDIDATE_RECOVERY_DELAY_MS)); + return createExecutionRuntimeHostComposition(context, compositionOptions); + }, + }, +}); diff --git a/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts b/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts new file mode 100644 index 0000000000..95791aeea6 --- /dev/null +++ b/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * The recovery delay `owned-candidate-delayed-recovery-main.ts` injects before + * creating its composition. The owner-loss lifecycle test kills the launch + * owner while the Host is still inside this window, and a launch-owner Client + * is admitted before the guard that closes the Host on owner loss is bound — + * so the remaining recovery time is part of that test's exit bound and has no + * kernel deadline. Pinning it turns the bound into a sum of contracts: this + * delay, the kernel's `shutdownGraceMs`, and margin. + */ +export const OWNED_CANDIDATE_RECOVERY_DELAY_MS = 5_000; From 2ee447b0f3018a3d2a3610bfe74f3865e1c7c4c0 Mon Sep 17 00:00:00 2001 From: ggbdpq Date: Mon, 7 Sep 2026 23:28:36 +0800 Subject: [PATCH 3/6] style(cli): collapse the recovery-wait call to one line format:check on CI wants waitForProcessExit on a single line; the previous round verified build and tests but skipped the format gate. Generated-by: GLM-5.3-Flash (ZCode) --- packages/runtime-host/src/__tests__/host-kernel.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 407a78356b..305167472b 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -2255,10 +2255,7 @@ describe('non-serving Runtime Host kernel', () => { // the guard binds; Windows locally terminates such grandchildren // abruptly without a JS exit event, so only CI verdicts count as // cross-platform evidence for this assertion. - await waitForProcessExit( - launchedPid, - OWNED_CANDIDATE_RECOVERY_DELAY_MS + 15_000, - ); + await waitForProcessExit(launchedPid, OWNED_CANDIDATE_RECOVERY_DELAY_MS + 15_000); await withTimeout( connected.connection.closed, 5_000, From f6f239b4cfcfd9346bfb7981b931ed7181995ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=B3=E5=A4=A9=E8=B1=AA?= Date: Tue, 8 Sep 2026 09:38:21 +0800 Subject: [PATCH 4/6] test(runtime-host): gate the owner-loss kill behind an explicit stall signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The P2 follow-up on #4814: a fixed 5 s sleep cannot decide whether the kill lands before the launch-owner guard binds — a test-side pause longer than the sleep silently turns the scenario post-bind, and the sleep is additive to the real composition startup rather than a cap on it. Replace the fixed delay with an explicit gate: the test-only entry now parks composition creation behind a release file and writes a stall marker when it reaches the gated window. The test waits for that marker (so the kill provably lands pre-bind), kills the launcher, releases the gate, and asserts the OS process exits within shutdownGraceMs + margin — observing the real PID rather than the kernel's `closed` promise. The registration-read error handling from the previous commit is kept. Generated-by: GLM-5.3-Flash (ZCode) --- .../src/__tests__/host-kernel.test.ts | 65 ++++++++++++------- ...=> owned-candidate-gated-recovery-main.ts} | 28 +++++--- .../owned-candidate-recovery-delay.ts | 29 --------- 3 files changed, 61 insertions(+), 61 deletions(-) rename packages/runtime-host/src/test-only/{owned-candidate-delayed-recovery-main.ts => owned-candidate-gated-recovery-main.ts} (55%) delete mode 100644 packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 305167472b..4ca5e56968 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -21,7 +21,7 @@ import { deferred, withTimeout } from '@maka/core/test-only/async-primitives'; import { RuntimeHostProtocolError } from '../protocol/errors.js'; import { defineInteractiveRuntimeHostComposition } from '../server/host-composition.js'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync, writeFileSync } from 'node:fs'; import { execFile, fork, type ChildProcess } from 'node:child_process'; import { chmod, @@ -88,7 +88,6 @@ import type { RuntimeHostCompositionSource } from '../server/host-composition.js import { createUnavailableDomainOperationHandlers } from '../server/operation-dispatcher.js'; import { HostChangeFeed } from '../server/host-change-feed.js'; import { FramedTransport, RuntimeHostTransportError } from '../transport/framed-transport.js'; -import { OWNED_CANDIDATE_RECOVERY_DELAY_MS } from '../test-only/owned-candidate-recovery-delay.js'; import { prepareStorageRootControlDirectory, resolveRootControlNamespace, @@ -2205,16 +2204,22 @@ describe('non-serving Runtime Host kernel', () => { capability.rootId, join(paths.base, 'authority-lease-probe'), launchOwnerClientInstanceId, - // The owner-loss exit bound below covers the recovery window, so - // this run pins startup behind the delayed-recovery entry instead - // of leaving the bound at the mercy of however long an unassisted - // recovery takes under load. - '../../test-only/owned-candidate-delayed-recovery-main.js', + // The owner-loss exit bound below covers the gated recovery + // window, so this run pins startup behind the gated-recovery + // entry instead of leaving both the window and the kill's + // ordering to however scheduling resolves them. + '../../test-only/owned-candidate-gated-recovery-main.js', ], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }, ), ); const launchedPid = paths.resources.trackPid(await waitForLaunch(launcher)); + // The gated-recovery entry parks composition creation behind these + // markers under the same base directory; the stall marker proves the + // Candidate reached the gated window, and the release marker lets the + // test unblock it through a channel that survives the launcher. + const stallMarker = join(paths.base, 'authority-lease-probe.stalled'); + const releaseMarker = join(paths.base, 'authority-lease-probe.release'); const connected = await retryConnect(paths, CURRENT_PROTOCOL, { clientInstanceId: launchOwnerClientInstanceId, }); @@ -2230,6 +2235,23 @@ describe('non-serving Runtime Host kernel', () => { }); assert.equal(ordinary.kind, 'draining'); + // The owner-loss contract under test is recorded pre-bind: a + // launch-owner Client is admitted while the Host is still recovering, + // and the guard that closes the Host on owner loss binds only after + // startup returns. Kill timing alone cannot prove the loss was + // recorded pre-bind — a test-side pause longer than the candidate's + // startup would silently turn this into the post-bind scenario — so + // the gated-recovery entry holds startup behind a release file and + // marks the stall; waiting for that marker makes the kill land inside + // the gated window by construction rather than by luck. + const stallDeadline = Date.now() + 10_000; + while (!existsSync(stallMarker) && Date.now() < stallDeadline) { + await sleep(20); + } + assert.ok( + existsSync(stallMarker), + 'gated-recovery entry never reached its stall window', + ); launcher.kill('SIGKILL'); await waitForExit(launcher); // The process is the only thing that reports the claim. A Client's @@ -2239,23 +2261,18 @@ describe('non-serving Runtime Host kernel', () => { // running, and gating the exit assertion on it starts the exit budget at // a moment that has nothing to do with the Host's shutdown. // - // The bound is a sum of contracts the test controls rather than an - // interval this test could predict. A launch-owner Client is admitted - // while the Host is still recovering, and the guard that closes the - // Host on owner loss binds only after startup returns, so the - // remaining recovery time is part of the bound and has no kernel - // deadline. The delayed-recovery entry pins that window to - // OWNED_CANDIDATE_RECOVERY_DELAY_MS (5 s), and the shutdown that - // follows is bounded by `shutdownGraceMs` (10 s), after which the - // kernel force-terminates. The deadline is that delay plus the - // shutdown grace and margin — twenty seconds — which sits above every - // legitimate exit and below the launcher's 60 s idle grace, so it - // cannot be satisfied by a Candidate that merely went idle. The kill - // lands inside the pinned window, so this exercises owner loss before - // the guard binds; Windows locally terminates such grandchildren - // abruptly without a JS exit event, so only CI verdicts count as - // cross-platform evidence for this assertion. - await waitForProcessExit(launchedPid, OWNED_CANDIDATE_RECOVERY_DELAY_MS + 15_000); + // Releasing the gate lets startup return promptly, `bind()` fires the + // loss recorded above, and the kernel closes the Host under + // `shutdownGraceMs` (10 s), after which it force-terminates. The + // deadline is that grace plus margin — twenty seconds — which sits + // below the launcher's 60 s idle grace, so it cannot be satisfied by a + // Candidate that merely went idle. The assertion observes the real + // operating-system PID: the kernel resolving its `closed` promise does + // not by itself mean the OS process has exited. Local Windows runs + // terminate such grandchildren abruptly without a JS exit event, so + // only CI verdicts count as cross-platform evidence here. + writeFileSync(releaseMarker, String(Date.now())); + await waitForProcessExit(launchedPid, 20_000); await withTimeout( connected.connection.closed, 5_000, diff --git a/packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts similarity index 55% rename from packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts rename to packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts index d1f03f8a23..1ba90e4a64 100644 --- a/packages/runtime-host/src/test-only/owned-candidate-delayed-recovery-main.ts +++ b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts @@ -22,21 +22,33 @@ * Test-only Candidate entry for the owner-loss lifecycle tests. A * launch-owner Client is admitted while the Host is still recovering, and * the guard that closes the Host on owner loss binds only after startup - * returns, so the remaining recovery time is part of any owner-loss exit - * bound and has no kernel deadline. Delaying composition creation by a fixed - * interval turns that window into a number the test controls (see - * `OWNED_CANDIDATE_RECOVERY_DELAY_MS`) instead of however long an unassisted - * recovery happens to take under load. The run still goes through the real - * Runtime Host composition — only its start is deferred. + * returns — so whether the test's kill lands before or after the bind is a + * scheduling race a fixed sleep cannot decide. This entry instead gates + * composition creation behind a release file: writing the stall marker + * proves the Candidate is parked before the bind, the test kills the + * launcher at that point, and releasing the gate afterwards lets startup + * return promptly so the recorded loss closes the Host under the kernel's + * `shutdownGraceMs`. The run still goes through the real Runtime Host + * composition — only its start is held at the gate. */ +import { existsSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; import { runExecutionCandidateEntry } from '../candidate-entry.js'; import { createExecutionRuntimeHostComposition } from '../server/execution-composition.js'; -import { OWNED_CANDIDATE_RECOVERY_DELAY_MS } from './owned-candidate-recovery-delay.js'; + +const rootArgumentIndex = process.argv.indexOf('--root') + 1; +const rootPath = process.argv[rootArgumentIndex]; +if (!rootPath) throw new Error('gated-recovery entry requires --root'); +const stallMarker = join(dirname(rootPath), 'authority-lease-probe.stalled'); +const releaseMarker = join(dirname(rootPath), 'authority-lease-probe.release'); await runExecutionCandidateEntry(process.argv.slice(2), import.meta.url, { dependencies: { createComposition: async (context, compositionOptions) => { - await new Promise((resolve) => setTimeout(resolve, OWNED_CANDIDATE_RECOVERY_DELAY_MS)); + writeFileSync(stallMarker, String(Date.now())); + while (!existsSync(releaseMarker)) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } return createExecutionRuntimeHostComposition(context, compositionOptions); }, }, diff --git a/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts b/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts deleted file mode 100644 index 95791aeea6..0000000000 --- a/packages/runtime-host/src/test-only/owned-candidate-recovery-delay.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -/** - * The recovery delay `owned-candidate-delayed-recovery-main.ts` injects before - * creating its composition. The owner-loss lifecycle test kills the launch - * owner while the Host is still inside this window, and a launch-owner Client - * is admitted before the guard that closes the Host on owner loss is bound — - * so the remaining recovery time is part of that test's exit bound and has no - * kernel deadline. Pinning it turns the bound into a sum of contracts: this - * delay, the kernel's `shutdownGraceMs`, and margin. - */ -export const OWNED_CANDIDATE_RECOVERY_DELAY_MS = 5_000; From 11207629dad639d692be429ab886aed5bc8a36b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=B3=E5=A4=A9=E8=B1=AA?= Date: Tue, 8 Sep 2026 10:08:36 +0800 Subject: [PATCH 5/6] style(runtime-host): collapse the stall assertion to Biome's single-line form Generated-by: GLM-5.3-Flash (ZCode) --- packages/runtime-host/src/__tests__/host-kernel.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 4ca5e56968..7f9055c569 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -2248,10 +2248,7 @@ describe('non-serving Runtime Host kernel', () => { while (!existsSync(stallMarker) && Date.now() < stallDeadline) { await sleep(20); } - assert.ok( - existsSync(stallMarker), - 'gated-recovery entry never reached its stall window', - ); + assert.ok(existsSync(stallMarker), 'gated-recovery entry never reached its stall window'); launcher.kill('SIGKILL'); await waitForExit(launcher); // The process is the only thing that reports the claim. A Client's From c10d35c48a2116c0a02e178f6590389f6c275e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=85=B3=E5=A4=A9=E8=B1=AA?= Date: Tue, 8 Sep 2026 12:46:52 +0800 Subject: [PATCH 6/6] test(runtime-host): start the owner-loss exit budget at the guard-bind boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on #4814 (Astro-Han P2): the gated-recovery fixture parks composition creation behind the release marker, so real composition creation and recovery run after the release — inside the 20-second exit budget — while the launch-owner guard only binds once startup returns. A legitimately slow startup (25 s) would exhaust the assertion before the owner-loss shutdown it claims to bound even begins; the kernel's 10 s shutdown grace does not constrain that startup phase. The fixture's `onWon` hook now writes a bound marker: `candidate-entry` fires it immediately after `launchOwnerGuard.bind()`, so the marker is an explicit guard-bound boundary rather than a Client-transport signal (a liveness-aborted `connection.closed` can resolve while a busy Host still runs). The test releases the gate, waits for that marker, and only then starts the 20-second real-PID exit budget; the shutdown now has the full grace-plus-margin the deadline documents. The stall/kill pre-bind window is unchanged, so the regression this test pins still holds. Generated-by: GLM-5.3-Flash (ZCode) --- .../src/__tests__/host-kernel.test.ts | 43 ++++++++++++++----- .../owned-candidate-gated-recovery-main.ts | 14 +++++- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/packages/runtime-host/src/__tests__/host-kernel.test.ts b/packages/runtime-host/src/__tests__/host-kernel.test.ts index 7f9055c569..f6b5307f58 100644 --- a/packages/runtime-host/src/__tests__/host-kernel.test.ts +++ b/packages/runtime-host/src/__tests__/host-kernel.test.ts @@ -2255,20 +2255,41 @@ describe('non-serving Runtime Host kernel', () => { // `connection.closed` does not: it is that Client's own transport, and // the Client aborts it after its liveness probe goes unanswered for two // seconds. A Host that is merely busy therefore resolves it while still - // running, and gating the exit assertion on it starts the exit budget at - // a moment that has nothing to do with the Host's shutdown. + // running, so it is used only as a post-exit consistency check below. // - // Releasing the gate lets startup return promptly, `bind()` fires the - // loss recorded above, and the kernel closes the Host under - // `shutdownGraceMs` (10 s), after which it force-terminates. The - // deadline is that grace plus margin — twenty seconds — which sits - // below the launcher's 60 s idle grace, so it cannot be satisfied by a - // Candidate that merely went idle. The assertion observes the real + // Startup — composition creation and recovery included — runs after the + // release and is not bounded by the kernel's shutdown grace, so the exit + // budget must not start at the release. The entry's `onWon` marker is + // the explicit guard-bound boundary that starts it instead: the + // launch-owner guard has bound and the pre-bind recorded loss is being + // acted on, so everything the 20-second deadline covers (the + // `shutdownGraceMs` close plus margin — which sits below the launcher's + // 60 s idle grace, so it cannot be satisfied by a Candidate that merely + // went idle) happens after the marker. + // + // The race below keeps that boundary honest without breaking local + // Windows runs: there the Candidate can be terminated abruptly the + // moment its launcher dies — no JS exit event, so no bind and no marker + // — and `isProcessAlive` releasing the wait only records that platform + // limitation, while a Candidate still alive without a marker past the + // deadline is a failure. The assertion observes the real // operating-system PID: the kernel resolving its `closed` promise does - // not by itself mean the OS process has exited. Local Windows runs - // terminate such grandchildren abruptly without a JS exit event, so - // only CI verdicts count as cross-platform evidence here. + // not by itself mean the OS process has exited, so only CI verdicts + // count as cross-platform evidence here. writeFileSync(releaseMarker, String(Date.now())); + const boundMarker = join(paths.base, 'authority-lease-probe.bound'); + const boundDeadline = Date.now() + 10_000; + while ( + !existsSync(boundMarker) && + isProcessAlive(launchedPid) && + Date.now() < boundDeadline + ) { + await sleep(20); + } + assert.ok( + existsSync(boundMarker) || !isProcessAlive(launchedPid), + 'gated-recovery entry never reached its guard bind', + ); await waitForProcessExit(launchedPid, 20_000); await withTimeout( connected.connection.closed, diff --git a/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts index 1ba90e4a64..6afcbf9fc4 100644 --- a/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts +++ b/packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts @@ -29,7 +29,10 @@ * launcher at that point, and releasing the gate afterwards lets startup * return promptly so the recorded loss closes the Host under the kernel's * `shutdownGraceMs`. The run still goes through the real Runtime Host - * composition — only its start is held at the gate. + * composition — only its start is held at the gate. The `onWon` hook adds + * the second boundary the test needs: a marker written right after the + * guard binds, so the exit budget starts at the bind rather than at the + * release, which only unblocks startup. */ import { existsSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; @@ -41,8 +44,17 @@ const rootPath = process.argv[rootArgumentIndex]; if (!rootPath) throw new Error('gated-recovery entry requires --root'); const stallMarker = join(dirname(rootPath), 'authority-lease-probe.stalled'); const releaseMarker = join(dirname(rootPath), 'authority-lease-probe.release'); +const boundMarker = join(dirname(rootPath), 'authority-lease-probe.bound'); await runExecutionCandidateEntry(process.argv.slice(2), import.meta.url, { + // `onWon` fires right after the launch-owner guard binds (candidate-entry + // binds before invoking it), so this marker is the test's explicit + // guard-bound boundary: the pre-bind recorded loss starts acting only past + // it, which is where the exit budget under test actually begins. + onWon: () => { + writeFileSync(boundMarker, String(Date.now())); + return () => undefined; + }, dependencies: { createComposition: async (context, compositionOptions) => { writeFileSync(stallMarker, String(Date.now()));