test(runtime-host): bound owned-Host lifecycle tests to kernel contracts

packages/runtime-host/src/__tests__/fixtures/owned-authority-launcher.ts

@@ -20,19 +20,28 @@
 import { openSync } from 'node:fs';
 import { launchOwnedRuntimeHostCandidate } from '../../client/launcher.js';
 
-const [rootPath, expectedRootId, leasePath, clientInstanceId] = process.argv.slice(2);
+const [rootPath, expectedRootId, leasePath, clientInstanceId, entrypointOverride] =
+  process.argv.slice(2);
 if (!rootPath || !expectedRootId || !leasePath || !clientInstanceId) {
   throw new Error(
-    'usage: owned-authority-launcher <root> <expected-root-id> <lease-path> <client-id>',
+    'usage: owned-authority-launcher <root> <expected-root-id> <lease-path> <client-id> [entrypoint]',
   );
 }
 
 const leaseFd = openSync(leasePath, 'a+');
 const attempt = await launchOwnedRuntimeHostCandidate({
   rootPath,
   expectedRootId,
-  entrypoint: new URL('../../execution-candidate-main.js', import.meta.url),
-  idleGraceMs: 10_000,
+  // Tests that bound an owner-loss exit pass a test-only entry whose
+  // startup window they control; the production entry stays the default.
+  entrypoint: new URL(entrypointOverride ?? '../../execution-candidate-main.js', import.meta.url),
+  // The idle grace only has to outlast the test, and it has to stay clear of
+  // any bound a test puts on an owner-loss exit: a Candidate that exits
+  // because it went idle must never be mistaken for one that exited because
+  // its launch owner died. The first-connection deadline stays short so a
+  // Candidate no Client ever reaches still exits on its own.
+  idleGraceMs: 60_000,
+  initialConnectionTimeoutMs: 10_000,
   inheritableAuthorityLeaseFd: leaseFd,
   launchOwnerClientInstanceId: clientInstanceId,
 }).spawned;

packages/runtime-host/src/__tests__/host-kernel.test.ts

@@ -21,7 +21,7 @@ import { deferred, withTimeout } from '@maka/core/test-only/async-primitives';
 import { RuntimeHostProtocolError } from '../protocol/errors.js';
 import { defineInteractiveRuntimeHostComposition } from '../server/host-composition.js';
 import assert from 'node:assert/strict';
-import { readFileSync } from 'node:fs';
+import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { execFile, fork, type ChildProcess } from 'node:child_process';
 import {
   chmod,
@@ -2204,11 +2204,22 @@ describe('non-serving Runtime Host kernel', () => {
             capability.rootId,
             join(paths.base, 'authority-lease-probe'),
             launchOwnerClientInstanceId,
+            // The owner-loss exit bound below covers the gated recovery
+            // window, so this run pins startup behind the gated-recovery
+            // entry instead of leaving both the window and the kill's
+            // ordering to however scheduling resolves them.
+            '../../test-only/owned-candidate-gated-recovery-main.js',
           ],
           { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] },
         ),
       );
       const launchedPid = paths.resources.trackPid(await waitForLaunch(launcher));
+      // The gated-recovery entry parks composition creation behind these
+      // markers under the same base directory; the stall marker proves the
+      // Candidate reached the gated window, and the release marker lets the
+      // test unblock it through a channel that survives the launcher.
+      const stallMarker = join(paths.base, 'authority-lease-probe.stalled');
+      const releaseMarker = join(paths.base, 'authority-lease-probe.release');
       const connected = await retryConnect(paths, CURRENT_PROTOCOL, {
         clientInstanceId: launchOwnerClientInstanceId,
       });
@@ -2224,14 +2235,67 @@ describe('non-serving Runtime Host kernel', () => {
       });
       assert.equal(ordinary.kind, 'draining');
 
+      // The owner-loss contract under test is recorded pre-bind: a
+      // launch-owner Client is admitted while the Host is still recovering,
+      // and the guard that closes the Host on owner loss binds only after
+      // startup returns. Kill timing alone cannot prove the loss was
+      // recorded pre-bind — a test-side pause longer than the candidate's
+      // startup would silently turn this into the post-bind scenario — so
+      // the gated-recovery entry holds startup behind a release file and
+      // marks the stall; waiting for that marker makes the kill land inside
+      // the gated window by construction rather than by luck.
+      const stallDeadline = Date.now() + 10_000;
+      while (!existsSync(stallMarker) && Date.now() < stallDeadline) {
+        await sleep(20);
+      }
+      assert.ok(existsSync(stallMarker), 'gated-recovery entry never reached its stall window');
       launcher.kill('SIGKILL');
       await waitForExit(launcher);
+      // The process is the only thing that reports the claim. A Client's
+      // `connection.closed` does not: it is that Client's own transport, and
+      // the Client aborts it after its liveness probe goes unanswered for two
+      // seconds. A Host that is merely busy therefore resolves it while still
+      // running, so it is used only as a post-exit consistency check below.
+      //
+      // Startup — composition creation and recovery included — runs after the
+      // release and is not bounded by the kernel's shutdown grace, so the exit
+      // budget must not start at the release. The entry's `onWon` marker is
+      // the explicit guard-bound boundary that starts it instead: the
+      // launch-owner guard has bound and the pre-bind recorded loss is being
+      // acted on, so everything the 20-second deadline covers (the
+      // `shutdownGraceMs` close plus margin — which sits below the launcher's
+      // 60 s idle grace, so it cannot be satisfied by a Candidate that merely
+      // went idle) happens after the marker.
+      //
+      // The race below keeps that boundary honest without breaking local
+      // Windows runs: there the Candidate can be terminated abruptly the
+      // moment its launcher dies — no JS exit event, so no bind and no marker
+      // — and `isProcessAlive` releasing the wait only records that platform
+      // limitation, while a Candidate still alive without a marker past the
+      // deadline is a failure. The assertion observes the real
+      // operating-system PID: the kernel resolving its `closed` promise does
+      // not by itself mean the OS process has exited, so only CI verdicts
+      // count as cross-platform evidence here.
+      writeFileSync(releaseMarker, String(Date.now()));
+      const boundMarker = join(paths.base, 'authority-lease-probe.bound');
+      const boundDeadline = Date.now() + 10_000;
+      while (
+        !existsSync(boundMarker) &&
+        isProcessAlive(launchedPid) &&
+        Date.now() < boundDeadline
+      ) {
+        await sleep(20);
+      }
+      assert.ok(
+        existsSync(boundMarker) || !isProcessAlive(launchedPid),
+        'gated-recovery entry never reached its guard bind',
+      );
+      await waitForProcessExit(launchedPid, 20_000);
       await withTimeout(
         connected.connection.closed,
         5_000,
-        'authority-supervised Candidate survived its launch owner',
+        'authority-supervised Candidate exited without closing its Client connection',
       );
-      await waitForProcessExit(launchedPid);
       paths.resources.forgetPid(launchedPid);
     });
   });

packages/runtime-host/src/__tests__/owned-candidate.test.ts

@@ -37,6 +37,11 @@ import {
   type CandidateExitDetails,
   type OwnedCandidateAttempt,
 } from '../client/launcher.js';
+import {
+  resolveExistingStorageRoot,
+  resolveExistingStorageRootControlDirectory,
+} from '@maka/storage/root-authority';
+import { readHostRegistration } from '../control/registration.js';
 
 test('owned connection keeps a fresh Host alive for its full election window', async () => {
   const rootPath = await mkdtemp(join(tmpdir(), 'maka-owned-first-connection-'));
@@ -257,12 +262,20 @@ test('owned Host exits promptly after its first connection closes', async () =>
 
   assert.equal(result.kind, 'connected', connectFailure(result));
   if (result.kind !== 'connected') return;
+  const controlDirectory = await resolveHostControlDirectory(rootPath, result.connection.rootId);
   await result.connection.close();
-  // Prompt means the owned launch's idleGraceMs of 0, as opposed to the 30 s
-  // default grace, so the bound only has to sit well below that. Shutdown takes
-  // about 30 ms on an idle machine and stretches past 500 ms under a full CI
-  // suite while still exiting cleanly: the Host is starved, not stuck.
-  assert.equal(await result.host.settle(5_000), true);
+  // Promptness is when the Host starts shutting down, not how long shutting
+  // down takes: the owned launch's idleGraceMs is 0 against a 30 s default.
+  // The kernel publishes its draining registration as the first step of
+  // shutdown, so the registration reports the idle grace directly. Reading it
+  // from `settle` alone could not separate the two, which is why a loaded
+  // machine that only made the shutdown itself slow failed this assertion.
+  await waitForHostShutdownStart(controlDirectory, 10_000);
+  // The exit is a second claim with a bound of its own, and the kernel sets
+  // it: shutdown gets `shutdownGraceMs` (10 s) to close every resource before
+  // the kernel force-terminates the process. Anything below that fails a Host
+  // that is starved rather than stuck.
+  assert.equal(await result.host.settle(15_000), true);
 });
 
 test('an exited owned Candidate permits one real successor in the same election', {
@@ -436,6 +449,38 @@ test('pre-cancelled hosted execution does not start a Runtime Host', async () =>
   assert.deepEqual(await readdir(rootPath), []);
 });
 
+async function resolveHostControlDirectory(rootPath: string, rootId: string): Promise<string> {
+  const capability = await resolveExistingStorageRoot({
+    path: rootPath,
+    kind: 'interactive',
+    expectedRootId: rootId,
+  });
+  const { controlDirectory } = await resolveExistingStorageRootControlDirectory(capability);
+  return controlDirectory;
+}
+
+/**
+ * Resolves once the Host has begun shutting down. `draining` is the state the
+ * kernel publishes before it does any shutdown work, and the registration is
+ * removed near the end of that work, so either observation proves shutdown
+ * started; the Host was serving this Client, so its registration existed.
+ */
+async function waitForHostShutdownStart(
+  controlDirectory: string,
+  timeoutMs: number,
+): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    // readHostRegistration already maps a missing file to `undefined`; the
+    // unguarded await lets real I/O or decode errors fail this wait loudly
+    // instead of masquerading as "shutdown started".
+    const registration = await readHostRegistration(controlDirectory);
+    if (!registration || registration.state === 'draining') return;
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  }
+  throw new Error('owned Host did not begin shutting down after its first connection closed');
+}
+
 function connectFailure(
   result:
     | Awaited<ReturnType<typeof connectOwnedRuntimeHostWithDependencies>>

packages/runtime-host/src/test-only/owned-candidate-gated-recovery-main.ts

@@ -0,0 +1,67 @@
+#!/usr/bin/env node
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Test-only Candidate entry for the owner-loss lifecycle tests. A
+ * launch-owner Client is admitted while the Host is still recovering, and
+ * the guard that closes the Host on owner loss binds only after startup
+ * returns — so whether the test's kill lands before or after the bind is a
+ * scheduling race a fixed sleep cannot decide. This entry instead gates
+ * composition creation behind a release file: writing the stall marker
+ * proves the Candidate is parked before the bind, the test kills the
+ * launcher at that point, and releasing the gate afterwards lets startup
+ * return promptly so the recorded loss closes the Host under the kernel's
+ * `shutdownGraceMs`. The run still goes through the real Runtime Host
+ * composition — only its start is held at the gate. The `onWon` hook adds
+ * the second boundary the test needs: a marker written right after the
+ * guard binds, so the exit budget starts at the bind rather than at the
+ * release, which only unblocks startup.
+ */
+import { existsSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { runExecutionCandidateEntry } from '../candidate-entry.js';
+import { createExecutionRuntimeHostComposition } from '../server/execution-composition.js';
+
+const rootArgumentIndex = process.argv.indexOf('--root') + 1;
+const rootPath = process.argv[rootArgumentIndex];
+if (!rootPath) throw new Error('gated-recovery entry requires --root');
+const stallMarker = join(dirname(rootPath), 'authority-lease-probe.stalled');
+const releaseMarker = join(dirname(rootPath), 'authority-lease-probe.release');
+const boundMarker = join(dirname(rootPath), 'authority-lease-probe.bound');
+
+await runExecutionCandidateEntry(process.argv.slice(2), import.meta.url, {
+  // `onWon` fires right after the launch-owner guard binds (candidate-entry
+  // binds before invoking it), so this marker is the test's explicit
+  // guard-bound boundary: the pre-bind recorded loss starts acting only past
+  // it, which is where the exit budget under test actually begins.
+  onWon: () => {
+    writeFileSync(boundMarker, String(Date.now()));
+    return () => undefined;
+  },
+  dependencies: {
+    createComposition: async (context, compositionOptions) => {
+      writeFileSync(stallMarker, String(Date.now()));
+      while (!existsSync(releaseMarker)) {
+        await new Promise((resolve) => setTimeout(resolve, 20));
+      }
+      return createExecutionRuntimeHostComposition(context, compositionOptions);
+    },
+  },
+});