From 6530b261bb15f9dd26da4dea2c2a04006582cb4a Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:06:11 -0400 Subject: [PATCH] fix(runtime): retire authored starts refused before journal reservation A native consumer that refuses before reserving its runtime journal (for example native_graph_admission for a repository@sha256 image, the M3 next.5 repro) leaves no receipt, so Removed retirement could never succeed and the start intent stayed pending with no supported recovery. graph native inspect now refuses the distinct native_graph_unreserved on macOS only when the run's journal directory is exactly absent under private state directories and no foreground owner is published. Startup retires the exact start intent and source for that refusal after its consumer settled, reporting outcome `unreserved` with the original native code. `down --recover` retires a pending start the same way under fresh startup admission. Every other inspection result keeps all evidence; receipt and ownership checks are unchanged. Refs HACK-1272 Co-Authored-By: Claude Opus 5.5 --- docs/reference/native-authored-commands.md | 35 +++- packages/runtime-core/README.md | 6 +- .../provider/graph/native/foreground/tests.rs | 41 +++++ .../src/provider/graph/native/journal.rs | 34 ++++ .../src/provider/graph/native/runtime.rs | 13 ++ src/backends/native-authored-project-run.ts | 146 +++++++++++----- src/backends/native-authored-project-start.ts | 72 +++++++- .../native-authored-unreserved-start.ts | 112 +++++++++++++ src/backends/native-runtime-client.ts | 8 + src/lib/native-authored-command.ts | 35 +++- tests/native-authored-command.test.ts | 79 +++++++++ tests/native-authored-project-start.test.ts | 156 +++++++++++++++++- 12 files changed, 688 insertions(+), 49 deletions(-) create mode 100644 src/backends/native-authored-unreserved-start.ts diff --git a/docs/reference/native-authored-commands.md b/docs/reference/native-authored-commands.md index e27d7f184..8c42abcd4 100644 --- a/docs/reference/native-authored-commands.md +++ b/docs/reference/native-authored-commands.md @@ -68,7 +68,7 @@ provider contact. A refusal names no option value, path or project detail. | --- | --- | --- | | `up` | Whole-project foreground; `--branch`, `--profile` and `--env` (including `--env base`) keep their compiler contracts | `--detach`, `--json`, `--recover`, service subsets, single-service, command, workdir, follow, tail and format options, other unsupported options | | `down` | Whole-project owner-mediated stop of a ready generation through its live foreground owner or private no-host endpoint | `--json`, `--detach`, `--profile` and `--env` overrides, service subsets including an empty selection, single-service, command, workdir, follow, tail and format options | -| `down --recover` | Explicit retirement of a complete stored generation whose owners are dead; omit `--profile` | `--json`, `--detach`, explicit empty or named `--profile`, `--env` overrides, service subsets, single-service, command, workdir, follow, tail and format options | +| `down --recover` | Explicit retirement of a complete stored generation whose owners are dead, or of a pending startup intent that never reserved runtime state; omit `--profile` | `--json`, `--detach`, explicit empty or named `--profile`, `--env` overrides, service subsets, single-service, command, workdir, follow, tail and format options | | `ps` | `ps` and `ps --json` for the exact saved project and explicit `--branch` | `--profile`, `--env`, service subsets or a single service, `--recover`, `--detach`, command, workdir, follow, tail and format options | | `logs` | `logs SERVICE --no-follow` with `--tail` from 1 through 1000 (default 200), plain or `--json` | Follow mode (the default), omitted, multiple or malformed service selection, non-integer or out-of-range `--tail`, pretty format, time and query filters, `--profile`, `--env`, `--recover`, `--detach`, command and workdir options | | `exec` | `exec SERVICE -- COMMAND ARG...` with closed stdin, no TTY and an optional absolute `--workdir` | Empty command or empty first argument, NUL bytes, more than 256 arguments, an argument over 16 KiB or 64 KiB in total, relative `--workdir`, interactive or TTY input, `--json`, `--profile`, `--env`, service subsets, follow, tail and format options, `--recover`, `--detach` | @@ -90,6 +90,21 @@ inspection also refuses, the captured native startup code takes precedence over the inspection code. That preserves the failure's owning boundary; it does not prove cleanup. Retained bindings and recovery requirements stay in force. These diagnostics do not grant retry or cleanup authority. + +A native consumer can refuse after the startup intent is reserved but before it +reserves any runtime journal. For example, a `repository@sha256:` image reaches +execution and refuses `native_graph_admission`. Such a run has no receipt, so the +Removed retirement inspection can never succeed. After the consumer process has +settled, startup asks the native runtime to inspect the exact run. Startup retires +the startup intent and its source envelope only when the runtime refuses +`native_graph_unreserved`: the run's journal directory is exactly absent under +private state directories, and no foreground owner is published for the run. The +failure then reports outcome `unreserved` with the original native code, and `ps` +reports `not_started`. Any other inspection result keeps the attempt retained. +Examples include `native_graph_receipt` for damaged or foreign evidence, +`native_graph_foreground` for a published owner, and a stopped runtime. This +applies only to attempts without lifecycle hooks, host processes or a live-stop +owner. A canceled `unreserved` startup exits 130, like confirmed removal. Readiness requires authenticated current observations and unchanged input before publication. The foreground owner remains attached until owned shutdown finishes; Ctrl-C requests that shutdown. Exact durable Removed evidence retires the tagged @@ -107,7 +122,10 @@ Only fresh runtime observations supply state and health. Stored readiness and ph do not prove that a service is still live. A project with no saved attempt reports `not_started`; a retained startup intent -without a Ready mapping reports `pending` with no confirmed service rows. When a +without a Ready mapping reports `pending` with no confirmed service rows. That +state covers a startup in progress and a retained incomplete attempt. When no +startup is running, `down --recover` can retire the attempt if it never reserved +runtime state (see below). When a Ready generation is saved but the candidate runtime is stopped (native code `runtime_not_running`, or the authenticated `native_graph_foreground` control code with that exact cause), `ps` exits 0 and reports `runtime_not_running` with the @@ -184,6 +202,19 @@ nor acquires managed values or starts a provider. Omit `--profile` to recover the stored generation; explicit empty or named profile overrides both refuse. +When the selection has a retained startup intent but no Ready mapping and no +recovery intent, `down --recover` takes a different path. It handles only an +incomplete startup that never reserved runtime state. One example is an attempt +retained by an earlier CLI before this retirement existed. The command first +takes a fresh startup admission. A live or interrupted foreground CLI still holds +that admission, so the command refuses as busy in that case. It also requires that +no Ready mapping, lifecycle-hook owner or live-stop owner exists. It then inspects +the exact run. It retires the start intent and the validated source envelope only +when the native runtime refuses `native_graph_unreserved`. It removes the source +first, so an interrupted retirement keeps the start intent blocking. Any receipt, +any other refusal, or a successful inspection retains every file. This path does +not compile input, acquire values, serve a run or touch guest resources. + The closed version 3 publication and version 1 recovery selector remain supported with their original calendar boot qualifier; clock drift conservatively refuses that legacy recovery. New version 4 owners use version 2 UUID selectors. Missing, diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index eb308245d..db04d5344 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -264,7 +264,11 @@ ordinary frontend selection yet and does not implement reactive health or hooks. The optional feature exposes this bounded consumer through a distinct public CLI: `graph native plan --source-file FILE --json`, then `graph native run --source-file FILE --expect-review SHA --json`. -`graph native inspect|cleanup --run-id RUN --json` use its versioned native journal. These commands +`graph native inspect|cleanup --run-id RUN --json` use its versioned native journal. On macOS, +`inspect` refuses `native_graph_unreserved` before guest access only when the run's journal +directory is exactly absent under private state directories and no foreground owner is +published for the run. It is a point-in-time observation, not proof that no consumer can +still start. A partial, public or present journal keeps the ordinary receipt refusal. These commands are explicit bounded prerequisites, not full ordinary project parity. Their project network does not publish ports or grant host-service access. Normal native `hack up` continues to require the full source/storage/routing/host and foreground-owner contract. diff --git a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs index af74ff024..165e4a529 100644 --- a/packages/runtime-core/src/provider/graph/native/foreground/tests.rs +++ b/packages/runtime-core/src/provider/graph/native/foreground/tests.rs @@ -586,6 +586,47 @@ fn abandoned_publication_is_evidence_and_never_implicit_recovery_authority() { assert!(fixture.owner_root().join("owner.json").exists()); } +#[test] +fn inspect_names_only_an_exactly_absent_unpublished_journal_as_unreserved() { + // HACK-1272: a startup refused before reservation must be distinguishable from + // damaged or foreign receipt evidence, without weakening either refusal. + let fixture = Fixture::new(); + let code = |fixture: &Fixture| inspect(&fixture.candidate, RUN).unwrap_err().code; + assert_eq!(code(&fixture), "native_graph_unreserved"); + assert!( + !journal::directory(&fixture.candidate, RUN) + .unwrap() + .exists() + ); + + // A published (live or abandoned) owner is evidence, never "unreserved". + let receipt = fixture.receipt(); + let mut publication = owner::Publication::bind(&fixture.candidate, &receipt.review).unwrap(); + assert_eq!(code(&fixture), "native_graph_foreground"); + publication.finish().unwrap(); + assert_eq!(code(&fixture), "native_graph_unreserved"); + + // A partial journal directory keeps the ordinary receipt refusal. + let root = journal::directory(&fixture.candidate, RUN).unwrap(); + fs::DirBuilder::new() + .mode(0o700) + .recursive(true) + .create(&root) + .unwrap(); + assert_ne!(code(&fixture), "native_graph_unreserved"); + fs::remove_dir(&root).unwrap(); + + // A public ancestor refuses instead of proving absence. + let parent = root.parent().unwrap(); + fs::set_permissions(parent, fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(code(&fixture), "native_graph_receipt"); + fs::set_permissions(parent, fs::Permissions::from_mode(0o700)).unwrap(); + + // A reserved journal is validated as before. + journal::reserve(&fixture.candidate, &receipt).unwrap(); + assert_ne!(code(&fixture), "native_graph_unreserved"); +} + #[test] fn direct_operation_guard_refuses_replaced_lock_path() { let fixture = Fixture::new(); diff --git a/packages/runtime-core/src/provider/graph/native/journal.rs b/packages/runtime-core/src/provider/graph/native/journal.rs index 576f85a77..75908f5f4 100644 --- a/packages/runtime-core/src/provider/graph/native/journal.rs +++ b/packages/runtime-core/src/provider/graph/native/journal.rs @@ -555,6 +555,40 @@ fn valid_network_name(value: &str) -> bool { || matches!(*byte, b'-' | b'_' | b'.') }) } +/// Stable refusal for a run that never reserved native runtime state. +#[cfg(target_os = "macos")] +pub(super) fn unreserved() -> CandidateError { + error( + "native_graph_unreserved", + "Native runtime state for this run was never reserved: its journal is absent and no foreground owner is published. This point-in-time observation does not prove that no consumer can still start; no state was created, repaired or adopted.", + ) +} +/// True only when this run's journal directory is exactly absent beneath private +/// existing ancestors. A present, partial or unreadable path returns false so the +/// ordinary receipt validation still owns it. Read-only; nothing is created. +#[cfg(target_os = "macos")] +pub(super) fn absent(candidate: &Candidate, run: &str) -> Result { + let root = directory(candidate, run)?; + match fs::symlink_metadata(&root) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + _ => return Ok(false), + } + for path in root + .ancestors() + .skip(1) + .take_while(|path| path.starts_with(&candidate.state_root)) + { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => state::check_private_directory(path).map_err(|_| refused())?, + Err(_) => return Err(refused()), + } + } + Ok( + fs::symlink_metadata(&root) + .is_err_and(|error| error.kind() == std::io::ErrorKind::NotFound), + ) +} pub(super) fn directory(candidate: &Candidate, run: &str) -> Result { if !hex(run, 32) { return Err(refused()); diff --git a/packages/runtime-core/src/provider/graph/native/runtime.rs b/packages/runtime-core/src/provider/graph/native/runtime.rs index aa8acd468..074c93ea7 100644 --- a/packages/runtime-core/src/provider/graph/native/runtime.rs +++ b/packages/runtime-core/src/provider/graph/native/runtime.rs @@ -1286,7 +1286,20 @@ fn logs_with( } /// Read-only same-incarnation inspection. A historical ready phase is not current health. +/// +/// A run whose journal is exactly absent and whose foreground owner is not +/// published refuses `native_graph_unreserved` before any guest access. Absence +/// is a point-in-time observation: only a caller that separately proved its +/// consumer settled may treat it as "never reserved". Every other journal state +/// keeps the ordinary receipt validation below. pub fn inspect(candidate: &Candidate, run: &str) -> Result { + #[cfg(target_os = "macos")] + if journal::absent(candidate, run)? { + super::foreground::require_unpublished(candidate, run)?; + if journal::absent(candidate, run)? { + return Err(journal::unreserved()); + } + } let engine = Engine::connect_cleanup(candidate)?; let (receipt, _) = journal::load( candidate, diff --git a/src/backends/native-authored-project-run.ts b/src/backends/native-authored-project-run.ts index 316d8f7b7..308311c52 100644 --- a/src/backends/native-authored-project-run.ts +++ b/src/backends/native-authored-project-run.ts @@ -45,6 +45,10 @@ import { requestNativeAuthoredLiveStop, } from "./native-authored-live-stop.ts"; import type { NativeAuthoredProcessIncarnation } from "./native-authored-process-incarnation.ts"; +import { + NATIVE_GRAPH_UNRESERVED, + NativeRuntimeRequestError, +} from "./native-runtime-client.ts"; const LIMIT = 64 * 1024; const SOURCE_LIMIT = 1024 * 1024; @@ -140,6 +144,19 @@ export type NativeAuthoredProjectAdmission = { readonly expectedRun?: NativeAuthoredProjectRunSelection; readonly cleaned: NativeAuthoredReceipt; }) => Promise; + /** + * Retire a start intent whose native consumer never reserved runtime state + * (HACK-1272). `refusal` must be the fresh `native_graph_unreserved` inspection + * refusal for this exact run, obtained while this admission is held and after + * the caller proved no consumer for the run can still start. The ready mapping, + * hook owner and live-stop owner must be absent. The run's retained source is + * removed first (absence is accepted) so an interruption keeps the start + * intent blocking. No receipt, journal or guest state is read or changed here. + */ + readonly retireUnreserved: (opts: { + readonly expectedStart: NativeAuthoredProjectStartSelection; + readonly refusal: unknown; + }) => Promise; }; function refused(): never { throw new Error( @@ -540,43 +557,7 @@ export async function withNativeAuthoredProjectRecoveryStorage( ); }, async readSource(run) { - const current = await readPrivate(sourcePath(run), SOURCE_LIMIT); - const value: unknown = JSON.parse(current.text); - if ( - !( - isRecord(value) && - keys( - value, - value.version === 3 || value.version === 5 - ? "branch,env_metadata,hook_permit,kind,overlay,profiles,project,run,version" - : "branch,env_metadata,kind,overlay,profiles,project,run,version" - ) && - (value.version === 2 || - ((value.version === 3 || value.version === 5) && - validHookPin(value.hook_permit))) - ) || - value.kind !== "native-graph-source" || - value.project !== store.identity.projectRoot || - value.branch !== store.identity.branch || - value.run !== run || - !Array.isArray(value.profiles) || - value.profiles.length > 64 || - value.profiles.some( - (profile: unknown) => - typeof profile !== "string" || !validBranch(profile) - ) || - !( - value.overlay === "inherit" || - value.overlay === "base" || - (isRecord(value.overlay) && - keys(value.overlay, "named") && - typeof value.overlay.named === "string" && - validBranch(value.overlay.named)) - ) || - !parseNativeEnvMetadata(value.env_metadata) - ) { - return refused(); - } + const current = await readRetainedSource(store, sourcePath(run), run); await store.check(); return { dev: current.info.dev, @@ -587,6 +568,70 @@ export async function withNativeAuthoredProjectRecoveryStorage( }); }); } +/** Read one retained private source envelope and require its exact kind, scope and run. */ +async function readRetainedSource(store: Store, path: string, run: string) { + const current = await readPrivate(path, SOURCE_LIMIT); + const value: unknown = JSON.parse(current.text); + if ( + !( + isRecord(value) && + keys( + value, + value.version === 3 || value.version === 5 + ? "branch,env_metadata,hook_permit,kind,overlay,profiles,project,run,version" + : "branch,env_metadata,kind,overlay,profiles,project,run,version" + ) && + (value.version === 2 || + ((value.version === 3 || value.version === 5) && + validHookPin(value.hook_permit))) + ) || + value.kind !== "native-graph-source" || + value.project !== store.identity.projectRoot || + value.branch !== store.identity.branch || + value.run !== run || + !Array.isArray(value.profiles) || + value.profiles.length > 64 || + value.profiles.some( + (profile: unknown) => typeof profile !== "string" || !validBranch(profile) + ) || + !( + value.overlay === "inherit" || + value.overlay === "base" || + (isRecord(value.overlay) && + keys(value.overlay, "named") && + typeof value.overlay.named === "string" && + validBranch(value.overlay.named)) + ) || + !parseNativeEnvMetadata(value.env_metadata) + ) { + return refused(); + } + return current; +} +/** Remove this run's exact validated source envelope; an absent source is accepted. */ +async function removeRetainedSource( + store: Store, + path: string, + run: string, + check: () => Promise +): Promise { + let source: Awaited>; + try { + source = await readRetainedSource(store, path, run); + } catch (error) { + if (hasCode(error, "ENOENT")) { + return; + } + return refused(); + } + await check(); + const current = await readPrivate(path, SOURCE_LIMIT); + if (!sameFile(source.info, current.info) || current.text !== source.text) { + return refused(); + } + await unlink(path); + await synchronizeDirectories(store.held); +} async function read(store: Store): Promise { return await readRecord(store, store.file, selected); } @@ -1097,6 +1142,33 @@ export async function withNativeAuthoredProjectAdmission( await synchronizeDirectories(store.held); }); }, + async retireUnreserved(input) { + const expected = selection(input.expectedStart, started); + const run = expected.record.review.provenance.run; + if ( + !( + input.refusal instanceof NativeRuntimeRequestError && + input.refusal.nativeCode === NATIVE_GRAPH_UNRESERVED && + HEX32.test(run) + ) || + (await loadNativeAuthoredHookOwner(hookStore)) !== null + ) { + return refused(); + } + const sourcePath = join(store.root.path, `${run}.source.json`); + await withMutation(async (check) => { + await unchanged(store, store.startFile, expected, started); + await absent(store.file); + await absent(liveStopPath(store)); + await removeRetainedSource(store, sourcePath, run, check); + // If retirement stops here, the durable start continues to block a new run. + await unchanged(store, store.startFile, expected, started); + await absent(store.file); + await check(); + await unlink(store.startFile); + await synchronizeDirectories(store.held); + }); + }, }; try { return await action(Object.freeze(capability)); diff --git a/src/backends/native-authored-project-start.ts b/src/backends/native-authored-project-start.ts index 6133d9cd9..b2b28a943 100644 --- a/src/backends/native-authored-project-start.ts +++ b/src/backends/native-authored-project-start.ts @@ -63,6 +63,7 @@ import { } from "./native-project-process.ts"; import { invokeNativeRuntime, + NATIVE_GRAPH_UNRESERVED, NativeRuntimeRequestError, type NativeRuntimeSelection, } from "./native-runtime-client.ts"; @@ -71,7 +72,12 @@ const RUN = /^[a-f0-9]{32}$/; const CODE = /^[a-z][a-z0-9_]{0,63}$/; const PRIVATE_LIMIT = 256 * 1024; type Inputs = Awaited>; -type Outcome = "not-started" | "removed" | "retained"; +/** + * `unreserved`: the native consumer settled after refusing before it reserved + * any runtime journal or published a receipt, and the exact startup intent and + * source were retired. Nothing is retained. + */ +type Outcome = "not-started" | "removed" | "retained" | "unreserved"; const STAGES = [ "selection", "admission", @@ -137,6 +143,8 @@ export class NativeAuthoredProjectStartError extends Error { const detail = { "not-started": "no native consumer was started", removed: "exact native cleanup was confirmed", + unreserved: + "native runtime state was never reserved; the startup intent was retired and nothing is retained", retained: "startup evidence is retained; inspect owned state before retrying", }[opts.outcome]; @@ -428,6 +436,42 @@ async function retireAttempt(opts: { await attempt.source.remove(); } +/** + * HACK-1272: a consumer that refused before journal reservation (for example + * `native_graph_admission` for a non-content-ID image) leaves no receipt, so + * Removed retirement can never succeed. Retire the exact start intent and + * source only when the settled consumer's failure was never observed as a + * receipt and fresh inspection names the run `native_graph_unreserved` (an + * absent journal and no published foreground owner). Every other inspection + * refusal keeps all evidence. + */ +async function retireUnreservedAttempt(opts: { + readonly admission: NativeAuthoredProjectAdmission; + readonly attempt: Attempt; + readonly failure: unknown; + readonly refusal: unknown; +}): Promise { + if ( + opts.failure === undefined || + opts.attempt.observed !== undefined || + opts.attempt.ready !== undefined || + !( + opts.refusal instanceof NativeRuntimeRequestError && + opts.refusal.nativeCode === NATIVE_GRAPH_UNRESERVED + ) + ) { + return false; + } + // The exact source this attempt wrote is still unchanged; retirement removes + // it before the start intent so an interruption keeps the intent blocking. + await opts.attempt.source.assertFresh(); + await opts.admission.retireUnreserved({ + expectedStart: opts.attempt.start, + refusal: opts.refusal, + }); + return true; +} + const HOOK_STAGES: Record = { "up.before": "hook-up-before", "up.after": "hook-up-after", @@ -534,6 +578,7 @@ async function runPreparedLifecycle(ctx: { }) => void; readonly published: () => void; readonly removed: () => void; + readonly unreserved: () => void; readonly hookRetired: () => void; readonly nativeCode: (code: string | undefined) => void; }): Promise { @@ -677,7 +722,25 @@ async function runPreparedLifecycle(ctx: { await ctx.hookOwner.retire(); ctx.hookRetired(); } - await retireAttempt({ ...retirement, beforeRetire: liveStop?.retire }); + try { + await retireAttempt({ ...retirement, beforeRetire: liveStop?.retire }); + } catch (refusal) { + if ( + ctx.hookOwner !== undefined || + liveStop !== undefined || + !(await retireUnreservedAttempt({ + admission: ctx.admission, + attempt: ctx.admitted, + failure, + refusal, + })) + ) { + throw refusal; + } + ctx.unreserved(); + ctx.completeStop(false); + throw failure; + } ctx.removed(); ctx.completeStop(failure === undefined); if (failure) { @@ -1065,6 +1128,9 @@ export async function serveNativeAuthoredProject( removed: () => { outcome = "removed"; }, + unreserved: () => { + outcome = "unreserved"; + }, hookRetired: () => { hookOwner = undefined; }, @@ -1101,7 +1167,7 @@ export async function serveNativeAuthoredProject( host: hostProcesses, hookStop, storageTool, - retained: outcome !== "removed", + retained: outcome !== "removed" && outcome !== "unreserved", }); } } diff --git a/src/backends/native-authored-unreserved-start.ts b/src/backends/native-authored-unreserved-start.ts new file mode 100644 index 000000000..51e71ac31 --- /dev/null +++ b/src/backends/native-authored-unreserved-start.ts @@ -0,0 +1,112 @@ +import { + loadNativeAuthoredProjectRun, + type NativeAuthoredProjectRunScope, + withNativeAuthoredProjectAdmission, + withNativeAuthoredProjectStatus, +} from "./native-authored-project-run.ts"; +import { + invokeNativeRuntime, + type NativeRuntimeSelection, +} from "./native-runtime-client.ts"; + +function refused(): never { + throw new Error( + "Native startup intent is not a provably unreserved attempt; selected state is retained. Values omitted." + ); +} + +/** + * Read-only selection for explicit recovery: true only when a startup intent is + * retained without a Ready mapping or a frontend recovery intent. Any refusal + * (including a saved recovery intent) selects the stored-generation owner, which + * keeps its own refusals. Nothing is created, locked or repaired here. + */ +export async function nativeAuthoredUnreservedStartSelected( + scope: NativeAuthoredProjectRunScope +): Promise { + try { + return await withNativeAuthoredProjectStatus( + { ...scope }, + async (saved) => await Promise.resolve(saved.pending) + ); + } catch { + return false; + } +} + +/** + * Explicit `down --recover` for a startup intent whose native consumer never + * reserved runtime state (HACK-1272). It holds a fresh startup admission, so no + * live or interrupted foreground CLI (and therefore no consumer it spawned) can + * still own this attempt; a stale admission lock refuses as busy. Under that + * admission it requires no Ready mapping, hook owner or live-stop owner, asks the + * native runtime to inspect the exact run, and retires the start intent and its + * source only for the exact `native_graph_unreserved` refusal (journal absent and + * no foreground owner published). Any receipt, any other refusal, or a successful + * inspection retains every file. No input is acquired and nothing is served, + * replayed or cleaned in the guest. + */ +export async function retireNativeAuthoredUnreservedStart(opts: { + readonly scope: NativeAuthoredProjectRunScope; + readonly runtime: NativeRuntimeSelection; + readonly timeoutMs: number; + readonly signal?: AbortSignal; + /** Exact public native-request seam for isolated owner tests. */ + readonly request?: typeof invokeNativeRuntime; +}): Promise<{ readonly run: string }> { + const scope = { ...opts.scope }; + const runtime = { ...opts.runtime }; + const { timeoutMs, signal } = opts; + const request = opts.request ?? invokeNativeRuntime; + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 1 || + timeoutMs > 300_000 || + signal?.aborted || + runtime.home !== scope.nativeHome + ) { + return refused(); + } + const deadline = performance.now() + timeoutMs; + const remaining = () => { + const value = Math.floor(deadline - performance.now()); + if (value < 1 || signal?.aborted) { + return refused(); + } + return value; + }; + return await withNativeAuthoredProjectAdmission(scope, async (admission) => { + const start = await admission.loadStart(); + if ( + start === null || + (await admission.hooksRetained()) || + (await admission.liveStopRetained()) || + (await loadNativeAuthoredProjectRun(scope)) !== null + ) { + return refused(); + } + const run = start.record.review.provenance.run; + remaining(); + let refusal: unknown; + try { + await request({ + runtime, + cwd: scope.projectRoot, + args: ["graph", "native", "inspect", "--run-id", run, "--json"], + timeoutMs: remaining(), + signal, + boundNativeAuthoredReadDrain: true, + }); + } catch (error) { + refusal = error; + } + if (refusal === undefined) { + // A journal exists; this attempt belongs to receipt-owned recovery. + return refused(); + } + remaining(); + await admission.assertHeld(); + await admission.retireUnreserved({ expectedStart: start, refusal }); + return { run }; + }); +} diff --git a/src/backends/native-runtime-client.ts b/src/backends/native-runtime-client.ts index 765b5d77e..7d9e8a9be 100644 --- a/src/backends/native-runtime-client.ts +++ b/src/backends/native-runtime-client.ts @@ -41,6 +41,14 @@ export class NativeRuntimeRequestError extends ClosedCliError { } } +/** + * Native inspection code for a run whose journal is exactly absent and whose + * foreground owner is not published: no runtime state was ever reserved. It is + * point-in-time only; a caller must separately prove the run's consumer settled + * (or cannot exist) before treating it as retirement authority. + */ +export const NATIVE_GRAPH_UNRESERVED = "native_graph_unreserved"; + /** Native code for a read against a candidate runtime that is not running. */ export const NATIVE_RUNTIME_NOT_RUNNING = "runtime_not_running"; diff --git a/src/lib/native-authored-command.ts b/src/lib/native-authored-command.ts index 0187256bb..6690afa85 100644 --- a/src/lib/native-authored-command.ts +++ b/src/lib/native-authored-command.ts @@ -25,6 +25,10 @@ import { NativeAuthoredProjectStartError, serveNativeAuthoredProject, } from "../backends/native-authored-project-start.ts"; +import { + nativeAuthoredUnreservedStartSelected, + retireNativeAuthoredUnreservedStart, +} from "../backends/native-authored-unreserved-start.ts"; import { resolveNativeAuthoredExecStaleEnvWarning } from "../backends/native-exec-stale-env.ts"; import { parseNativeHttpsSelection } from "../backends/native-project-start.ts"; import { @@ -205,7 +209,10 @@ function assertForegroundOptions(options: NativeComposeCommandOptions): void { function startupFailure(error: unknown): number { if (error instanceof NativeAuthoredProjectStartError) { - if (error.canceled && error.outcome === "removed") { + if ( + error.canceled && + (error.outcome === "removed" || error.outcome === "unreserved") + ) { return 130; } throw new HackCliError({ @@ -296,7 +303,7 @@ const STATUS_NOTES: Readonly< > = { not_started: "Native project has not been started.", pending: - "Native project startup is pending; no current service status is confirmed.", + "Native project startup is pending or incomplete; no current service status is confirmed. If no startup is running, `hack down --recover` retires an attempt that never reserved runtime state.", runtime_not_running: "Native runtime is not running; no services are running.", }; @@ -369,6 +376,9 @@ type Seams = { /** Execution seam for command-selection controls; the CLI always uses the real owner. */ readonly serve?: typeof serveNativeAuthoredProject; readonly recover?: typeof recoverNativeAuthoredProject; + /** Pending-start selection and unreserved retirement seams (HACK-1272). */ + readonly unreservedSelected?: typeof nativeAuthoredUnreservedStartSelected; + readonly retireUnreserved?: typeof retireNativeAuthoredUnreservedStart; readonly observe?: typeof nativeAuthoredProjectPs; readonly logs?: typeof nativeAuthoredProjectLogs; readonly stop?: typeof stopNativeAuthoredProject; @@ -408,6 +418,27 @@ async function runStatus(input: LifecycleInput): Promise { } async function runRecovery(input: LifecycleInput): Promise { + if ( + await ( + input.seams.unreservedSelected ?? nativeAuthoredUnreservedStartSelected + )(input.scope) + ) { + // A startup intent without a Ready generation: only a provably unreserved + // attempt is retired; every other retained attempt refuses unchanged. + await (input.seams.retireUnreserved ?? retireNativeAuthoredUnreservedStart)( + { + runtime: input.runtime, + scope: input.scope, + timeoutMs: input.startupTimeoutMs, + signal: input.signal, + } + ); + logger.info({ + message: + "Retired an incomplete native startup that never reserved runtime state; nothing else was changed.", + }); + return 0; + } await (input.seams.recover ?? recoverNativeAuthoredProject)({ runtime: input.runtime, scope: input.scope, diff --git a/tests/native-authored-command.test.ts b/tests/native-authored-command.test.ts index cbb484264..b491bef7c 100644 --- a/tests/native-authored-command.test.ts +++ b/tests/native-authored-command.test.ts @@ -226,6 +226,85 @@ macTest( } ); +macTest( + "HACK-1272: down --recover retires a pending unreserved start without the generation owner", + async () => { + const selected = await fixture(); + const options = { + ...selected.options, + operation: "down" as const, + recover: true, + instance: "alpha", + }; + const scope = { + projectRoot: selected.root, + projectDir: join(selected.root, ".hack"), + nativeHome: join(selected.root, "candidate"), + branch: "alpha", + }; + const retired: unknown[] = []; + const outcome = (retire: () => Promise<{ readonly run: string }>) => + tryNativeAuthoredCommand({ + ...selected, + options, + serve: () => { + throw new Error("Recovery cannot invoke startup."); + }, + recover: () => { + throw new Error("A pending start has no stored generation."); + }, + unreservedSelected: (value) => { + expect(value).toEqual(scope); + return Promise.resolve(true); + }, + retireUnreserved: (value) => { + retired.push(value); + return retire(); + }, + }).catch((error: unknown) => error); + expect(await outcome(() => Promise.resolve({ run: "a".repeat(32) }))).toBe( + 0 + ); + expect(retired).toHaveLength(1); + expect(retired[0]).toMatchObject({ scope, timeoutMs: 1500 }); + const refused = await outcome(() => + Promise.reject(new Error("Private retained detail.")) + ); + expect(refused).toHaveProperty("code", "E_LIFECYCLE_FAILED"); + expect(String(refused)).not.toContain("Private retained detail"); + expect(await readdir(join(selected.root, ".hack"))).toEqual([]); + } +); + +macTest( + "HACK-1272: a canceled unreserved startup exits 130 like confirmed removal", + async () => { + const selected = await fixture(); + for (const [outcome, expected] of [ + ["unreserved", 130], + ["retained", "E_STARTUP_INCOMPLETE"], + ] as const) { + const result = await tryNativeAuthoredCommand({ + ...selected, + serve: () => + Promise.reject( + new NativeAuthoredProjectStartError({ + outcome, + canceled: true, + stage: "runtime", + nativeCode: "native_graph_canceled", + }) + ), + }).catch((error: unknown) => error); + if (typeof expected === "number") { + expect(result).toBe(expected); + } else { + expect(result).toHaveProperty("code", expected); + } + } + } +); + macTest( "recovery refuses env/profile/effect overrides before runtime pin or stored owner selection", async () => { diff --git a/tests/native-authored-project-start.test.ts b/tests/native-authored-project-start.test.ts index 437adc5ec..3fcfd9d0f 100644 --- a/tests/native-authored-project-start.test.ts +++ b/tests/native-authored-project-start.test.ts @@ -6,12 +6,14 @@ import { mkdir, mkdtemp, readdir, + readFile, realpath, rm, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { nativeAuthoredProjectPs } from "../src/backends/native-authored-project-observe.ts"; import { loadNativeAuthoredProjectRun, stopNativeAuthoredProject, @@ -26,6 +28,14 @@ import { NATIVE_STORAGE_TOOL_MANIFEST, nativeStorageToolManifest, } from "../src/backends/native-authored-storage-tool.ts"; +import { + nativeAuthoredUnreservedStartSelected, + retireNativeAuthoredUnreservedStart, +} from "../src/backends/native-authored-unreserved-start.ts"; +import { + type invokeNativeRuntime, + NativeRuntimeRequestError, +} from "../src/backends/native-runtime-client.ts"; import { type ComposeEnvFingerprintRecord, evaluateComposeServiceEnvFreshness, @@ -198,6 +208,10 @@ type FixtureOptions = { readonly semanticAfterHook?: boolean; readonly planFailure?: boolean; readonly failBeforeReceipt?: boolean; + /** Native startup code for failBeforeReceipt; defaults to graph_network_intent. */ + readonly failBeforeReceiptCode?: string; + /** Inspection after failBeforeReceipt proves the journal was never reserved. */ + readonly unreservedInspection?: boolean; readonly foreignPlan?: boolean; readonly foreignNamespace?: boolean; readonly branch?: string | null; @@ -444,7 +458,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ const digests=Object.fromEntries(Object.entries(services).map(([name,values])=>[name,Object.fromEntries(Object.entries(values).map(([key,value])=>[key,createHash('sha256').update(value).digest('hex')]))])); await writeFile(${JSON.stringify(delivery)},JSON.stringify({version:input.version,kind:input.kind,review:input.review,run:input.run,lifetime_seconds:input.lifetime_seconds,keys,digests,expectedDigest:${JSON.stringify(digest)}})); } - if(${options.failBeforeReceipt === true}){console.error(JSON.stringify({code:'graph_network_intent',message:${JSON.stringify(CANARY)}}));process.exit(2)} + if(${options.failBeforeReceipt === true}){console.error(JSON.stringify({code:${JSON.stringify(options.failBeforeReceiptCode ?? "graph_network_intent")},message:${JSON.stringify(CANARY)}}));process.exit(2)} const receipt={version:2,kind:'native-graph-runtime',owner:'f'.repeat(32),boot:'00000000-0000-0000-0000-000000000001',review,phase:'ready-observed',readiness:{web:'started'},resources:{'network:default':{kind:'network',key:'default',name:'hkn-'+input.run+'-network-0',id:'1'.repeat(64),image:null,phase:'created',outbound:true},'container:web':{kind:'container',key:'web',name:'hkn-'+input.run+'-container-0',id:'2'.repeat(64),image:'sha256:'+ '3'.repeat(64),phase:'started',networks:['default']}}}; if(${options.storage === true && options.liveStop === true}){ const namespace=provenance.namespace, owner='9'.repeat(32); @@ -469,7 +483,7 @@ if(['plan','serve','frontend-plan','frontend-serve'].includes(action)){ } await finish(); }else{ - if(action==='inspect'&&${options.failBeforeReceipt === true}){console.error(JSON.stringify({code:'native_graph_receipt',message:${JSON.stringify(CANARY)}}));process.exit(2)} + if(action==='inspect'&&${options.failBeforeReceipt === true}){console.error(JSON.stringify({code:${JSON.stringify(options.unreservedInspection === true ? "native_graph_unreserved" : "native_graph_receipt")},message:${JSON.stringify(CANARY)}}));process.exit(2)} if(action==='inspect'&&${JSON.stringify(options.cleanup)}==='missing'){console.error(JSON.stringify({code:'graph_owner_recovery',message:'private-missing-journal-detail'}));process.exit(2)} const receipt=JSON.parse(await readFile(journal,'utf8')); const observations={web:receipt.phase==='removed'?null:{state:'running',health:'none'}}; @@ -505,7 +519,7 @@ async function artifacts(scope: Awaited>["scope"]) { } async function failure( value: Promise, - outcome: "not-started" | "removed" | "retained" + outcome: "not-started" | "removed" | "retained" | "unreserved" ) { const error: unknown = await value.catch((caught: unknown) => caught); expect(error).toBeInstanceOf(NativeAuthoredProjectStartError); @@ -534,7 +548,141 @@ macTest( expect(retained).toHaveLength(2); expect(retained.some((name) => name.endsWith(".start.json"))).toBe(true); expect(retained.some((name) => name.endsWith(".source.json"))).toBe(true); - } + }, + 30_000 +); + +macTest( + "HACK-1272: a consumer refused before journal reservation retires its exact start intent", + async () => { + // M3 next.5 repro: a repository@digest image passes planning, then the + // native consumer refuses native_graph_admission before reserving a journal. + // Retirement inspection can never authenticate a Removed receipt; only the + // exact native_graph_unreserved refusal may retire the attempt. + const selected = await fixture({ + failBeforeReceipt: true, + failBeforeReceiptCode: "native_graph_admission", + unreservedInspection: true, + }); + const error = await failure( + serveNativeAuthoredProject({ + ...selected, + run, + startupTimeoutMs: 60_000, + }), + "unreserved" + ); + expect(error).toHaveProperty("nativeCode", "native_graph_admission"); + expect(error).toHaveProperty("stage", "runtime"); + expect(String(error)).toContain("never reserved"); + expect(await artifacts(selected.scope)).toEqual([]); + const status = await nativeAuthoredProjectPs({ + runtime: selected.runtime, + scope: selected.scope, + invoke: () => { + throw new Error("No runtime request for an absent attempt."); + }, + }); + expect(status.status).toBe("not_started"); + const calls = (await readFile(selected.calls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)[4]); + expect(calls).toEqual(["plan", "serve", "inspect"]); + }, + 90_000 +); + +macTest( + "HACK-1272: down --recover retires only a provably unreserved retained start", + async () => { + // Reproduce the next.5 retained state: inspection named the absent journal + // native_graph_receipt, so start and source were retained and ps is pending. + const selected = await fixture({ + failBeforeReceipt: true, + failBeforeReceiptCode: "native_graph_admission", + }); + await failure( + serveNativeAuthoredProject({ + ...selected, + run, + startupTimeoutMs: 60_000, + }), + "retained" + ); + expect(await artifacts(selected.scope)).toHaveLength(2); + const ps = () => + nativeAuthoredProjectPs({ + runtime: selected.runtime, + scope: selected.scope, + invoke: () => { + throw new Error("No runtime request without a Ready mapping."); + }, + }); + expect((await ps()).status).toBe("pending"); + expect(await nativeAuthoredUnreservedStartSelected(selected.scope)).toBe( + true + ); + const requests: string[][] = []; + const refusing = + (code: string | null): typeof invokeNativeRuntime => + (input) => { + requests.push([...input.args]); + if (code === null) { + return Promise.resolve({ receipt: {}, observations: {} }); + } + return Promise.reject( + new NativeRuntimeRequestError({ + message: "synthetic refusal", + nativeCode: code, + }) + ); + }; + const recover = (code: string | null) => + retireNativeAuthoredUnreservedStart({ + scope: selected.scope, + runtime: selected.runtime, + timeoutMs: 10_000, + request: refusing(code), + }); + // Damaged/foreign receipt evidence, a published owner, a stopped runtime + // and a present journal all keep every file. + for (const code of [ + "native_graph_receipt", + "native_graph_foreground", + "runtime_not_running", + null, + ]) { + expect( + await recover(code).catch((caught: unknown) => caught) + ).toBeInstanceOf(Error); + expect(await artifacts(selected.scope)).toHaveLength(2); + expect((await ps()).status).toBe("pending"); + } + expect(await recover("native_graph_unreserved")).toEqual({ run }); + expect(requests.at(-1)).toEqual([ + "graph", + "native", + "inspect", + "--run-id", + run, + "--json", + ]); + expect(await artifacts(selected.scope)).toEqual([]); + expect((await ps()).status).toBe("not_started"); + expect(await nativeAuthoredUnreservedStartSelected(selected.scope)).toBe( + false + ); + // Nothing remains to retire; a second recovery refuses without a request. + const before = requests.length; + expect( + await recover("native_graph_unreserved").catch( + (caught: unknown) => caught + ) + ).toBeInstanceOf(Error); + expect(requests).toHaveLength(before); + }, + 90_000 ); test("startup diagnostics accept only closed compiler codes", () => {