diff --git a/docs/reference/native-compose-generation.md b/docs/reference/native-compose-generation.md index 01d2373c..a6bb9af0 100644 --- a/docs/reference/native-compose-generation.md +++ b/docs/reference/native-compose-generation.md @@ -73,7 +73,12 @@ are never diagnostic fields. Copied error properties do not acquire provenance. The diagnostic does not grant recovery, clear intent or change the original error code, checks, child lifetime or deadline. Missing detail leaves the cause unknown; returned incomplete outcomes retain their existing reporting contract. Historical -failures without this detail remain unexplained. +failures without this detail remain unexplained. An unclassified native failure +(fixed `E_COMPOSE_FAILED` text, for example a storage-carrier refusal during +`exec`) carries the same detail when its owner issued one. Without `--json`, the +CLI prints the same pair after the error line as +`Refusal boundary: (); values omitted.`; detail with any other +shape prints nothing extra. `publishRunProjection` derives a one-off document from the verified saved artifact inside the mutation lock. It accepts a target service, never caller-supplied JSON. diff --git a/docs/reference/native-compose-storage-xattr-carrier.md b/docs/reference/native-compose-storage-xattr-carrier.md index e9a000ef..9aeb0876 100644 --- a/docs/reference/native-compose-storage-xattr-carrier.md +++ b/docs/reference/native-compose-storage-xattr-carrier.md @@ -381,7 +381,8 @@ publishes and synchronizes that identity, then repeats identity, boot, executabl and admission checks before a one-use continuation of the captured child. A missing or failed publication cancels that original invocation; an armed record alone proves neither spawn nor absence. The command's original budget includes -the handshake. No guessed PID or retry is used to rescue a fast command. +the handshake, and each process identity read shares that remaining budget rather +than a tighter fixed bound. No guessed PID or retry is used to rescue a fast command. Live handshake and settlement budgets use a captured monotonic deadline; the saved wall deadline is a record field only. Executable device/inode identities use exact decimal strings from bigint metadata, including sealed macOS binaries. @@ -392,9 +393,14 @@ The carrier uses held regular capture files, not pipes. Version 2 therefore name its capture evidence `held-files-quiescent`, never EOF. The original exit callback, strict ESRCH group absence, held file/path identity, both synchronized bounded captures and their hashes precede settlement publication. EPERM/EIO and unknown -exit or capture disposition refuse. Settlement publication shares the existing -three-second post-command group-observation budget; timeout/cancellation remain -refused results and cannot grant a subsequent daemon command. +exit or capture disposition refuse. Settlement publication shares one finite +15-second post-command budget with group observation and its identity reads; +timeout/cancellation remain refused results and cannot grant a subsequent daemon +command. These budgets must outlast an ordinary multi-second host stall: a stalled +owner resumes with its timer already due, and a refusal after the daemon effect +leaves durable uncertainty that only explicit recovery can report. Earlier +one-second identity reads and a three-second settlement budget refused healthy +commands that way on loaded hosts. Only the original in-memory writer can advance its exact sequence. Publication failure or deadline permanently invalidates that writer and retains its private diff --git a/src/cli/run.ts b/src/cli/run.ts index 375d8377..8c83a23f 100644 --- a/src/cli/run.ts +++ b/src/cli/run.ts @@ -187,6 +187,16 @@ export async function handleRunCliError(opts: { if (error instanceof HackCliError) { logger.error({ message: `${error.code}: ${error.message}` }); + if (error.detail !== undefined) { + // Loaded only for a detailed error, off the ordinary CLI start path. + const { nativeComposeEffectRefusalSummary } = await import( + "../lib/native-compose-effect-diagnostics.ts" + ); + const summary = nativeComposeEffectRefusalSummary(error.detail); + if (summary) { + logger.error({ message: summary }); + } + } logDebugStack(error); return 1; } diff --git a/src/lib/native-compose-command.ts b/src/lib/native-compose-command.ts index 87c510fa..fd69115c 100644 --- a/src/lib/native-compose-command.ts +++ b/src/lib/native-compose-command.ts @@ -2456,9 +2456,15 @@ function throwNativeComposeCommandError(error: unknown): never { if (error instanceof HackCliError || error instanceof CliUsageError) { throw error; } + // An unclassified failure (for example a storage-carrier refusal) keeps the + // closed boundary its owner issued; the code and fixed text are unchanged. + const diagnostic = nativeComposeEffectRefusal(error); throw new HackCliError({ code: "E_COMPOSE_FAILED", message: "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + ...(diagnostic + ? { detail: { native_compose_effect_refusal: diagnostic } } + : {}), }); } diff --git a/src/lib/native-compose-effect-diagnostics.ts b/src/lib/native-compose-effect-diagnostics.ts index 9d0913f6..7e4a5a84 100644 --- a/src/lib/native-compose-effect-diagnostics.ts +++ b/src/lib/native-compose-effect-diagnostics.ts @@ -255,6 +255,38 @@ export function nativeComposeEffectReason( return "unclassified"; } +function ownData(value: unknown, key: string): unknown { + if (typeof value !== "object" || value === null) { + return undefined; + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + return descriptor && Object.hasOwn(descriptor, "value") + ? descriptor.value + : undefined; +} + +/** + * One fixed human line for the closed boundary carried in public CLI error + * detail, so a command without `--json` (such as `exec`) still names it. + * Anything but a known stage and reason pair yields null; accessors are not read. + */ +export function nativeComposeEffectRefusalSummary( + detail: unknown +): string | null { + try { + const refusal = ownData(detail, "native_compose_effect_refusal"); + const stage = stages.find((value) => value === ownData(refusal, "stage")); + const reason = reasons.find( + (value) => value === ownData(refusal, "reason") + ); + return stage && reason + ? `Refusal boundary: ${stage} (${reason}); values omitted.` + : null; + } catch { + return null; + } +} + /** Keep the original issued boundary through a deliberate fixed-code translation. */ export function copyNativeComposeEffectRefusal( source: unknown, diff --git a/src/lib/native-compose-storage-witness-docker.ts b/src/lib/native-compose-storage-witness-docker.ts index c8e06218..edfe267f 100644 --- a/src/lib/native-compose-storage-witness-docker.ts +++ b/src/lib/native-compose-storage-witness-docker.ts @@ -105,6 +105,15 @@ const PROCESS = const TICKS = /^[1-9][0-9]*$/; const BOOT = /^[a-f0-9]{8}-(?:[a-f0-9]{4}-){3}[a-f0-9]{12}$/; const CAPTURE_LIMIT = 4096; +/** + * Post-exit settlement budget of a recorded command (group absence, boot and + * executable identity, capture sync and settlement publication). It and the + * identity reads must outlast an ordinary multi-second host stall: a stalled + * owner resumes with its timer already due, so a sub-second or three-second + * bound refused healthy commands after their daemon effect and left durable + * uncertainty (HACK-1281). It stays finite and grants nothing. + */ +const SETTLEMENT_BUDGET_MS = 15_000; // A late/unconfirmed publication cannot release its original held directory FDs. const unsettledCommandFiles = new Set(); type Selection = { readonly name: string; readonly storage: string }; @@ -810,7 +819,7 @@ async function groupAbsent( return refuse(); } if (now() >= deadline) { - return refuse(); + return refuseCommand("probe-budget"); } await new Promise((resolve) => setTimeout(resolve, 20)); } @@ -903,6 +912,42 @@ async function commandBoot( } /** No callback can reject before the shared owner awaits exit. A rejected owner or * unproven group retains the journal; daemon effects are never inferred from exit. */ +type StorageCommandRefusal = "probe-timeout" | "probe-cancel" | "probe-budget"; +const storageCommandRefusals = new WeakMap(); +/** The same fixed refusal, keeping only its closed command cause (a timed-out or + * cancelled child, or an exhausted publication/settlement budget) for the effect + * diagnostic. It grants nothing and changes no check, deadline or retention. */ +function refuseCommand(reason: StorageCommandRefusal | undefined): never { + const error = new Error( + "Native storage xattr proof refused; values omitted." + ); + if (reason) { + storageCommandRefusals.set(error, reason); + } + throw error; +} +function storageCommandRefusal( + error: unknown +): StorageCommandRefusal | undefined { + return typeof error === "object" && error !== null + ? storageCommandRefusals.get(error) + : undefined; +} +function exitRefusal( + exit: RunExitEvent | null +): StorageCommandRefusal | undefined { + if (exit?.timedOut) { + return "probe-timeout"; + } + return exit?.cancelled ? "probe-cancel" : undefined; +} +/** Effect reason for a carrier boundary: a classified command cause first. */ +export function nativeComposeStorageCarrierReason( + error: unknown +): NativeComposeEffectRefusal["reason"] { + return storageCommandRefusal(error) ?? nativeComposeEffectReason(error); +} + export async function runNativeComposeStorageDockerCommand(supplied: { readonly context: Pick; readonly files: Pick< @@ -958,6 +1003,7 @@ export async function runNativeComposeStorageDockerCommand(supplied: { exit: RunExitEvent | null = null; let armed: NativeComposeStorageArmedCommand | null = null; let publicationRefused = false; + let publicationCause: StorageCommandRefusal | undefined; const cancellation = new AbortController(); try { await opts.assertAdmitted(); @@ -965,23 +1011,31 @@ export async function runNativeComposeStorageDockerCommand(supplied: { ? opts.timeoutMs : Math.min(opts.timeoutMs, opts.context.deadline - Date.now()); if (!(timeoutMs > 0)) { - return refuse(); + return refuseCommand("probe-budget"); } const deadline = Date.now() + timeoutMs; let boundaryDeadline = performance.now() + timeoutMs; let recordingSettlement = false; const record = opts.originalCommand; const guard = () => { + if (performance.now() >= boundaryDeadline) { + return refuseCommand("probe-budget"); + } + if (publicationRefused) { + return refuseCommand(publicationCause); + } if ( - performance.now() >= boundaryDeadline || - publicationRefused || - (!(recordingSettlement || opts.cleanup) && opts.context.signal.aborted) + !(recordingSettlement || opts.cleanup) && + opts.context.signal.aborted ) { - return refuse(); + return refuseCommand("probe-cancel"); } }; const bounded = async (work: Promise): Promise => { let timer: ReturnType | undefined; + // An exhausted budget refuses below before the race observes the work; + // its own rejection must not escape as an unhandled process error. + work.catch(() => undefined); try { guard(); const result = await Promise.race([ @@ -990,6 +1044,7 @@ export async function runNativeComposeStorageDockerCommand(supplied: { timer = setTimeout( () => { publicationRefused = true; + publicationCause ??= "probe-budget"; if (record) { invalidateNativeComposeStorageCommandOwner(record.owner); } @@ -1006,13 +1061,14 @@ export async function runNativeComposeStorageDockerCommand(supplied: { ]); guard(); return result; - } catch { + } catch (error) { publicationRefused = true; + publicationCause ??= storageCommandRefusal(error); if (record) { invalidateNativeComposeStorageCommandOwner(record.owner); } cancellation.abort(); - return refuse(); + return refuseCommand(publicationCause); } finally { if (timer !== undefined) { clearTimeout(timer); @@ -1036,7 +1092,8 @@ export async function runNativeComposeStorageDockerCommand(supplied: { guard(); return Promise.resolve(); }, - timeoutMs: Math.min(1000, boundaryDeadline - performance.now()), + // Shares the enclosing handshake or settlement budget; no tighter bound. + timeoutMs: boundaryDeadline - performance.now(), }); if (result.exitCode !== 0) { return refuse(); @@ -1156,8 +1213,9 @@ export async function runNativeComposeStorageDockerCommand(supplied: { if (!event.resumeSuspended?.()) { return refuse(); } - } catch { + } catch (error) { publicationRefused = true; + publicationCause ??= storageCommandRefusal(error); invalidateNativeComposeStorageCommandOwner(record.owner); cancellation.abort(); } @@ -1168,12 +1226,15 @@ export async function runNativeComposeStorageDockerCommand(supplied: { }, } ); - boundaryDeadline = performance.now() + 3000; + boundaryDeadline = performance.now() + SETTLEMENT_BUDGET_MS; recordingSettlement = true; await groupAbsent(pid, record ? boundaryDeadline : undefined); settled = true; if (record) { - if (publicationRefused || !armed || !exit) { + if (publicationRefused) { + return refuseCommand(publicationCause); + } + if (!(armed && exit)) { return refuse(); } if ( @@ -1208,13 +1269,13 @@ export async function runNativeComposeStorageDockerCommand(supplied: { !(exit as RunExitEvent).cancelled ) ) { - return refuse(); + return refuseCommand(exitRefusal(exit)); } return { exitCode: code, stdout: io.read().stdout }; } catch (error) { if (opts.originalCommand) { invalidateNativeComposeStorageCommandOwner(opts.originalCommand.owner); - return refuse(); + return refuseCommand(storageCommandRefusal(error)); } throw error; } finally { @@ -1334,7 +1395,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1375,7 +1436,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage: "storage-target", - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1535,7 +1596,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -1689,7 +1750,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage: "storage-helper-policy", - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } @@ -2026,7 +2087,7 @@ export async function createNativeComposeDockerStorageXattrCarrier( } catch (error) { retainNativeComposeEffectRefusal(error, { stage, - reason: nativeComposeEffectReason(error), + reason: nativeComposeStorageCarrierReason(error), }); throw error; } diff --git a/tests/cli-closed-error-output.test.ts b/tests/cli-closed-error-output.test.ts index a31432fa..34a7fcce 100644 --- a/tests/cli-closed-error-output.test.ts +++ b/tests/cli-closed-error-output.test.ts @@ -101,6 +101,62 @@ test("an unexpected exception keeps its stack", async () => { expect(output).toMatch(STACK_FRAME); }); +test("a coded native refusal also names its closed boundary without --json", async () => { + const coded = new HackCliError({ + code: "E_COMPOSE_FAILED", + message: "Native Compose operation failed. Values omitted.", + detail: { + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }, + }); + expect(await handleRunCliError({ error: coded, jsonRequested: false })).toBe( + 1 + ); + expect(output).toBe( + "ERROR: E_COMPOSE_FAILED: Native Compose operation failed. Values omitted.\nERROR: Refusal boundary: storage-helper-start (probe-timeout); values omitted.\n" + ); +}); + +test("unknown or accessor-backed boundary detail prints nothing extra", async () => { + const canary = "private-boundary-canary"; + for (const detail of [ + { + native_compose_effect_refusal: { stage: canary, reason: "probe-timeout" }, + }, + { + native_compose_effect_refusal: { + stage: "storage-target", + reason: canary, + }, + }, + { native_compose_effect_refusal: "storage-target" }, + { + native_compose_effect_refusal: { + get stage() { + return "storage-target"; + }, + reason: "unclassified", + }, + }, + { other: { stage: "storage-target", reason: "unclassified" } }, + canary, + ]) { + output = ""; + const coded = new HackCliError({ + code: "E_COMPOSE_FAILED", + message: "Closed refusal.", + detail, + }); + expect( + await handleRunCliError({ error: coded, jsonRequested: false }) + ).toBe(1); + expect(output).toBe("ERROR: E_COMPOSE_FAILED: Closed refusal.\n"); + } +}); + test("--json keeps one envelope for a closed refusal", async () => { const error = nativeRefusal(); expect(await handleRunCliError({ error, jsonRequested: true })).toBe(1); diff --git a/tests/e2e/native-config-networks-acceptance.ts b/tests/e2e/native-config-networks-acceptance.ts index 633cb044..1b23d104 100644 --- a/tests/e2e/native-config-networks-acceptance.ts +++ b/tests/e2e/native-config-networks-acceptance.ts @@ -19,6 +19,7 @@ import type { } from "../../packages/config-compiler/generated/native-config.ts"; import { isRecord } from "../../src/lib/guards.ts"; import { + NativeComposeGenerationError, type NativeComposeIdentity, openNativeComposeGenerationStore, readNativeComposeNetworkTopology, @@ -39,6 +40,7 @@ const GENERATION = "io.hack.native-config.generation"; const STORAGE = "io.hack.native-config.storage"; const GLOBAL_PROJECT = "hack-dev-proxy"; const FIXTURE = "hack.e2e.native-config-networks"; +const CARRIER_OWNER = "io.hack.storage-witness.owner"; const OUTPUT_LIMIT = 2 * 1024 * 1024; const COMMAND_TIMEOUT = 180_000; // Witness-bearing exec/up verification runs three fresh helper carriers per proof; on the @@ -102,6 +104,411 @@ export function nativeNetworkFixtureRefusalDiagnostic(opts: { dockerInvoked: opts.dockerInvoked, })}`; } +const HACK_VERBS = [ + "up", + "down", + "exec", + "run", + "restart", + "ps", + "config", +] as const; +const DOCKER_VERBS = [ + "container", + "network", + "volume", + "image", + "info", + "compose", + "exec", +] as const; +const DOCKER_ACTIONS = [ + "create", + "start", + "stop", + "rm", + "inspect", + "ls", + "version", +] as const; +export const NATIVE_NETWORK_FIXTURE_ROLES = [ + "primary", + "sibling-a", + "sibling-b", + "fixture", +] as const; +type FixtureRole = (typeof NATIVE_NETWORK_FIXTURE_ROLES)[number]; +const CONTAINER_STATES = [ + "created", + "running", + "paused", + "restarting", + "removing", + "exited", + "dead", +] as const; +const CLOSED_CODE = /^E_[A-Z0-9_]{1,62}$/; +const CLOSED_TOKEN = /^[a-z][a-z0-9-]{0,62}$/; + +/** Fixed command class only: executable role, one closed verb and one closed action. */ +export function nativeNetworkFixtureStep( + argv: readonly string[], + executables: { readonly cli: string; readonly engine: string } +): string { + const [executable, first, second] = argv; + if (executable === executables.cli) { + const verb = HACK_VERBS.find((value) => value === first) ?? "other"; + return `hack:${verb}${argv.includes("--recover") ? "+recover" : ""}`; + } + if (executable === executables.engine) { + const verb = DOCKER_VERBS.find((value) => value === first) ?? "other"; + const action = DOCKER_ACTIONS.find((value) => value === second); + return action ? `docker:${verb}:${action}` : `docker:${verb}`; + } + return "other"; +} + +type ClosedFailure = { + readonly code: string | null; + readonly stage: string | null; + readonly reason: string | null; + readonly message: string | null; +}; + +/** + * Exact fixed product sentences, each named by a closed label. A failure that + * issues no refusal boundary (for example an incomplete saved stop) is still + * classified; any other text, or a changed sentence, yields null. + */ +const FIXED_MESSAGES: ReadonlyMap = new Map([ + [ + "Native Compose stop is incomplete; saved ownership and persistent data are retained.", + "stop-incomplete", + ], + [ + "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + "operation-failed", + ], + ...( + [ + ["E_NATIVE_COMPOSE_STATE", "native-state"], + ["E_NATIVE_COMPOSE_BUSY", "native-busy"], + ["E_NATIVE_COMPOSE_UNCERTAIN", "native-uncertain"], + ["E_NATIVE_COMPOSE_STALE", "native-stale"], + ] as const + ).map( + ([code, label]) => + [new NativeComposeGenerationError(code).message, label] as const + ), +]); + +function fixedMessage(value: unknown): string | null { + return typeof value === "string" ? (FIXED_MESSAGES.get(value) ?? null) : null; +} + +function closedToken(value: unknown, pattern: RegExp): string | null { + return typeof value === "string" && pattern.test(value) ? value : null; +} + +/** The JSON envelope's closed code and issued boundary; null when stdout is not JSON. */ +function jsonClosedFailure(stdout: string): ClosedFailure | null { + let value: unknown; + try { + value = JSON.parse(stdout); + } catch { + return null; + } + const error = isRecord(value) ? own(value, "error") : undefined; + if (!isRecord(error)) { + return { code: null, stage: null, reason: null, message: null }; + } + const detail = own(error, "detail"); + const refusal = isRecord(detail) + ? own(detail, "native_compose_effect_refusal") + : undefined; + return { + code: closedToken(own(error, "code"), CLOSED_CODE), + stage: isRecord(refusal) + ? closedToken(own(refusal, "stage"), CLOSED_TOKEN) + : null, + reason: isRecord(refusal) + ? closedToken(own(refusal, "reason"), CLOSED_TOKEN) + : null, + message: fixedMessage(own(error, "message")), + }; +} + +function closedFailure( + result: Pick +): ClosedFailure { + const json = jsonClosedFailure(result.stdout); + if (json) { + return json; + } + const boundary = humanBoundary(result.stdout) ?? humanBoundary(result.stderr); + return { + code: humanClosedCode(result.stdout) ?? humanClosedCode(result.stderr), + stage: boundary?.stage ?? null, + reason: boundary?.reason ?? null, + message: + humanFixedMessage(result.stdout) ?? humanFixedMessage(result.stderr), + }; +} + +const HUMAN_MESSAGE = /^[^\w\n]*(?:ERROR:?[ \t]+)?E_[A-Z0-9_]{1,62}: (.+)$/m; + +/** The closed label of a human error line's exact fixed sentence; never its text. */ +export function humanFixedMessage(text: string): string | null { + return fixedMessage(HUMAN_MESSAGE.exec(text.replace(ANSI, ""))?.[1]); +} + +const HUMAN_BOUNDARY = + /^[^\w\n]*(?:ERROR:?[ \t]+)?Refusal boundary: ([a-z][a-z0-9-]{0,62}) \(([a-z][a-z0-9-]{0,62})\); values omitted\.$/m; + +/** The closed refusal boundary line a human CLI error prints after its code. */ +export function humanBoundary( + text: string +): { readonly stage: string; readonly reason: string } | null { + const match = HUMAN_BOUNDARY.exec(text.replace(ANSI, "")); + return match?.[1] && match[2] ? { stage: match[1], reason: match[2] } : null; +} + +const ANSI = /\u001b\[[0-9;]*m/g; +// A human error line opens with the logger's glyph or level, then `E_CODE: `: +// clack writes `■ E_CODE: ` to stdout, gum `ERROR E_CODE: ` and console +// `ERROR: E_CODE: ` to stderr. Only the closed code is kept. +const HUMAN_CODE = /^[^\w\n]*(?:ERROR:?[ \t]+)?(E_[A-Z0-9_]{1,62}): /m; + +/** The leading closed error code of human CLI output; nothing else of the text. */ +export function humanClosedCode(text: string): string | null { + return HUMAN_CODE.exec(text.replace(ANSI, ""))?.[1] ?? null; +} + +/** + * Closed exit codes of the in-container probe scripts. Every script failure + * exits with one of them, so a failing `hack exec` that exits 1 is the product + * (or the runtime), never an uncaught probe error. + */ +export const NATIVE_NETWORK_PROBE_EXIT = { + environment: 23, + response: 24, + marker: 25, + rejected: 26, + timeout: 27, + unreadable: 28, + unwritable: 29, +} as const; +const PROBE = NATIVE_NETWORK_PROBE_EXIT; + +/** Fetches the endpoint origin and its alias; both must answer the exact marker. */ +export function nativeNetworkFixturePeerProbe(opts: { + readonly origin: string; + readonly alias: string; + readonly expected: string; + readonly timeoutMs?: number; +}): string { + const timeout = opts.timeoutMs ?? 4000; + return `if(process.env.PEER_ORIGIN!==${JSON.stringify(opts.origin)})process.exit(${PROBE.environment});for(const url of [process.env.PEER_ORIGIN,${JSON.stringify(opts.alias)}]){let ok=false;try{const r=await fetch(url,{signal:AbortSignal.timeout(${timeout})});ok=r.ok&&await r.text()===${JSON.stringify(opts.expected)}}catch(e){process.exit(e?.name==="TimeoutError"?${PROBE.timeout}:${PROBE.rejected})}if(!ok)process.exit(${PROBE.response})}`; +} + +/** Reads the retained marker; a missing or unreadable file has its own code. */ +export function nativeNetworkFixtureMarkerRead(opts: { + readonly path: string; + readonly marker: string; +}): string { + return `let t;try{t=await Bun.file(${JSON.stringify(opts.path)}).text()}catch{process.exit(${PROBE.unreadable})}if(t!==${JSON.stringify(opts.marker)})process.exit(${PROBE.marker})`; +} + +/** Writes the retained marker. */ +export function nativeNetworkFixtureMarkerWrite(opts: { + readonly path: string; + readonly marker: string; +}): string { + return `try{await Bun.write(${JSON.stringify(opts.path)},${JSON.stringify(opts.marker)})}catch{process.exit(${PROBE.unwritable})}`; +} + +/** Fixed failing-command evidence: command class, checkout role, sequence, exit and closed product codes. Never argv, output text, values or paths. */ +export function nativeNetworkFixtureCommandDiagnostic(opts: { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + readonly expected: number; + readonly result: Pick< + CliResult, + "exitCode" | "timedOut" | "stdout" | "stderr" | "durationMs" + >; +}): string { + const step = /^(hack|docker):[a-z]{1,16}(?::[a-z]{1,16}|\+recover)?$|^other$/; + if ( + !( + step.test(opts.step) && + NATIVE_NETWORK_FIXTURE_ROLES.includes(opts.role) && + Number.isSafeInteger(opts.sequence) && + Number.isSafeInteger(opts.expected) && + Number.isSafeInteger(opts.result.exitCode) && + typeof opts.result.timedOut === "boolean" && + Number.isSafeInteger(opts.result.durationMs) && + typeof opts.result.stdout === "string" && + typeof opts.result.stderr === "string" + ) + ) { + return "Network command diagnostic unavailable"; + } + return `Network command diagnostic ${JSON.stringify({ + step: opts.step, + role: opts.role, + sequence: opts.sequence, + expected: opts.expected, + exit: opts.result.exitCode, + timedOut: opts.result.timedOut, + durationMs: opts.result.durationMs, + ...closedFailure(opts.result), + })}`; +} + +const RUNNER_FAILURES = [ + [ + "Fixture command timed out; uncertain engine outcome and captures retained", + "timeout", + ], + [ + "Fixture command exceeded its output bound; bounded captures retained", + "output-bound", + ], + [ + "Fixture could not confirm owned group cleanup; captures retained", + "group-unconfirmed", + ], + [ + "Fixture capture failed after owned command reap; captures retained", + "capture", + ], +] as const; + +/** + * A command the runner could not return (a timeout kills it) still names its + * closed class, role, sequence, duration and runner outcome; never its output. + */ +export function nativeNetworkFixtureRunnerDiagnostic(opts: { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + readonly durationMs: number; + readonly error: unknown; +}): string { + const step = /^(hack|docker):[a-z]{1,16}(?::[a-z]{1,16}|\+recover)?$|^other$/; + if ( + !( + step.test(opts.step) && + NATIVE_NETWORK_FIXTURE_ROLES.includes(opts.role) && + Number.isSafeInteger(opts.sequence) && + Number.isSafeInteger(opts.durationMs) + ) + ) { + return "Network command diagnostic unavailable"; + } + const message = opts.error instanceof Error ? opts.error.message : null; + const outcome = + RUNNER_FAILURES.find(([text]) => text === message)?.[1] ?? "other"; + return `Network command diagnostic ${JSON.stringify({ + step: opts.step, + role: opts.role, + sequence: opts.sequence, + durationMs: opts.durationMs, + runner: outcome, + })}`; +} + +/** A fixture-authored assertion is a fixed sentence; anything else (paths, values, product text) is withheld. */ +export function nativeNetworkFixtureFailureLabel(error: unknown): string { + const message = error instanceof Error ? error.message : null; + return typeof message === "string" && + message.length <= 160 && + /^[A-Z](?:[A-Za-z ,;'-]|(?<=[a-z])\/(?=[a-z]))*$/.test(message) + ? `Network failure ${JSON.stringify(message)}` + : "Network failure withheld"; +} + +const CENSUS_KINDS = [ + "service", + "oneoff", + "stray", + "carrier", + "fixture", +] as const; + +/** A project-labelled container is a canonical service, a Compose one-off, or a stray (for example a recreate temporary). */ +export function nativeNetworkFixtureProjectKind(opts: { + readonly project: string; + readonly name: string; + readonly service: string; + readonly oneoff: string; +}): "service" | "oneoff" | "stray" { + if (opts.oneoff === "True") { + return "oneoff"; + } + return opts.oneoff === "False" && + NAMES.includes(opts.service as Name) && + opts.name === `${opts.project}-${opts.service}-1` + ? "service" + : "stray"; +} + +/** Counts exact owned engine objects by fixed role, kind and closed state; foreign rows are never selected. */ +export function nativeNetworkFixtureCensus( + rows: readonly { + readonly role: FixtureRole; + readonly kind: (typeof CENSUS_KINDS)[number]; + readonly state: string; + }[] +): string { + const counts: Record = {}; + for (const row of rows) { + const state = CONTAINER_STATES.find((value) => value === row.state); + const key = `${NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown"}:${CENSUS_KINDS.includes(row.kind) ? row.kind : "unknown"}:${state ?? "unknown"}`; + counts[key] = (counts[key] ?? 0) + 1; + } + return `Network failure census ${JSON.stringify( + Object.fromEntries( + Object.entries(counts).sort(([a], [b]) => a.localeCompare(b)) + ) + )}`; +} +const PENDING_OPERATIONS = ["up", "restart", "run", "down"] as const; + +/** Saved intent per started checkout: closed booleans and the pending operation only. */ +export function nativeNetworkFixtureSavedEvidence( + rows: readonly { + readonly role: FixtureRole; + readonly stopped: unknown; + readonly pending: unknown; + readonly storageWitnessesPending: unknown; + readonly beforeHooksPending: unknown; + }[] +): string { + const flag = (value: unknown) => + typeof value === "boolean" ? value : "unknown"; + const entries = rows.map((row) => { + const operation = + isRecord(row.pending) && typeof row.pending.operation === "string" + ? row.pending.operation + : null; + return [ + NATIVE_NETWORK_FIXTURE_ROLES.includes(row.role) ? row.role : "unknown", + { + stopped: flag(row.stopped), + pending: + row.pending === null + ? "none" + : (PENDING_OPERATIONS.find((value) => value === operation) ?? + "unknown"), + storageWitnessesPending: flag(row.storageWitnessesPending), + beforeHooksPending: flag(row.beforeHooksPending), + }, + ] as const; + }); + return `Network failure saved state ${JSON.stringify(Object.fromEntries(entries))}`; +} function object(text: string): Record { let value: unknown; try { @@ -1166,6 +1573,18 @@ export const nativeConfigNetworksScenario: Scenario = { "Fixture requires absolute Docker and Git executables" ); let cleanupStarted: number | null = null; + // Closed failure evidence: hosted captures stay private, so the log carries only fixed fields. + const roles = new Map(); + const commands = new WeakMap< + CliResult, + { + readonly step: string; + readonly role: FixtureRole; + readonly sequence: number; + } + >(); + let sequence = 0; + let stepExecutables = { cli: "", engine: engine ?? "" }; const execute = async ( argv: readonly string[], cwd = ctx.tempRoot, @@ -1179,15 +1598,51 @@ export const nativeConfigNetworksScenario: Scenario = { remaining > 0, "Network scenario exceeded its whole-run budget" ); - return await runNativeNetworkFixtureCommand({ - argv, - cwd, - env: { ...env, ...extra }, - captures, - timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), - }); + sequence += 1; + const command = { + step: nativeNetworkFixtureStep(argv, stepExecutables), + role: roles.get(cwd) ?? ("fixture" as const), + sequence, + }; + const begun = Date.now(); + let result: CliResult; + try { + result = await runNativeNetworkFixtureCommand({ + argv, + cwd, + env: { ...env, ...extra }, + captures, + timeoutMs: Math.min(COMMAND_TIMEOUT, remaining), + }); + } catch (error: unknown) { + // A killed or unreturned command has no result for ok(); name it here. + ctx.log( + nativeNetworkFixtureRunnerDiagnostic({ + ...command, + durationMs: Date.now() - begun, + error, + }) + ); + throw error; + } + commands.set(result, command); + return result; + }; + const ok = (result: CliResult, expected = 0): void => { + const command = commands.get(result); + if (command && !(result.exitCode === expected && !result.timedOut)) { + ctx.log( + nativeNetworkFixtureCommandDiagnostic({ + ...command, + expected, + result, + }) + ); + } + resultOk(result, expected); }; const manifest = await qualifyArtifacts(ctx, execute); + stepExecutables = { cli: manifest.cli.path, engine: engine ?? "" }; const composePlugin = await provisionNativeNetworkFixtureComposePlugin({ path: manifest.composePlugin.path, expectedHash: manifest.composePlugin.sha256, @@ -1203,7 +1658,7 @@ export const nativeConfigNetworksScenario: Scenario = { "version", "--short", ]); - resultOk(pluginVersion); + ok(pluginVersion); const composeVersion = pluginVersion.stdout.trim(); requireValue( /^v?\d+\.\d+\.\d+(?:[-+][A-Za-z0-9._-]+)?$/.test(composeVersion), @@ -1211,7 +1666,7 @@ export const nativeConfigNetworksScenario: Scenario = { ); const docker: Docker = async (args) => { const result = await execute([engine, ...args]); - resultOk(result); + ok(result); return result.stdout.trim(); }; const raw = ( @@ -1224,7 +1679,7 @@ export const nativeConfigNetworksScenario: Scenario = { args: readonly string[] ): Promise => { const result = await raw(checkout, args); - resultOk(result); + ok(result); return result; }; requireValue( @@ -1257,6 +1712,7 @@ export const nativeConfigNetworksScenario: Scenario = { const name = `e2e-net-${token.slice(0, 12)}`; const root = join(await realpath(ctx.tempRoot), name); await mkdir(join(root, ".hack"), { recursive: true, mode: 0o700 }); + roles.set(root, "primary"); const baseText = `${JSON.stringify(authored({ name, image, marker: "primary" }))}\n`; await Bun.write(join(root, ".hack", "hack.project.json"), baseText); await Bun.write( @@ -1280,7 +1736,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Fixture native networks", ], ]) { - resultOk(await execute([git, ...args], root)); + ok(await execute([git, ...args], root)); } const checkouts: Checkout[] = [ { @@ -1291,11 +1747,9 @@ export const nativeConfigNetworksScenario: Scenario = { text: baseText, }, ]; - for (const branch of ["sibling-a", "sibling-b"]) { + for (const branch of ["sibling-a", "sibling-b"] as const) { const linked = join(ctx.tempRoot, `${name}-${branch}`); - resultOk( - await execute([git, "worktree", "add", "-qb", branch, linked], root) - ); + ok(await execute([git, "worktree", "add", "-qb", branch, linked], root)); const text = `${JSON.stringify(authored({ name, image, marker: branch }))}\n`; const checkout = { root: await realpath(linked), @@ -1304,6 +1758,7 @@ export const nativeConfigNetworksScenario: Scenario = { path: join(linked, ".hack", "hack.project.json"), text, }; + roles.set(checkout.root, branch); await Bun.write(checkout.path, text); checkouts.push(checkout); } @@ -1365,9 +1820,7 @@ export const nativeConfigNetworksScenario: Scenario = { service: Name, script: string ): Promise => { - resultOk( - await raw(checkout, ["exec", service, "--", "bun", "-e", script]) - ); + ok(await raw(checkout, ["exec", service, "--", "bun", "-e", script])); }; const markers = async (checkout: Checkout): Promise => { for (const [service, target, alias] of [ @@ -1376,17 +1829,23 @@ export const nativeConfigNetworksScenario: Scenario = { ["vault", "reader", "reader-alias"], ["reader", "vault", "vault-alias"], ] as const) { - const expected = JSON.stringify(`${checkout.marker}:${target}`); await exec( checkout, service, - `if(process.env.PEER_ORIGIN!==${JSON.stringify(`http://${target}:3000`)})process.exit(23);for(const url of [process.env.PEER_ORIGIN,${JSON.stringify(`http://${alias}:3000/`)}]){const r=await fetch(url,{signal:AbortSignal.timeout(4000)});if(!r.ok||await r.text()!==${expected})process.exit(24)}` + nativeNetworkFixturePeerProbe({ + origin: `http://${target}:3000`, + alias: `http://${alias}:3000/`, + expected: `${checkout.marker}:${target}`, + }) ); } await exec( checkout, "web", - `if(await Bun.file("/state/marker").text()!==${JSON.stringify(checkout.marker)})process.exit(25)` + nativeNetworkFixtureMarkerRead({ + path: "/state/marker", + marker: checkout.marker, + }) ); }; const siblings = checkouts.slice(1); @@ -1423,7 +1882,7 @@ export const nativeConfigNetworksScenario: Scenario = { receipt, }); const result = await raw(checkout, args, { PATH: `${shim}:${env.PATH}` }); - resultOk(result, 1); + ok(result, 1); const fragmentPresent = result.combined.includes(fragment); const dockerInvoked = await Bun.file(receipt).exists(); if (!fragmentPresent || dockerInvoked) { @@ -1580,6 +2039,96 @@ export const nativeConfigNetworksScenario: Scenario = { "Proxy cleanup requires exact no-port/tmpfs/readonly-socket fixture identity" ); }; + // Owned objects only, selected by exact saved identity or fixture token; logs counts by closed state. + const census = async (): Promise => { + try { + const rows: Parameters[0][number][] = + []; + const listed = async (filter: string): Promise => + ( + await docker([ + "container", + "ls", + "-a", + "--no-trunc", + "--filter", + filter, + "--format", + `{{.State}}|{{.Names}}|{{.Label "${SERVICE}"}}|{{.Label "com.docker.compose.oneoff"}}`, + ]) + ) + .split("\n") + .filter(Boolean) + .map((line) => line.split("|")); + const states = async (filter: string): Promise => + (await listed(filter)).map(([state]) => state ?? "unknown"); + const savedRows: Parameters< + typeof nativeNetworkFixtureSavedEvidence + >[0][number][] = []; + for (const checkout of checkouts) { + if (!startedRoots.has(checkout.root)) { + continue; + } + const store = await openNativeComposeGenerationStore({ + projectRoot: checkout.root, + instance: null, + mode: "saved", + }); + const identity = store.identity; + const role = roles.get(checkout.root) ?? "fixture"; + try { + // storageWitnessesPending is nonenumerable; read each field explicitly. + const state = await store.loadCurrent(); + savedRows.push({ + role, + stopped: state.stopped, + pending: state.pending, + storageWitnessesPending: state.storageWitnessesPending, + beforeHooksPending: state.beforeHooksPending, + }); + } catch { + savedRows.push({ + role, + stopped: null, + pending: undefined, + storageWitnessesPending: null, + beforeHooksPending: null, + }); + } finally { + await store.close(); + } + for (const [ + state = "unknown", + name = "", + service = "", + oneoff = "", + ] of await listed(`label=${PROJECT}=${identity.composeProject}`)) { + rows.push({ + role, + kind: nativeNetworkFixtureProjectKind({ + project: identity.composeProject, + name, + service, + oneoff, + }), + state, + }); + } + for (const state of await states( + `label=${CARRIER_OWNER}=${identity.ownerToken}` + )) { + rows.push({ role, kind: "carrier", state }); + } + } + for (const state of await states(`label=${FIXTURE}=${token}`)) { + rows.push({ role: "fixture", kind: "fixture", state }); + } + ctx.log(nativeNetworkFixtureCensus(rows)); + ctx.log(nativeNetworkFixtureSavedEvidence(savedRows)); + } catch { + ctx.log("Network failure census unavailable"); + } + }; const runPhases = async (): Promise => { const store = await openNativeComposeGenerationStore({ projectRoot: primary.root, @@ -1598,7 +2147,7 @@ export const nativeConfigNetworksScenario: Scenario = { receipt: createReceipt, }); startedRoots.add(primary.root); - resultOk( + ok( await raw(primary, ["up", "--detach", "--json"], { PATH: `${shim}:${env.PATH}`, }), @@ -1629,7 +2178,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Controlled create must expose the actual empty NetworkID recovery window" ); } - resultOk(await raw(primary, ["up", "--detach", "--json"]), 1); + ok(await raw(primary, ["up", "--detach", "--json"]), 1); await cli(primary, ["down", "--recover", "--json"]); await resourceAbsent(primary); ctx.log( @@ -1642,7 +2191,10 @@ export const nativeConfigNetworksScenario: Scenario = { await exec( checkout, "web", - `await Bun.write("/state/marker",${JSON.stringify(checkout.marker)})` + nativeNetworkFixtureMarkerWrite({ + path: "/state/marker", + marker: checkout.marker, + }) ); await markers(checkout); await cli(checkout, ["ps", "--json"]); @@ -1805,7 +2357,7 @@ export const nativeConfigNetworksScenario: Scenario = { "Foreign endpoint must be running on the pinned bridge before admission" ); const foreign = await raw(primary, ["up", "--detach", "--json"]); - resultOk(foreign, 1); + ok(foreign, 1); const foreignState = await saved(primary.root); const foreignSnapshot = await observe(primary); requireValue( @@ -1859,7 +2411,7 @@ export const nativeConfigNetworksScenario: Scenario = { internal: true, }; const beforeTamper = await saved(primary.root); - resultOk(await raw(primary, ["up", "--detach", "--json"]), 1); + ok(await raw(primary, ["up", "--detach", "--json"]), 1); requireValue( (await inventory(docker, "container", original.identity.composeProject)) .length === 0 && @@ -2083,6 +2635,7 @@ export const nativeConfigNetworksScenario: Scenario = { await runPhases(); } catch (error: unknown) { failure = error; + await census(); } cleanupStarted = Date.now(); @@ -2199,6 +2752,8 @@ export const nativeConfigNetworksScenario: Scenario = { ctx.log( "Cleanup also failed; original failure retained in private captures" ); + ctx.log(nativeNetworkFixtureFailureLabel(cleanupError)); + await census(); } } if (failure !== undefined) { diff --git a/tests/native-compose-effect-diagnostics.test.ts b/tests/native-compose-effect-diagnostics.test.ts index 0f6ed455..b3afdb0d 100644 --- a/tests/native-compose-effect-diagnostics.test.ts +++ b/tests/native-compose-effect-diagnostics.test.ts @@ -12,6 +12,7 @@ import { copyNativeComposeEffectRefusal, nativeComposeEffectReason, nativeComposeEffectRefusal, + nativeComposeEffectRefusalSummary, retainNativeComposeEffectRefusal, } from "../src/lib/native-compose-effect-diagnostics.ts"; import * as generation from "../src/lib/native-compose-generation.ts"; @@ -229,6 +230,105 @@ test("reason classification preserves an original issued probe refusal without c expect(nativeComposeEffectReason(copied)).toBe("unclassified"); }); +test.each([ + "issued", + "unissued", +] as const)("an unclassified native failure keeps only an %s boundary under its fixed E_COMPOSE_FAILED text", async (kind) => { + const root = await realpath( + await mkdtemp(join(tmpdir(), "native-effect-fallback-")) + ); + const oldBackend = process.env.HACK_RUNTIME_BACKEND; + // A storage carrier refuses with a plain Error whose text is never public. + const error = new Error("private-carrier-canary"); + const diagnostic = { + stage: "storage-helper-policy", + reason: "unclassified", + } as const; + if (kind === "issued") { + retainNativeComposeEffectRefusal(error, diagnostic); + } + const opened = spyOn( + generation, + "openNativeComposeGenerationStore" + ).mockImplementation(() => Promise.reject(error)); + try { + process.env.HACK_RUNTIME_BACKEND = "compose"; + await mkdir(join(root, ".hack")); + await Bun.write( + join(root, ".hack/hack.project.json"), + JSON.stringify({ schema_version: 1, name: "fixture" }) + ); + let caught: unknown; + try { + await tryNativeComposeCommand({ + cwd: root, + operation: "down", + json: true, + }); + } catch (value) { + caught = value; + } + expect(opened).toHaveBeenCalledTimes(1); + const result = errorResultFromUnknown({ error: caught }); + expect(result).toEqual({ + ok: false, + error: { + code: "E_COMPOSE_FAILED", + message: + "Native Compose operation failed; saved ownership and persistent data are retained. Values omitted.", + ...(kind === "issued" + ? { detail: { native_compose_effect_refusal: diagnostic } } + : {}), + }, + }); + expect(JSON.stringify(result)).not.toContain("private-carrier-canary"); + } finally { + opened.mockRestore(); + restoreEnv("HACK_RUNTIME_BACKEND", oldBackend); + await rm(root, { recursive: true, force: true }); + } +}); + +test("the human boundary summary names only a known stage and reason pair", () => { + expect( + nativeComposeEffectRefusalSummary({ + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }) + ).toBe( + "Refusal boundary: storage-helper-start (probe-timeout); values omitted." + ); + const canary = "private-summary-canary"; + for (const detail of [ + undefined, + null, + canary, + { + native_compose_effect_refusal: { stage: canary, reason: "unclassified" }, + }, + { + native_compose_effect_refusal: { + stage: "storage-target", + reason: canary, + }, + }, + { + native_compose_effect_refusal: new Proxy( + {}, + { + getOwnPropertyDescriptor: () => { + throw new Error(canary); + }, + } + ), + }, + ]) { + expect(nativeComposeEffectRefusalSummary(detail)).toBeNull(); + } +}); + test.each([ "guard-fresh-before", "guard-ownership", diff --git a/tests/native-compose-storage-command-record.test.ts b/tests/native-compose-storage-command-record.test.ts index 2bc06d10..7a306e54 100644 --- a/tests/native-compose-storage-command-record.test.ts +++ b/tests/native-compose-storage-command-record.test.ts @@ -31,7 +31,10 @@ import { publishNativeComposeStorageCommandChild, settleNativeComposeStorageCommand, } from "../src/lib/native-compose-storage-command-record.ts"; -import { runNativeComposeStorageDockerCommand } from "../src/lib/native-compose-storage-witness-docker.ts"; +import { + nativeComposeStorageCarrierReason, + runNativeComposeStorageDockerCommand, +} from "../src/lib/native-compose-storage-witness-docker.ts"; import * as shell from "../src/lib/shell.ts"; const hash = "a".repeat(64); @@ -389,7 +392,8 @@ test.each(["timedOut", "cancelled"] as const)( async function transport( f: Awaited>, marker: string, - timeoutMs = 3000 + timeoutMs = 3000, + script = 'printf complete > "$1"; printf value' ) { try { return await runNativeComposeStorageDockerCommand({ @@ -398,13 +402,7 @@ async function transport( deadline: Date.now() + timeoutMs, }, files: f.files, - args: [ - "/bin/sh", - "-c", - 'printf complete > "$1"; printf value', - "owned", - marker, - ], + args: ["/bin/sh", "-c", script, "owned", marker], timeoutMs, beforeSpawn: () => {}, assertAdmitted: async () => {}, @@ -520,7 +518,11 @@ test( const f = await fixture(), marker = join(f.root, "effect"); f.failChild(); - await expect(transport(f, marker)).rejects.toThrow("values omitted"); + const caught = await transport(f, marker).catch((error: unknown) => error); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + // An ordinary refusal keeps no command cause. + expect(nativeComposeStorageCarrierReason(caught)).toBe("unclassified"); expect(existsSync(marker)).toBe(false); expect((await f.read()).commands[0]?.child?.pid).toBeGreaterThan(1); expect((await f.read()).commands[0]?.settlement).toBeNull(); @@ -567,6 +569,162 @@ test( }), 30_000 ); +// HACK-1281 replay: a stalled owner resumes with its timer already due. A +// one-second identity read refused this healthy command after its stopped child +// was published, leaving the durable uncertainty that blocks saved recovery. +test( + "a host stall in a child identity read stays within the command budget", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + run = shell.run; + let stalled = 0; + const observer = spyOn(shell, "run").mockImplementation((args, opts) => { + const pid = args[args.indexOf("-p") + 1]; + if ( + stalled === 0 && + args[4] === "/bin/ps" && + pid !== undefined && + pid !== String(process.pid) + ) { + stalled++; + // The read itself stalls past the former one-second bound. + return run( + [ + args[0] ?? "", + args[1] ?? "", + `sleep 1.2; ${args[2]}`, + ...args.slice(3), + ], + opts + ); + } + return run(args, opts); + }); + try { + expect(await transport(f, marker, 15_000)).toEqual({ + exitCode: 0, + stdout: "value", + }); + expect(stalled).toBe(1); + expect(await readFile(marker, "utf8")).toBe("complete"); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + exitCode: 0, + timedOut: false, + groupAbsent: true, + }); + } finally { + if (confirmedTestLifetime()) { + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a multi-second host stall after exit still settles the original command", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + sync = f.directory.file.sync.bind(f.directory.file); + let stalled = false; + // One post-exit sync stalls past the former three-second settlement budget. + const observer = spyOn(f.directory.file, "sync").mockImplementation( + async () => { + if (existsSync(marker) && !stalled) { + stalled = true; + await Bun.sleep(3500); + } + await sync(); + } + ); + try { + expect(await transport(f, marker)).toEqual({ + exitCode: 0, + stdout: "value", + }); + expect(stalled).toBe(true); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + exitCode: 0, + groupAbsent: true, + }); + expect(nativeComposeStorageCommandOwnerConfirmed(f.owner)).toBe(true); + } finally { + if (confirmedTestLifetime()) { + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a settlement that outlasts its post-exit budget refuses as probe-budget and stays unsettled", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"), + kill = process.kill, + monotonic = performance.now.bind(performance); + let elapsed = 0; + // The post-exit group observation sees 16 s of monotonic time pass: past + // the finite settlement budget, without a real 16 s wait. + const clock = spyOn(performance, "now").mockImplementation( + () => monotonic() + elapsed + ); + const observer = spyOn(process, "kill").mockImplementation( + (pid, signal) => { + if ( + signal === 0 && + typeof pid === "number" && + pid < 0 && + existsSync(marker) + ) { + elapsed = 16_000; + } + return kill(pid, signal); + } + ); + try { + const caught = await transport(f, marker).catch( + (error: unknown) => error + ); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + expect(nativeComposeStorageCarrierReason(caught)).toBe("probe-budget"); + expect(await readFile(marker, "utf8")).toBe("complete"); + expect((await f.read()).commands[0]?.settlement).toBeNull(); + expect(nativeComposeStorageCommandOwnerConfirmed(f.owner)).toBe(false); + retained.add(f.root); + } finally { + if (confirmedTestLifetime()) { + clock.mockRestore(); + observer.mockRestore(); + } + } + }), + 30_000 +); +test( + "a recorded child that outlives its command budget refuses as probe-timeout", + ownedCase(async () => { + const f = await fixture(), + marker = join(f.root, "effect"); + const caught = await transport( + f, + marker, + 1500, + 'printf started > "$1"; exec sleep 10' + ).catch((error: unknown) => error); + expect(caught).toBeInstanceOf(Error); + expect((caught as Error).message).toContain("values omitted"); + expect(nativeComposeStorageCarrierReason(caught)).toBe("probe-timeout"); + expect(await readFile(marker, "utf8")).toBe("started"); + expect((await f.read()).commands[0]?.settlement).toMatchObject({ + timedOut: true, + groupAbsent: true, + }); + }), + 30_000 +); test( "missing original exit callback cannot be replaced by capture bytes or group absence", ownedCase(async () => { diff --git a/tests/native-networks-fixture.test.ts b/tests/native-networks-fixture.test.ts index 6386d6d8..b831b34d 100644 --- a/tests/native-networks-fixture.test.ts +++ b/tests/native-networks-fixture.test.ts @@ -16,16 +16,29 @@ import { import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { + humanBoundary, + humanClosedCode, + NATIVE_NETWORK_PROBE_EXIT, nativeNetworkFixtureAttachmentMatches, + nativeNetworkFixtureCensus, + nativeNetworkFixtureCommandDiagnostic, nativeNetworkFixtureCreateArgs, + nativeNetworkFixtureFailureLabel, nativeNetworkFixtureHasNoPublication, nativeNetworkFixtureInventory, + nativeNetworkFixtureMarkerRead, + nativeNetworkFixtureMarkerWrite, nativeNetworkFixtureNetworkMatches, + nativeNetworkFixturePeerProbe, nativeNetworkFixturePreservedContainerMatches, + nativeNetworkFixtureProjectKind, nativeNetworkFixtureProtocolMatches, nativeNetworkFixtureRefusalDiagnostic, + nativeNetworkFixtureRunnerDiagnostic, + nativeNetworkFixtureSavedEvidence, nativeNetworkFixtureShim, nativeNetworkFixtureStateRefused, + nativeNetworkFixtureStep, nativeNetworkFixtureVolumeMatches, nativeNetworkFixtureVolumeSelectionMatches, provisionNativeNetworkFixtureComposePlugin, @@ -141,6 +154,485 @@ test("refusal diagnostics never print untrusted labels or non-boolean values", ( } }); +test("failing command evidence names only its closed class, role and product codes", () => { + const secret = "/private/fixture/secret-path"; + const executables = { cli: "/pinned/hack", engine: "/pinned/docker" }; + expect( + nativeNetworkFixtureStep( + ["/pinned/hack", "down", "--recover", "--json"], + executables + ) + ).toBe("hack:down+recover"); + expect( + nativeNetworkFixtureStep( + ["/pinned/hack", "exec", "web", "--", "bun", "-e", secret], + executables + ) + ).toBe("hack:exec"); + expect( + nativeNetworkFixtureStep( + ["/pinned/docker", "container", "start", secret], + executables + ) + ).toBe("docker:container:start"); + expect( + nativeNetworkFixtureStep(["/pinned/docker", "exec", secret], executables) + ).toBe("docker:exec"); + expect(nativeNetworkFixtureStep([secret, "up"], executables)).toBe("other"); + expect(nativeNetworkFixtureStep(["/pinned/hack", secret], executables)).toBe( + "hack:other" + ); + + const json = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:down+recover", + role: "primary", + sequence: 412, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 2345, + stdout: JSON.stringify({ + ok: false, + error: { + code: "E_CONFIG_INVALID", + message: `values ${secret}`, + detail: { + native_compose_effect_refusal: { + stage: "storage-helper-start", + reason: "probe-timeout", + }, + }, + }, + }), + stderr: secret, + }, + }); + expect(json).toBe( + 'Network command diagnostic {"step":"hack:down+recover","role":"primary","sequence":412,"expected":0,"exit":1,"timedOut":false,"durationMs":2345,"code":"E_CONFIG_INVALID","stage":"storage-helper-start","reason":"probe-timeout","message":null}' + ); + const human = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:exec", + role: "sibling-b", + sequence: 7, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 7, + // The clack backend (no gum on the hosted runner) writes errors to stdout. + stdout: `\u2502\n\u25a0 E_COMPOSE_FAILED: Native Compose operation failed ${secret}\n\u2502\n\u25a0 Refusal boundary: storage-helper-policy (unclassified); values omitted.\n`, + stderr: secret, + }, + }); + expect(human).toContain( + '"code":"E_COMPOSE_FAILED","stage":"storage-helper-policy","reason":"unclassified"' + ); + const unbounded = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:exec", + role: "sibling-b", + sequence: 8, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 7, + stdout: `\u25a0 E_CONFIG_INVALID: Refused ${secret}\n`, + stderr: "", + }, + }); + expect(unbounded).toContain( + '"code":"E_CONFIG_INVALID","stage":null,"reason":null' + ); + for (const result of [json, human, unbounded]) { + expect(result).not.toContain(secret); + expect(result).not.toContain("values"); + } +}); + +test("human error output yields only its leading closed code for every logger backend", () => { + const secret = "/private/fixture/secret-path"; + for (const [text, code] of [ + // clack (hosted runner, NO_COLOR) on stdout + [`│\n■ E_CONFIG_INVALID: Refused ${secret}\n`, "E_CONFIG_INVALID"], + // clack with color + [ + `\u001b[90m│\u001b[39m\n\u001b[31m■\u001b[39m E_COMPOSE_FAILED: ${secret}\n`, + "E_COMPOSE_FAILED", + ], + // gum and console backends on stderr + [`ERROR E_NATIVE_COMPOSE_STATE: ${secret}`, "E_NATIVE_COMPOSE_STATE"], + [`ERROR: E_LIFECYCLE_FAILED: ${secret}\n`, "E_LIFECYCLE_FAILED"], + ] as const) { + expect(humanClosedCode(text)).toBe(code); + } + for (const text of [ + "", + secret, + "ERROR MissingProjectContextError: E_FAKE: nested", + `value E_INLINE: ${secret}`, + `■ e_lower: ${secret}`, + `■ E_${"X".repeat(70)}: ${secret}`, + ]) { + expect(humanClosedCode(text)).toBeNull(); + } + expect( + humanBoundary( + "ERROR: Refusal boundary: storage-target (probe-timeout); values omitted.\n" + ) + ).toEqual({ stage: "storage-target", reason: "probe-timeout" }); + for (const text of [ + `■ Refusal boundary: ${secret} (probe-timeout); values omitted.`, + "■ Refusal boundary: storage-target (Probe Timeout); values omitted.", + "■ Refusal boundary: storage-target (probe-timeout)", + "value Refusal boundary: storage-target (probe-timeout); values omitted.", + ]) { + expect(humanBoundary(text)).toBeNull(); + } +}); + +test("probe scripts exit with a closed code for every in-container failure", async () => { + const root = await mkdtemp(join(tmpdir(), "hack-network-probe-")); + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const path = new URL(request.url).pathname; + if (path === "/hang") { + return new Promise(() => undefined); + } + return new Response(path === "/wrong" ? "other" : "primary:peer"); + }, + }); + const closed = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(""), + }); + const refused = `http://127.0.0.1:${closed.port}/`; + closed.stop(true); + const origin = `http://127.0.0.1:${server.port}`; + const run = async (script: string, peer = origin): Promise => { + const child = Bun.spawn([process.execPath, "-e", script], { + cwd: root, + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", PEER_ORIGIN: peer }, + stdout: "ignore", + stderr: "ignore", + }); + return await child.exited; + }; + try { + const probe = (alias: string, timeoutMs = 4000) => + nativeNetworkFixturePeerProbe({ + origin, + alias, + expected: "primary:peer", + timeoutMs, + }); + expect(await run(probe(`${origin}/alias`))).toBe(0); + expect(await run(probe(`${origin}/alias`), `${origin}/x`)).toBe( + NATIVE_NETWORK_PROBE_EXIT.environment + ); + expect(await run(probe(`${origin}/wrong`))).toBe( + NATIVE_NETWORK_PROBE_EXIT.response + ); + expect(await run(probe(refused))).toBe(NATIVE_NETWORK_PROBE_EXIT.rejected); + expect(await run(probe(`${origin}/hang`, 200))).toBe( + NATIVE_NETWORK_PROBE_EXIT.timeout + ); + + const marker = join(root, "marker"); + const read = nativeNetworkFixtureMarkerRead({ path: marker, marker: "m" }); + expect(await run(read)).toBe(NATIVE_NETWORK_PROBE_EXIT.unreadable); + expect( + await run(nativeNetworkFixtureMarkerWrite({ path: marker, marker: "m" })) + ).toBe(0); + expect(await run(read)).toBe(0); + expect( + await run(nativeNetworkFixtureMarkerRead({ path: marker, marker: "n" })) + ).toBe(NATIVE_NETWORK_PROBE_EXIT.marker); + expect( + await run( + nativeNetworkFixtureMarkerWrite({ + // Bun.write creates directories; a regular-file parent cannot be one. + path: join(marker, "child"), + marker: "m", + }) + ) + ).toBe(NATIVE_NETWORK_PROBE_EXIT.unwritable); + // No probe code collides with an ordinary product failure exit. + expect(Object.values(NATIVE_NETWORK_PROBE_EXIT)).not.toContain(1); + } finally { + server.stop(true); + await rm(root, { recursive: true, force: true }); + } +}); + +test("saved-state evidence reports closed flags and pending operation only", () => { + const secret = "secret-generation-token"; + expect( + nativeNetworkFixtureSavedEvidence([ + { + role: "primary", + stopped: false, + pending: null, + storageWitnessesPending: false, + beforeHooksPending: false, + }, + { + role: "sibling-b", + stopped: false, + pending: { operation: "up", token: secret, generationId: secret }, + storageWitnessesPending: true, + beforeHooksPending: false, + }, + { + role: "sibling-a", + stopped: secret, + pending: { operation: secret }, + storageWitnessesPending: undefined, + beforeHooksPending: null, + }, + ]) + ).toBe( + 'Network failure saved state {"primary":{"stopped":false,"pending":"none","storageWitnessesPending":false,"beforeHooksPending":false},"sibling-b":{"stopped":false,"pending":"up","storageWitnessesPending":true,"beforeHooksPending":false},"sibling-a":{"stopped":"unknown","pending":"unknown","storageWitnessesPending":"unknown","beforeHooksPending":"unknown"}}' + ); +}); + +test("an incomplete saved stop without an issued boundary is still named by its fixed sentence", () => { + const secret = "/private/fixture/secret-path"; + const stop = + "Native Compose stop is incomplete; saved ownership and persistent data are retained."; + const diagnostic = (stdout: string, stderr = "") => + nativeNetworkFixtureCommandDiagnostic({ + step: "hack:down+recover", + role: "sibling-a", + sequence: 942, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 2750, + stdout, + stderr, + }, + }); + // The sampled HACK-1281 cleanup failure: E_COMPOSE_FAILED with no stage or reason. + const json = diagnostic( + JSON.stringify({ + ok: false, + error: { code: "E_COMPOSE_FAILED", message: stop }, + }) + ); + expect(json).toContain( + '"code":"E_COMPOSE_FAILED","stage":null,"reason":null,"message":"stop-incomplete"' + ); + expect(diagnostic(`│\n■ E_COMPOSE_FAILED: ${stop}\n`, secret)).toContain( + '"message":"stop-incomplete"' + ); + expect( + diagnostic( + JSON.stringify({ + ok: false, + error: { + code: "E_CONFIG_INVALID", + message: + "Native Compose operation has an uncertain outcome; inspect the saved generation or explicitly stop its owned resources before retrying.", + }, + }) + ) + ).toContain('"message":"native-uncertain"'); + // Only the exact sentence is classified; appended values are withheld. + for (const changed of [`${stop} ${secret}`, stop.toUpperCase(), ""]) { + const result = diagnostic( + JSON.stringify({ + ok: false, + error: { code: "E_COMPOSE_FAILED", message: changed }, + }) + ); + expect(result).toContain('"message":null'); + expect(result).not.toContain(secret); + } + expect(diagnostic(`■ E_COMPOSE_FAILED: ${stop} ${secret}\n`)).toContain( + '"message":null' + ); +}); + +test("a command the runner killed or could not return still names its class and runner outcome", async () => { + const root = await mkdtemp(join(tmpdir(), "native-network-runner-")); + try { + const caught = await runNativeNetworkFixtureCommand({ + argv: ["/bin/sh", "-c", "sleep 5"], + cwd: root, + env: {}, + captures: join(root, "captures"), + timeoutMs: 150, + }).catch((error: unknown) => error); + const timeout = nativeNetworkFixtureRunnerDiagnostic({ + step: "hack:exec", + role: "sibling-a", + sequence: 941, + durationMs: 180_000, + error: caught, + }); + expect(timeout).toBe( + 'Network command diagnostic {"step":"hack:exec","role":"sibling-a","sequence":941,"durationMs":180000,"runner":"timeout"}' + ); + const secret = "/private/fixture/secret-path"; + expect( + nativeNetworkFixtureRunnerDiagnostic({ + step: "hack:exec", + role: "primary", + sequence: 1, + durationMs: 1, + error: new Error(secret), + }) + ).toBe( + 'Network command diagnostic {"step":"hack:exec","role":"primary","sequence":1,"durationMs":1,"runner":"other"}' + ); + for (const value of [ + { step: secret, role: "primary" }, + { step: "hack:exec", role: secret }, + ]) { + expect( + nativeNetworkFixtureRunnerDiagnostic({ + ...(value as { step: string; role: "primary" }), + sequence: 1, + durationMs: 1, + error: caught, + }) + ).toBe("Network command diagnostic unavailable"); + } + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("failing command evidence drops non-closed codes and malformed fields", () => { + const secret = "secret-value-must-not-appear"; + const forged = nativeNetworkFixtureCommandDiagnostic({ + step: "hack:up", + role: "primary", + sequence: 1, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 1, + stdout: JSON.stringify({ + ok: false, + error: { + code: `E_${secret}`, + detail: { + native_compose_effect_refusal: { + stage: `/${secret}`, + reason: { value: secret }, + }, + }, + }, + }), + stderr: "", + }, + }); + expect(forged).toContain('"code":null,"stage":null,"reason":null'); + expect(forged).not.toContain(secret); + for (const value of [ + { step: secret, role: "primary" }, + { step: "hack:up", role: secret }, + { step: "hack:up --json", role: "primary" }, + ]) { + const result = nativeNetworkFixtureCommandDiagnostic({ + ...(value as { step: string; role: "primary" }), + sequence: 1, + expected: 0, + result: { + exitCode: 1, + timedOut: false, + durationMs: 1, + stdout: "", + stderr: "", + }, + }); + expect(result).toBe("Network command diagnostic unavailable"); + } +}); + +test("fixture failure label prints only fixed assertion sentences", () => { + expect( + nativeNetworkFixtureFailureLabel( + new Error( + "Product saved down must retire all exact fixture containers/bridges and clear intent" + ) + ) + ).toBe( + 'Network failure "Product saved down must retire all exact fixture containers/bridges and clear intent"' + ); + for (const error of [ + new Error("Refused /private/fixture/root"), + new Error("Value 42 omitted"), + new Error("lowercase start"), + new Error(`Long ${"x".repeat(200)}`), + "Fixture string throw", + { message: "Fixture object" }, + ]) { + expect(nativeNetworkFixtureFailureLabel(error)).toBe( + "Network failure withheld" + ); + } +}); + +test("failure census separates canonical services from one-offs and strays", () => { + const project = "hack-nc-0123456789abcdef0123456789abcdef"; + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `${project}-web-1`, + service: "web", + oneoff: "False", + }) + ).toBe("service"); + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `0123456789ab_${project}-web-1`, + service: "web", + oneoff: "False", + }) + ).toBe("stray"); + expect( + nativeNetworkFixtureProjectKind({ + project, + name: `${project}-run-0123`, + service: "web", + oneoff: "True", + }) + ).toBe("oneoff"); + for (const value of [ + { name: `${project}-other-1`, service: "other", oneoff: "False" }, + { name: `${project}-web-2`, service: "web", oneoff: "False" }, + { name: `${project}-web-1`, service: "web", oneoff: "" }, + ]) { + expect(nativeNetworkFixtureProjectKind({ project, ...value })).toBe( + "stray" + ); + } +}); + +test("failure census counts owned roles and kinds by closed engine state only", () => { + expect( + nativeNetworkFixtureCensus([ + { role: "primary", kind: "service", state: "running" }, + { role: "primary", kind: "service", state: "running" }, + { role: "primary", kind: "carrier", state: "exited" }, + { role: "fixture", kind: "fixture", state: "created" }, + { role: "sibling-a", kind: "service", state: "secret-state" }, + ]) + ).toBe( + 'Network failure census {"fixture:fixture:created":1,"primary:carrier:exited":1,"primary:service:running":2,"sibling-a:service:unknown":1}' + ); + expect(nativeNetworkFixtureCensus([])).toBe("Network failure census {}"); +}); + async function withPluginFixture( run: (opts: { path: string;