From ebbdb9ce30c2a289cb6113d026c4fb4b9db862b4 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:38:08 -0400 Subject: [PATCH 01/21] test(agents): fix slow onboarding inventory tests Three onboarding/inventory tests timed out at Bun's default 5 s in the full local suite: `hack agent onboard` on an empty repo, and the Python-only and other-ecosystem inventory fixtures. None of these fixtures has a package.json, so findRepoRootForInit walked up past the temp dir. With TMPDIR=/tmp and a stray /tmp/package.json on the host, the repo root resolved to /tmp. The inventory then globbed all of /tmp and described the wrong repo. Bound each fixture with an empty `.git` (tests/helpers/repo-boundary.ts, matching the existing retained-* fixtures), assert the onboard prompt's inventory root is the fixture, and add a host-independent guard. The guard nests a bounded and an unbounded fixture under an ancestor package.json and checks where the walk-up stops. Timeouts and product code are unchanged. HACK-1137 Co-Authored-By: Claude Opus 5.5 --- tests/agent-onboard-command.test.ts | 4 +++ tests/helpers/repo-boundary.ts | 19 +++++++++++ tests/init-assistant.test.ts | 49 ++++++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 tests/helpers/repo-boundary.ts diff --git a/tests/agent-onboard-command.test.ts b/tests/agent-onboard-command.test.ts index d8adcb17d..f00664835 100644 --- a/tests/agent-onboard-command.test.ts +++ b/tests/agent-onboard-command.test.ts @@ -2,6 +2,7 @@ import { afterEach, expect, test } from "bun:test"; import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { markRepoBoundary } from "./helpers/repo-boundary.ts"; type CapturedRunResult = { readonly exitCode: number; @@ -17,8 +18,10 @@ afterEach(async () => { } }); +/** An empty temp repo bounded by its own `.git` (no host ancestor adopted). */ async function setupTempRepo(): Promise { tempDir = await mkdtemp(join(tmpdir(), "hack-onboard-cmd-")); + await markRepoBoundary(tempDir); return tempDir; } @@ -50,6 +53,7 @@ test("hack agent onboard prints the new-project prompt outside a hack project", ]); expect(result.exitCode).toBe(0); + expect(result.stdout).toContain(`- Root: ${repoRoot}`); expect(result.stdout).toContain("stand up hack in this repo"); expect(result.stdout).toContain("hack init --auto"); expect(result.stdout).toContain("## Phase 5 — Verify the requested workflow"); diff --git a/tests/helpers/repo-boundary.ts b/tests/helpers/repo-boundary.ts new file mode 100644 index 000000000..44266d301 --- /dev/null +++ b/tests/helpers/repo-boundary.ts @@ -0,0 +1,19 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +/** + * Make `root` the repository root for repo-root discovery. + * + * `findRepoRootForInit` walks up from its start directory to the nearest + * `package.json` or `.git`. A temp fixture without its own `package.json` + * (an empty repo, a Python-only repo) otherwise resolves to whatever host + * ancestor has one. With a stray `/tmp/package.json` on a shared machine and + * `TMPDIR=/tmp`, the onboarding inventory scanned all of `/tmp`, so the tests + * hit their timeout while describing the wrong repo. + * + * An empty `.git` directory stops the walk at the fixture. The inventory never + * reads dot directories, so it reports the same signals. + */ +export async function markRepoBoundary(root: string): Promise { + await mkdir(join(root, ".git"), { recursive: true }); +} diff --git a/tests/init-assistant.test.ts b/tests/init-assistant.test.ts index 640a80b46..58a37fe16 100644 --- a/tests/init-assistant.test.ts +++ b/tests/init-assistant.test.ts @@ -7,7 +7,11 @@ import { buildInitAssistantReport, type InitAssistantReport, } from "../src/agents/init-assistant.ts"; -import { buildOnboardingPrompt } from "../src/agents/onboarding-prompt.ts"; +import { + buildOnboardingPrompt, + collectOnboardingContext, +} from "../src/agents/onboarding-prompt.ts"; +import { markRepoBoundary } from "./helpers/repo-boundary.ts"; const ALWAYS_PRESENT_LIMITATION = "Inventory reads file names and manifest keys only; env files, secrets, lockfile contents, and pin values were not read."; @@ -20,15 +24,27 @@ afterEach(async () => { } }); +/** + * A temp repo bounded by its own `.git`, so the walk-up in + * `buildOnboardingPrompt` stops here even without a `package.json`. + */ async function createRepo(files: Record): Promise { const repoRoot = await mkdtemp(join(tmpdir(), "hack-init-assistant-")); tempDirs.push(repoRoot); - for (const [rel, content] of Object.entries(files)) { - const abs = join(repoRoot, rel); + await markRepoBoundary(repoRoot); + await writeFiles({ root: repoRoot, files }); + return repoRoot; +} + +async function writeFiles(opts: { + readonly root: string; + readonly files: Record; +}): Promise { + for (const [rel, content] of Object.entries(opts.files)) { + const abs = join(opts.root, rel); await mkdir(dirname(abs), { recursive: true }); await Bun.write(abs, content); } - return repoRoot; } function packageJson(value: Record): string { @@ -244,3 +260,28 @@ test("prompt embeds the canonical onboarding guidance after the inventory", asyn expect(onboardingIndex).toBeGreaterThan(inventoryIndex); expect(prompt).toContain("bun install --frozen-lockfile"); }); + +test("a fixture without package.json is inventoried at its own boundary, not a host ancestor", async () => { + // A host ancestor with a package.json, like a stray /tmp/package.json. + const host = await mkdtemp(join(tmpdir(), "hack-init-assistant-host-")); + tempDirs.push(host); + const files = { + "package.json": packageJson({ name: "host-ancestor" }), + "unbounded/pyproject.toml": '[project]\nname = "svc"\n', + "bounded/pyproject.toml": '[project]\nname = "svc"\n', + }; + await writeFiles({ root: host, files }); + const bounded = join(host, "bounded"); + await markRepoBoundary(bounded); + + const rootOf = async (startDir: string): Promise => { + const context = await collectOnboardingContext({ startDir }); + return context.inventory.status === "collected" + ? context.inventory.report.repoRoot + : null; + }; + // Negative control: without a boundary the walk-up adopts the ancestor + // and the inventory describes (and scans) the wrong tree. + expect(await rootOf(join(host, "unbounded"))).toBe(host); + expect(await rootOf(bounded)).toBe(bounded); +}); From e43075ccac87c6cfa701faea65f8688b57d0e854 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:38:12 -0400 Subject: [PATCH 02/21] fix(release): lift the prerelease installer archive member limit safely The prerelease installer bounded archive members by the sum of its allowlist sizes, so the 17-member next.4/next.5 layout filled the bound exactly. Replace it with a fixed 64-member hard ceiling (MAX_ARCHIVE_MEMBERS) and a distinct "Too many archive entries." refusal. Every other check is unchanged: each member must still be a distinct allowlisted, non-traversing, bounded regular file in a complete group, with checksums, modes, identity and signatures verified before selection. Add the manager shipped by 5.0.0-next.3 through next.5 (e2e2bcba) to MANAGER_PREDECESSORS with the fixture prerelease-manager-storage-v5.py, so those channels can adopt this installer through upgrade-manager. HACK-1124 Co-Authored-By: Claude Opus 5.5 --- docs/guides/candidate-install.md | 5 +- docs/guides/prereleases.md | 8 + scripts/install-prerelease.py | 16 +- .../fixtures/prerelease-manager-storage-v5.py | 971 ++++++++++++++++++ tests/prerelease-plan.test.ts | 6 + tests/python/test_prerelease_install.py | 174 ++++ 6 files changed, 1175 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/prerelease-manager-storage-v5.py diff --git a/docs/guides/candidate-install.md b/docs/guides/candidate-install.md index 23b77109d..19cacd84b 100644 --- a/docs/guides/candidate-install.md +++ b/docs/guides/candidate-install.md @@ -75,8 +75,9 @@ from the retained manager, before publishing a version or changing selection. Th older manager itself refuses the newer layout as foreign files. To update a reviewed predecessor manager, including the managers shipped with -`5.0.0-next.1` and `5.0.0-next.2`, review the newer installer from the tag you are -upgrading to, then run that saved file explicitly. For example, before upgrading a +`5.0.0-next.1`, `5.0.0-next.2` and `5.0.0-next.3` through `5.0.0-next.5`, review +the newer installer from the tag you are upgrading to, then run that saved file +explicitly. For example, before upgrading a `5.0.0-next.2` channel to `5.0.0-next.3`: ```sh diff --git a/docs/guides/prereleases.md b/docs/guides/prereleases.md index 2b39e0073..c951ed42a 100644 --- a/docs/guides/prereleases.md +++ b/docs/guides/prereleases.md @@ -155,6 +155,14 @@ against the existing root, then run `upgrade`. See [upgrade and return to a retained candidate](candidate-install.md#upgrade-and-return-to-a-retained-candidate). A fresh `--root` does not need this step. +Channels created by the `5.0.0-next.3`, `next.4` or `next.5` installer share one +manager. It accepts at most 17 archive members, the exact size of the layout those +releases ship, and no file names beyond it. Later installers use a fixed 64-member +ceiling, but every member must still be a reviewed name in a complete group. A +later bundle that adds a file therefore needs the same `upgrade-manager` step +first. Upgrading with that newer installer against such a channel also refuses +until you run it, and leaves the channel unchanged. + A truncated, interrupted or oversized official download is a refused installation. The installer removes only the staging directory that attempt created. It leaves the current selection, retained bundles and homes unchanged. Retry the same version diff --git a/scripts/install-prerelease.py b/scripts/install-prerelease.py index dafdab645..c85b4a7a8 100644 --- a/scripts/install-prerelease.py +++ b/scripts/install-prerelease.py @@ -43,11 +43,21 @@ "backend": "hack-mcp-backend"} MCP_MEMBER = re.compile(r"mcp/([0-9a-f]{64})/(manifest\.json|hack-mcp-adapter|hack-mcp-owner|hack-mcp-backend)\Z") MAX_ARCHIVE = 512 * 1024 * 1024 +# Hard ceiling on tar members read before the payload inventory is checked. Every +# member must still be a distinct allowlisted name with a complete group, so this +# only bounds header parsing, chiefly for the MCP group whose directory is a digest +# wildcard. The largest reviewed inventory is 17 members (seven core files plus +# SHA256SUMS, three compiler, two storage, four MCP). The ceiling is a fixed number +# rather than that sum, so a reviewed addition needs only its allowlist entry, mode +# and group rule; a hostile archive still refuses after at most 64 headers. +MAX_ARCHIVE_MEMBERS = 64 METADATA_KEYS = {"schema", "version", "tag", "source_revision", "platform"} REPOSITORY = "hack-dance/hack" DOWNLOAD_HOSTS = {"api.github.com", "github.com", "release-assets.githubusercontent.com", "objects.githubusercontent.com"} -# Reviewed flat-layout, shared-MCP, compiler, batched-signature, and verified-launcher managers. +# Reviewed flat-layout, shared-MCP, compiler, batched-signature, verified-launcher and +# storage-witness managers. e2e2bcba is the manager the 5.0.0-next.3, next.4 and +# next.5 tags shipped (fixture prerelease-manager-storage-v5.py). # Keep this an explicit allowlist; a matching user-written receipt is not provenance. MANAGER_PREDECESSORS = frozenset({ "b459ffc4f227482119b48e357c91e4607aa4f7d1f88c7ccf5b7d684f66f2c0cd", @@ -55,6 +65,7 @@ "c7562953361b6b37141e126df31edefabc05e4441e3c6e0a38b1c3cda6b645da", "ca432b7fc6562bb091d17d3217f5d1daf9f91621a6919c8964ca51bee2111d0c", "d39e77623be1876567e9db66c468e8da96067aeac0d88089df28efc11f850ef8", + "e2e2bcbacf58b21d1086047c94727e5e814776384439700adcf5aabe5c49c2fa", }) MANAGER_UPGRADE = ".manager-upgrade.json" @@ -349,8 +360,7 @@ def extract_archive(archive, checksum, destination, version, release_metadata=No seen = set() total = 0 for entry in source: - require(len(entries) < len(BUNDLE_FILES) + len(COMPILER_PAYLOAD) - + len(STORAGE_TOOL_PAYLOAD) + 4, "Incomplete or duplicate archive entries.") + require(len(entries) < MAX_ARCHIVE_MEMBERS, "Too many archive entries.") require((entry.name in BUNDLE_FILES | COMPILER_PAYLOAD | STORAGE_TOOL_PAYLOAD or MCP_MEMBER.fullmatch(entry.name)) and entry.name not in seen, "Foreign, duplicate or traversing archive entry.") diff --git a/tests/fixtures/prerelease-manager-storage-v5.py b/tests/fixtures/prerelease-manager-storage-v5.py new file mode 100644 index 000000000..dafdab645 --- /dev/null +++ b/tests/fixtures/prerelease-manager-storage-v5.py @@ -0,0 +1,971 @@ +#!/usr/bin/env python3 +"""Explicit, side-by-side native prerelease selection (Python 3.9+, standard library). + +Selection changes commit through one atomic receipt. Explicit manager upgrades +use a recoverable journal. Bundles and homes are retained by version; this manager +never migrates runtime state or edits shell PATH. +""" + +import argparse +import contextlib +import fcntl +import hashlib +import json +import os +from pathlib import Path +import platform +import re +import signal +import stat +import subprocess +import sys +import uuid + + +VERSION = re.compile(r"5\.0\.0-next\.([1-9][0-9]*)\Z") +SHA256 = re.compile(r"[0-9a-f]{64}\Z") +REVISION = re.compile(r"[0-9a-f]{40}\Z") +PAYLOAD = frozenset(("hack-native", "hack-relay-guest", "hack-cli", "hack-v5", + "provider-pins.json", "README.md", "prerelease.json")) +LEGACY_COMPILER_PAYLOAD = frozenset(("hack-config-compiler", "hack.project.schema.json")) +COMPILER_PAYLOAD = LEGACY_COMPILER_PAYLOAD | {"hack.local.schema.json"} +# Guest-only Linux ARM64 witness for authored persistent mounts and its byte +# manifest. The CLI resolves both as siblings of hack-native; older bundles omit both. +STORAGE_TOOL = "hack-storage-root-witness" +STORAGE_TOOL_MANIFEST = "native-storage-tool.json" +STORAGE_TOOL_PAYLOAD = frozenset((STORAGE_TOOL, STORAGE_TOOL_MANIFEST)) +STORAGE_TOOL_KEYS = {"version", "kind", "platform", "architecture", "file", "sha256", "bytes", + "sourceRevision"} +EXECUTABLES = frozenset(("hack-native", "hack-relay-guest", "hack-cli", "hack-v5", + "hack-config-compiler")) +BUNDLE_FILES = PAYLOAD | {"SHA256SUMS"} +MCP_FILES = {"adapter": "hack-mcp-adapter", "owner": "hack-mcp-owner", + "backend": "hack-mcp-backend"} +MCP_MEMBER = re.compile(r"mcp/([0-9a-f]{64})/(manifest\.json|hack-mcp-adapter|hack-mcp-owner|hack-mcp-backend)\Z") +MAX_ARCHIVE = 512 * 1024 * 1024 +METADATA_KEYS = {"schema", "version", "tag", "source_revision", "platform"} +REPOSITORY = "hack-dance/hack" +DOWNLOAD_HOSTS = {"api.github.com", "github.com", "release-assets.githubusercontent.com", + "objects.githubusercontent.com"} +# Reviewed flat-layout, shared-MCP, compiler, batched-signature, and verified-launcher managers. +# Keep this an explicit allowlist; a matching user-written receipt is not provenance. +MANAGER_PREDECESSORS = frozenset({ + "b459ffc4f227482119b48e357c91e4607aa4f7d1f88c7ccf5b7d684f66f2c0cd", + "b7c49e3fec6b06790e833db1d2dcb441d2223c283b792713be46826aa2eef877", + "c7562953361b6b37141e126df31edefabc05e4441e3c6e0a38b1c3cda6b645da", + "ca432b7fc6562bb091d17d3217f5d1daf9f91621a6919c8964ca51bee2111d0c", + "d39e77623be1876567e9db66c468e8da96067aeac0d88089df28efc11f850ef8", +}) +MANAGER_UPGRADE = ".manager-upgrade.json" + + +class Refusal(Exception): + """Untrusted, changed or ambiguous input: preserve the current selection.""" + + +def require(condition, message): + if not condition: + raise Refusal(message) + + +def version_number(value): + match = VERSION.fullmatch(value) if isinstance(value, str) else None + require(match is not None, "Version must be 5.0.0-next.N (positive N, no leading zero).") + return int(match.group(1)) + + +def digest(path): + value = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def json_bytes(value): + return (json.dumps(value, sort_keys=True, indent=2) + "\n").encode("utf-8") + + +def parse_json(raw): + def unique(pairs): + result = {} + for key, value in pairs: + require(key not in result, "Duplicate JSON field.") + result[key] = value + return result + try: + value = json.loads(raw, object_pairs_hook=unique) + except (ValueError, UnicodeError) as error: + raise Refusal("Malformed JSON.") from error + require(isinstance(value, dict), "Expected a JSON object.") + return value + + +def metadata(raw, version): + value = parse_json(raw) + require(set(value) == METADATA_KEYS, "Unexpected prerelease metadata fields.") + require(value["schema"] == "hack.prerelease/v1" and value["version"] == version + and value["tag"] == "v" + version and value["platform"] == "darwin-arm64" + and isinstance(value["source_revision"], str) + and REVISION.fullmatch(value["source_revision"]), "Prerelease identity mismatch.") + return value + + +def canonical(path): + path = Path(path) + require(path.is_absolute() and path == path.resolve(), + "Use a canonical absolute path without symlinks or traversal.") + return path + + +def owned(path, directory=False, mode=None): + info = path.lstat() + expected = stat.S_ISDIR(info.st_mode) if directory else stat.S_ISREG(info.st_mode) + require(expected and info.st_uid == os.getuid(), "Foreign or aliased path: " + str(path)) + if not directory: + require(info.st_nlink == 1, "Hard-linked file: " + str(path)) + if mode is not None: + require(stat.S_IMODE(info.st_mode) == mode, "Unsafe mode: " + str(path)) + return info + + +def private_json(path): + owned(path, mode=0o600) + require(path.stat().st_size <= 1024 * 1024, "Oversized installation receipt.") + return parse_json(path.read_bytes()) + + +def sync_directory(path): + descriptor = os.open(str(path), os.O_RDONLY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + + +def write_file(path, contents, mode=0o600): + descriptor = os.open(str(path), os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode) + with os.fdopen(descriptor, "wb") as target: + os.fchmod(target.fileno(), mode) + target.write(contents) + target.flush() + os.fsync(target.fileno()) + + +def atomic_file(path, contents): + temporary = path.parent / (".selection-" + uuid.uuid4().hex) + try: + write_file(temporary, contents) + os.replace(str(temporary), str(path)) + sync_directory(path.parent) + finally: + if temporary.exists(): + temporary.unlink() + + +def atomic_json(path, value): + atomic_file(path, json_bytes(value)) + + +def launcher_bytes(): + return ("#!/bin/sh\nset -eu\nroot=$(CDPATH= cd -- \"$(dirname -- \"$0\")/..\" && pwd -P)\n" + "exec /usr/bin/python3 -I -S \"$root/manager.py\" --root \"$root\" run -- \"$@\"\n").encode() + + +def checksums(raw, expected): + try: + lines = raw.decode("ascii").splitlines() + except UnicodeError as error: + raise Refusal("Malformed SHA256SUMS.") from error + result = {} + for line in lines: + match = re.fullmatch(r"([0-9a-f]{64}) ([A-Za-z0-9./-]+)", line) + require(match is not None, "Malformed SHA256SUMS entry.") + checksum, name = match.groups() + require(name in expected and name not in result, "Duplicate or foreign checksum entry.") + result[name] = checksum + require(set(result) == set(expected), "Incomplete checksum manifest.") + return result + + +def payload_inventory(names): + """Accept complete optional compiler, storage tool and MCP groups beside the original payload.""" + names = set(names) + require(BUNDLE_FILES <= names, "Incomplete candidate bundle.") + compiler = names & COMPILER_PAYLOAD + require(not compiler or compiler in (LEGACY_COMPILER_PAYLOAD, COMPILER_PAYLOAD), + "Incomplete config compiler payload.") + storage = names & STORAGE_TOOL_PAYLOAD + require(not storage or storage == STORAGE_TOOL_PAYLOAD, "Incomplete storage tool payload.") + extra = names - BUNDLE_FILES - COMPILER_PAYLOAD - STORAGE_TOOL_PAYLOAD + if extra: + matches = [MCP_MEMBER.fullmatch(name) for name in extra] + require(all(matches), "Foreign candidate MCP payload.") + identities = {match.group(1) for match in matches} + require(len(identities) == 1, "Candidate requires one MCP bundle identity.") + prefix = "mcp/" + next(iter(identities)) + "/" + require(extra == {prefix + name for name in (*MCP_FILES.values(), "manifest.json")}, + "Incomplete candidate MCP payload.") + return names - {"SHA256SUMS"} + + +def bundle_inventory(bundle): + names = {entry.name for entry in bundle.iterdir()} + if "mcp" in names: + owned(bundle / "mcp", directory=True, mode=0o700) + identities = list((bundle / "mcp").iterdir()) + require(len(identities) == 1 and SHA256.fullmatch(identities[0].name), + "Candidate requires one MCP bundle identity.") + directory = identities[0] + owned(directory, directory=True, mode=0o700) + names.remove("mcp") + names.update("mcp/" + directory.name + "/" + entry.name for entry in directory.iterdir()) + return payload_inventory(names) + + +def payload_mode(name): + if MCP_MEMBER.fullmatch(name): + return 0o400 if name.endswith("/manifest.json") else 0o500 + if name == STORAGE_TOOL: + return 0o500 + return 0o755 if name in EXECUTABLES else 0o600 + + +def verify_storage_tool(bundle, payload, verified_digests, identity): + """Bind the witness manifest to bytes hashed in this verification and to provenance. + + Runtime admission parses the same fields and refuses any mismatch; checking here + keeps an unusable pair from being selected. The ELF itself never runs on macOS. + """ + if STORAGE_TOOL not in payload: + return + path = bundle / STORAGE_TOOL_MANIFEST + require(path.stat().st_size <= 4096, "Oversized storage tool manifest.") + manifest = parse_json(path.read_bytes()) + require(set(manifest) == STORAGE_TOOL_KEYS, "Invalid storage tool manifest fields.") + require(type(manifest["version"]) is int and manifest["version"] == 1 + and manifest["kind"] == "native-storage-root-witness" + and manifest["platform"] == "linux" and manifest["architecture"] == "arm64" + and manifest["file"] == STORAGE_TOOL, "Incompatible storage tool manifest.") + size = (bundle / STORAGE_TOOL).stat().st_size + require(type(manifest["bytes"]) is int and 64 <= manifest["bytes"] <= 2 * 1024 * 1024 + and manifest["bytes"] == size + and isinstance(manifest["sha256"], str) and SHA256.fullmatch(manifest["sha256"]) + and manifest["sha256"] == verified_digests[STORAGE_TOOL], + "Storage tool fingerprint mismatch.") + require(manifest["sourceRevision"] == identity["source_revision"], + "Storage tool source revision mismatch.") + + +def verify_mcp_bundle(bundle, payload, verified_digests): + nested = sorted(name for name in payload if MCP_MEMBER.fullmatch(name)) + if not nested: + return + prefix = str(Path(nested[0]).parent) + path = bundle / prefix / "manifest.json" + require(path.stat().st_size <= 8192, "Oversized MCP manifest.") + manifest = parse_json(path.read_bytes()) + require(set(manifest) == {"schemaVersion", "startupProtocol", "wireProtocol", "platform", + "architecture", "files", "bundleId"}, "Invalid MCP manifest fields.") + require(all(type(manifest[field]) in (int, float) + for field in ("schemaVersion", "startupProtocol", "wireProtocol")) + and manifest["schemaVersion"] == 1 and manifest["startupProtocol"] == 2 + and manifest["wireProtocol"] == 1 and manifest["platform"] == "darwin" + and manifest["architecture"] == "arm64", "Incompatible MCP protocol or host.") + files = manifest["files"] + require(isinstance(files, dict) and set(files) == set(MCP_FILES), "Invalid MCP asset inventory.") + canonical_files = {} + for role, name in MCP_FILES.items(): + entry = files[role] + require(isinstance(entry, dict) and set(entry) == {"sha256", "bytes"} + and isinstance(entry["sha256"], str) and SHA256.fullmatch(entry["sha256"]) + and type(entry["bytes"]) is int and 0 < entry["bytes"] <= 256 * 1024 * 1024, + "Invalid MCP asset fingerprint.") + asset = bundle / prefix / name + # Compare the nested identity against bytes hashed in this verification, + # not just the outer manifest's claims. Never retain this across calls. + require(asset.stat().st_size == entry["bytes"] + and verified_digests[prefix + "/" + name] == entry["sha256"], + "MCP asset fingerprint mismatch.") + canonical_files[role] = {"sha256": entry["sha256"], "bytes": entry["bytes"]} + body = {"schemaVersion": 1, "startupProtocol": 2, "wireProtocol": 1, + "platform": "darwin", "architecture": "arm64", "files": canonical_files} + identity = hashlib.sha256(json.dumps(body, separators=(",", ":")).encode()).hexdigest() + require(manifest["bundleId"] == identity and Path(prefix).name == identity, + "MCP bundle identity mismatch.") + + +def verify_bundle(bundle, version): + owned(bundle, directory=True, mode=0o700) + payload = bundle_inventory(bundle) + for name in payload | {"SHA256SUMS"}: + owned(bundle / name, mode=payload_mode(name)) + require((bundle / "SHA256SUMS").stat().st_size <= 64 * 1024 + and (bundle / "prerelease.json").stat().st_size <= 64 * 1024, + "Oversized bundle metadata or checksum manifest.") + manifest = checksums((bundle / "SHA256SUMS").read_bytes(), payload) + verified_digests = {} + for name, checksum in manifest.items(): + actual = digest(bundle / name) + require(actual == checksum, "Candidate checksum mismatch: " + name) + verified_digests[name] = actual + identity = metadata((bundle / "prerelease.json").read_bytes(), version) + verify_mcp_bundle(bundle, payload, verified_digests) + verify_storage_tool(bundle, payload, verified_digests, identity) + return identity, manifest + + +def verify_signatures(bundle): + # Linux guest ELFs (hack-relay-guest, the storage witness) carry no macOS code + # signature; checksums and, for the witness, its bound manifest pin them. + names = ["hack-native", "hack-cli"] + payload = bundle_inventory(bundle) + if "hack-config-compiler" in payload: + names.append("hack-config-compiler") + names.extend(sorted(name for name in payload + if MCP_MEMBER.fullmatch(name) and not name.endswith("/manifest.json"))) + # codesign verifies every supplied path and fails if any is invalid. Keep all + # executables and strict validation in one bounded, output-redacted process. + try: + result = subprocess.run(["/usr/bin/codesign", "--verify", "--strict", + *(str(bundle / name) for name in names)], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + raise Refusal("Cannot verify candidate code signature.") from error + require(result.returncode == 0, "Candidate code signature failed.") + + +def extract_archive(archive, checksum, destination, version, release_metadata=None): + import tarfile + + owned(archive) + require(SHA256.fullmatch(checksum) is not None, "Expected archive SHA-256 is required.") + require(archive.stat().st_size <= MAX_ARCHIVE and digest(archive) == checksum, + "Archive checksum or size mismatch.") + try: + with tarfile.open(str(archive), "r:gz") as source: + entries = [] + seen = set() + total = 0 + for entry in source: + require(len(entries) < len(BUNDLE_FILES) + len(COMPILER_PAYLOAD) + + len(STORAGE_TOOL_PAYLOAD) + 4, "Incomplete or duplicate archive entries.") + require((entry.name in BUNDLE_FILES | COMPILER_PAYLOAD | STORAGE_TOOL_PAYLOAD + or MCP_MEMBER.fullmatch(entry.name)) and entry.name not in seen, + "Foreign, duplicate or traversing archive entry.") + require(entry.isfile() and not entry.issparse() and entry.size > 0 + and entry.size <= MAX_ARCHIVE, "Only bounded regular archive files are accepted.") + seen.add(entry.name) + total += entry.size + require(total <= MAX_ARCHIVE, "Oversized archive.") + entries.append(entry) + payload = payload_inventory(seen) + by_name = {entry.name: entry for entry in entries} + require(by_name["prerelease.json"].size <= 64 * 1024 + and by_name["SHA256SUMS"].size <= 64 * 1024, + "Oversized archive metadata or checksum manifest.") + with source.extractfile(by_name["prerelease.json"]) as incoming: + identity = metadata(incoming.read(), version) + require(release_metadata is None or identity == release_metadata, + "Archive metadata differs from the pinned release.") + with source.extractfile(by_name["SHA256SUMS"]) as incoming: + manifest = checksums(incoming.read(), payload) + # Preflight hashes before writing any payload. Revalidation after + # copying also catches a source changed between these two passes. + for entry in entries: + name = entry.name + if name not in payload: + continue + value = hashlib.sha256() + with source.extractfile(entry) as incoming: + for chunk in iter(lambda: incoming.read(1024 * 1024), b""): + value.update(chunk) + require(value.hexdigest() == manifest[name], "Candidate checksum mismatch: " + name) + destination.mkdir(mode=0o700) + nested = {name for name in payload if MCP_MEMBER.fullmatch(name)} + if nested: + prefix = Path(next(iter(nested))).parent + (destination / "mcp").mkdir(mode=0o700) + (destination / prefix).mkdir(mode=0o700) + for entry in entries: + incoming = source.extractfile(entry) + require(incoming is not None, "Unreadable archive payload.") + with incoming: + descriptor = os.open(str(destination / entry.name), + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + payload_mode(entry.name)) + with os.fdopen(descriptor, "wb") as target: + os.fchmod(target.fileno(), payload_mode(entry.name)) + remaining = entry.size + while remaining: + chunk = incoming.read(min(1024 * 1024, remaining)) + require(bool(chunk), "Truncated archive payload.") + target.write(chunk) + remaining -= len(chunk) + target.flush() + os.fsync(target.fileno()) + except (tarfile.TarError, EOFError) as error: + raise Refusal("Invalid candidate archive.") from error + identity, manifest = verify_bundle(destination, version) + require(release_metadata is None or identity == release_metadata, + "Archive metadata differs from the pinned release.") + verify_signatures(destination) + sync_directory(destination) + return identity, manifest + + +def secure_url(url): + import urllib.parse + + parsed = urllib.parse.urlsplit(url) + require(parsed.scheme == "https" and parsed.hostname in DOWNLOAD_HOSTS + and parsed.port in (None, 443) and not parsed.username and not parsed.password + and not parsed.fragment, "Refusing nonofficial or non-HTTPS download.") + + +def official_redirect_handler(): + # Local launches do not need the TLS/network/archive import graph. Load the + # same restricted redirect handler only when an official download is needed. + import urllib.request + + class OfficialRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, request, response, code, message, headers, url): + secure_url(url) + return super().redirect_request(request, response, code, message, headers, url) + + return OfficialRedirect() + + +def download(url, target, limit): + """Write one complete official response, or refuse. + + A short body, a stream interrupted by the transport, or bytes past the budget + refuse. The caller owns `target` and its staging directory; this function never + publishes or selects what it wrote. + """ + import http.client + import urllib.request + + secure_url(url) + request = urllib.request.Request(url, headers={"Accept": "application/vnd.github+json", + "User-Agent": "hack-prerelease-installer"}) + opener = urllib.request.build_opener(official_redirect_handler()) + with opener.open(request, timeout=30) as response: + secure_url(response.geturl()) + # http.client returns a short body as an ordinary end of stream, so the + # declared length is checked here rather than trusted to the transport. + declared = response.headers.get("Content-Length") + require(declared is None or isinstance(declared, str) and re.fullmatch(r"[0-9]{1,12}", declared), + "Malformed release download length.") + expected = None if declared is None else int(declared) + require(expected is None or expected <= limit, "Release download exceeds its size budget.") + descriptor = os.open(str(target), os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "wb") as output: + size = 0 + while True: + try: + chunk = response.read(1024 * 1024) + except (http.client.HTTPException, OSError) as error: + raise Refusal("Release download was interrupted.") from error + if not chunk: + break + size += len(chunk) + require(size <= limit, "Release download exceeds its size budget.") + require(expected is None or size <= expected, + "Release download exceeds its declared length.") + output.write(chunk) + require(expected is None or size == expected, "Release download is incomplete.") + output.flush() + os.fsync(output.fileno()) + + +def official_archive(stage, version): + tag = "v" + version + api_path = stage / "release-api.json" + download("https://api.github.com/repos/" + REPOSITORY + "/releases/tags/" + tag, + api_path, 2 * 1024 * 1024) + release = parse_json(api_path.read_bytes()) + require(release.get("tag_name") == tag and release.get("prerelease") is True + and release.get("draft") is False, "Expected a published, pinned official prerelease.") + archive_name = "hack-" + version + "-darwin-arm64-native.tar.gz" + wanted = {archive_name, "SHA256SUMS", "prerelease.json"} + assets = release.get("assets") + require(isinstance(assets, list), "Missing release assets.") + selected = {} + for asset in assets: + require(isinstance(asset, dict), "Malformed release asset.") + name = asset.get("name") + require(isinstance(name, str), "Malformed release asset name.") + if name in wanted: + require(name not in selected, "Duplicate release asset.") + expected = "https://github.com/" + REPOSITORY + "/releases/download/" + tag + "/" + name + require(asset.get("browser_download_url") == expected, "Foreign release asset URL.") + selected[name] = expected + require(set(selected) == wanted, "Incomplete official prerelease assets.") + download(selected["prerelease.json"], stage / "release-metadata.json", 64 * 1024) + identity = metadata((stage / "release-metadata.json").read_bytes(), version) + download("https://api.github.com/repos/" + REPOSITORY + "/git/ref/tags/" + tag, + stage / "release-tag.json", 64 * 1024) + reference = parse_json((stage / "release-tag.json").read_bytes()) + commit = reference.get("object") + require(reference.get("ref") == "refs/tags/" + tag and isinstance(commit, dict) + and commit.get("type") == "commit" and commit.get("sha") == identity["source_revision"], + "Official prerelease tag does not match its source revision.") + download(selected["SHA256SUMS"], stage / "release-checksums", 64 * 1024) + checksum = checksums((stage / "release-checksums").read_bytes(), {archive_name})[archive_name] + archive = stage / archive_name + download(selected[archive_name], archive, MAX_ARCHIVE) + return archive, checksum, identity + + +def remove_created_stage(stage, created): + """Remove the one install stage whose identity this call recorded at creation. + + A missing, replaced or foreign path is left for inspection. Other stages, + including a manager upgrade's journaled stage, are never touched. + """ + import shutil + + try: + info = stage.lstat() + except FileNotFoundError: + return + if (stat.S_ISDIR(info.st_mode) and info.st_uid == os.getuid() + and (info.st_dev, info.st_ino) == (created.st_dev, created.st_ino)): + shutil.rmtree(str(stage)) + sync_directory(stage.parent) + + +class Channel: + """Owned private layout; the selection file records immutable receipt hashes.""" + + def __init__(self, root): + self.root = canonical(root) + self.state = None + + def initialize(self): + if not self.root.exists(): + owned(self.root.parent, directory=True) + self.root.mkdir(mode=0o700) + owned(self.root, directory=True, mode=0o700) + if (self.root / ".channel.json").exists(): + return + require(not list(self.root.iterdir()), "Refusing to adopt a nonempty installation root.") + (self.root / "bin").mkdir(mode=0o700) + (self.root / "versions").mkdir(mode=0o700) + write_file(self.root / "manager.py", Path(__file__).read_bytes()) + write_file(self.root / "bin/hack-next", launcher_bytes(), 0o755) + root_info = self.root.stat() + write_file(self.root / ".channel.json", json_bytes({ + "schema": "hack.prerelease-install/v1", "root": str(self.root), "uid": os.getuid(), + "device": root_info.st_dev, "inode": root_info.st_ino, + "manager_sha256": digest(self.root / "manager.py"), + "launcher_sha256": digest(self.root / "bin/hack-next")})) + write_file(self.root / ".selection.json", json_bytes({ + "schema": "hack.prerelease-selection/v1", "installed": {}, + "selected": None, "previous": None})) + write_file(self.root / ".manager.lock", b"") + sync_directory(self.root) + + @contextlib.contextmanager + def lock(self, shared=False, manager_upgrade=False): + owned(self.root, directory=True, mode=0o700) + owned(self.root / ".manager.lock", mode=0o600) + descriptor = os.open(str(self.root / ".manager.lock"), os.O_RDWR | os.O_NOFOLLOW) + with os.fdopen(descriptor, "rb") as handle: + try: + fcntl.flock(handle, (fcntl.LOCK_SH if shared else fcntl.LOCK_EX) | fcntl.LOCK_NB) + except BlockingIOError as error: + raise Refusal("Another candidate manager or launcher is active.") from error + self.validate(manager_upgrade=manager_upgrade) + yield + + def validate(self, manager_upgrade=False): + root_info = owned(self.root, directory=True, mode=0o700) + pending = os.path.lexists(self.root / MANAGER_UPGRADE) + require(not pending or manager_upgrade, + "Manager upgrade is pending; rerun upgrade-manager with the same reviewed installer.") + transition = self.read_manager_upgrade() if pending else None + marker = private_json(self.root / ".channel.json") + require(set(marker) == {"schema", "root", "uid", "device", "inode", + "manager_sha256", "launcher_sha256"} + and marker["schema"] == "hack.prerelease-install/v1" + and marker["root"] == str(self.root) and marker["uid"] == os.getuid() + and marker["device"] == root_info.st_dev and marker["inode"] == root_info.st_ino, + "Foreign or malformed installation receipt.") + owned(self.root / "manager.py", mode=0o600) + owned(self.root / "bin", directory=True, mode=0o700) + owned(self.root / "bin/hack-next", mode=0o755) + manager_hash = digest(self.root / "manager.py") + require((manager_hash == marker["manager_sha256"] or transition is not None + and manager_hash == transition["to_sha256"] + and marker["manager_sha256"] == transition["from_sha256"]) + and digest(self.root / "bin/hack-next") == marker["launcher_sha256"], + "Installation manager or launcher changed.") + require({p.name for p in (self.root / "bin").iterdir()} == {"hack-next"}, + "Foreign launcher entry.") + state = private_json(self.root / ".selection.json") + require(set(state) == {"schema", "installed", "selected", "previous"} + and state["schema"] == "hack.prerelease-selection/v1" + and isinstance(state["installed"], dict), "Malformed selection receipt.") + installed = state["installed"] + require(len(installed) <= 256, "Too many candidate versions; inspection required.") + for version, checksum in installed.items(): + version_number(version) + require(isinstance(checksum, str) and SHA256.fullmatch(checksum), "Malformed receipt hash.") + self.verify_version(version, checksum) + for key in ("selected", "previous"): + require(state[key] is None or isinstance(state[key], str) and state[key] in installed, + "Unknown selected or previous candidate.") + require(state["selected"] is None or state["selected"] != state["previous"], + "Malformed candidate history.") + owned(self.root / "versions", directory=True, mode=0o700) + for entry in (self.root / "versions").iterdir(): + if entry.name not in installed: + version_number(entry.name) + # Rename can precede selection by a power-loss boundary. Validate + # the retained directory, but never select/adopt it automatically. + private_json(entry / ".receipt.json") + self.verify_version(entry.name, digest(entry / ".receipt.json")) + allowed = {".channel.json", ".selection.json", ".manager.lock", "manager.py", "bin", "versions"} + if transition is not None: + allowed.add(MANAGER_UPGRADE) + for entry in self.root.iterdir(): + if entry.name in allowed: + continue + if re.fullmatch(r"\.stage-[0-9a-f]{32}", entry.name): + owned(entry, directory=True, mode=0o700) + continue + if re.fullmatch(r"\.selection-[0-9a-f]{32}", entry.name): + owned(entry, mode=0o600) + continue + raise Refusal("Foreign installation entry: " + entry.name) + self.state = state + + def read_manager_upgrade(self): + """Verify both sides of the only supported two-file transition before recovery. + + The journal closes ordinary launch admission before either file changes. + Recovery may finish this exact transition, never adopt changed receipts or + select software. The staged originals remain available for inspection. + """ + plan = private_json(self.root / MANAGER_UPGRADE) + require(set(plan) == {"schema", "stage", "from_sha256", "to_sha256", + "channel_sha256", "selection_sha256"} + and plan["schema"] == "hack.prerelease-manager-upgrade/v1" + and isinstance(plan["stage"], str) + and re.fullmatch(r"\.stage-[0-9a-f]{32}", plan["stage"]), + "Malformed manager upgrade journal.") + require(all(isinstance(plan[key], str) and SHA256.fullmatch(plan[key]) + for key in ("from_sha256", "to_sha256", "channel_sha256", "selection_sha256")), + "Malformed manager upgrade digest.") + require(plan["from_sha256"] in MANAGER_PREDECESSORS + and plan["to_sha256"] == digest(Path(__file__)), + "Manager recovery requires the same reviewed installer and a known predecessor.") + stage = self.root / plan["stage"] + owned(stage, directory=True, mode=0o700) + require({p.name for p in stage.iterdir()} == { + "manager-before.py", "manager-after.py", "channel-before.json", "channel-after.json"}, + "Foreign manager upgrade staging entry.") + for entry in stage.iterdir(): + owned(entry, mode=0o600) + before = private_json(stage / "channel-before.json") + after = private_json(stage / "channel-after.json") + require(digest(stage / "manager-before.py") == plan["from_sha256"] + and digest(stage / "manager-after.py") == plan["to_sha256"] + and digest(stage / "channel-before.json") == plan["channel_sha256"] + and before.get("manager_sha256") == plan["from_sha256"] + and after == dict(before, manager_sha256=plan["to_sha256"]), + "Manager upgrade staging changed.") + owned(self.root / "manager.py", mode=0o600) + private_json(self.root / ".channel.json") + private_json(self.root / ".selection.json") + old_receipt = (stage / "channel-before.json").read_bytes() + new_receipt = (stage / "channel-after.json").read_bytes() + current_receipt = (self.root / ".channel.json").read_bytes() + current_hash = digest(self.root / "manager.py") + require((current_hash == plan["from_sha256"] and current_receipt == old_receipt) + or (current_hash == plan["to_sha256"] and current_receipt in (old_receipt, new_receipt)), + "Manager upgrade publication state changed.") + require(digest(self.root / ".selection.json") == plan["selection_sha256"], + "Selection changed during manager upgrade; inspection required.") + owned(self.root / "bin", directory=True, mode=0o700) + owned(self.root / "bin/hack-next", mode=0o755) + require((self.root / "bin/hack-next").read_bytes() == launcher_bytes(), + "Custom launcher cannot be upgraded.") + return plan + + def upgrade_manager(self): + """Explicitly replace a known manager, with recoverable fail-closed publication.""" + self.validate(manager_upgrade=True) + pending = os.path.lexists(self.root / MANAGER_UPGRADE) + source = canonical(Path(__file__).absolute()) + owned(source) + target = source.read_bytes() + target_hash = hashlib.sha256(target).hexdigest() + if not pending: + current_hash = digest(self.root / "manager.py") + if current_hash == target_hash: + return + require(current_hash in MANAGER_PREDECESSORS, + "Unknown or customized manager; explicit upgrade supports only reviewed predecessors.") + require((self.root / "bin/hack-next").read_bytes() == launcher_bytes(), + "Custom launcher cannot be upgraded.") + # All retained versions share this manager. Keep the existing executor + # ownership checks, including status/down/status, for each of them. + for version in self.state["installed"]: + self.require_quiescent(version, manager_upgrade=True) + self.validate(manager_upgrade=True) + if not pending: + stage = self.root / (".stage-" + uuid.uuid4().hex) + stage.mkdir(mode=0o700) + before = (self.root / ".channel.json").read_bytes() + after = dict(parse_json(before), manager_sha256=target_hash) + write_file(stage / "manager-before.py", (self.root / "manager.py").read_bytes()) + write_file(stage / "manager-after.py", target) + write_file(stage / "channel-before.json", before) + write_file(stage / "channel-after.json", json_bytes(after)) + plan = {"schema": "hack.prerelease-manager-upgrade/v1", "stage": stage.name, + "from_sha256": current_hash, "to_sha256": target_hash, + "channel_sha256": hashlib.sha256(before).hexdigest(), + "selection_sha256": digest(self.root / ".selection.json")} + write_file(stage / "journal.json", json_bytes(plan)) + sync_directory(stage) + self.validate() + os.rename(str(stage / "journal.json"), str(self.root / MANAGER_UPGRADE)) + sync_directory(stage) + sync_directory(self.root) + self.validate(manager_upgrade=True) + plan = self.read_manager_upgrade() + stage = self.root / plan["stage"] + if digest(self.root / "manager.py") != target_hash: + atomic_file(self.root / "manager.py", (stage / "manager-after.py").read_bytes()) + self.validate(manager_upgrade=True) + if (self.root / ".channel.json").read_bytes() != (stage / "channel-after.json").read_bytes(): + atomic_file(self.root / ".channel.json", (stage / "channel-after.json").read_bytes()) + self.validate(manager_upgrade=True) + (self.root / MANAGER_UPGRADE).unlink() + sync_directory(self.root) + self.validate() + + def verify_version(self, version, checksum): + directory = self.root / "versions" / version + owned(directory, directory=True, mode=0o700) + require({p.name for p in directory.iterdir()} == {"bundle", "native-home", "cli-home", ".receipt.json"}, + "Incomplete or foreign version installation.") + receipt_path = directory / ".receipt.json" + receipt = private_json(receipt_path) + require(digest(receipt_path) == checksum, "Candidate installation receipt changed.") + require(set(receipt) == {"schema", "metadata", "archive_sha256", "checksums", "homes"} + and receipt["schema"] == "hack.prerelease-version/v1" + and isinstance(receipt["archive_sha256"], str) + and SHA256.fullmatch(receipt["archive_sha256"]), "Malformed version receipt.") + identity, manifest = verify_bundle(directory / "bundle", version) + require(receipt["metadata"] == identity and receipt["checksums"] == manifest, + "Candidate identity or checksum receipt changed.") + require(isinstance(receipt["homes"], dict) and set(receipt["homes"]) == {"native-home", "cli-home"}, + "Malformed candidate home receipt.") + for name in ("native-home", "cli-home"): + info = owned(directory / name, directory=True, mode=0o700) + require(receipt["homes"][name] == {"device": info.st_dev, "inode": info.st_ino}, + "Candidate home identity changed.") + return directory + + def environment(self, version): + directory = self.root / "versions" / version + environment = dict(os.environ) + environment.pop("HACK_GLOBAL_CONFIG_PATH", None) + environment.update({"HACK_RUNTIME_BACKEND": "native", + "HACK_NATIVE_BINARY": str(directory / "bundle/hack-native"), + "HACK_NATIVE_HOME": str(directory / "native-home"), + "HACK_HOME": str(directory / "cli-home")}) + return environment + + def require_quiescent(self, version, manager_upgrade=False): + if version is None: + return + self.validate(manager_upgrade=manager_upgrade) + directory = self.verify_version(version, self.state["installed"][version]) + verify_signatures(directory / "bundle") + for action in ("status", "down", "status"): + command = [str(directory / "bundle/hack-native"), "--candidate-root", + str(directory / "native-home"), "runtime", action, "--json"] + try: + result = subprocess.run(command, env=self.environment(version), stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=30, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + raise Refusal("Candidate quiescence is unavailable; selection preserved.") from error + require(result.returncode == 0 and len(result.stdout) <= 1024 * 1024, + "Candidate quiescence check failed; failure never means idle.") + status = parse_json(result.stdout) + phase = status.get("phase") + require(phase in ("uninitialized", "stopped") and status.get("process_alive") is False, + "Candidate runtime is active or ambiguous; stop it through its supported CLI first.") + if phase == "uninitialized": + require(not list((directory / "native-home").iterdir()), + "Uninitialized home contains state; inspect it through its owning CLI.") + self.validate(manager_upgrade=manager_upgrade) + + def select(self, version, installed=None): + self.require_quiescent(self.state["selected"]) + if version is not None: + self.require_quiescent(version) + # Recheck receipts at the effect boundary, then commit exactly one pointer. + self.validate() + state = dict(self.state) + if installed is not None: + state["installed"] = installed + if version != state["selected"]: + state["previous"] = state["selected"] + state["selected"] = version + atomic_json(self.root / ".selection.json", state) + self.state = state + + def install(self, version, archive=None, checksum=None, upgrade=False): + version_number(version) + current = self.state["selected"] + if version in self.state["installed"]: + require(version == current, "Version is already installed; use rollback to select it.") + if archive is not None: + archive = canonical(archive) + owned(archive) + require(archive is None or digest(archive) == checksum + == private_json(self.root / "versions" / version / ".receipt.json")["archive_sha256"], + "Installed version differs from the supplied archive.") + return + require(upgrade or current is None, "A candidate is selected; use upgrade with a newer version.") + require(current is None or version_number(version) > version_number(current), + "Upgrade must increase the prerelease version; use rollback for an installed version.") + self.require_quiescent(current) + stage = self.root / (".stage-" + uuid.uuid4().hex) + stage.mkdir(mode=0o700) + created = stage.lstat() + try: + if archive is None: + archive, checksum, release_metadata = official_archive(stage, version) + else: + archive = canonical(archive) + release_metadata = None + installation = stage / "installation" + installation.mkdir(mode=0o700) + identity, manifest = extract_archive(archive, checksum, installation / "bundle", version, + release_metadata) + # The retained launcher always executes its recorded manager. Never select + # a new layout that an older manager cannot subsequently validate/rollback. + # Manager replacement is a separate explicit, recoverable operation. + require(set(manifest) <= PAYLOAD + or digest(self.root / "manager.py") == digest(Path(__file__)), + "Optional candidate payloads require this channel's retained manager to match the installer. " + "Run upgrade-manager with this reviewed installer, or use a fresh --root; " + "the existing channel and its selection are unchanged.") + homes = {} + for name in ("native-home", "cli-home"): + (installation / name).mkdir(mode=0o700) + info = (installation / name).stat() + homes[name] = {"device": info.st_dev, "inode": info.st_ino} + write_file(installation / ".receipt.json", json_bytes({ + "schema": "hack.prerelease-version/v1", "metadata": identity, + "archive_sha256": checksum, "checksums": manifest, "homes": homes})) + sync_directory(installation) + # Final refusal happens before publication. A power loss after rename retains + # an uncommitted directory for inspection, while the old selection stays valid. + self.require_quiescent(current) + self.validate() + destination = self.root / "versions" / version + require(not destination.exists(), "Refusing to overwrite a candidate version.") + os.rename(str(installation), str(destination)) + sync_directory(destination.parent) + installed = dict(self.state["installed"]) + installed[version] = digest(destination / ".receipt.json") + state = dict(self.state, installed=installed, selected=version, previous=current) + atomic_json(self.root / ".selection.json", state) + self.state = state + finally: + # Downloads, partial extraction and an unpublished installation live only + # in this call's stage. Cleanup never decides the outcome: validate() + # tolerates a stage that could not be removed, and the selection above + # is either committed or unchanged. + with contextlib.suppress(OSError): + remove_created_stage(stage, created) + + def run(self, arguments): + version = self.state["selected"] + require(version is not None, "No candidate selected. Use your ordinary stable hack command.") + directory = self.verify_version(version, self.state["installed"][version]) + verify_signatures(directory / "bundle") + # Shared launcher locks permit ps/down alongside foreground up, while an + # exclusive selection switch refuses any in-flight launcher. Background + # runtime state is gated by the owning executor on later switches. + child = subprocess.Popen([str(directory / "bundle/hack-v5"), *arguments], + env=self.environment(version)) + handlers = {} + def forward(signum, _frame): + child.send_signal(signum) + try: + for signum in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + handlers[signum] = signal.signal(signum, forward) + code = child.wait() + return code if code >= 0 else 128 - code + finally: + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + +def parser(): + arguments = argparse.ArgumentParser(description=__doc__) + arguments.add_argument("--root", type=Path, default=Path.home() / ".hack-next", + help="private canonical installation root (default ~/.hack-next)") + commands = arguments.add_subparsers(dest="command", required=True) + for command in ("install", "upgrade"): + install = commands.add_parser(command) + install.add_argument("--version", required=True) + install.add_argument("--archive", type=Path, help="explicit local archive instead of GitHub") + install.add_argument("--sha256", help="required expected digest for --archive") + rollback = commands.add_parser("rollback") + rollback.add_argument("--version", help="retained version; default previous selection") + commands.add_parser("stable", help="deselect candidate; retain all bundles and homes") + commands.add_parser("status") + commands.add_parser("upgrade-manager", help="explicitly upgrade a known channel manager, or finish its interrupted upgrade") + run = commands.add_parser("run") + run.add_argument("arguments", nargs=argparse.REMAINDER) + return arguments + + +def main(argv=None): + args = parser().parse_args(argv) + os.umask(0o077) + require(platform.system() == "Darwin" and platform.machine() == "arm64", + "Native prereleases support Apple Silicon macOS only.") + channel = Channel(args.root) + if args.command in ("install", "upgrade"): + version_number(args.version) + require(bool(args.archive) == bool(args.sha256), "--archive and --sha256 must be supplied together.") + channel.initialize() + with channel.lock(shared=args.command == "run", manager_upgrade=args.command == "upgrade-manager"): + if args.command in ("install", "upgrade"): + channel.install(args.version, args.archive, args.sha256, args.command == "upgrade") + elif args.command == "upgrade-manager": + channel.upgrade_manager() + elif args.command == "rollback": + version = args.version or channel.state["previous"] + require(version is not None and version in channel.state["installed"], "No retained rollback version.") + channel.select(version) + elif args.command == "stable": + channel.select(None) + elif args.command == "run": + arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments + return channel.run(arguments) + print(json.dumps({"selected": channel.state["selected"], "previous": channel.state["previous"], + "installed": sorted(channel.state["installed"], key=version_number), + "launcher": str(channel.root / "bin/hack-next")})) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (Refusal, OSError, ValueError) as error: + print("hack-next: " + str(error), file=sys.stderr) + sys.exit(1) diff --git a/tests/prerelease-plan.test.ts b/tests/prerelease-plan.test.ts index 72527fc5c..db952f0a6 100644 --- a/tests/prerelease-plan.test.ts +++ b/tests/prerelease-plan.test.ts @@ -1351,6 +1351,7 @@ test("installer accepts exactly the versioned bundle contract with the packaged "print(json.dumps({", " 'accepted': sorted(m.payload_inventory(names | {'SHA256SUMS'})),", " 'top_level': sorted(m.PAYLOAD | m.COMPILER_PAYLOAD | m.STORAGE_TOOL_PAYLOAD),", + " 'member_ceiling': m.MAX_ARCHIVE_MEMBERS,", " 'modes': {name: m.payload_mode(name) for name in sorted(names)},", " 'foreign_refused': refused(names | {'SHA256SUMS', 'native-storage-tool.sig'}),", " 'storage_partial_refused': [refused(names - {name} | {'SHA256SUMS'})", @@ -1367,6 +1368,7 @@ test("installer accepts exactly the versioned bundle contract with the packaged const result = JSON.parse(probe.stdout.toString()) as { accepted: string[]; top_level: string[]; + member_ceiling: number; modes: Record; foreign_refused: boolean; storage_partial_refused: boolean[]; @@ -1380,6 +1382,10 @@ test("installer accepts exactly the versioned bundle contract with the packaged } expect(result.foreign_refused).toBe(true); expect(result.storage_partial_refused).toEqual([true, true]); + // The packaged archive holds the versioned payload plus SHA256SUMS. Its member + // count must stay below the installer's fixed ceiling, leaving headroom for a + // reviewed addition, instead of filling it exactly. + expect(result.member_ceiling).toBeGreaterThan(names.length + 1); }); /** GitHub's heading anchor: lowercase, punctuation removed, spaces as hyphens. */ diff --git a/tests/python/test_prerelease_install.py b/tests/python/test_prerelease_install.py index 8d41a0c1f..1bf8db4be 100644 --- a/tests/python/test_prerelease_install.py +++ b/tests/python/test_prerelease_install.py @@ -81,6 +81,34 @@ def with_current_payload(entries): return with_storage_tool(with_mcp(with_compiler(entries))) +# A hypothetical reviewed top-level bundle file. Tests patch it into the installer's +# allowlist to stand in for the next reviewed installer that ships a new file. +EXTENSION = "hack-reviewed-extension.json" +EXTENSION_BYTES = b'{"reviewed": true}\n' + + +def with_extended_payload(entries): + """The current 17-member layout plus one reviewed addition: 18 archive members.""" + payload = {name: value for name, value, _ in entries if name != "SHA256SUMS"} + payload[EXTENSION] = EXTENSION_BYTES + # with_current_payload recomputes SHA256SUMS over every member, including this one. + return with_current_payload([(name, value, tarfile.REGTYPE) for name, value in payload.items()]) + + +def reviewed_extension(): + """Admit EXTENSION as a required core file, as a later reviewed installer would.""" + stack = contextlib.ExitStack() + stack.enter_context(mock.patch.object(installer, "PAYLOAD", installer.PAYLOAD | {EXTENSION})) + stack.enter_context(mock.patch.object(installer, "BUNDLE_FILES", + installer.BUNDLE_FILES | {EXTENSION})) + return stack + + +def archive_members(archive): + with tarfile.open(archive, "r:gz") as source: + return [entry.name for entry in source] + + class OfficialResponse: """One HTTP response read through the installer's real download loop. @@ -742,6 +770,152 @@ def test_released_and_reviewed_managers_upgrade_before_accepting_storage_tool(se self.assertEqual((self.channel.root / "versions" / version / home / "marker").read_text(), version + home) + def test_member_ceiling_is_fixed_above_the_largest_reviewed_inventory(self): + archive, _ = self.archive(mutate=with_current_payload) + reviewed = len(installer.BUNDLE_FILES | installer.COMPILER_PAYLOAD + | installer.STORAGE_TOOL_PAYLOAD) + len(installer.MCP_FILES) + 1 + self.assertEqual(len(archive_members(archive)), reviewed) + self.assertGreater(installer.MAX_ARCHIVE_MEMBERS, reviewed) + with reviewed_extension(): + # The bound no longer follows the allowlist sizes; growing them leaves it unchanged. + self.assertEqual(installer.MAX_ARCHIVE_MEMBERS, 64) + + def test_reviewed_addition_beyond_seventeen_members_installs_and_rolls_back(self): + with reviewed_extension(): + for number in (1, 2): + version = "5.0.0-next." + str(number) + archive, checksum = self.archive(version, with_extended_payload) + self.assertEqual(len(archive_members(archive)), 18) + with self.channel.lock(): + self.channel.install(version, archive, checksum, number == 2) + bundle = self.channel.root / "versions" / version / "bundle" + _, manifest = installer.verify_bundle(bundle, version) + self.assertEqual(len(manifest), 17) + self.assertIn(EXTENSION, manifest) + self.assertEqual((bundle / EXTENSION).read_bytes(), EXTENSION_BYTES) + self.assertEqual((bundle / EXTENSION).stat().st_mode & 0o777, 0o600) + self.assertEqual((bundle / installer.STORAGE_TOOL).stat().st_mode & 0o777, 0o500) + for prior in ("5.0.0-next.1", "5.0.0-next.2"): + with self.channel.lock(): + self.channel.select(prior) + self.assertEqual(self.selection()["selected"], prior) + + def test_member_ceiling_refuses_excess_headers_before_inventory(self): + # MCP members are the only digest-wildcard names, so distinct identities are + # allowlisted one by one. The ceiling bounds them; the inventory refuses below it. + def mcp_identities(count): + return lambda entries: [*entries, *(("mcp/%064x/manifest.json" % index, b"{}\n", + tarfile.REGTYPE) for index in range(count))] + flat = 8 + self.rejects_install(mcp_identities(installer.MAX_ARCHIVE_MEMBERS - flat + 1), + r"\AToo many archive entries\.\Z") + self.rejects_install(mcp_identities(installer.MAX_ARCHIVE_MEMBERS - flat), + "one MCP bundle identity") + + def test_larger_reviewed_inventory_keeps_member_refusals(self): + with reviewed_extension(): + archive, checksum = self.archive(mutate=with_extended_payload) + with self.channel.lock(): + self.channel.install("5.0.0-next.1", archive, checksum) + + def extended(change): + return lambda entries: change(with_extended_payload(entries)) + + def replace(name=None, value=None, kind=None): + return extended(lambda entries: [ + (name if name and key == EXTENSION else key, + value if value is not None and key == EXTENSION else data, + kind if kind is not None and key == EXTENSION else entry_kind) + for key, data, entry_kind in entries]) + + foreign = "Foreign, duplicate or traversing archive entry" + cases = [ + (extended(lambda entries: [*entries, next(entry for entry in entries + if entry[0] == EXTENSION)]), foreign), + (replace(name="../" + EXTENSION), foreign), + (replace(name="/tmp/" + EXTENSION), foreign), + (replace(name="bundle/" + EXTENSION), foreign), + (replace(name="hack-unreviewed-extension.json"), foreign), + (replace(kind=tarfile.SYMTYPE), "regular archive files"), + (replace(kind=tarfile.LNKTYPE), "regular archive files"), + (replace(value=b'{"reviewed": false}\n'), "checksum mismatch"), + (extended(lambda entries: [entry for entry in entries if entry[0] != EXTENSION]), + "Incomplete candidate bundle"), + ] + for mutate, message in cases: + with self.subTest(message=message, mutate=mutate): + self.rejects_install(mutate, message) + limit = 4096 + with mock.patch.object(installer, "MAX_ARCHIVE", limit): + for mutate, message in ( + (replace(value=bytes(limit + 1)), "Only bounded regular archive files"), + (extended(lambda entries: [ + (key, bytes(limit // 2) if key in (EXTENSION, "hack-cli") else value, kind) + for key, value, kind in entries]), "Oversized archive")): + with self.subTest(message=message): + archive, _ = self.archive("5.0.0-next.2", mutate) + self.assertLessEqual(archive.stat().st_size, limit) + self.rejects_install(mutate, message) + + def test_released_storage_manager_upgrades_before_accepting_new_installer_payloads(self): + # 5.0.0-next.3, next.4 and next.5 all shipped this manager byte for byte. + source = SOURCE.parent.parent / "tests/fixtures/prerelease-manager-storage-v5.py" + predecessor = "e2e2bcbacf58b21d1086047c94727e5e814776384439700adcf5aabe5c49c2fa" + self.assertEqual(installer.digest(source), predecessor) + self.assertIn(predecessor, installer.MANAGER_PREDECESSORS) + spec = importlib.util.spec_from_file_location("storage_prerelease_manager", source) + previous = importlib.util.module_from_spec(spec) + spec.loader.exec_module(previous) + self.channel = previous.Channel(self.root / "storage-v5-channel") + self.channel.initialize() + self.install() + for version in ("5.0.0-next.4", "5.0.0-next.5"): + archive, checksum = self.archive(version, with_current_payload) + with self.channel.lock(): + self.channel.install(version, archive, checksum, True) + for version in ("5.0.0-next.1", "5.0.0-next.4", "5.0.0-next.5"): + for home in ("native-home", "cli-home"): + (self.channel.root / "versions" / version / home / "marker").write_text(version + home) + before = self.retained_snapshot() + # The retained manager filled its derived 17-member bound and knows no new name, + # so it refuses a larger layout instead of selecting files it cannot verify. + larger, larger_checksum = self.archive("5.0.0-next.6", with_extended_payload) + self.assertEqual(len(archive_members(larger)), 18) + with self.channel.lock(): + with self.assertRaisesRegex(previous.Refusal, "archive entr"): + self.channel.install("5.0.0-next.6", larger, larger_checksum, True) + self.assertEqual(self.retained_snapshot(), before) + archive, checksum = self.archive("5.0.0-next.6", with_current_payload) + current = installer.Channel(self.channel.root) + with current.lock(): + with self.assertRaisesRegex(installer.Refusal, "upgrade-manager"): + current.install("5.0.0-next.6", archive, checksum, True) + self.assertEqual(self.retained_snapshot(), before) + self.channel = self.upgrade_manager() + self.assertEqual(self.retained_snapshot(), before) + self.assertEqual((self.channel.root / "manager.py").read_bytes(), SOURCE.read_bytes()) + with self.channel.lock(): + self.assertEqual(self.channel.state["selected"], "5.0.0-next.5") + self.channel.install("5.0.0-next.6", archive, checksum, True) + for prior in ("5.0.0-next.5", "5.0.0-next.1", "5.0.0-next.4", "5.0.0-next.6"): + with self.channel.lock(): + self.channel.select(prior) + self.assertEqual(self.selection()["selected"], prior) + bundle = self.channel.root / "versions/5.0.0-next.6/bundle" + child = mock.Mock() + child.wait.return_value = 0 + with self.channel.lock(shared=True), \ + mock.patch.object(installer.subprocess, "Popen", return_value=child) as spawn: + self.assertEqual(self.channel.run(["--version"]), 0) + self.assertEqual(Path(spawn.call_args.args[0][0]), bundle / "hack-v5") + for version in ("5.0.0-next.1", "5.0.0-next.4", "5.0.0-next.5"): + for home in ("native-home", "cli-home"): + self.assertEqual((self.channel.root / "versions" / version / home / "marker").read_text(), + version + home) + # Repeating the upgrade with the installed manager is a no-op. + self.upgrade_manager() + self.assertEqual((self.channel.root / "manager.py").read_bytes(), SOURCE.read_bytes()) + def test_bundle_verification_hashes_each_payload_once_without_cross_call_cache(self): archive, checksum = self.archive(mutate=with_current_payload) with self.channel.lock(): From 254a7bf0a0bbe152668a9bb90790e1f243f789a2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:43:37 -0400 Subject: [PATCH 03/21] test(runtime-core): cover job journal durability and cancellation negatives Add WU04 negative-case coverage for the node job journal: - real SQLITE_FULL on terminal publication and cancellation leaves the prior receipt, generation and operation set untouched; the same requests apply exactly once after space returns - cancel and completion serialize under real SQLite write-lock contention; terminal receipts are final and an early cancel is never reported as an achieved cancellation - the same operation ID with a changed fixture, deadline, capability set, generation or action conflicts without effect, and a cancel operation ID cannot be re-pointed at another job - a supervisor crash after its claim, before or after spawn persistence, is quarantined without replay or signalling the recorded PID - terminal receipts, generation and retry identities survive both an in-process journal reopen and a node service restart Refs HACK-1117. Co-Authored-By: Claude Opus 5.5 --- packages/runtime-core/src/node/journal.rs | 306 +++++++++++++++++++ packages/runtime-core/tests/node_contract.rs | 200 ++++++++++++ 2 files changed, 506 insertions(+) diff --git a/packages/runtime-core/src/node/journal.rs b/packages/runtime-core/src/node/journal.rs index efad292cb..7d07e2cde 100644 --- a/packages/runtime-core/src/node/journal.rs +++ b/packages/runtime-core/src/node/journal.rs @@ -784,4 +784,310 @@ mod tests { drop(store); std::fs::remove_dir_all(root).unwrap(); } + + static ROOTS: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + + /// A private journal directory owned by one test; removed by the caller. + fn test_root(label: &str) -> PathBuf { + std::env::temp_dir().canonicalize().unwrap().join(format!( + "hack-node-{label}-{}-{}", + std::process::id(), + ROOTS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + )) + } + + fn mutation(store: &Store, operation_id: &str, generation: i64) -> Mutation { + Mutation { + operation_id: operation_id.into(), + expected_generation: generation, + target: store.target.clone(), + principal: unsafe { libc::geteuid() }, + request_digest: String::new(), + required_capabilities: vec!["fixture_jobs_v1".into()], + } + } + + fn submit(store: &Store, operation_id: &str, generation: i64) -> Request { + Request::Submit { + version: 1, + mutation: mutation(store, operation_id, generation), + fixture: "success".into(), + queue_timeout_ms: 30_000, + execution_timeout_ms: 1000, + } + .seal() + .unwrap() + } + + fn cancel(store: &Store, operation_id: &str, generation: i64, job_id: &str) -> Request { + Request::Cancel { + version: 1, + mutation: mutation(store, operation_id, generation), + job_id: job_id.into(), + } + .seal() + .unwrap() + } + + fn generation(store: &mut Store) -> i64 { + store + .handle(&Request::Status { version: 1 }, unsafe { libc::geteuid() }) + .unwrap()["generation"] + .as_i64() + .unwrap() + } + + fn operations(store: &Store) -> i64 { + store + .connection + .query_row("SELECT count(*) FROM operations", [], |r| r.get(0)) + .unwrap() + } + + fn snapshot(store: &Store, id: &str) -> serde_json::Value { + serde_json::to_value(store.get(id).unwrap()).unwrap() + } + + /// Simulates the scheduler claim and the supervisor's spawn persistence for a + /// fixture job without starting a process: only journal transitions are tested. + fn mark_running(store: &mut Store, id: &str) { + store + .update(id, |r| { + r.state = "running".into(); + r.supervisor_pid = Some(1); + r.child_pid = Some(1); + r.starts += 1; + Ok(()) + }) + .unwrap(); + } + + #[test] + fn sqlite_full_cancel_and_terminal_publication_leave_prior_receipts_intact() { + let uid = unsafe { libc::geteuid() }; + let root = test_root("full-effects"); + let mut store = Store::open(&root, true).unwrap(); + let mut jobs = Vec::new(); + for index in 0..100 { + let accepted = store + .handle(&submit(&store, &format!("job-{index}"), index), uid) + .unwrap(); + jobs.push(accepted["job_id"].as_str().unwrap().to_owned()); + } + let running = jobs.pop().unwrap(); + mark_running(&mut store, &running); + // No free pages: every growth below must allocate, so SQLITE_FULL comes from + // SQLite's real allocation failure rather than an injected trigger. + let free: i64 = store + .connection + .pragma_query_value(None, "freelist_count", |r| r.get(0)) + .unwrap(); + assert_eq!(free, 0); + let pages: i64 = store + .connection + .pragma_query_value(None, "page_count", |r| r.get(0)) + .unwrap(); + store + .connection + .pragma_update(None, "max_page_count", pages) + .unwrap(); + + // Terminal publication: retained output needs overflow pages. The failed + // write must not publish a partial result over the persisted running state. + let before = snapshot(&store, &running); + let failure = store + .update(&running, |r| { + r.state = "succeeded".into(); + r.exit_code = Some(0); + r.stdout = "o".repeat(OUTPUT_LIMIT); + r.stderr = "e".repeat(OUTPUT_LIMIT); + Ok(()) + }) + .unwrap_err(); + assert!(failure.message.contains("full"), "{failure:?}"); + assert_eq!(snapshot(&store, &running), before); + + // Cancellation: each accepted cancel adds an operation row, so the journal + // fills within the queued set. The refused cancel leaves no trace at all. + let mut refused = None; + for (index, job) in jobs.iter().enumerate() { + let expected = 100 + index as i64; + let request = cancel(&store, &format!("cancel-{index}"), expected, job); + let (before, recorded) = (snapshot(&store, job), operations(&store)); + match store.handle(&request, uid) { + Ok(response) => { + assert_eq!(store.get(job).unwrap().state, "cancelled", "{response}"); + } + Err(failure) => { + assert!(failure.message.contains("full"), "{failure:?}"); + assert_eq!(snapshot(&store, job), before); + assert_eq!(store.get(job).unwrap().state, "queued"); + assert!(!store.get(job).unwrap().cancel_requested); + assert_eq!(operations(&store), recorded); + assert_eq!(generation(&mut store), expected); + refused = Some((request, job.clone(), expected)); + break; + } + } + } + let (request, job, expected) = + refused.expect("Test must exercise SQLite's actual SQLITE_FULL path for cancel."); + + // Space returns: the same requests now apply exactly once. + store + .connection + .pragma_update(None, "max_page_count", 100_000) + .unwrap(); + let accepted = store.handle(&request, uid).unwrap(); + assert_eq!(accepted["generation"], expected + 1); + assert_eq!(store.handle(&request, uid).unwrap(), accepted); + assert_eq!(store.get(&job).unwrap().state, "cancelled"); + let published = store + .update(&running, |r| { + r.state = "succeeded".into(); + r.exit_code = Some(0); + r.stdout = "o".repeat(OUTPUT_LIMIT); + Ok(()) + }) + .unwrap(); + assert_eq!(published.state, "succeeded"); + assert_eq!(published.starts, 1); + drop(store); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn cancel_and_completion_serialize_and_terminal_receipts_are_final() { + let uid = unsafe { libc::geteuid() }; + let root = test_root("cancel-race"); + // Two connections stand in for the node service and a job supervisor process. + let mut node = Store::open(&root, true).unwrap(); + let mut supervisor = Store::open(&root, false).unwrap(); + let mut ids = Vec::new(); + for (index, name) in ["completes-first", "cancel-first", "queued"] + .iter() + .enumerate() + { + let accepted = node + .handle(&submit(&node, name, index as i64), uid) + .unwrap(); + ids.push(accepted["job_id"].as_str().unwrap().to_owned()); + } + let (first, second, queued) = (ids[0].as_str(), ids[1].as_str(), ids[2].as_str()); + mark_running(&mut supervisor, first); + mark_running(&mut supervisor, second); + + // Completion holds the write lock while cancellation arrives. SQLite + // serializes them: the cancel waits, then observes a terminal receipt. + supervisor + .connection + .execute_batch("BEGIN IMMEDIATE") + .unwrap(); + let mut completed = read_receipt(&supervisor.connection, first).unwrap(); + completed.state = "succeeded".into(); + completed.exit_code = Some(0); + write_receipt(&supervisor.connection, &completed).unwrap(); + let request = cancel(&node, "late-cancel", 3, first); + let waiting = std::thread::spawn(move || { + let response = node.handle(&request, uid); + (node, request, response) + }); + std::thread::sleep(std::time::Duration::from_millis(200)); + supervisor.connection.execute_batch("COMMIT").unwrap(); + let (mut node, request, response) = waiting.join().unwrap(); + let response = response.unwrap(); + assert_eq!(response["generation"], 4); + let terminal = snapshot(&node, first); + assert_eq!(terminal["state"], "succeeded"); + assert_eq!(terminal["cancel_requested"], false, "{terminal}"); + assert_eq!(node.handle(&request, uid).unwrap(), response); + let late = supervisor + .update(first, |r| { + r.state = "cancelled".into(); + Ok(()) + }) + .unwrap(); + assert_eq!(late.state, "succeeded"); + assert_eq!(snapshot(&node, first), terminal); + + // Cancellation commits first while the child runs: it is recorded as a + // request, never as an achieved cancellation. The owner's natural-exit + // result stays truthful and later cancels cannot rewrite it. + node.handle(&cancel(&node, "early-cancel", 4, second), uid) + .unwrap(); + let requested = node.get(second).unwrap(); + assert_eq!(requested.state, "running"); + assert!(requested.cancel_requested); + let natural = supervisor + .update(second, |r| { + r.state = "succeeded".into(); + r.exit_code = Some(0); + Ok(()) + }) + .unwrap(); + assert_eq!(natural.state, "succeeded"); + node.handle(&cancel(&node, "after-exit", 5, second), uid) + .unwrap(); + assert_eq!( + snapshot(&node, second), + serde_json::to_value(&natural).unwrap() + ); + + // A queued cancel is terminal at once; a racing scheduler claim cannot + // move it to preparing or start it. + node.handle(&cancel(&node, "queued-cancel", 6, queued), uid) + .unwrap(); + let claim = supervisor + .update(queued, |r| { + r.state = "preparing".into(); + Ok(()) + }) + .unwrap(); + assert_eq!(claim.state, "cancelled"); + assert_eq!(node.get(queued).unwrap().starts, 0); + assert_eq!(generation(&mut node), 7); + drop((node, supervisor)); + std::fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn reopened_journal_preserves_receipts_generation_and_retry_identity() { + let uid = unsafe { libc::geteuid() }; + let root = test_root("reopen"); + let mut store = Store::open(&root, true).unwrap(); + let target = store.target.clone(); + let request = submit(&store, "survives", 0); + let accepted = store.handle(&request, uid).unwrap(); + let id = accepted["job_id"].as_str().unwrap().to_owned(); + mark_running(&mut store, &id); + store + .update(&id, |r| { + r.state = "failed".into(); + r.exit_code = Some(23); + r.stderr = "fixture failure\n".into(); + Ok(()) + }) + .unwrap(); + let cancel_request = cancel(&store, "late-cancel", 1, &id); + let cancelled = store.handle(&cancel_request, uid).unwrap(); + let receipt = snapshot(&store, &id); + drop(store); + + // An in-process restart: the only surviving state is the committed journal. + let mut reopened = Store::open(&root, false).unwrap(); + assert_eq!(reopened.target, target); + assert_eq!(snapshot(&reopened, &id), receipt); + assert_eq!(generation(&mut reopened), 2); + assert_eq!(reopened.handle(&request, uid).unwrap(), accepted); + assert_eq!(reopened.handle(&cancel_request, uid).unwrap(), cancelled); + assert_eq!(snapshot(&reopened, &id), receipt); + assert_eq!(generation(&mut reopened), 2); + assert_eq!(reopened.get(&id).unwrap().starts, 1); + drop(reopened); + let mut inspect = Store::inspect(&root).unwrap(); + assert_eq!(snapshot(&inspect, &id), receipt); + assert_eq!(generation(&mut inspect), 2); + drop(inspect); + std::fs::remove_dir_all(root).unwrap(); + } } diff --git a/packages/runtime-core/tests/node_contract.rs b/packages/runtime-core/tests/node_contract.rs index 28cb17bdd..9d6c19ee3 100644 --- a/packages/runtime-core/tests/node_contract.rs +++ b/packages/runtime-core/tests/node_contract.rs @@ -650,3 +650,203 @@ fn a_new_journal_incarnation_rejects_old_target_and_copied_state() { .unwrap(); assert!(Store::open(&other.root, false).is_err()); } + +#[test] +fn same_operation_id_with_a_different_request_conflicts_without_effect() { + let root = std::env::temp_dir().canonicalize().unwrap().join(format!( + "hkl-operation-conflict-{}-{}", + std::process::id(), + SERIAL.fetch_add(1, Ordering::SeqCst) + )); + let uid = unsafe { libc::geteuid() }; + let mut store = Store::open(&root, true).unwrap(); + let target = store.target.clone(); + let mutation = |id: &str, generation: i64| Mutation { + operation_id: id.into(), + expected_generation: generation, + target: target.clone(), + principal: uid, + request_digest: String::new(), + required_capabilities: vec!["fixture_jobs_v1".into()], + }; + let submit = |id: &str, generation: i64| Request::Submit { + version: 1, + mutation: mutation(id, generation), + fixture: "success".into(), + queue_timeout_ms: 5000, + execution_timeout_ms: 2000, + }; + let original = submit("same-id", 0); + let mut variants = Vec::new(); + for edit in 0..5 { + let mut variant = original.clone(); + if let Request::Submit { + mutation, + fixture, + queue_timeout_ms, + execution_timeout_ms, + .. + } = &mut variant + { + match edit { + 0 => *fixture = "failure".into(), + 1 => *queue_timeout_ms += 1, + 2 => *execution_timeout_ms += 1, + 3 => mutation + .required_capabilities + .push("cooperative_process_groups".into()), + // A retry that refreshes its generation is a different request. + _ => mutation.expected_generation = 1, + } + } + variants.push(variant); + } + let accepted = store + .handle(&original.clone().seal().unwrap(), uid) + .unwrap(); + let job = accepted["job_id"].as_str().unwrap().to_owned(); + // The job ID is derived from the operation ID, so every variant names this job. + variants.push(Request::Cancel { + version: 1, + mutation: mutation("same-id", 0), + job_id: job.clone(), + }); + let receipt = serde_json::to_value(store.get(&job).unwrap()).unwrap(); + for variant in variants { + let refusal = store + .handle(&variant.clone().seal().unwrap(), uid) + .unwrap_err(); + assert_eq!(refusal.code, "operation_conflict", "{variant:?}"); + } + assert_eq!( + serde_json::to_value(store.get(&job).unwrap()).unwrap(), + receipt + ); + assert_eq!(store.list().unwrap().len(), 1); + assert_eq!( + store + .handle(&original.clone().seal().unwrap(), uid) + .unwrap(), + accepted + ); + + // A cancel operation ID cannot be re-pointed at a different job. + let other = store + .handle(&submit("other", 1).seal().unwrap(), uid) + .unwrap(); + let other = other["job_id"].as_str().unwrap().to_owned(); + let cancel = |job_id: &str| Request::Cancel { + version: 1, + mutation: mutation("cancel-once", 2), + job_id: job_id.into(), + }; + store.handle(&cancel(&job).seal().unwrap(), uid).unwrap(); + assert_eq!( + store + .handle(&cancel(&other).seal().unwrap(), uid) + .unwrap_err() + .code, + "operation_conflict" + ); + assert_eq!(store.get(&job).unwrap().state, "cancelled"); + let untouched = store.get(&other).unwrap(); + assert_eq!(untouched.state, "queued"); + assert!(!untouched.cancel_requested); + let status = store.handle(&Request::Status { version: 1 }, uid).unwrap(); + assert_eq!(status["generation"], 3); + drop(store); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn crash_between_claim_spawn_and_result_persistence_quarantines_without_replay() { + // The supervisor persisted its claim ("running") and then died either before + // recording its spawned child or after it, but before any result was written. + // The recorded PIDs belong to an unrelated live process, as after PID reuse. + for child_recorded in [false, true] { + let mut n = Node::new(); + let request = n.submit("spawn-window", "success", 5000, 2000); + n.stop(); + let mut store = Store::open(&n.root, false).unwrap(); + let accepted = store + .handle(&request.clone().seal().unwrap(), unsafe { libc::geteuid() }) + .unwrap(); + let id = accepted["job_id"].as_str().unwrap(); + let mut sentinel = Command::new("/bin/sleep").arg("10").spawn().unwrap(); + let mut receipt = store.get(id).unwrap(); + receipt.state = "running".into(); + receipt.supervisor_pid = Some(sentinel.id() as i32); + if child_recorded { + receipt.child_pid = Some(sentinel.id() as i32); + receipt.starts = 1; + } + let connection = rusqlite::Connection::open(n.root.join("journal.sqlite")).unwrap(); + connection + .execute( + "UPDATE jobs SET receipt=? WHERE id=?", + rusqlite::params![serde_json::to_string(&receipt).unwrap(), id], + ) + .unwrap(); + drop((connection, store)); + n.start(); + let result = n.terminal(id); + assert_eq!(result.state, "quarantined", "{result:?}"); + assert_eq!(result.starts, u32::from(child_recorded)); + assert_eq!(result.exit_code, None); + assert!(result.stdout.is_empty()); + assert_eq!(n.ask(&request), accepted); + // Several scheduling passes later, quarantine has neither replayed the job + // nor signalled the recorded PID. + std::thread::sleep(Duration::from_millis(300)); + assert_eq!( + serde_json::to_value(n.receipt(id)).unwrap(), + serde_json::to_value(&result).unwrap() + ); + assert!(sentinel.try_wait().unwrap().is_none()); + sentinel.kill().unwrap(); + sentinel.wait().unwrap(); + } +} + +#[test] +fn terminal_receipts_and_retry_identity_survive_node_restart() { + let mut n = Node::new(); + let success = n.submit("restart-success", "success", 5000, 3000); + let accepted = n.ask(&success); + let success_id = accepted["job_id"].as_str().unwrap().to_owned(); + let succeeded = n.terminal(&success_id); + let failure = n.submit("restart-failure", "failure", 5000, 3000); + let rejected = n.ask(&failure); + let failure_id = rejected["job_id"].as_str().unwrap().to_owned(); + let failed = n.terminal(&failure_id); + let cancel = Request::Cancel { + version: 1, + mutation: n.mutation("restart-late-cancel"), + job_id: success_id.clone(), + }; + let cancelled = n.ask(&cancel); + let status = n.ask(&Request::Status { version: 1 }); + // The service process is gone; the journal is the only durable owner. + n.stop(); + n.start(); + assert_eq!(n.ask(&Request::Status { version: 1 }), status); + for (id, expected) in [(&success_id, &succeeded), (&failure_id, &failed)] { + assert_eq!( + serde_json::to_value(n.receipt(id)).unwrap(), + serde_json::to_value(expected).unwrap() + ); + } + assert_eq!(succeeded.state, "succeeded"); + assert!(!succeeded.cancel_requested); + assert_eq!( + (failed.state.as_str(), failed.exit_code), + ("failed", Some(23)) + ); + assert_eq!(n.ask(&success), accepted); + assert_eq!(n.ask(&failure), rejected); + assert_eq!(n.ask(&cancel), cancelled); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(n.ask(&Request::Status { version: 1 }), status); + assert_eq!(n.receipt(&success_id).starts, 1); + assert_eq!(n.receipt(&failure_id).starts, 1); +} From 56e20fe265c9c14b836be202e77db7d60da22e80 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:45:47 -0400 Subject: [PATCH 04/21] test(e2e): qualify the P01, P02 and P07 pattern fixtures Register Docker-tier scenarios that start copies of the checked-in examples/patterns fixtures in the hosted docker-e2e shards: - pattern-p01-dependency-bootstrap (HACK-1145): cold and warm starts, a linked worktree, a bun.lock change and an installer killed mid-run; running consumers only ever read complete payloads. - pattern-p02-portable-toolchain (HACK-1146): hack run --profile from the primary checkout and a linked worktree with a .git pointer file; failing checks and exact exit statuses propagate. - pattern-p07-capability-overlays (HACK-1151): new laptop/runner overlay fixture; value-free env explain sources, container metadata, upstream reachability and no browser opener. Cards name their scenario in e2eScenario; static tests require it to be a registered, hinted Docker-tier scenario and check the P07 overlay split. Co-Authored-By: Claude Opus 5.5 --- .../p01-dependency-bootstrap/README.md | 15 +- .../patterns/p02-portable-toolchain/README.md | 12 + .../.hack/docker-compose.yml | 32 ++ .../.hack/hack.config.json | 10 + .../.hack/hack.env.default.yaml | 9 + .../.hack/hack.env.laptop.yaml | 9 + .../.hack/hack.env.runner.yaml | 9 + .../p07-capability-overlays/README.md | 81 +++ .../p07-capability-overlays/src/api.ts | 48 ++ .../p07-capability-overlays/src/upstream.ts | 11 + .../skills/hack-cli/references/patterns.md | 4 +- src/agents/pattern-catalogue.ts | 26 +- tests/e2e/README.md | 39 ++ tests/e2e/fixture.ts | 3 +- tests/e2e/run.ts | 6 + tests/e2e/scenarios/pattern-fixture.ts | 457 ++++++++++++++++ .../pattern-p01-dependency-bootstrap.ts | 498 ++++++++++++++++++ .../pattern-p02-portable-toolchain.ts | 195 +++++++ .../pattern-p07-capability-overlays.ts | 390 ++++++++++++++ tests/e2e/shard.ts | 6 + tests/helpers/pattern-fixtures.ts | 112 ++++ tests/pattern-examples.test.ts | 199 ++++++- 22 files changed, 2157 insertions(+), 14 deletions(-) create mode 100644 examples/patterns/p07-capability-overlays/.hack/docker-compose.yml create mode 100644 examples/patterns/p07-capability-overlays/.hack/hack.config.json create mode 100644 examples/patterns/p07-capability-overlays/.hack/hack.env.default.yaml create mode 100644 examples/patterns/p07-capability-overlays/.hack/hack.env.laptop.yaml create mode 100644 examples/patterns/p07-capability-overlays/.hack/hack.env.runner.yaml create mode 100644 examples/patterns/p07-capability-overlays/README.md create mode 100644 examples/patterns/p07-capability-overlays/src/api.ts create mode 100644 examples/patterns/p07-capability-overlays/src/upstream.ts create mode 100644 tests/e2e/scenarios/pattern-fixture.ts create mode 100644 tests/e2e/scenarios/pattern-p01-dependency-bootstrap.ts create mode 100644 tests/e2e/scenarios/pattern-p02-portable-toolchain.ts create mode 100644 tests/e2e/scenarios/pattern-p07-capability-overlays.ts diff --git a/examples/patterns/p01-dependency-bootstrap/README.md b/examples/patterns/p01-dependency-bootstrap/README.md index 636ff42a7..c6f9e9396 100644 --- a/examples/patterns/p01-dependency-bootstrap/README.md +++ b/examples/patterns/p01-dependency-bootstrap/README.md @@ -57,8 +57,19 @@ If an initializer attempt fails or is interrupted, find the cause, then incremen writers. - Installers that need credentials or environment values. `locked-v1` refuses explicit `environment` and `env_file` on the initializer. -- Two-worktree or interrupted-writer qualification. Those runtime gates belong to - WU07/WU11. +- Automatic reclamation of failed or superseded generations. Those runtime gates + belong to WU07/WU11. + +## How this example is checked The static checks in `tests/pattern-examples.test.ts` validate this config and its commands. They do not start containers. + +The Docker-tier scenario `pattern-p01-dependency-bootstrap` in `tests/e2e/` +starts a copy of this example in the hosted `docker-e2e` job. It runs a cold and +a warm start, a second worktree, a `bun.lock` change, and an installer killed +mid-run. Every running consumer must read a complete old or complete new +payload. The killed attempt must stay unready, a retry under the same +generation must be refused, and a generation bump must recover. The copy uses +the runner's cached `oven/bun:1.4.2-slim` digest and the daemon's platform +instead of the pinned Alpine image, and never pulls. diff --git a/examples/patterns/p02-portable-toolchain/README.md b/examples/patterns/p02-portable-toolchain/README.md index f092d68d1..f3785f2f6 100644 --- a/examples/patterns/p02-portable-toolchain/README.md +++ b/examples/patterns/p02-portable-toolchain/README.md @@ -44,5 +44,17 @@ the named cache volumes. A test-only source identity override must never reach release evidence. +## How this example is checked + The static checks in `tests/pattern-examples.test.ts` validate this config and its commands. They do not start containers. + +The Docker-tier scenario `pattern-p02-portable-toolchain` in `tests/e2e/` runs +both commands above in the hosted `docker-e2e` job, from the primary checkout +and from a linked worktree. In the linked worktree `/app/.git` is a pointer file +whose target is not mounted, and the checks still pass because they do not need +Git. A failing check exits non-zero, Hack returns the command's exact exit +status, and an ordinary down keeps both instances' cache volumes. Each instance gets +its own caches; cross-worktree reuse is still open. The copy uses the runner's +cached `oven/bun:1.4.2-slim` digest instead of the pinned Alpine image, and +never pulls. diff --git a/examples/patterns/p07-capability-overlays/.hack/docker-compose.yml b/examples/patterns/p07-capability-overlays/.hack/docker-compose.yml new file mode 100644 index 000000000..b10e4c0d4 --- /dev/null +++ b/examples/patterns/p07-capability-overlays/.hack/docker-compose.yml @@ -0,0 +1,32 @@ +name: pattern-p07 +# One shared topology for laptops and headless runners (HACK-P07). The env +# overlays change only auth and helper capabilities, never these services. +services: + upstream: + image: oven/bun:1.4.2-alpine@sha256:d888c0ae6c86d7866ff10c5aafdd9077b36aee6455b33dd270fb93c0dd5cef6f + working_dir: /app + command: ["bun", "src/upstream.ts"] + volumes: + - ..:/app:ro + healthcheck: + test: ["CMD", "bun", "-e", "const r = await fetch('http://127.0.0.1:4000/healthz'); process.exit(r.ok ? 0 : 1)"] + interval: 2s + timeout: 3s + retries: 30 + + api: + image: oven/bun:1.4.2-alpine@sha256:d888c0ae6c86d7866ff10c5aafdd9077b36aee6455b33dd270fb93c0dd5cef6f + working_dir: /app + command: ["bun", "src/api.ts"] + depends_on: + upstream: + condition: service_healthy + volumes: + - ..:/app:ro + # Readiness includes the dependency: the API is healthy only when it can + # reach the upstream through the shared topology. + healthcheck: + test: ["CMD", "bun", "-e", "const r = await fetch('http://127.0.0.1:3000/healthz'); process.exit(r.ok ? 0 : 1)"] + interval: 2s + timeout: 3s + retries: 30 diff --git a/examples/patterns/p07-capability-overlays/.hack/hack.config.json b/examples/patterns/p07-capability-overlays/.hack/hack.config.json new file mode 100644 index 000000000..07b72aace --- /dev/null +++ b/examples/patterns/p07-capability-overlays/.hack/hack.config.json @@ -0,0 +1,10 @@ +{ + "$schema": "https://schemas.hack/hack.config.schema.json", + "name": "pattern-p07", + "dev_host": "pattern-p07.hack.local", + "logs": { + "follow_backend": "compose", + "snapshot_backend": "compose", + "clear_on_down": false + } +} diff --git a/examples/patterns/p07-capability-overlays/.hack/hack.env.default.yaml b/examples/patterns/p07-capability-overlays/.hack/hack.env.default.yaml new file mode 100644 index 000000000..d45755dfa --- /dev/null +++ b/examples/patterns/p07-capability-overlays/.hack/hack.env.default.yaml @@ -0,0 +1,9 @@ +version: 1 +environment: default +secretsprovider: project_key +values: + # Shared topology. Overlays must not redefine these keys. + global: + UPSTREAM_URL: http://upstream:4000 + api: + UPSTREAM_TIMEOUT_MS: "3000" diff --git a/examples/patterns/p07-capability-overlays/.hack/hack.env.laptop.yaml b/examples/patterns/p07-capability-overlays/.hack/hack.env.laptop.yaml new file mode 100644 index 000000000..a1492e5c8 --- /dev/null +++ b/examples/patterns/p07-capability-overlays/.hack/hack.env.laptop.yaml @@ -0,0 +1,9 @@ +version: 1 +environment: laptop +secretsprovider: project_key +values: + # A developer laptop: a person can complete an interactive sign-in. + global: + AUTH_MODE: browser-session + BROWSER_SIGN_IN: allowed + HELPER_MODE: host-helper diff --git a/examples/patterns/p07-capability-overlays/.hack/hack.env.runner.yaml b/examples/patterns/p07-capability-overlays/.hack/hack.env.runner.yaml new file mode 100644 index 000000000..957e39489 --- /dev/null +++ b/examples/patterns/p07-capability-overlays/.hack/hack.env.runner.yaml @@ -0,0 +1,9 @@ +version: 1 +environment: runner +secretsprovider: project_key +values: + # A headless runner: nobody can open a browser, so sign-in is disabled. + global: + AUTH_MODE: machine-identity + BROWSER_SIGN_IN: disabled + HELPER_MODE: none diff --git a/examples/patterns/p07-capability-overlays/README.md b/examples/patterns/p07-capability-overlays/README.md new file mode 100644 index 000000000..7c7c25c96 --- /dev/null +++ b/examples/patterns/p07-capability-overlays/README.md @@ -0,0 +1,81 @@ +# HACK-P07 example: laptop and headless-runner capability overlays + +Minimal project for catalogue card HACK-P07 (tracked in HACK-1151). It shows the +part of the pattern Hack supports today. The card's disposition is **partial**; +see "What this example does not prove" below. + +## Layout + +`.hack/docker-compose.yml` is the one shared topology: an `api` service that +depends on a healthy `upstream` service. Neither overlay changes it. + +The env files express only what differs between machines: + +- `.hack/hack.env.default.yaml` holds the shared topology values, such as the + upstream URL. Overlays must not redefine them. +- `.hack/hack.env.laptop.yaml` allows an interactive browser sign-in. +- `.hack/hack.env.runner.yaml` disables browser sign-in and selects a + non-interactive auth mode for a headless runner. + +The `api` health check passes only when the API can reach `upstream`, so +readiness includes the dependency on both machines. + +## Commands + +Inspect where a value comes from on each machine. The output names the source +file, precedence and delivery target, never the value: + +```sh +hack env explain AUTH_MODE --env laptop --json +hack env explain AUTH_MODE --env runner --target compose --json +``` + +Start the shared topology with the overlay for the machine you are on: + +```sh +hack up --env laptop --json +hack up --env runner --json +``` + +Check what the running containers actually received: + +```sh +hack exec api -- bun -e "console.log(await (await fetch('http://127.0.0.1:3000/')).text())" +``` + +Read the API's bounded log: + +```sh +hack logs --no-follow --tail 100 api +``` + +A worktree-local file such as `.hack/hack.env.runner.local.yaml` overrides the +shared overlay. In a linked worktree, Hack also reads the primary checkout's +local files. `hack env explain` shows that file as the source, so an inherited +override stays visible. + +## What this example does not prove + +- A laptop/headless capability contract that Hack enforces. The overlays here + are labels for env files; Hack does not check which machine selected them. +- A headless machine identity. The runner overlay only selects a + non-interactive auth mode; issuing that identity belongs to WU27/WU30. + +Host env is not evidence of container delivery: check the container, as the +command above does, instead of the shell that ran `hack up`. + +## How this example is checked + +The static checks in `tests/pattern-examples.test.ts` validate the config, the +commands above and the overlay split: overlays declare the same keys, never +redefine a shared value, and hold no secrets. + +The Docker-tier scenario `pattern-p07-capability-overlays` in `tests/e2e/` +starts the laptop overlay in the primary checkout and the runner overlay in a +linked worktree in the hosted `docker-e2e` job. It checks that `hack env explain` +names each source file without printing values, including an inherited +primary-local runner override. It also checks that the containers' metadata +carries each overlay's keys, that both APIs reach `upstream`, and that no Hack +command tries to open a browser. The copy uses the runner's cached +`oven/bun:1.4.2-slim` digest instead of the pinned Alpine image, and never +pulls. diff --git a/examples/patterns/p07-capability-overlays/src/api.ts b/examples/patterns/p07-capability-overlays/src/api.ts new file mode 100644 index 000000000..f96f8589f --- /dev/null +++ b/examples/patterns/p07-capability-overlays/src/api.ts @@ -0,0 +1,48 @@ +/** + * Application service shared by every overlay. It reads its capabilities from + * env and never starts an interactive sign-in itself: a runner overlay must + * provide a non-interactive auth mode. + */ +const authMode = process.env.AUTH_MODE; +const browserSignIn = process.env.BROWSER_SIGN_IN === "allowed"; +const upstreamUrl = process.env.UPSTREAM_URL; +const timeoutMs = Number(process.env.UPSTREAM_TIMEOUT_MS ?? "3000"); + +if (!(authMode && upstreamUrl)) { + console.error("Select an env overlay that sets AUTH_MODE and UPSTREAM_URL."); + process.exit(2); +} +if (authMode === "browser-session" && !browserSignIn) { + console.error("browser-session auth needs an overlay that allows sign-in."); + process.exit(2); +} + +async function upstreamReachable(): Promise { + try { + const response = await fetch(`${upstreamUrl}/healthz`, { + signal: AbortSignal.timeout(timeoutMs), + }); + return response.ok; + } catch { + return false; + } +} + +Bun.serve({ + port: 3000, + async fetch(request) { + const { pathname } = new URL(request.url); + const reachable = await upstreamReachable(); + if (pathname === "/healthz") { + return new Response(reachable ? "ok" : "upstream unreachable", { + status: reachable ? 200 : 503, + }); + } + return Response.json({ + authMode, + browserSignIn, + helperMode: process.env.HELPER_MODE ?? null, + upstreamReachable: reachable, + }); + }, +}); diff --git a/examples/patterns/p07-capability-overlays/src/upstream.ts b/examples/patterns/p07-capability-overlays/src/upstream.ts new file mode 100644 index 000000000..df8d5810a --- /dev/null +++ b/examples/patterns/p07-capability-overlays/src/upstream.ts @@ -0,0 +1,11 @@ +/** Shared dependency that both laptop and runner overlays must reach. */ +Bun.serve({ + port: 4000, + fetch(request) { + const { pathname } = new URL(request.url); + if (pathname === "/healthz") { + return new Response("ok"); + } + return Response.json({ service: "upstream" }); + }, +}); diff --git a/plugins/hack/skills/hack-cli/references/patterns.md b/plugins/hack/skills/hack-cli/references/patterns.md index b191f8189..c65735eff 100644 --- a/plugins/hack/skills/hack-cli/references/patterns.md +++ b/plugins/hack/skills/hack-cli/references/patterns.md @@ -23,7 +23,7 @@ missing part with ad hoc scripts. ## HACK-P01: One-shot dependency and generated-artifact bootstrap - Use when: Services need installed dependencies or generated artifacts before the application starts. -- Disposition: Partial. Missing: locked-v1 covers one immutable payload per volume. Atomic publication of worktree-specific generated artifacts, env-dependent installers, and two-worktree or interrupted-writer qualification remain open. +- Disposition: Partial. Missing: locked-v1 covers one immutable payload per volume. Atomic publication of worktree-specific generated artifacts across concurrent writers, env-dependent installers, and automatic reclamation of failed generations remain open. - Owner: WU07/WU11; tracked in HACK-1145. - Commands: `hack up --json`, `hack ps --json`, `hack logs --no-follow --tail 100` - Prerequisites: A Compose initializer service that consumers require through `service_completed_successfully`; for the opt-in `hack.dependencies.cache-protocol: locked-v1` label, a pinned installer image and an immutable dependency payload. @@ -214,4 +214,4 @@ missing part with ad hoc scripts. --- -Content revision: `091b4e80a9b6`. Generated by Hack; edit the canonical source and regenerate the plugin. +Content revision: `ff1faf16f047`. Generated by Hack; edit the canonical source and regenerate the plugin. diff --git a/src/agents/pattern-catalogue.ts b/src/agents/pattern-catalogue.ts index 2aa0b783a..84ec04b4f 100644 --- a/src/agents/pattern-catalogue.ts +++ b/src/agents/pattern-catalogue.ts @@ -13,6 +13,9 @@ * - A card's `example` directory exists and is statically validated by * tests/pattern-examples.test.ts (config schema, Compose contract, README * commands); every directory under `examples/patterns/` belongs to one card. + * - A card with an `example` names the Docker-tier E2E scenario that starts + * that fixture against a real engine; the scenario is registered for the + * hosted docker-e2e shards with a duration hint. * * Consumer names stay in the Linear issues; the shipped reference records * only how many consumers a card generalizes from. @@ -65,6 +68,13 @@ export type PatternCard = { * plugin ships without the repository's `examples/` tree. */ readonly example: string | null; + /** + * Name of the Docker-tier scenario in `tests/e2e/run.ts` that runs `example` + * against a real engine in the hosted docker-e2e job, or `null` when the + * card has no example. Contributor reference only; not rendered. A + * registered scenario is a maintained gate, not a recorded pass. + */ + readonly e2eScenario: string | null; }; /** The complete pattern register, in id order. */ @@ -93,9 +103,10 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "A populated cache directory is not a completion marker. Concurrent writers to one volume and failed publication need ownership and recovery; consumers must see a complete old or complete new artifact.", domainOwner: "WU07/WU11", missingContract: - "locked-v1 covers one immutable payload per volume. Atomic publication of worktree-specific generated artifacts, env-dependent installers, and two-worktree or interrupted-writer qualification remain open.", + "locked-v1 covers one immutable payload per volume. Atomic publication of worktree-specific generated artifacts across concurrent writers, env-dependent installers, and automatic reclamation of failed generations remain open.", provenance: { consumers: 4, source: "HACK-P01 research register" }, example: "examples/patterns/p01-dependency-bootstrap", + e2eScenario: "pattern-p01-dependency-bootstrap", }, { id: "HACK-P02", @@ -120,6 +131,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Cross-worktree cache reuse with bounded eviction, a source receipt for linked worktrees, and owned cache accounting remain open.", provenance: { consumers: 4, source: "HACK-P02 research register" }, example: "examples/patterns/p02-portable-toolchain", + e2eScenario: "pattern-p02-portable-toolchain", }, { id: "HACK-P03", @@ -147,6 +159,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack has no credential availability, expiry or scope check and no machine-identity path for headless runners; the WU27 env contract owns both.", provenance: { consumers: 3, source: "HACK-P03 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P04", @@ -171,6 +184,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Today's adoption is port-only. Health- and upstream-checked adoption with a recorded adopted-versus-owned state is the WU29 lifecycle ownership contract.", provenance: { consumers: 1, source: "HACK-P04 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P05", @@ -194,6 +208,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack has no branch-scoped external claim/release contract with bounded retry, stop-during-retry and PID-reuse safety.", provenance: { consumers: 1, source: "HACK-P05 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P06", @@ -218,6 +233,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack has no opt-in tunnel lifecycle with external-route readiness and owned-child shutdown.", provenance: { consumers: 1, source: "HACK-P06 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P07", @@ -245,7 +261,8 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ missingContract: "Hack has env overlays and value-free inspection but no laptop/headless capability contract or headless machine identity; WU27 and WU30 own them.", provenance: { consumers: 1, source: "HACK-P07 research register" }, - example: null, + example: "examples/patterns/p07-capability-overlays", + e2eScenario: "pattern-p07-capability-overlays", }, { id: "HACK-P08", @@ -269,6 +286,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack has no semantic no-op detection or atomic machine-local import contract for env hydration.", provenance: { consumers: 1, source: "HACK-P08 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P09", @@ -295,6 +313,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Generation/run ownership, active references, holds and interrupted-reclamation recovery are not implemented for run workspaces.", provenance: { consumers: 1, source: "HACK-P09 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P10", @@ -322,6 +341,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack does not yet distinguish L1, L2 and cold misses, detect a tampered cache, or emit stage timing and provenance receipts.", provenance: { consumers: 3, source: "HACK-P10 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P11", @@ -351,6 +371,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "A guidance freeze during execution and a typed terminal result for resumed background operations remain open.", provenance: { consumers: 2, source: "HACK-P11 research register" }, example: null, + e2eScenario: null, }, { id: "HACK-P12", @@ -374,6 +395,7 @@ export const PATTERN_CATALOGUE: readonly PatternCard[] = [ "Hack has no multi-lane admission, lease or durable run-provenance contract.", provenance: { consumers: 2, source: "HACK-P12 research register" }, example: null, + e2eScenario: null, }, ]; diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 6ae45edea..d62abb8da 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -205,6 +205,45 @@ data survive that cleanup. The canary is then removed separately. PostgreSQL use fixture-only trust on this unexposed network. This qualifies the actual execution host, not Codex Cloud capabilities, browser access or native VM support elsewhere. +## Pattern fixture qualification + +`pattern-p01-dependency-bootstrap` (HACK-1145), `pattern-p02-portable-toolchain` +(HACK-1146) and `pattern-p07-capability-overlays` (HACK-1151) start copies of the +checked-in `examples/patterns/` fixtures against a real engine. Each catalogue card +names its scenario in `e2eScenario`, and `tests/pattern-examples.test.ts` fails when +that scenario is not registered here as Docker-tier with a duration hint. +`tests/e2e/scenarios/pattern-fixture.ts` makes the copy. It changes only the +project name, the image (the cached `oven/bun:1.4.2-slim` repository digest with +`pull_policy: never` instead of the pinned Alpine image), a declared platform +(the daemon's architecture) and exact owner labels. Commands, mounts, dependency +labels, gates and health checks stay as the fixture authored them. + +- P01 runs a cold and a warm `hack up --json`, a linked worktree, a `bun.lock` + change and an installer killed mid-run under a fresh generation. Every running + consumer must read a complete payload with a ready marker and never mount the + killed attempt's volume. That volume must stay populated but unready, a retry + of the same generation must be refused, and a generation bump must recover. +- P02 runs the README's `hack run --profile toolchain` commands from the primary + checkout and a linked worktree, whose `/app/.git` is a pointer to an unmounted + `gitdir`. A failing check must exit non-zero, `sh -c "exit 7"` must return 7, and + a pipe filter must show how filtered output hides the failure. Ordinary + `hack down` must keep both instances' cache volumes. +- P07 runs the laptop overlay in the primary checkout and the runner overlay in a + linked worktree. `hack env explain` must name each source file, including an + inherited primary-local override, without printing a value. Container metadata + must carry each overlay's keys, both APIs must reach `upstream`, `hack exec` + with the other overlay must be refused, and no browser opener on `PATH` or in + `BROWSER` may run. + +```sh +HACK_E2E_CLI_BIN=./dist/hack HACK_E2E_DOCKER=1 HACK_E2E_REQUIRE_DOCKER=1 bun tests/e2e/run.ts --only=pattern-p01-dependency-bootstrap,pattern-p02-portable-toolchain,pattern-p07-capability-overlays +``` + +These are maintained gates for the hosted Linux runner, not recorded passes on +other hosts. They do not qualify the pinned Alpine image, registry access, +credential-dependent installers, cross-worktree toolchain cache reuse or a +headless machine identity. + ## Compose config-hash drift (F03) `compose-config-hash-drift` (HACK-1124) checks that `hack run` and `hack exec` diff --git a/tests/e2e/fixture.ts b/tests/e2e/fixture.ts index d014f1ea1..35583a0a8 100644 --- a/tests/e2e/fixture.ts +++ b/tests/e2e/fixture.ts @@ -460,7 +460,8 @@ function renderComposeYaml(opts: { ].join("\n"); } -async function gitInitAndCommit(opts: { +/** Initialize a repository on `main` and commit every file under `root`. */ +export async function gitInitAndCommit(opts: { readonly root: string; }): Promise { await git({ root: opts.root, args: ["init", "-q", "-b", "main"] }); diff --git a/tests/e2e/run.ts b/tests/e2e/run.ts index 7fdfb7ff4..8357b3502 100644 --- a/tests/e2e/run.ts +++ b/tests/e2e/run.ts @@ -48,6 +48,9 @@ import { nativeConfigProcessPolicyScenario } from "./scenarios/native-config-pro import { nativeConfigProtectedFilesScenario } from "./scenarios/native-config-protected-files.ts"; import { nativeConfigRoutingScenario } from "./scenarios/native-config-routing.ts"; import { nativeConfigVmFilesScenario } from "./scenarios/native-config-vm-files.ts"; +import { patternP01DependencyBootstrapScenario } from "./scenarios/pattern-p01-dependency-bootstrap.ts"; +import { patternP02PortableToolchainScenario } from "./scenarios/pattern-p02-portable-toolchain.ts"; +import { patternP07CapabilityOverlaysScenario } from "./scenarios/pattern-p07-capability-overlays.ts"; import { portableMultiserviceScenario } from "./scenarios/portable-multiservice.ts"; import { upDownScenario } from "./scenarios/up-down.ts"; import { worktreeBranchDefaultScenario } from "./scenarios/worktree-branch-default.ts"; @@ -93,6 +96,9 @@ export const ALL_SCENARIOS: readonly Scenario[] = [ dependencyCacheBootstrapScenario, dependencyCacheProtocolScenario, dependencyCachePackageScenario, + patternP01DependencyBootstrapScenario, + patternP02PortableToolchainScenario, + patternP07CapabilityOverlaysScenario, domainMigrationScenario, upDownScenario, execStaleEnvScenario, diff --git a/tests/e2e/scenarios/pattern-fixture.ts b/tests/e2e/scenarios/pattern-fixture.ts new file mode 100644 index 000000000..796377742 --- /dev/null +++ b/tests/e2e/scenarios/pattern-fixture.ts @@ -0,0 +1,457 @@ +import { cp, rename } from "node:fs/promises"; +import { join, relative, sep } from "node:path"; +import { YAML } from "bun"; + +import { PATTERN_CATALOGUE } from "../../../src/agents/pattern-catalogue.ts"; +import { isRecord } from "../../../src/lib/guards.ts"; +import { + gitInitAndCommit, + type MonorepoFixture, + randomProjectName, +} from "../fixture.ts"; +import { + type CliResult, + expect, + expectExit, + REPO_ROOT, + runCommand, + type ScenarioContext, +} from "../harness.ts"; + +/** + * Shared setup for the Docker-tier scenarios that start a HACK-P example + * fixture (`examples/patterns/`) against a real engine. + * + * The scenario copies the checked-in fixture and changes only what a + * disposable, offline CI run needs: + * - the project name and dev host become a random `e2e-` name; + * - the digest-pinned Alpine Bun image becomes the repository digest of the + * runner's cached `oven/bun:1.4.2-slim` (same Bun release), with + * `pull_policy: never`, so nothing is downloaded during qualification; + * - a declared `platform` becomes the daemon's architecture; + * - every service and named volume gets an exact owner label for cleanup. + * Commands, mounts, labels, dependency gates and health checks stay as the + * fixture authored them. A fixture whose image no longer matches + * `PATTERN_FIXTURE_IMAGE` fails instead of being silently rewritten. + */ + +/** The image every pattern fixture declares today. */ +export const PATTERN_FIXTURE_IMAGE = + "oven/bun:1.4.2-alpine@sha256:d888c0ae6c86d7866ff10c5aafdd9077b36aee6455b33dd270fb93c0dd5cef6f"; +/** Pre-pulled by the docker-e2e job; resolved to its repository digest. */ +export const CACHED_BUN_IMAGE = "oven/bun:1.4.2-slim"; + +const DOCKER_TIMEOUT_MS = 60_000; +const ARCHITECTURES: Readonly> = { + arm64: "arm64", + aarch64: "arm64", + amd64: "amd64", + x86_64: "amd64", +}; +/** Never copied from a contributor checkout into the disposable fixture. */ +const UNCOPIED = new Set(["node_modules", ".internal", ".branch", ".env"]); + +export type PatternEngine = { + /** Docker architecture name, `arm64` or `amd64`. */ + readonly arch: string; + /** Immutable `oven/bun@sha256:...` reference of the cached image. */ + readonly image: string; +}; + +export type DockerResource = { + readonly Id?: string; + readonly Name?: string; + readonly Labels?: Record | null; + readonly Config?: { + readonly Labels?: Record | null; + readonly Env?: readonly string[] | null; + }; + readonly State?: { + readonly Running?: boolean; + readonly ExitCode?: number; + readonly Health?: { readonly Status?: string } | null; + }; + readonly Mounts?: readonly { + readonly Type: string; + readonly Name?: string; + readonly Source?: string; + readonly Destination: string; + readonly RW: boolean; + }[]; +}; + +/** Run a docker client command; the scenario owns argument selection. */ +export async function docker(opts: { + readonly ctx: ScenarioContext; + readonly args: readonly string[]; + readonly timeoutMs?: number; +}): Promise { + return await runCommand({ + argv: ["docker", ...opts.args], + cwd: opts.ctx.tempRoot, + timeoutMs: opts.timeoutMs ?? DOCKER_TIMEOUT_MS, + }); +} + +/** Run a docker client command that must succeed. */ +export async function checkedDocker(opts: { + readonly ctx: ScenarioContext; + readonly args: readonly string[]; + readonly timeoutMs?: number; +}): Promise { + const result = await docker(opts); + expectExit({ + result, + codes: [0], + message: `docker ${opts.args[0]} pattern fixture operation`, + }); + return result; +} + +/** + * Resolve the daemon architecture and the cached Bun image. An unavailable + * daemon or unsupported architecture skips; a missing cached image fails, + * because the scenario never pulls. + */ +export async function requirePatternEngine( + ctx: ScenarioContext +): Promise { + const info = await docker({ + ctx, + args: ["info", "--format", "{{.Architecture}}"], + }); + if (info.exitCode !== 0) { + ctx.skip("Docker daemon unavailable"); + } + const arch = ARCHITECTURES[info.stdout.trim()]; + if (!arch) { + ctx.skip("unsupported Docker daemon architecture"); + } + const inspected = await checkedDocker({ + ctx, + args: [ + "image", + "inspect", + CACHED_BUN_IMAGE, + "--format", + "{{json .RepoDigests}}", + ], + }); + const digests: unknown = JSON.parse(inspected.stdout); + const image = Array.isArray(digests) + ? digests.find( + (value): value is string => + typeof value === "string" && value.startsWith("oven/bun@sha256:") + ) + : undefined; + expect({ + that: image !== undefined, + message: `${CACHED_BUN_IMAGE} must already be cached with a repository digest; no pulls`, + }); + return { arch: arch as string, image: image as string }; +} + +function patternExample(cardId: string): string { + const card = PATTERN_CATALOGUE.find((entry) => entry.id === cardId); + if (!card?.example) { + throw new Error(`${cardId} has no example fixture`); + } + return join(REPO_ROOT, card.example); +} + +function withOwner(opts: { + readonly value: unknown; + readonly owner: string; + readonly name: string; +}): Record { + const labels = isRecord(opts.value) ? { ...opts.value } : {}; + expect({ + that: opts.value === undefined || isRecord(opts.value), + message: "pattern fixture labels must be a mapping", + }); + labels[opts.owner] = opts.name; + return labels; +} + +function rewriteCompose(opts: { + readonly compose: unknown; + readonly name: string; + readonly owner: string; + readonly engine: PatternEngine; +}): Record { + expect({ + that: isRecord(opts.compose) && isRecord(opts.compose.services), + message: "pattern fixture Compose file must declare services", + }); + const compose = structuredClone(opts.compose) as Record; + compose.name = opts.name; + for (const service of Object.values( + compose.services as Record + )) { + expect({ + that: isRecord(service) && service.image === PATTERN_FIXTURE_IMAGE, + message: + "pattern fixture image changed; update PATTERN_FIXTURE_IMAGE and the cached CI image together", + }); + const entry = service as Record; + entry.image = opts.engine.image; + entry.pull_policy = "never"; + if (entry.platform !== undefined) { + entry.platform = `linux/${opts.engine.arch}`; + } + entry.labels = withOwner({ + value: entry.labels, + owner: opts.owner, + name: opts.name, + }); + } + if (isRecord(compose.volumes)) { + for (const [volume, definition] of Object.entries(compose.volumes)) { + const declared = isRecord(definition) ? { ...definition } : {}; + declared.labels = withOwner({ + value: declared.labels, + owner: opts.owner, + name: opts.name, + }); + compose.volumes[volume] = declared; + } + } + return compose; +} + +/** + * Copy a card's example into `ctx.tempRoot`, apply the substitutions listed + * above, and commit it as the primary checkout of a fresh repository. + */ +export async function copyPatternFixture(opts: { + readonly ctx: ScenarioContext; + readonly cardId: string; + readonly owner: string; + readonly engine: PatternEngine; +}): Promise { + const source = patternExample(opts.cardId); + const name = randomProjectName(); + const root = join(opts.ctx.tempRoot, name); + await cp(source, root, { + recursive: true, + filter: (path) => + !relative(source, path) + .split(sep) + .some((part) => UNCOPIED.has(part)), + }); + const hackDir = join(root, ".hack"); + const devHost = `${name}.hack.local`; + const config: unknown = await Bun.file( + join(hackDir, "hack.config.json") + ).json(); + expect({ + that: isRecord(config), + message: "pattern fixture hack.config.json must be an object", + }); + await Bun.write( + join(hackDir, "hack.config.json"), + `${JSON.stringify({ ...(config as Record), name, dev_host: devHost }, null, 2)}\n` + ); + const compose = rewriteCompose({ + compose: YAML.parse( + await Bun.file(join(hackDir, "docker-compose.yml")).text() + ), + name, + owner: opts.owner, + engine: opts.engine, + }); + await writeComposeFile({ root, compose }); + await gitInitAndCommit({ root }); + return { root, name, devHost, hackDir }; +} + +/** Read a checkout's (already rewritten, JSON-encoded) Compose file. */ +export async function readComposeFile( + root: string +): Promise> { + const parsed: unknown = YAML.parse( + await Bun.file(join(root, ".hack/docker-compose.yml")).text() + ); + if (!isRecord(parsed)) { + throw new Error("pattern fixture Compose root must be a mapping"); + } + return parsed; +} + +/** Publish a Compose file on a fresh inode, as Docker bind readers expect. */ +export async function writeComposeFile(opts: { + readonly root: string; + readonly compose: Record; +}): Promise { + await replaceFile({ + path: join(opts.root, ".hack/docker-compose.yml"), + content: `${JSON.stringify(opts.compose, null, 2)}\n`, + }); +} + +/** Write a file by rename so a bind-mounted reader never sees a partial file. */ +export async function replaceFile(opts: { + readonly path: string; + readonly content: string; +}): Promise { + const pending = `${opts.path}.pending`; + await Bun.write(pending, opts.content); + await rename(pending, opts.path); +} + +/** Labels of a container or volume after exact fixture ownership checks. */ +export function ownedLabels(opts: { + readonly resource: DockerResource; + readonly owner: string; + readonly name: string; + readonly projects: readonly string[]; +}): Record { + const labels = opts.resource.Config?.Labels ?? opts.resource.Labels ?? {}; + expect({ + that: + labels[opts.owner] === opts.name && + opts.projects.includes(labels["com.docker.compose.project"] ?? ""), + message: "refuse operation outside exact pattern fixture ownership", + }); + return labels; +} + +/** Inspect owned containers, optionally narrowed to one project and service. */ +export async function ownedContainers(opts: { + readonly ctx: ScenarioContext; + readonly owner: string; + readonly name: string; + readonly projects: readonly string[]; + readonly project?: string; + readonly service?: string; +}): Promise { + const listed = await checkedDocker({ + ctx: opts.ctx, + args: [ + "ps", + "-aq", + "--filter", + `label=${opts.owner}=${opts.name}`, + ...(opts.project + ? ["--filter", `label=com.docker.compose.project=${opts.project}`] + : []), + ...(opts.service + ? ["--filter", `label=com.docker.compose.service=${opts.service}`] + : []), + ], + }); + const ids = listed.stdout.trim().split(/\s+/).filter(Boolean); + if (ids.length === 0) { + return []; + } + const inspected = await checkedDocker({ + ctx: opts.ctx, + args: ["inspect", ...ids], + }); + const resources = JSON.parse(inspected.stdout) as DockerResource[]; + for (const resource of resources) { + ownedLabels({ ...opts, resource }); + } + return resources; +} + +/** Inspect owned named volumes. */ +export async function ownedVolumes(opts: { + readonly ctx: ScenarioContext; + readonly owner: string; + readonly name: string; + readonly projects: readonly string[]; +}): Promise { + const listed = await checkedDocker({ + ctx: opts.ctx, + args: [ + "volume", + "ls", + "-q", + "--filter", + `label=${opts.owner}=${opts.name}`, + ], + }); + const names = listed.stdout.trim().split(/\s+/).filter(Boolean); + if (names.length === 0) { + return []; + } + const inspected = await checkedDocker({ + ctx: opts.ctx, + args: ["volume", "inspect", ...names], + }); + const resources = JSON.parse(inspected.stdout) as DockerResource[]; + for (const resource of resources) { + ownedLabels({ ...opts, resource }); + } + return resources; +} + +/** Poll `check` every 200 ms until it holds or `timeoutMs` passes. */ +export async function waitFor(opts: { + readonly check: () => Promise; + readonly timeoutMs: number; + readonly message: string; +}): Promise { + const deadline = Date.now() + opts.timeoutMs; + while (Date.now() < deadline) { + if (await opts.check()) { + return; + } + await Bun.sleep(200); + } + throw new Error(opts.message); +} + +/** + * Stop each checkout through Hack, then remove only exactly owned containers, + * volumes and the projects' Compose networks, and verify their absence. + */ +export async function cleanupPatternFixture(opts: { + readonly ctx: ScenarioContext; + readonly owner: string; + readonly name: string; + readonly projects: readonly string[]; + readonly checkouts: readonly string[]; +}): Promise { + for (const cwd of opts.checkouts) { + // Best effort: the owned-resource sweep below is the cleanup authority. + await opts.ctx.cli({ args: ["down"], cwd, timeoutMs: 180_000 }); + } + for (const resource of await ownedContainers(opts)) { + await checkedDocker({ + ctx: opts.ctx, + args: ["rm", "-f", resource.Id ?? ""], + }); + } + for (const volume of await ownedVolumes(opts)) { + await checkedDocker({ + ctx: opts.ctx, + args: ["volume", "rm", volume.Name ?? ""], + }); + } + for (const project of opts.projects) { + const networks = await checkedDocker({ + ctx: opts.ctx, + args: [ + "network", + "ls", + "-q", + "--filter", + `label=com.docker.compose.project=${project}`, + ], + }); + for (const id of networks.stdout.trim().split(/\s+/).filter(Boolean)) { + await checkedDocker({ ctx: opts.ctx, args: ["network", "rm", id] }); + } + } + expect({ + that: + (await ownedContainers(opts)).length === 0 && + (await ownedVolumes(opts)).length === 0, + message: "all owned pattern fixture containers and volumes must be gone", + }); + opts.ctx.log( + "owned pattern fixture containers, volumes and networks removed; absence verified" + ); +} diff --git a/tests/e2e/scenarios/pattern-p01-dependency-bootstrap.ts b/tests/e2e/scenarios/pattern-p01-dependency-bootstrap.ts new file mode 100644 index 000000000..51df1dafd --- /dev/null +++ b/tests/e2e/scenarios/pattern-p01-dependency-bootstrap.ts @@ -0,0 +1,498 @@ +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import { addLinkedWorktree } from "../fixture.ts"; +import { + type CliResult, + expect, + expectExit, + extractJsonObject, + type Scenario, +} from "../harness.ts"; +import { + checkedDocker, + cleanupPatternFixture, + copyPatternFixture, + docker, + ownedContainers, + readComposeFile, + replaceFile, + requirePatternEngine, + waitFor, + writeComposeFile, +} from "./pattern-fixture.ts"; + +const OWNER = "hack.e2e.pattern-p01-owner"; +const BRANCH = "p01-linked"; +const COMMAND_MS = 180_000; +const CACHE = "/app/node_modules"; +const META = `${CACHE}/.hack-dependency-cache-v1`; +const BEGAN = `${CACHE}/.e2e-installer-began`; +/** + * Test-only initializer used for the interrupted attempt: it writes into the + * cache, then blocks before the real installer so the scenario can kill it. + */ +const INTERRUPTED_INSTALL = [ + "sh", + "-c", + `touch ${BEGAN} && sleep 120 && exec bun install --frozen-lockfile`, +]; +/** + * Consumer readback inside a running `app`: the generated artifact parses, + * the dependency payload carries the protocol's ready marker, the installed + * `greeting` package works, and the served response is complete. Prints which + * payload the consumer sees (`farewell` exists only after the lock change). + */ +const READ_ARTIFACTS = `const fs = await import("node:fs"); +const info = JSON.parse(fs.readFileSync("/generated/build-info.json", "utf8")); +if (typeof info.generatedAt !== "string") process.exit(41); +if (!fs.existsSync("${META}/ready")) process.exit(42); +const { greet } = await import("greeting"); +if (greet("x") !== "hello, x") process.exit(43); +const body = await (await fetch("http://127.0.0.1:3000/")).json(); +if (body.message !== "hello, pattern-p01" || body.generatedAt !== info.generatedAt) process.exit(44); +let farewell = false; +try { await import("farewell"); farewell = true; } catch {} +console.log(JSON.stringify({ farewell, generatedAt: info.generatedAt }));`; +/** The bun.lock change: one more local package, resolved by Bun 1.4. */ +const CHANGED_PACKAGE_JSON = `${JSON.stringify( + { + name: "pattern-p01", + private: true, + type: "module", + dependencies: { + farewell: "file:./packages/farewell", + greeting: "file:./packages/greeting", + }, + }, + null, + 2 +)}\n`; +const CHANGED_LOCK = `{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "pattern-p01", + "dependencies": { + "farewell": "file:./packages/farewell", + "greeting": "file:./packages/greeting", + }, + }, + }, + "packages": { + "farewell": ["farewell@file:packages/farewell", {}], + + "greeting": ["greeting@file:packages/greeting", {}], + } +} +`; + +type Readback = { readonly farewell: boolean; readonly generatedAt: string }; + +/** + * HACK-P01 (HACK-1145): the checked-in dependency-bootstrap fixture against a + * real engine. Covers cold and warm runs, two worktrees, a bun.lock change and + * an installer killed mid-run; every running consumer must see a complete old + * or complete new artifact, never the interrupted one. + */ +export const patternP01DependencyBootstrapScenario: Scenario = { + name: "pattern-p01-dependency-bootstrap", + tier: "docker", + summary: + "P01 fixture: cold/warm, two worktrees, lock change, killed installer", + run: async (ctx) => { + const engine = await requirePatternEngine(ctx); + const fixture = await copyPatternFixture({ + ctx, + cardId: "HACK-P01", + owner: OWNER, + engine, + }); + const linked = await addLinkedWorktree({ fixture, branch: BRANCH }); + const primaryProject = fixture.name; + const linkedProject = `${fixture.name}--${BRANCH}`; + const owned = { + ctx, + owner: OWNER, + name: fixture.name, + projects: [primaryProject, linkedProject], + }; + const pending: Promise[] = []; + const hack = (cwd: string, args: readonly string[]) => { + const task = ctx.cli({ + cwd, + args, + env: { HACK_COMPOSE_STARTUP_TIMEOUT_MS: "150000" }, + timeoutMs: COMMAND_MS, + }); + pending.push(task); + return task; + }; + const succeed = async (cwd: string, args: readonly string[]) => { + const result = await hack(cwd, args); + expectExit({ + result, + codes: [0], + message: `hack ${args.join(" ")} (P01 fixture)`, + }); + return result; + }; + const refused = (result: CliResult, message: string) => + expect({ + that: !result.timedOut && result.exitCode !== 0, + message, + result, + }); + const up = async (cwd: string) => { + const result = await succeed(cwd, ["up", "--json"]); + expect({ + that: + extractJsonObject<{ ok?: boolean }>({ text: result.stdout })?.ok === + true, + message: "hack up --json must report a successful envelope", + result, + }); + }; + const only = async (project: string, service: string) => { + const resources = await ownedContainers({ ...owned, project, service }); + expect({ + that: resources.length === 1, + message: `exactly one owned ${service} container in ${project}`, + }); + return resources[0] as NonNullable<(typeof resources)[number]>; + }; + const installerVolume = async (project: string) => { + const mount = (await only(project, "install")).Mounts?.find( + (entry) => entry.Destination === CACHE + ); + expect({ + that: + mount?.Type === "volume" && + mount.RW && + mount.Name?.startsWith(`hack-cache-${fixture.name}-dependencies-`) === + true, + message: "installer must write one fingerprinted dependency volume", + }); + return mount?.Name as string; + }; + const consumerMounts = async (project: string) => { + const app = await only(project, "app"); + const deps = app.Mounts?.find((entry) => entry.Destination === CACHE); + const generated = app.Mounts?.find( + (entry) => entry.Destination === "/generated" + ); + expect({ + that: + app.State?.Running === true && + deps?.Type === "volume" && + !deps.RW && + generated?.Type === "volume" && + !generated.RW, + message: + "running app must mount dependencies and generated output read-only", + }); + return { + dependencies: deps?.Name as string, + generated: generated?.Name as string, + }; + }; + const installerSaid = async (project: string, outcome: string) => { + const logs = await checkedDocker({ + ctx, + args: ["logs", (await only(project, "install")).Id as string], + }); + expect({ + that: logs.stdout.includes(`Dependency cache ${outcome} (locked-v1).`), + message: `installer in ${project} must report ${outcome}`, + result: logs, + }); + }; + const read = async (cwd: string, farewell: boolean) => { + const result = await succeed(cwd, [ + "exec", + "app", + "--", + "bun", + "-e", + READ_ARTIFACTS, + ]); + const payload = extractJsonObject({ text: result.stdout }); + expect({ + that: payload !== null && payload.farewell === farewell, + message: `consumer must read a complete ${farewell ? "new" : "old"} artifact`, + result, + }); + return payload as Readback; + }; + /** + * Every running consumer, in either instance, reads a complete artifact + * from a ready payload and never mounts `excluded`. + */ + const runningConsumersComplete = async (excluded: string) => { + const apps = (await ownedContainers({ ...owned, service: "app" })).filter( + (app) => app.State?.Running === true + ); + for (const app of apps) { + const mount = app.Mounts?.find((entry) => entry.Destination === CACHE); + expect({ + that: mount?.Name !== excluded, + message: "no consumer may mount the interrupted dependency volume", + }); + const result = await docker({ + ctx, + args: ["exec", app.Id as string, "bun", "-e", READ_ARTIFACTS], + }); + expectExit({ + result, + codes: [0], + message: "running consumer must read a complete artifact", + }); + } + return apps.length; + }; + const inspectVolume = async (volume: string, script: string) => + await docker({ + ctx, + args: [ + "run", + "--rm", + "--pull=never", + "--network=none", + "--label", + `${OWNER}=${fixture.name}`, + "--label", + `com.docker.compose.project=${linkedProject}`, + "--volume", + `${volume}:/deps:ro`, + engine.image, + "sh", + "-c", + script, + ], + }); + const incomplete = async (volume: string) => { + const result = await inspectVolume( + volume, + "test -f /deps/.e2e-installer-began && test -f /deps/.hack-dependency-cache-v1/attempt && test ! -e /deps/.hack-dependency-cache-v1/ready && test ! -e /deps/.hack-dependency-cache-v1/ready.pending" + ); + expectExit({ + result, + codes: [0], + message: + "interrupted volume must be populated but carry no ready marker", + }); + }; + const setInstaller = async (opts: { + readonly command: readonly string[] | null; + readonly generation: string; + }) => { + const compose = await readComposeFile(linked); + const services = compose.services as Record< + string, + Record + >; + const install = services.install as Record; + if (opts.command !== null) { + install.command = [...opts.command]; + } + (install.labels as Record)[ + "hack.dependencies.cache-generation" + ] = opts.generation; + await writeComposeFile({ root: linked, compose }); + }; + try { + // Cold start, then the fixture README's inspection commands. + await up(fixture.root); + await installerSaid(primaryProject, "initialized"); + const cold = await consumerMounts(primaryProject); + const coldRead = await read(fixture.root, false); + const ps = await succeed(fixture.root, ["ps", "--json"]); + let psDocument: unknown = null; + try { + psDocument = JSON.parse(ps.stdout); + } catch { + psDocument = null; + } + expect({ + that: typeof psDocument === "object" && psDocument !== null, + message: "hack ps --json must print one JSON document on stdout", + result: ps, + }); + await succeed(fixture.root, [ + "logs", + "--no-follow", + "--tail", + "100", + "install", + ]); + ctx.log("cold start installed, generated and served a complete artifact"); + + // Warm restart: ordinary down keeps the payload; generation reruns. + await succeed(fixture.root, ["down"]); + await up(fixture.root); + await installerSaid(primaryProject, "ready"); + const warm = await consumerMounts(primaryProject); + expect({ + that: warm.dependencies === cold.dependencies, + message: "warm start must reuse the same dependency volume", + }); + const warmRead = await read(fixture.root, false); + expect({ + that: warmRead.generatedAt !== coldRead.generatedAt, + message: "the always-run generation step must publish a new artifact", + }); + ctx.log("warm start reused the payload and regenerated the artifact"); + + // Second worktree: shares the ready payload, owns its generated output. + await up(linked); + await installerSaid(linkedProject, "ready"); + const second = await consumerMounts(linkedProject); + expect({ + that: + second.dependencies === cold.dependencies && + second.generated !== cold.generated, + message: + "linked worktree must share the payload but own its generated volume", + }); + await read(linked, false); + ctx.log("linked worktree reused the payload with its own generation"); + + // bun.lock change in the linked worktree selects a fresh payload. + await mkdir(join(linked, "packages/farewell"), { recursive: true }); + await replaceFile({ + path: join(linked, "packages/farewell/package.json"), + content: `${JSON.stringify({ name: "farewell", version: "1.0.0", type: "module", main: "index.js" }, null, 2)}\n`, + }); + await replaceFile({ + path: join(linked, "packages/farewell/index.js"), + content: + "export function farewell(name) {\n return `goodbye, ${name}`;\n}\n", + }); + await replaceFile({ + path: join(linked, "package.json"), + content: CHANGED_PACKAGE_JSON, + }); + await replaceFile({ + path: join(linked, "bun.lock"), + content: CHANGED_LOCK, + }); + await up(linked); + await installerSaid(linkedProject, "initialized"); + const changed = await consumerMounts(linkedProject); + expect({ + that: + changed.dependencies !== cold.dependencies && + changed.generated === second.generated, + message: + "a lock change must select a fresh payload and keep the instance's generated volume", + }); + await read(linked, true); + await read(fixture.root, false); + expect({ + that: + (await consumerMounts(primaryProject)).dependencies === + cold.dependencies, + message: "the primary consumer must keep its complete old payload", + }); + ctx.log("lock change installed a new payload; primary kept the old one"); + + // Kill the installer mid-run under a fresh generation. + await setInstaller({ command: INTERRUPTED_INSTALL, generation: "1" }); + const interrupted = hack(linked, ["up", "--json"]); + let killedId = ""; + await waitFor({ + timeoutMs: 90_000, + message: "interrupted installer never began writing its payload", + check: async () => { + const installers = await ownedContainers({ + ...owned, + project: linkedProject, + service: "install", + }); + const running = installers.find((entry) => entry.State?.Running); + if (!running?.Id) { + return false; + } + const began = await docker({ + ctx, + args: ["exec", running.Id, "test", "-f", BEGAN], + }); + if (began.exitCode !== 0) { + return false; + } + killedId = running.Id; + return true; + }, + }); + const interruptedVolume = await installerVolume(linkedProject); + expect({ + that: (await runningConsumersComplete(interruptedVolume)) >= 1, + message: "consumers stay on complete payloads while the writer runs", + }); + await checkedDocker({ ctx, args: ["kill", "--signal=KILL", killedId] }); + refused( + await interrupted, + "a killed installer must fail startup rather than wait for a timeout" + ); + await incomplete(interruptedVolume); + await runningConsumersComplete(interruptedVolume); + await read(fixture.root, false); + ctx.log("killed installer left an incomplete payload no consumer mounts"); + + // Negative control: the populated, unready payload is never reused. + refused( + await hack(linked, ["up", "--json"]), + "a populated cache without a ready marker must not count as complete" + ); + expect({ + that: (await installerVolume(linkedProject)) === interruptedVolume, + message: "the same generation must address the quarantined volume", + }); + await incomplete(interruptedVolume); + await runningConsumersComplete(interruptedVolume); + ctx.log("same-generation retry refused the incomplete payload"); + + // Recovery: the declared installer under a new generation. + await setInstaller({ + command: ["bun", "install", "--frozen-lockfile"], + generation: "2", + }); + await up(linked); + await installerSaid(linkedProject, "initialized"); + const recovered = await consumerMounts(linkedProject); + expect({ + that: + recovered.dependencies !== interruptedVolume && + recovered.dependencies !== changed.dependencies && + recovered.dependencies !== cold.dependencies, + message: "a generation bump must select a fresh, complete payload", + }); + await read(linked, true); + await read(fixture.root, false); + ctx.log("generation bump recovered with a complete new payload"); + } catch (failure) { + try { + for (const resource of await ownedContainers(owned)) { + const labels = resource.Config?.Labels ?? {}; + const logs = await docker({ + ctx, + args: ["logs", "--tail", "40", resource.Id as string], + }); + ctx.log( + `${labels["com.docker.compose.project"]}/${labels["com.docker.compose.service"]} exit=${resource.State?.ExitCode}:\n${logs.combined.slice(-3000)}` + ); + } + } catch (diagnosticError) { + ctx.log(`P01 diagnostics unavailable: ${String(diagnosticError)}`); + } + throw failure; + } finally { + await Promise.allSettled(pending); + await cleanupPatternFixture({ + ...owned, + checkouts: [fixture.root, linked], + }); + } + }, +}; diff --git a/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts b/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts new file mode 100644 index 000000000..54c483bea --- /dev/null +++ b/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts @@ -0,0 +1,195 @@ +import { join } from "node:path"; + +import { addLinkedWorktree } from "../fixture.ts"; +import { + type CliResult, + expect, + expectExit, + type Scenario, +} from "../harness.ts"; +import { + cleanupPatternFixture, + copyPatternFixture, + ownedVolumes, + replaceFile, + requirePatternEngine, +} from "./pattern-fixture.ts"; + +const OWNER = "hack.e2e.pattern-p02-owner"; +const BRANCH = "p02-linked"; +const COMMAND_MS = 180_000; +const CACHE_VOLUMES = ["toolchain_dependencies", "toolchain_bun_cache"]; +/** + * Prints which checkout the toolchain mounted. In a linked worktree `/app/.git` + * is a pointer file whose host `gitdir` target does not exist in the + * container, so a check that assumed a `.git` directory would fail there. + */ +const CHECKOUT_PROBE = `set -eu +if [ -d /app/.git ]; then + echo "git=directory" +else + test -f /app/.git + IFS= read -r pointer < /app/.git + case "$pointer" in "gitdir: "*) ;; *) exit 3;; esac + test ! -e "\${pointer#gitdir: }" + echo "git=pointer-unresolved" +fi +cat /app/.e2e-checkout`; +const FAILING_TEST = `import { expect, test } from "bun:test"; +import { sum } from "./sum.ts"; + +test("a deliberately failing check in the linked worktree", () => { + expect(sum([1, 2])).toBe(4); +}); +`; + +/** + * HACK-P02 (HACK-1146): the checked-in portable-toolchain fixture against a + * real engine, from the primary checkout and a linked worktree. A failing + * check must exit non-zero through `hack run`, and the linked worktree's + * `.git` pointer must not be mistaken for a repository directory. + */ +export const patternP02PortableToolchainScenario: Scenario = { + name: "pattern-p02-portable-toolchain", + tier: "docker", + summary: + "P02 fixture: hack run --profile from primary and linked worktree, exit status", + run: async (ctx) => { + const engine = await requirePatternEngine(ctx); + const fixture = await copyPatternFixture({ + ctx, + cardId: "HACK-P02", + owner: OWNER, + engine, + }); + const linked = await addLinkedWorktree({ fixture, branch: BRANCH }); + const primaryProject = fixture.name; + const linkedProject = `${fixture.name}--${BRANCH}`; + const owned = { + ctx, + owner: OWNER, + name: fixture.name, + projects: [primaryProject, linkedProject], + }; + // Untracked checkout identities: each run must mount its own source. + await Bun.write(join(fixture.root, ".e2e-checkout"), "checkout=primary\n"); + await Bun.write(join(linked, ".e2e-checkout"), "checkout=linked\n"); + const run = async (cwd: string, command: readonly string[]) => + await ctx.cli({ + cwd, + args: ["run", "--profile", "toolchain", "toolchain", "--", ...command], + timeoutMs: COMMAND_MS, + }); + const passes = async (cwd: string, command: readonly string[]) => { + const result = await run(cwd, command); + expectExit({ + result, + codes: [0], + message: `hack run toolchain -- ${command.join(" ")} (P02 fixture)`, + }); + return result; + }; + const checksPass = async (cwd: string, count: number) => { + const result = await passes(cwd, ["bun", "test"]); + expect({ + that: + result.combined.includes(`${count} pass`) && + result.combined.includes("0 fail"), + message: `bun test must report ${count} passing and no failing checks`, + result, + }); + }; + const failed = (result: CliResult, message: string) => + expect({ + that: !result.timedOut && result.exitCode !== 0, + message, + result, + }); + const checkout = async (cwd: string, expected: string) => { + const result = await passes(cwd, ["sh", "-c", CHECKOUT_PROBE]); + expect({ + that: result.stdout.trim() === expected, + message: "toolchain must mount the invoking checkout", + result, + }); + }; + const cacheVolumes = async () => + new Set( + (await ownedVolumes(owned)).map( + (volume) => + `${volume.Labels?.["com.docker.compose.project"]}/${volume.Labels?.["com.docker.compose.volume"]}` + ) + ); + const expectCaches = async (message: string) => { + const present = await cacheVolumes(); + for (const project of owned.projects) { + for (const volume of CACHE_VOLUMES) { + expect({ that: present.has(`${project}/${volume}`), message }); + } + } + }; + try { + // The fixture README's two commands, from the primary checkout. + await passes(fixture.root, ["bun", "install", "--frozen-lockfile"]); + await checksPass(fixture.root, 1); + await checkout(fixture.root, "git=directory\ncheckout=primary"); + ctx.log("primary checkout installed and passed its checks"); + + // The same commands from a linked worktree with a .git pointer file. + await passes(linked, ["bun", "install", "--frozen-lockfile"]); + await checksPass(linked, 1); + await checkout(linked, "git=pointer-unresolved\ncheckout=linked"); + await expectCaches( + "each instance must keep its own named toolchain cache volumes" + ); + ctx.log("linked worktree ran its own source with its own caches"); + + // A failing check in the linked worktree propagates its exit status. + await replaceFile({ + path: join(linked, "src/regression.test.ts"), + content: FAILING_TEST, + }); + failed( + await run(linked, ["bun", "test"]), + "a failing check must make hack run exit non-zero" + ); + const exact = await run(linked, ["sh", "-c", "exit 7"]); + expect({ + that: !exact.timedOut && exact.exitCode === 7, + message: "hack run must return the toolchain command's exit status", + result: exact, + }); + // Negative control: a pipe filter hides the failure, as the README warns. + const filtered = await run(linked, [ + "sh", + "-c", + "bun test 2>&1 | tail -n 3", + ]); + expectExit({ + result: filtered, + codes: [0], + message: "a pipe filter reports the filter's status, not the check's", + }); + await checksPass(fixture.root, 1); + ctx.log( + "failing check exited non-zero; exact status propagated; primary still passes" + ); + + // Ordinary down keeps both instances' cache volumes. + for (const cwd of [fixture.root, linked]) { + expectExit({ + result: await ctx.cli({ cwd, args: ["down"], timeoutMs: COMMAND_MS }), + codes: [0], + message: "hack down of the one-shot toolchain project", + }); + } + await expectCaches("hack down must keep the named toolchain caches"); + ctx.log("hack down kept the named cache volumes"); + } finally { + await cleanupPatternFixture({ + ...owned, + checkouts: [fixture.root, linked], + }); + } + }, +}; diff --git a/tests/e2e/scenarios/pattern-p07-capability-overlays.ts b/tests/e2e/scenarios/pattern-p07-capability-overlays.ts new file mode 100644 index 000000000..bffce2d73 --- /dev/null +++ b/tests/e2e/scenarios/pattern-p07-capability-overlays.ts @@ -0,0 +1,390 @@ +import { chmod, mkdir } from "node:fs/promises"; +import { join } from "node:path"; + +import { addLinkedWorktree } from "../fixture.ts"; +import { + type CliResult, + expect, + expectExit, + extractJsonObject, + type Scenario, +} from "../harness.ts"; +import { + cleanupPatternFixture, + copyPatternFixture, + ownedContainers, + requirePatternEngine, + waitFor, +} from "./pattern-fixture.ts"; + +const OWNER = "hack.e2e.pattern-p07-owner"; +const BRANCH = "p07-runner"; +const COMMAND_MS = 180_000; +/** Fixture values that value-free inspection must never print. */ +const FIXTURE_VALUES = [ + "browser-session", + "machine-identity", + "host-helper", + "http://upstream:4000", + "e2e-runner-pool", +]; +const RUNNER_LOCAL = `version: 1 +environment: runner +secretsprovider: project_key +values: + global: + RUNNER_POOL: e2e-runner-pool +`; +const BROWSER_OPENERS = [ + "open", + "xdg-open", + "sensible-browser", + "x-www-browser", + "www-browser", +]; +const CAPABILITIES = `const response = await fetch("http://127.0.0.1:3000/"); +console.log(await response.text());`; + +type Explain = { + readonly key?: string; + readonly available?: boolean; + readonly secret?: boolean; + readonly environment?: string | null; + readonly target?: string; + readonly delivered_to?: readonly string[]; + readonly source?: string | null; + readonly precedence?: readonly string[]; +}; +type Capabilities = { + readonly authMode?: string; + readonly browserSignIn?: boolean; + readonly helperMode?: string | null; + readonly upstreamReachable?: boolean; +}; + +/** + * HACK-P07 (HACK-1151): the checked-in laptop/headless-runner overlay fixture + * against a real engine. The primary checkout runs the laptop overlay and a + * linked worktree runs the runner overlay on the same topology. Value-free + * `env explain` must name each overlay's source file, the expected keys must + * reach the containers, the shared dependency must be reachable, and no CLI + * step may try to open a browser. + */ +export const patternP07CapabilityOverlaysScenario: Scenario = { + name: "pattern-p07-capability-overlays", + tier: "docker", + summary: + "P07 fixture: laptop/runner overlays, value-free explain, container env", + run: async (ctx) => { + const engine = await requirePatternEngine(ctx); + const fixture = await copyPatternFixture({ + ctx, + cardId: "HACK-P07", + owner: OWNER, + engine, + }); + const linked = await addLinkedWorktree({ fixture, branch: BRANCH }); + const laptopProject = fixture.name; + const runnerProject = `${fixture.name}--${BRANCH}`; + const owned = { + ctx, + owner: OWNER, + name: fixture.name, + projects: [laptopProject, runnerProject], + }; + // An untracked primary-local runner override that the linked worktree + // inherits; inspection must show where it came from. + const inheritedOverride = join( + fixture.hackDir, + "hack.env.runner.local.yaml" + ); + await Bun.write(inheritedOverride, RUNNER_LOCAL); + + // Every browser opener on PATH, and BROWSER, records an attempt. + const sentinelBin = join(ctx.tempRoot, "browser-sentinel"); + const browserAttempts = join(ctx.tempRoot, "browser-attempts.log"); + await mkdir(sentinelBin, { recursive: true }); + for (const opener of BROWSER_OPENERS) { + const path = join(sentinelBin, opener); + await Bun.write( + path, + `#!/bin/sh\necho "${opener}" >> "${browserAttempts}"\nexit 0\n` + ); + await chmod(path, 0o755); + } + const env = { + PATH: `${sentinelBin}:${process.env.PATH ?? "/usr/bin:/bin"}`, + BROWSER: join(sentinelBin, "xdg-open"), + HACK_COMPOSE_STARTUP_TIMEOUT_MS: "150000", + }; + const hack = async (cwd: string, args: readonly string[]) => + await ctx.cli({ cwd, args, env, timeoutMs: COMMAND_MS }); + const succeed = async (cwd: string, args: readonly string[]) => { + const result = await hack(cwd, args); + expectExit({ + result, + codes: [0], + message: `hack ${args.join(" ")} (P07 fixture)`, + }); + return result; + }; + const valueFree = (result: CliResult) => { + for (const value of FIXTURE_VALUES) { + expect({ + that: !result.combined.includes(value), + message: "env explain must not print a resolved value", + result, + }); + } + }; + const explain = async (opts: { + readonly cwd: string; + readonly key: string; + readonly overlay: string; + readonly compose: boolean; + }) => { + const result = await succeed(opts.cwd, [ + "env", + "explain", + opts.key, + "--env", + opts.overlay, + ...(opts.compose ? ["--target", "compose"] : []), + "--json", + ]); + valueFree(result); + const payload = extractJsonObject({ text: result.stdout }); + expect({ + that: payload?.key === opts.key, + message: "env explain --json must describe the requested key", + result, + }); + return { payload: payload as Explain, result }; + }; + const explainedFrom = async (opts: { + readonly cwd: string; + readonly key: string; + readonly overlay: string; + readonly file: string; + }) => { + const { payload, result } = await explain({ ...opts, compose: true }); + expect({ + that: + payload.available === true && + payload.secret === false && + payload.environment === opts.overlay && + payload.target === "compose" && + payload.delivered_to?.join(",") === "compose" && + payload.source === `global:${opts.file}` && + payload.precedence?.includes(opts.file) === true, + message: `${opts.key} under --env ${opts.overlay} must come from its expected file`, + result, + }); + }; + const healthyApi = async (project: string) => { + let api: Awaited>[number] | undefined; + await waitFor({ + timeoutMs: 120_000, + message: `api in ${project} never became healthy`, + check: async () => { + api = ( + await ownedContainers({ ...owned, project, service: "api" }) + )[0]; + return api?.State?.Health?.Status === "healthy"; + }, + }); + return api as NonNullable; + }; + const containerEnv = ( + api: Awaited> + ): Map => + new Map( + (api.Config?.Env ?? []).map((entry) => { + const index = entry.indexOf("="); + return [entry.slice(0, index), entry.slice(index + 1)] as const; + }) + ); + const capabilities = async (cwd: string, overlay: string) => { + const result = await succeed(cwd, [ + "exec", + "--env", + overlay, + "api", + "--", + "bun", + "-e", + CAPABILITIES, + ]); + const payload = extractJsonObject({ text: result.stdout }); + expect({ + that: payload?.upstreamReachable === true, + message: "the api container must reach the shared upstream", + result, + }); + return payload as Capabilities; + }; + try { + // Value-free inspection before anything starts. + const laptopFile = join(fixture.hackDir, "hack.env.laptop.yaml"); + const runnerFile = join(linked, ".hack/hack.env.runner.yaml"); + await explainedFrom({ + cwd: fixture.root, + key: "AUTH_MODE", + overlay: "laptop", + file: laptopFile, + }); + await explainedFrom({ + cwd: linked, + key: "AUTH_MODE", + overlay: "runner", + file: runnerFile, + }); + await explainedFrom({ + cwd: linked, + key: "BROWSER_SIGN_IN", + overlay: "runner", + file: runnerFile, + }); + await explainedFrom({ + cwd: linked, + key: "UPSTREAM_URL", + overlay: "runner", + file: join(linked, ".hack/hack.env.default.yaml"), + }); + await explainedFrom({ + cwd: linked, + key: "RUNNER_POOL", + overlay: "runner", + file: inheritedOverride, + }); + // Host-view inspection reports the same overlay source. + const hostView = await explain({ + cwd: fixture.root, + key: "AUTH_MODE", + overlay: "laptop", + compose: false, + }); + expect({ + that: hostView.payload.source === `global:${laptopFile}`, + message: "host-view explain must name the same overlay source", + result: hostView.result, + }); + // Negative control: the base layers alone provide no auth capability. + const base = await explain({ + cwd: fixture.root, + key: "AUTH_MODE", + overlay: "base", + compose: true, + }); + expect({ + that: + base.payload.available === false && + base.payload.source === null && + (base.payload.delivered_to ?? []).length === 0, + message: "without an overlay AUTH_MODE must be unavailable", + result: base.result, + }); + ctx.log( + "env explain named each overlay, base and inherited source without values" + ); + + // One topology, two overlays, two instances. + for (const [cwd, overlay] of [ + [fixture.root, "laptop"], + [linked, "runner"], + ] as const) { + const started = await succeed(cwd, ["up", "--env", overlay, "--json"]); + expect({ + that: + extractJsonObject<{ ok?: boolean }>({ text: started.stdout }) + ?.ok === true, + message: "hack up --json must report a successful envelope", + result: started, + }); + } + const laptopApi = await healthyApi(laptopProject); + const runnerApi = await healthyApi(runnerProject); + const laptopEnv = containerEnv(laptopApi); + const runnerEnv = containerEnv(runnerApi); + const expected: readonly [Map, Record][] = + [ + [ + laptopEnv, + { + AUTH_MODE: "browser-session", + BROWSER_SIGN_IN: "allowed", + HELPER_MODE: "host-helper", + UPSTREAM_URL: "http://upstream:4000", + UPSTREAM_TIMEOUT_MS: "3000", + }, + ], + [ + runnerEnv, + { + AUTH_MODE: "machine-identity", + BROWSER_SIGN_IN: "disabled", + HELPER_MODE: "none", + UPSTREAM_URL: "http://upstream:4000", + UPSTREAM_TIMEOUT_MS: "3000", + RUNNER_POOL: "e2e-runner-pool", + }, + ], + ]; + for (const [actual, values] of expected) { + for (const [key, value] of Object.entries(values)) { + expect({ + that: actual.get(key) === value, + message: `container metadata must carry the overlay's ${key}`, + }); + } + } + expect({ + that: !laptopEnv.has("RUNNER_POOL"), + message: "a runner-only local override must not reach the laptop stack", + }); + ctx.log("container metadata carries each overlay's expected keys"); + + const laptop = await capabilities(fixture.root, "laptop"); + const runner = await capabilities(linked, "runner"); + expect({ + that: + laptop.authMode === "browser-session" && + laptop.browserSignIn === true && + runner.authMode === "machine-identity" && + runner.browserSignIn === false && + runner.helperMode === "none", + message: "each api must report its overlay's capabilities", + }); + // Negative control: exec cannot silently switch a running stack's overlay. + const switched = await hack(linked, [ + "exec", + "--env", + "laptop", + "api", + "--", + "true", + ]); + expect({ + that: !switched.timedOut && switched.exitCode !== 0, + message: + "exec with a different overlay than the running stack must fail", + result: switched, + }); + await succeed(linked, ["logs", "--no-follow", "--tail", "100", "api"]); + ctx.log( + "both stacks reached the shared upstream with their own overlays" + ); + + expect({ + that: !(await Bun.file(browserAttempts).exists()), + message: "no Hack command may try to open a browser for sign-in", + }); + ctx.log("no browser opener was invoked"); + } finally { + await cleanupPatternFixture({ + ...owned, + checkouts: [fixture.root, linked], + }); + } + }, +}; diff --git a/tests/e2e/shard.ts b/tests/e2e/shard.ts index 907b364b6..966d72f61 100644 --- a/tests/e2e/shard.ts +++ b/tests/e2e/shard.ts @@ -65,6 +65,12 @@ export const E2E_DURATION_HINTS_SECONDS: Readonly> = { "exec-stale-env": 20, "cache-prune": 12, "dependency-cache-protocol": 11, + // Estimates until a hosted docker-e2e summary measures them: P01 runs six + // locked-v1 starts plus a killed installer; P02 seven one-shot runs; P07 + // two health-gated stacks plus value-free explain calls. + "pattern-p01-dependency-bootstrap": 90, + "pattern-p02-portable-toolchain": 45, + "pattern-p07-capability-overlays": 45, "worktree-branch-default": 2, "agent-docs-sync": 2, "domain-migration-files": 1, diff --git a/tests/helpers/pattern-fixtures.ts b/tests/helpers/pattern-fixtures.ts index 84113d05b..b27fb6ac6 100644 --- a/tests/helpers/pattern-fixtures.ts +++ b/tests/helpers/pattern-fixtures.ts @@ -300,3 +300,115 @@ export function readmeServiceIssues(opts: { } return issues; } + +/** Keys per scope in one parsed `hack.env.*.yaml` document. */ +function envKeysByScope(document: unknown): Map> { + const scopes = new Map>(); + const values = + isRecord(document) && isRecord(document.values) ? document.values : {}; + for (const [scope, entries] of Object.entries(values)) { + scopes.set(scope, new Set(isRecord(entries) ? Object.keys(entries) : [])); + } + return scopes; +} + +/** Issues of one overlay document plus its sorted `scope.KEY` signature. */ +function overlayDocumentIssues(opts: { + readonly name: string; + readonly document: unknown; + readonly sharedKeys: ReadonlySet; +}): { readonly issues: string[]; readonly signature: string } { + const { name, document } = opts; + const issues: string[] = []; + if (!isRecord(document) || document.environment !== name) { + issues.push(`${name}: environment-mismatch`); + } + const values = + isRecord(document) && isRecord(document.values) ? document.values : {}; + const keys: string[] = []; + for (const [scope, entries] of Object.entries(values)) { + const scoped = isRecord(entries) ? entries : {}; + for (const [key, value] of Object.entries(scoped)) { + keys.push(`${scope}.${key}`); + if (opts.sharedKeys.has(key)) { + issues.push(`${name}: overrides-shared-key`); + } + if (typeof value !== "string") { + issues.push(`${name}: non-plaintext-capability`); + } + } + } + return { issues, signature: keys.sort().join(",") }; +} + +/** + * HACK-P07 static contract for capability overlays: the base env file holds + * the shared topology values, and every overlay names its own environment, + * holds plaintext capability values only, declares the same keys as every + * other overlay (so switching never leaves a stale capability), and never + * redefines a base key (an overlay must not change the shared topology). + * Findings are closed codes plus the overlay name the fixture authored. + */ +export function overlayIssues(opts: { + readonly base: unknown; + readonly overlays: Readonly>; +}): string[] { + const issues: string[] = []; + const sharedKeys = new Set( + [...envKeysByScope(opts.base).values()].flatMap((keys) => [...keys]) + ); + if (sharedKeys.size === 0) { + issues.push("base-without-shared-values"); + } + const signatures = new Set(); + const overlays = Object.entries(opts.overlays); + for (const [name, document] of overlays) { + const checked = overlayDocumentIssues({ name, document, sharedKeys }); + issues.push(...checked.issues); + signatures.add(checked.signature); + } + if (overlays.length < 2) { + issues.push("fewer-than-two-overlays"); + } + if (signatures.size > 1) { + issues.push("overlay-key-sets-differ"); + } + return [...new Set(issues)]; +} + +/** + * Check that every card with an example names a registered Docker-tier E2E + * scenario with a checked-in duration hint, and that no card without an + * example claims one. Findings are closed codes plus the card id. + */ +export function e2eRegistrationIssues(opts: { + readonly cards: readonly PatternCard[]; + readonly scenarios: readonly { + readonly name: string; + readonly tier: string; + }[]; + readonly hints: Readonly>; +}): string[] { + const issues: string[] = []; + for (const card of opts.cards) { + if (card.example === null) { + if (card.e2eScenario !== null) { + issues.push(`${card.id}: scenario-without-example`); + } + continue; + } + const name = card.e2eScenario; + const scenario = opts.scenarios.find((entry) => entry.name === name); + if (name === null || scenario === undefined) { + issues.push(`${card.id}: scenario-not-registered`); + continue; + } + if (scenario.tier !== "docker") { + issues.push(`${card.id}: scenario-not-docker-tier`); + } + if (!Object.hasOwn(opts.hints, name)) { + issues.push(`${card.id}: scenario-without-duration-hint`); + } + } + return issues; +} diff --git a/tests/pattern-examples.test.ts b/tests/pattern-examples.test.ts index a9ec9d6ab..6a52f7b7c 100644 --- a/tests/pattern-examples.test.ts +++ b/tests/pattern-examples.test.ts @@ -15,29 +15,37 @@ import { findProjectContextAtRoot, readProjectConfig, } from "../src/lib/project.ts"; +import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; import { renderProjectConfigSchemaJson } from "../src/templates.ts"; +import { ALL_SCENARIOS } from "./e2e/run.ts"; +import { E2E_DURATION_HINTS_SECONDS } from "./e2e/shard.ts"; import { extractHackInvocations, findGuidanceCommandIssues, } from "./helpers/cli-guidance-commands.ts"; import { validateJsonSchemaSubset } from "./helpers/json-schema-subset.ts"; import { + e2eRegistrationIssues, exampleReferenceIssues, + overlayIssues, readmeServiceIssues, toolchainServiceIssues, } from "./helpers/pattern-fixtures.ts"; import { bannedRecipeFindings } from "./helpers/pattern-recipes.ts"; /** - * Static validation of the HACK-P01/P02 example fixtures. Nothing here starts - * containers or contacts Docker; the dependency-cache resolution runs against - * a temporary copy so generated `.hack/.internal` files never land in the - * repository. + * Static validation of the HACK-P01/P02/P07 example fixtures. Nothing here + * starts containers or contacts Docker; the dependency-cache and env + * resolution run against temporary copies so generated `.hack/.internal` + * files never land in the repository. The Docker-tier scenarios that start + * these fixtures live in tests/e2e/scenarios/pattern-*.ts. */ const REPO_ROOT = join(import.meta.dir, ".."); const P01_DIR = "examples/patterns/p01-dependency-bootstrap"; const P02_DIR = "examples/patterns/p02-portable-toolchain"; +const P07_DIR = "examples/patterns/p07-capability-overlays"; +const EXAMPLE_CARDS = ["HACK-P01", "HACK-P02", "HACK-P07"] as const; const roots: string[] = []; afterEach(async () => { @@ -130,9 +138,10 @@ async function resolveP01Cache(opts: { }); } -test("P01 and P02 reference their example fixtures and every fixture belongs to a card", async () => { +test("P01, P02 and P07 reference their example fixtures and every fixture belongs to a card", async () => { expect(card("HACK-P01").example).toBe(P01_DIR); expect(card("HACK-P02").example).toBe(P02_DIR); + expect(card("HACK-P07").example).toBe(P07_DIR); expect( await exampleReferenceIssues({ repoRoot: REPO_ROOT, @@ -184,7 +193,7 @@ test("negative control: a broken, shared or escaping example reference fails", a }); test("each fixture's hack.config.json validates against the project schema and parser", async () => { - for (const id of ["HACK-P01", "HACK-P02"]) { + for (const id of EXAMPLE_CARDS) { const dir = exampleDir(id); const config = await readConfigJson(dir); expect( @@ -392,7 +401,7 @@ test("negative control: breaking the P02 toolchain contract fails", async () => }); test("every hack command in the fixture READMEs resolves in CLI_SPEC and names real services", async () => { - for (const id of ["HACK-P01", "HACK-P02"]) { + for (const id of EXAMPLE_CARDS) { const dir = exampleDir(id); const readme = await readText(dir, "README.md"); const compose = await readCompose(dir); @@ -460,3 +469,179 @@ test("negative control: an invalid README command or service reference fails", a bannedRecipeFindings(`${readme}\n command: sleep infinity\n`) ).toEqual(["placeholder-liveness-sleep"]); }); + +async function readEnvFile(dir: string, overlay: string): Promise { + return YAML.parse(await readText(dir, `.hack/hack.env.${overlay}.yaml`)); +} + +async function readP07Overlays(dir: string) { + return { + base: await readEnvFile(dir, "default"), + overlays: { + laptop: await readEnvFile(dir, "laptop"), + runner: await readEnvFile(dir, "runner"), + }, + }; +} + +test("P07 overlays express only capability differences over one shared topology", async () => { + const dir = exampleDir("HACK-P07"); + expect(overlayIssues(await readP07Overlays(dir))).toEqual([]); + + // The real resolver sees the same split: shared values from the base file, + // capabilities only from the selected overlay. + const projectRoot = await copyExample("HACK-P07"); + const resolve = async (envName: string | null) => { + const resolved = await resolveProjectEnvConfig({ + projectRoot, + projectDir: join(projectRoot, ".hack"), + envName, + serviceNames: ["api", "upstream"], + }); + if (!resolved) { + throw new Error("missing resolved env"); + } + return resolved.serviceEnv.api ?? {}; + }; + const shared = { + UPSTREAM_URL: "http://upstream:4000", + UPSTREAM_TIMEOUT_MS: "3000", + }; + expect(await resolve("laptop")).toEqual({ + ...shared, + AUTH_MODE: "browser-session", + BROWSER_SIGN_IN: "allowed", + HELPER_MODE: "host-helper", + }); + expect(await resolve("runner")).toEqual({ + ...shared, + AUTH_MODE: "machine-identity", + BROWSER_SIGN_IN: "disabled", + HELPER_MODE: "none", + }); + // Base layers alone carry no auth capability, so the app refuses to start. + expect(await resolve(null)).toEqual(shared); + + // Both overlays start the same services with the same commands. + const compose = await readCompose(dir); + expect(Object.keys(compose.services as object).sort()).toEqual([ + "api", + "upstream", + ]); + expect(service(compose, "api").depends_on).toEqual({ + upstream: { condition: "service_healthy" }, + }); + expect(service(compose, "api").environment).toBeUndefined(); +}); + +test("negative control: an overlay that changes topology or drifts is refused", async () => { + const { base, overlays } = await readP07Overlays(exampleDir("HACK-P07")); + const globals = (document: unknown): Record => { + const values = isRecord(document) ? document.values : undefined; + const global = isRecord(values) ? values.global : undefined; + if (!isRecord(global)) { + throw new Error("overlay without global values"); + } + return global; + }; + const runner = (opts: { + readonly environment?: string; + readonly global: Record; + }) => ({ + base, + overlays: { + laptop: overlays.laptop, + runner: { + version: 1, + environment: opts.environment ?? "runner", + secretsprovider: "project_key", + values: { global: opts.global }, + }, + }, + }); + const runnerGlobal = globals(overlays.runner); + // Baseline: rebuilding the runner overlay unchanged passes. + expect(overlayIssues(runner({ global: { ...runnerGlobal } }))).toEqual([]); + expect( + overlayIssues( + runner({ + global: { ...runnerGlobal, UPSTREAM_URL: "http://localhost:4000" }, + }) + ) + ).toEqual(["runner: overrides-shared-key", "overlay-key-sets-differ"]); + const { BROWSER_SIGN_IN: _dropped, ...withoutSignIn } = runnerGlobal; + expect(overlayIssues(runner({ global: withoutSignIn }))).toEqual([ + "overlay-key-sets-differ", + ]); + expect( + overlayIssues( + runner({ + global: { ...runnerGlobal, AUTH_MODE: { secure: "v1:ciphertext" } }, + }) + ) + ).toEqual(["runner: non-plaintext-capability"]); + expect( + overlayIssues( + runner({ environment: "laptop", global: { ...runnerGlobal } }) + ) + ).toEqual(["runner: environment-mismatch"]); + expect( + overlayIssues({ base, overlays: { laptop: overlays.laptop } }) + ).toEqual(["fewer-than-two-overlays"]); + expect(overlayIssues({ base: {}, overlays })).toEqual([ + "base-without-shared-values", + ]); +}); + +test("each example card names a registered, hinted Docker-tier E2E scenario", () => { + expect( + EXAMPLE_CARDS.map((id) => card(id).e2eScenario).every( + (name) => typeof name === "string" + ) + ).toBe(true); + expect( + e2eRegistrationIssues({ + cards: PATTERN_CATALOGUE, + scenarios: ALL_SCENARIOS, + hints: E2E_DURATION_HINTS_SECONDS, + }) + ).toEqual([]); +}); + +test("negative control: an unregistered, local-tier or unhinted scenario is reported", () => { + const withScenario = (id: string, e2eScenario: string | null) => + PATTERN_CATALOGUE.map((entry) => + entry.id === id ? { ...entry, e2eScenario } : entry + ); + const check = (opts: { + readonly cards?: readonly PatternCard[]; + readonly scenarios?: readonly { name: string; tier: string }[]; + readonly hints?: Readonly>; + }) => + e2eRegistrationIssues({ + cards: opts.cards ?? PATTERN_CATALOGUE, + scenarios: opts.scenarios ?? ALL_SCENARIOS, + hints: opts.hints ?? E2E_DURATION_HINTS_SECONDS, + }); + expect( + check({ cards: withScenario("HACK-P01", "missing-scenario") }) + ).toEqual(["HACK-P01: scenario-not-registered"]); + expect(check({ cards: withScenario("HACK-P02", null) })).toEqual([ + "HACK-P02: scenario-not-registered", + ]); + expect(check({ cards: withScenario("HACK-P03", "init") })).toEqual([ + "HACK-P03: scenario-without-example", + ]); + const p07 = card("HACK-P07").e2eScenario ?? ""; + expect( + check({ + scenarios: ALL_SCENARIOS.map((entry) => + entry.name === p07 ? { ...entry, tier: "local" } : entry + ), + }) + ).toEqual(["HACK-P07: scenario-not-docker-tier"]); + const { [p07]: _removed, ...unhinted } = E2E_DURATION_HINTS_SECONDS; + expect(check({ hints: unhinted })).toEqual([ + "HACK-P07: scenario-without-duration-hint", + ]); +}); From d0eb3a665b3c814a0e064e697013d2cdd79c15a1 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:49:26 -0400 Subject: [PATCH 05/21] feat(setup): preview and roll back migration to the skills-only plugin Add `hack setup migrate` for moving one scope at a time from standalone Hack skill and rule files to the skills-only `hack` plugin (HACK-1132). - The default preview is read-only. It lists each standalone target as remove, blocked, preserve or absent, using the W11 ownership classifier. Customized, unknown and symlinked paths are preserved. - Replacement checks read client configuration per scope: Claude Code settings precedence plus an installed copy carrying the replacement skills; Codex `[plugins."hack@hack-dance"] enabled = true`. Cursor is not inspectable and needs an explicit --skip-plugin-check. - --apply is all or nothing. It refuses without writing when a target is blocked, commits a 0600 backup manifest with exact bytes and mode before removing anything, and rechecks bytes before each removal. - --rollback restores only into empty paths, never overwrites, refuses manifests whose paths, digests or bytes are not Hack-owned targets, and recovers an interrupted apply. - Hooks, MCP entries and AGENTS.md/CLAUDE.md sections are untouched. Co-Authored-By: Claude Opus 5.5 --- docs/cli.md | 4 + docs/integrations.md | 68 +- docs/reference/cli.md | 29 + src/agents/plugin-migration.ts | 790 ++++++++++++++++++++ src/agents/plugin-replacement.ts | 256 +++++++ src/agents/setup-sync.ts | 17 +- src/commands/setup-migrate.ts | 524 +++++++++++++ src/commands/setup.ts | 2 + tests/e2e/README.md | 9 +- tests/e2e/scenarios/agent-surface-freeze.ts | 23 +- tests/plugin-migration.test.ts | 557 ++++++++++++++ tests/setup-migrate-command.test.ts | 195 +++++ 12 files changed, 2464 insertions(+), 10 deletions(-) create mode 100644 src/agents/plugin-migration.ts create mode 100644 src/agents/plugin-replacement.ts create mode 100644 src/commands/setup-migrate.ts create mode 100644 tests/plugin-migration.test.ts create mode 100644 tests/setup-migrate-command.test.ts diff --git a/docs/cli.md b/docs/cli.md index 562f75097..48571f110 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -144,6 +144,10 @@ require installation. Repair only affected targets within the authorized scope; Hack-owned integrations; add `--install-missing` to create missing ones (see [integration setup](integrations.md#refresh-only-hack-setup-sync)). Read updated guidance after repair; restart only if the client cannot reload changed hooks or skills. +To move from standalone skill and rule files to the `hack` plugin, preview with +`hack setup migrate`, then run `hack setup migrate --apply` after the plugin loads; apply writes a +backup manifest that `hack setup migrate --rollback ` restores (see +[migrate standalone files](integrations.md#migrate-standalone-files-to-the-plugin)). Ordinary commands, `hack update`, and `hack doctor --fix` never render, repair, remove, or otherwise mutate integration files. CLI upgrades do not trigger integration updates. diff --git a/docs/integrations.md b/docs/integrations.md index e2d8b35b6..6afd07c6a 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -142,9 +142,71 @@ all, and why a repository `.mcp.json` counts as that client's MCP path. Installing a plugin does not migrate or remove existing rules, skills, hooks, or MCP configuration. If switching an existing installation, inspect the chosen client's project and user scopes, verify the plugin loads in a fresh session, then explicitly -remove only duplicate Hack-owned entries in that scope. Preserve customized content. -Do not use a broad sync or removal command as a migration step. If validation fails, -disable the plugin and keep using the standalone integration. +remove only duplicate Hack-owned entries in that scope with `hack setup migrate` +(below). Preserve customized content. Do not use a broad sync or removal command as +a migration step. If validation fails, disable the plugin and keep using the +standalone integration. + +### Migrate standalone files to the plugin + +`hack setup migrate` moves one scope at a time from standalone Hack skill and rule +files to the `hack` plugin. It covers the files the plugin replaces: the Codex +`hack-cli` and `hack-init` skills, the Claude Code `hack-init` skill, the Cursor +`hack.mdc` rule, and the shared `~/.ai/skills/hack-cli` skill. It never touches +Claude primer hooks, MCP entries, or `AGENTS.md`/`CLAUDE.md` sections; remove those +with their own `hack setup` command if you no longer want them. + +```bash +hack setup migrate # preview this project; changes nothing +hack setup migrate --global --codex # preview user-scope Codex files only +hack setup migrate --apply # back up and remove the previewed files +hack setup migrate --rollback ~/.hack/agent-migrations/.json +``` + +The preview (the default) is read-only. For each file it reports one action: + +| Action | Meaning | +| --- | --- | +| `remove` | Hack-owned (the current or an unedited earlier render), and the replacement plugin is verified or its check was skipped | +| `blocked` | Hack-owned, but the replacement plugin is not verified for this scope; `--apply` refuses | +| `preserve` | Customized, unrecognized, or reached through a symbolic link; never removed | +| `absent` | Not installed | + +Replacement checks read client configuration for the scope. Claude Code must enable +`hack@hack-dance` in the nearest settings file (project `settings.local.json`, then +project `settings.json`, then `~/.claude/settings.json`), and +`~/.claude/plugins/installed_plugins.json` must list an installed copy that carries +the `hack-cli` and `hack-init` skills. Codex must set +`[plugins."hack@hack-dance"] enabled = true` in the project or user `config.toml`. +For the user scope (`--global`), only user-level configuration counts. Hack cannot +inspect Cursor plugin installs, so Cursor files stay `blocked` until you pass +`--skip-plugin-check cursor`. A verified configuration is not proof that the plugin +loads: check that in a fresh session before applying, and use +`--skip-plugin-check ` (comma-separated `claude`, `codex`, `cursor`, or +`all`) only after you have. + +`--apply` is all or nothing. If any selected file is `blocked`, or a file cannot be +read for backup, it changes nothing and exits 1. Otherwise it first writes a backup +manifest under `$HACK_HOME/agent-migrations/` (default `~/.hack/agent-migrations/`) +with the exact bytes and mode of every file it will remove, then removes each file +only if its bytes still match the backup. A skill directory is removed only when it +is empty afterwards. A file edited in between is reported as `changed`, left in +place, and makes the command exit 1. With nothing Hack-owned to remove, apply writes +no manifest. + +`--rollback ` restores every recorded file whose path is now empty, with +its original bytes and mode. It never overwrites an existing file: identical bytes +are reported as `already-present`, and different content as `conflict` (exit 1), +left unchanged. Rollback can be repeated, and it also recovers an interrupted +apply. A manifest that names a path other than a known Hack target, or whose bytes +do not match their digest, is refused without restoring anything. + +Limit clients with `--claude`, `--codex`, or `--cursor` (default: all three). The +shared skill follows Claude Code or Codex. `--json` prints +`{ok, data: {action, dryRun, ...}}`: the preview carries `counts`, `plugins` +(per-scope check status) and `targets`; apply adds `status`, `reason` when refused, +`manifestPath`, and per-file `results`; rollback carries `status` and `results`. +Messages are fixed text and never include file contents. ### Upgrading from a plugin that bundled MCP diff --git a/docs/reference/cli.md b/docs/reference/cli.md index dd1f96c8c..5abc04f81 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1802,6 +1802,7 @@ hack setup [options] | `hack setup codex` | Install Codex skill for hack CLI usage | | `hack setup agents` | Install AGENTS.md / CLAUDE.md snippets for hack CLI usage | | `hack setup sync` | Refresh installed Hack-owned agent integrations | +| `hack setup migrate` | Preview, apply or roll back moving standalone agent files to the hack plugin | | `hack setup mcp` | Install MCP configs for hack CLI usage (no-shell only) | ### Options @@ -1947,6 +1948,34 @@ hack setup sync [options] | `--help, -h` | Show help | | `--version, -v` | Show version | +## `hack setup migrate` + +Preview, apply or roll back moving standalone agent files to the hack plugin + +### Usage + +```bash +hack setup migrate [options] +``` + +### Options + +| Option | Description | +| --- | --- | +| `--path, -p ` | Run a project command against a repo path (overrides cwd search) | +| `--global` | Migrate user-scope files under HOME instead of the project | +| `--all-scopes` | Migrate both project and user-scope files | +| `--claude` | Limit to Claude Code files (default: all clients) | +| `--codex` | Limit to Codex files (default: all clients) | +| `--cursor` | Limit to Cursor files (default: all clients) | +| `--skip-plugin-check ` | Treat the plugin as verified for these comma-separated clients (claude, codex, cursor, or all) after checking it loads yourself | +| `--apply` | Remove the previewed Hack-owned standalone files after backing them up (default: read-only preview) | +| `--rollback ` | Restore the exact files recorded in a migration backup manifest | +| `--json` | Output JSON (machine-readable) | +| `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | +| `--help, -h` | Show help | +| `--version, -v` | Show version | + ## `hack setup mcp` Install MCP configs for hack CLI usage (no-shell only) diff --git a/src/agents/plugin-migration.ts b/src/agents/plugin-migration.ts new file mode 100644 index 000000000..81441bde6 --- /dev/null +++ b/src/agents/plugin-migration.ts @@ -0,0 +1,790 @@ +import { createHash, randomBytes } from "node:crypto"; +import { + chmod, + lstat, + mkdir, + open, + readFile, + rename, + writeFile, +} from "node:fs/promises"; +import { dirname, isAbsolute, join, resolve } from "node:path"; + +import { readOptionalTextFile } from "../lib/fs.ts"; +import { isRecord } from "../lib/guards.ts"; +import { + classifyIntegrationContent, + classifyIntegrationFile, + describePreservedReason, + type IntegrationPreservedReason, + removeOwnedIntegrationFile, +} from "./integration-ownership.ts"; +import { + checkPluginReplacement, + type MigrationClient, + type PluginCheckStatus, +} from "./plugin-replacement.ts"; +import { + listOwnedFileTargets, + type OwnedFileTarget, + type SetupSyncScope, + type SetupSyncTarget, +} from "./setup-sync.ts"; + +/** + * Scoped migration from standalone Hack agent files to the skills-only `hack` + * plugin (HACK-1132). + * + * Contract: + * - `planPluginMigration` is read-only. It lists every standalone Hack file + * target in the selected scopes and clients with one action: `remove` + * (Hack-owned and the replacement plugin is verified or its check was + * explicitly skipped), `blocked` (Hack-owned but the replacement is not + * verified), `preserve` (customized, unknown, or reached through a symbolic + * link) or `absent`. Ownership comes from the W11 classifier + * (`integration-ownership.ts`); this module never widens it. + * - `applyPluginMigration` re-plans, refuses without writing anything when a + * target is `blocked` or a scope root is unresolved, and otherwise writes a + * backup manifest holding the exact bytes and mode of every file it will + * remove before removing any of them. The manifest write is the commit + * point: a file is removed only after its bytes are durable in the + * manifest, and only when its bytes still match the backup. + * - `rollbackPluginMigration` restores each manifest entry whose path is + * absent, byte-for-byte with its recorded mode. It never overwrites an + * existing path: identical bytes are `already-present`, anything else is a + * `conflict` left unchanged. Rollback is idempotent and also recovers an + * interrupted apply. + * + * Only standalone skill and rule files are in scope. Claude primer hooks, MCP + * entries and `AGENTS.md`/`CLAUDE.md` sections are never touched here. + * + * Replacement verification reads client configuration, not a live session: + * Claude Code must enable `hack@hack-dance` in a settings file that applies to + * the scope and list an installed copy that carries the replacement skills; + * Codex must enable it in a `config.toml` that applies to the scope. Hack + * cannot inspect Cursor plugin installs, so Cursor targets need an explicit + * skip. A verified configuration is not proof that a fresh session loads the + * plugin. + * + * Every path derives from explicit `root`, `home` and `backupDir` inputs. + * Diagnostics are closed strings with no file content or caller input. + * Planning, backup and removal are not atomic with respect to concurrent + * editors; each removal rechecks bytes and ownership immediately before it + * unlinks, which leaves the same one read-then-unlink window as W11. + */ + +/** + * Which plugin replaces a target. `shared` (`~/.ai/skills/hack-cli`) is a + * cross-client root, replaced when the Claude Code or Codex plugin is. + */ +export type ReplacementClient = MigrationClient | "shared"; + +export type PluginCheck = { + readonly scope: SetupSyncScope; + readonly client: MigrationClient; + readonly status: PluginCheckStatus; + readonly message: string; +}; + +/** Why migration leaves a file in place. Closed and value-free. */ +export type MigrationPreservedReason = + | IntegrationPreservedReason + | "symlinked-path"; + +export type MigrationPlanAction = + | "remove" + | "blocked" + | "preserve" + | "absent" + | "error"; + +export type MigrationPlanEntry = { + readonly scope: SetupSyncScope; + readonly target: SetupSyncTarget; + readonly replacement: ReplacementClient; + readonly path: string; + readonly action: MigrationPlanAction; + readonly reason?: MigrationPreservedReason; + readonly message?: string; +}; + +export type MigrationScopeRoot = { + readonly scope: SetupSyncScope; + /** Project root for `project`, home directory for `global`. */ + readonly root: string; +}; + +export type PluginMigrationInput = { + readonly scopes: readonly MigrationScopeRoot[]; + /** Home directory whose user-level client configuration applies. */ + readonly home: string; + readonly clients: readonly MigrationClient[]; + /** Clients whose replacement check the operator explicitly skipped. */ + readonly skipPluginCheck: readonly MigrationClient[]; +}; + +export type PluginMigrationPlan = { + readonly checks: readonly PluginCheck[]; + readonly entries: readonly MigrationPlanEntry[]; +}; + +const TARGET_REPLACEMENT: Readonly< + Partial> +> = { + "cursor-rules": "cursor", + "claude-hack-init-skill": "claude", + "codex-hack-cli-skill": "codex", + "codex-hack-init-skill": "codex", + "shared-hack-cli-skill": "shared", +}; + +const PLUGIN_CHECK_MESSAGES: Readonly> = { + verified: "The client configuration enables the hack plugin for this scope.", + skipped: "The replacement check was skipped explicitly.", + "not-installed": + "The client configuration does not enable the hack plugin for this scope. Install it, verify it loads in a fresh session, then rerun.", + disabled: + "The client configuration disables the hack plugin for this scope. Enable it, verify it loads in a fresh session, then rerun.", + incomplete: + "The installed hack plugin copy is missing a replacement skill. Update or reinstall the plugin, then rerun.", + unreadable: + "The client configuration could not be read or parsed; the replacement is not verified.", + "not-inspectable": + "Hack cannot inspect this client's plugin installs. Verify the plugin loads in a fresh session, then pass --skip-plugin-check for this client.", +}; + +const BLOCKED_MESSAGE = + "Hack-owned, but the replacement plugin is not verified; apply would refuse."; + +const SYMLINK_MESSAGE = + "Hack left this path unchanged because it is reached through a symbolic link, which rollback could not recreate. Remove it yourself if the plugin replaces it."; + +/** Fixed operator guidance for a migration-preserved file. */ +export function describeMigrationPreservedReason( + reason: MigrationPreservedReason +): string { + if (reason === "symlinked-path") { + return SYMLINK_MESSAGE; + } + return describePreservedReason({ reason, action: "remove" }); +} + +/** Targets in one scope whose replacement belongs to a selected client. */ +function selectTargets(opts: { + readonly scope: SetupSyncScope; + readonly clients: readonly MigrationClient[]; +}): readonly (OwnedFileTarget & { readonly replacement: ReplacementClient })[] { + const selected = new Set(opts.clients); + if (selected.has("claude") || selected.has("codex")) { + selected.add("shared"); + } + const out: (OwnedFileTarget & { readonly replacement: ReplacementClient })[] = + []; + for (const file of listOwnedFileTargets(opts.scope)) { + const replacement = TARGET_REPLACEMENT[file.target]; + if (replacement && selected.has(replacement)) { + out.push({ ...file, replacement }); + } + } + return out; +} + +/** Read-only migration preview for the selected scopes and clients. */ +export async function planPluginMigration( + input: PluginMigrationInput +): Promise { + const checks: PluginCheck[] = []; + const entries: MigrationPlanEntry[] = []; + const home = input.home.trim(); + for (const { scope, root: rawRoot } of input.scopes) { + const root = rawRoot.trim(); + const targets = selectTargets({ scope, clients: input.clients }); + if (!root || (scope === "project" && !home)) { + for (const target of targets) { + entries.push({ + scope, + target: target.target, + replacement: target.replacement, + path: "", + action: "error", + message: + scope === "global" || !home + ? "HOME is not set; cannot resolve integration paths." + : "Project root could not be resolved.", + }); + } + continue; + } + + const scopeChecks = new Map(); + for (const client of input.clients) { + const status = input.skipPluginCheck.includes(client) + ? "skipped" + : await checkPluginReplacement({ client, scope, root, home }); + const check: PluginCheck = { + scope, + client, + status, + message: PLUGIN_CHECK_MESSAGES[status], + }; + scopeChecks.set(client, check); + checks.push(check); + } + + for (const target of targets) { + const path = resolve(root, target.relativePath); + entries.push({ + scope, + target: target.target, + replacement: target.replacement, + path, + ...(await planTarget({ + target, + path, + replaced: isReplaced({ + replacement: target.replacement, + checks: scopeChecks, + }), + })), + }); + } + } + return { checks, entries }; +} + +function isSatisfied(check: PluginCheck | undefined): boolean { + return check?.status === "verified" || check?.status === "skipped"; +} + +function isReplaced(opts: { + readonly replacement: ReplacementClient; + readonly checks: ReadonlyMap; +}): boolean { + if (opts.replacement === "shared") { + return ( + isSatisfied(opts.checks.get("claude")) || + isSatisfied(opts.checks.get("codex")) + ); + } + return isSatisfied(opts.checks.get(opts.replacement)); +} + +async function planTarget(opts: { + readonly target: OwnedFileTarget; + readonly path: string; + readonly replaced: boolean; +}): Promise> { + const ownership = await classifyIntegrationFile({ + surface: opts.target.surface, + path: opts.path, + currentRender: opts.target.render(), + }); + if (ownership.kind === "absent") { + return { action: "absent" }; + } + if (ownership.kind === "customized" || ownership.kind === "unknown") { + return { + action: "preserve", + reason: ownership.reason, + message: describeMigrationPreservedReason(ownership.reason), + }; + } + if (await isSymlinkedPath({ path: opts.path })) { + return { + action: "preserve", + reason: "symlinked-path", + message: SYMLINK_MESSAGE, + }; + } + if (!opts.replaced) { + return { action: "blocked", message: BLOCKED_MESSAGE }; + } + return { action: "remove" }; +} + +/** True when the file or its containing directory is a symbolic link. */ +async function isSymlinkedPath(opts: { + readonly path: string; +}): Promise { + for (const candidate of [opts.path, dirname(opts.path)]) { + try { + if ((await lstat(candidate)).isSymbolicLink()) { + return true; + } + } catch { + // Absent components are judged by the classifier, not here. + } + } + return false; +} + +// --------------------------------------------------------------------------- +// Apply +// --------------------------------------------------------------------------- + +export const MIGRATION_MANIFEST_VERSION = 1; + +/** Per-file outcome of an apply. */ +export type MigrationApplyStatus = + | "removed" + | "preserved" + | "changed" + | "missing" + | "error"; + +export type MigrationApplyEntry = { + readonly scope: SetupSyncScope; + readonly target: SetupSyncTarget; + readonly path: string; + readonly status: MigrationApplyStatus; + readonly reason?: MigrationPreservedReason; + readonly message?: string; +}; + +export type PluginMigrationApplyResult = + | { + /** + * Nothing was written: a target is blocked, a scope root is unresolved, + * or a removable file could not be read for backup. + */ + readonly status: "refused"; + readonly reason: "blocked" | "scope-error" | "backup-failed"; + readonly plan: PluginMigrationPlan; + } + | { + /** Nothing Hack-owned to remove; no manifest was written. */ + readonly status: "nothing-to-migrate"; + readonly plan: PluginMigrationPlan; + } + | { + readonly status: "applied"; + readonly plan: PluginMigrationPlan; + readonly manifestPath: string; + readonly results: readonly MigrationApplyEntry[]; + }; + +type ManifestEntry = { + readonly scope: SetupSyncScope; + readonly root: string; + readonly target: SetupSyncTarget; + readonly relativePath: string; + readonly path: string; + readonly sha256: string; + readonly mode: number; + readonly contentBase64: string; + readonly outcome?: MigrationApplyStatus; +}; + +type MigrationManifest = { + readonly version: typeof MIGRATION_MANIFEST_VERSION; + readonly kind: "hack-plugin-migration"; + readonly id: string; + readonly createdAt: string; + readonly state: "applying" | "applied"; + readonly entries: readonly ManifestEntry[]; +}; + +/** Seams for tests; production uses the real clock, id and unlink. */ +export type PluginMigrationApplyHooks = { + readonly now?: () => Date; + readonly id?: string; + /** Called after the manifest commit and before each removal. */ + readonly beforeRemove?: (entry: { + readonly path: string; + readonly index: number; + }) => Promise; +}; + +/** + * Apply the migration: refuse on any blocked target, back up every removable + * file to a manifest under `backupDir`, then remove each one. + */ +export async function applyPluginMigration( + input: PluginMigrationInput & { + readonly backupDir: string; + readonly hooks?: PluginMigrationApplyHooks; + } +): Promise { + const plan = await planPluginMigration(input); + if (plan.entries.some((entry) => entry.action === "error")) { + return { status: "refused", reason: "scope-error", plan }; + } + if (plan.entries.some((entry) => entry.action === "blocked")) { + return { status: "refused", reason: "blocked", plan }; + } + const removable = plan.entries.filter((entry) => entry.action === "remove"); + if (removable.length === 0) { + return { status: "nothing-to-migrate", plan }; + } + + const roots = new Map(input.scopes.map((s) => [s.scope, s.root.trim()])); + const backups: ManifestEntry[] = []; + for (const entry of removable) { + const target = listOwnedFileTargets(entry.scope).find( + (file) => file.target === entry.target + ); + const root = roots.get(entry.scope) ?? ""; + const snapshot = await snapshotFile({ path: entry.path }); + if (!(target && snapshot)) { + // All or nothing: never remove a file set whose backup is incomplete. + return { status: "refused", reason: "backup-failed", plan }; + } + backups.push({ + scope: entry.scope, + root, + target: entry.target, + relativePath: target.relativePath, + path: entry.path, + sha256: snapshot.sha256, + mode: snapshot.mode, + contentBase64: snapshot.bytes.toString("base64"), + }); + } + + const now = input.hooks?.now?.() ?? new Date(); + const id = input.hooks?.id ?? manifestId(now); + const manifestPath = join(input.backupDir, `${id}.json`); + const manifest: MigrationManifest = { + version: MIGRATION_MANIFEST_VERSION, + kind: "hack-plugin-migration", + id, + createdAt: now.toISOString(), + state: "applying", + entries: backups, + }; + // Commit point: nothing is removed before this write succeeds. + await writeManifest({ path: manifestPath, manifest }); + + const results: MigrationApplyEntry[] = []; + const recorded: ManifestEntry[] = []; + for (const [index, backup] of backups.entries()) { + await input.hooks?.beforeRemove?.({ path: backup.path, index }); + const status = await removeBackedUpFile({ backup }); + results.push({ + scope: backup.scope, + target: backup.target, + path: backup.path, + ...status, + }); + recorded.push({ ...backup, outcome: status.status }); + } + await writeManifest({ + path: manifestPath, + manifest: { ...manifest, state: "applied", entries: recorded }, + }); + return { status: "applied", plan, manifestPath, results }; +} + +const ISO_SEPARATORS = /[-:]/g; +const ISO_FRACTION = /\.\d+Z$/; + +function manifestId(now: Date): string { + const stamp = now + .toISOString() + .replace(ISO_SEPARATORS, "") + .replace(ISO_FRACTION, "Z"); + return `plugin-migration-${stamp}-${randomBytes(4).toString("hex")}`; +} + +async function snapshotFile(opts: { readonly path: string }): Promise<{ + readonly bytes: Buffer; + readonly sha256: string; + readonly mode: number; +} | null> { + try { + const info = await lstat(opts.path); + if (!info.isFile()) { + return null; + } + const bytes = await readFile(opts.path); + return { bytes, sha256: sha256(bytes), mode: info.mode & 0o7777 }; + } catch { + return null; + } +} + +function sha256(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +async function writeManifest(opts: { + readonly path: string; + readonly manifest: MigrationManifest; +}): Promise { + await mkdir(dirname(opts.path), { recursive: true, mode: 0o700 }); + const temporary = `${opts.path}.${process.pid}.tmp`; + const handle = await open(temporary, "w", 0o600); + try { + await handle.writeFile(`${JSON.stringify(opts.manifest, null, 2)}\n`); + await handle.sync(); + } finally { + await handle.close(); + } + await rename(temporary, opts.path); +} + +async function removeBackedUpFile(opts: { + readonly backup: ManifestEntry; +}): Promise> { + const { backup } = opts; + const target = listOwnedFileTargets(backup.scope).find( + (file) => file.target === backup.target + ); + if (!target) { + return { status: "error", message: "Unknown migration target." }; + } + try { + if (await isSymlinkedPath({ path: backup.path })) { + return { + status: "preserved", + reason: "symlinked-path", + message: SYMLINK_MESSAGE, + }; + } + const current = await snapshotFile({ path: backup.path }); + if (!current) { + return { status: "missing" }; + } + if (current.sha256 !== backup.sha256) { + return { + status: "changed", + message: + "The file changed after it was backed up; left unchanged. Rerun the preview.", + }; + } + const outcome = await removeOwnedIntegrationFile({ + surface: target.surface, + path: backup.path, + currentRender: target.render(), + removeEmptyDirectory: target.removeEmptyDirectory, + }); + if (outcome.status === "preserved") { + return { + status: "preserved", + reason: outcome.reason, + message: describeMigrationPreservedReason(outcome.reason), + }; + } + return { status: outcome.status }; + } catch { + return { + status: "error", + message: + "The file could not be removed; it is recorded in the backup manifest.", + }; + } +} + +// --------------------------------------------------------------------------- +// Rollback +// --------------------------------------------------------------------------- + +export type MigrationRollbackStatus = + | "restored" + | "already-present" + | "conflict" + | "error"; + +export type MigrationRollbackEntry = { + readonly scope: SetupSyncScope; + readonly target: SetupSyncTarget; + readonly path: string; + readonly status: MigrationRollbackStatus; + readonly message?: string; +}; + +export type PluginMigrationRollbackResult = + | { + readonly status: "invalid-manifest"; + readonly message: string; + } + | { + readonly status: "rolled-back"; + readonly results: readonly MigrationRollbackEntry[]; + }; + +const CONFLICT_MESSAGE = + "A different file now exists at this path; left unchanged. Move it aside and rerun the rollback to restore the backup."; + +/** Restore the exact bytes recorded in a migration manifest. */ +export async function rollbackPluginMigration(opts: { + readonly manifestPath: string; +}): Promise { + const parsed = await readManifest({ path: opts.manifestPath }); + if (!parsed.ok) { + return { status: "invalid-manifest", message: parsed.message }; + } + const results: MigrationRollbackEntry[] = []; + for (const entry of parsed.entries) { + results.push({ + scope: entry.scope, + target: entry.target, + path: entry.path, + ...(await restoreEntry({ entry })), + }); + } + return { status: "rolled-back", results }; +} + +type ValidEntry = { + readonly scope: SetupSyncScope; + readonly target: SetupSyncTarget; + readonly path: string; + readonly bytes: Buffer; + readonly sha256: string; + readonly mode: number; +}; + +const INVALID_MANIFEST = + "The migration manifest is missing, unreadable, or not a valid Hack plugin migration backup; nothing was restored."; + +/** + * Parse and validate a manifest. Every entry must name a known target whose + * recorded path is exactly that target's path under the recorded root, its + * bytes must match the recorded digest, and those bytes must be a current or + * known earlier Hack render for the target; otherwise nothing is restored. + */ +async function readManifest(opts: { + readonly path: string; +}): Promise< + | { readonly ok: true; readonly entries: readonly ValidEntry[] } + | { readonly ok: false; readonly message: string } +> { + const invalid = { ok: false, message: INVALID_MANIFEST } as const; + const file = await readOptionalTextFile(opts.path); + if (file.kind !== "text") { + return invalid; + } + let parsed: unknown; + try { + parsed = JSON.parse(file.text); + } catch { + return invalid; + } + if ( + !isRecord(parsed) || + parsed.version !== MIGRATION_MANIFEST_VERSION || + parsed.kind !== "hack-plugin-migration" || + !Array.isArray(parsed.entries) + ) { + return invalid; + } + const entries: ValidEntry[] = []; + for (const raw of parsed.entries) { + const entry = validateManifestEntry(raw); + if (!entry) { + return invalid; + } + entries.push(entry); + } + return { ok: true, entries }; +} + +function validateManifestEntry(raw: unknown): ValidEntry | null { + if (!isRecord(raw)) { + return null; + } + const { + scope, + root, + target, + path, + sha256: digest, + mode, + contentBase64, + } = raw; + if ( + (scope !== "project" && scope !== "global") || + typeof root !== "string" || + !isAbsolute(root) || + typeof target !== "string" || + typeof path !== "string" || + typeof digest !== "string" || + typeof mode !== "number" || + !Number.isInteger(mode) || + mode < 0 || + mode > 0o7777 || + typeof contentBase64 !== "string" + ) { + return null; + } + const owned = listOwnedFileTargets(scope).find( + (file) => file.target === target + ); + if (!owned || resolve(root, owned.relativePath) !== path) { + return null; + } + const bytes = Buffer.from(contentBase64, "base64"); + if (sha256(bytes) !== digest) { + return null; + } + // Apply backs up only Hack-owned renders, so rollback restores only those. + const ownership = classifyIntegrationContent({ + surface: owned.surface, + content: bytes.toString("utf8"), + currentRender: owned.render(), + }); + if ( + ownership.kind !== "hack-owned-current" && + ownership.kind !== "hack-owned-prior" + ) { + return null; + } + return { scope, target: owned.target, path, bytes, sha256: digest, mode }; +} + +async function restoreEntry(opts: { + readonly entry: ValidEntry; +}): Promise> { + const { entry } = opts; + try { + const existing = await inspectRestorePath({ entry }); + if (existing === "identical") { + return { status: "already-present" }; + } + if (existing === "other") { + return { status: "conflict", message: CONFLICT_MESSAGE }; + } + await mkdir(dirname(entry.path), { recursive: true }); + // Exclusive create: never replace a file that appeared since the check. + await writeFile(entry.path, entry.bytes, { flag: "wx", mode: entry.mode }); + await chmod(entry.path, entry.mode); + return { status: "restored" }; + } catch (error: unknown) { + if (errorCode(error) === "EEXIST") { + return { status: "conflict", message: CONFLICT_MESSAGE }; + } + return { + status: "error", + message: "The backup could not be restored at this path.", + }; + } +} + +/** What occupies a restore path. Symlinks and directories are `other`. */ +async function inspectRestorePath(opts: { + readonly entry: ValidEntry; +}): Promise<"absent" | "identical" | "other"> { + let isFile: boolean; + try { + isFile = (await lstat(opts.entry.path)).isFile(); + } catch (error: unknown) { + if (errorCode(error) === "ENOENT") { + return "absent"; + } + throw error; + } + if (!isFile) { + return "other"; + } + const bytes = await readFile(opts.entry.path); + return sha256(bytes) === opts.entry.sha256 ? "identical" : "other"; +} + +function errorCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) { + return; + } + const { code } = error; + return typeof code === "string" ? code : undefined; +} diff --git a/src/agents/plugin-replacement.ts b/src/agents/plugin-replacement.ts new file mode 100644 index 000000000..59d060cfd --- /dev/null +++ b/src/agents/plugin-replacement.ts @@ -0,0 +1,256 @@ +import { isAbsolute, join } from "node:path"; + +import { readOptionalTextFile } from "../lib/fs.ts"; +import { isRecord } from "../lib/guards.ts"; +import type { SetupSyncScope } from "./setup-sync.ts"; + +/** + * Replacement verification for `hack setup migrate` (HACK-1132): whether a + * client's configuration enables the `hack` plugin for a scope. + * + * This reads configuration only; it never starts a client. Claude Code must + * enable `hack@hack-dance` in the nearest settings file that applies to the + * scope (project `settings.local.json`, project `settings.json`, then user + * `settings.json`) and list an installed copy carrying the replacement + * skills. Codex must set `enabled = true` for the plugin in the nearest + * `config.toml` (project, then user). A global-scope check reads user + * configuration only. Cursor installs are not inspectable. Results are closed + * statuses; file content and parser output are never surfaced. + */ + +/** Clients whose standalone files the plugin replaces. */ +export const MIGRATION_CLIENTS = ["claude", "codex", "cursor"] as const; +export type MigrationClient = (typeof MIGRATION_CLIENTS)[number]; + +/** Plugin identifier the clients record when the `hack` plugin is installed. */ +export const HACK_PLUGIN_ID = "hack@hack-dance"; + +/** Skills an installed Claude Code copy must carry to replace the targets. */ +const REQUIRED_PLUGIN_SKILLS: readonly string[] = ["hack-cli", "hack-init"]; + +export type PluginCheckStatus = + | "verified" + | "skipped" + | "not-installed" + | "disabled" + | "incomplete" + | "unreadable" + | "not-inspectable"; + +/** Check one client's replacement plugin for one scope. */ +export async function checkPluginReplacement(opts: { + readonly client: MigrationClient; + readonly scope: SetupSyncScope; + readonly root: string; + readonly home: string; +}): Promise { + switch (opts.client) { + case "claude": + return await checkClaudePlugin(opts); + case "codex": + return await checkCodexPlugin(opts); + default: + return "not-inspectable"; + } +} + +type EnabledLookup = + | { readonly kind: "unset" } + | { readonly kind: "unreadable" } + | { readonly kind: "set"; readonly enabled: boolean }; + +/** + * Claude Code: settings precedence is project local, project shared, user. + * The nearest file that sets the plugin decides. An enabled plugin must also + * appear in the user's plugin registry with an install that carries every + * replacement skill. + */ +async function checkClaudePlugin(opts: { + readonly scope: SetupSyncScope; + readonly root: string; + readonly home: string; +}): Promise { + const settingsFiles = [ + ...(opts.scope === "project" + ? [ + join(opts.root, ".claude", "settings.local.json"), + join(opts.root, ".claude", "settings.json"), + ] + : []), + join(opts.home, ".claude", "settings.json"), + ]; + const enabled = await firstSetting({ + paths: settingsFiles, + read: readClaudeEnabled, + }); + if (enabled !== "verified") { + return enabled; + } + return await checkClaudeInstalledCopy({ home: opts.home }); +} + +async function firstSetting(opts: { + readonly paths: readonly string[]; + readonly read: (path: string) => Promise; +}): Promise { + for (const path of opts.paths) { + const lookup = await opts.read(path); + if (lookup.kind === "unreadable") { + return "unreadable"; + } + if (lookup.kind === "set") { + return lookup.enabled ? "verified" : "disabled"; + } + } + return "not-installed"; +} + +async function readJsonRecord( + path: string +): Promise< + | { readonly kind: "absent" } + | { readonly kind: "unreadable" } + | { readonly kind: "ok"; readonly value: Record } +> { + const file = await readOptionalTextFile(path); + if (file.kind === "absent") { + return { kind: "absent" }; + } + if (file.kind === "unreadable") { + return { kind: "unreadable" }; + } + try { + const parsed: unknown = JSON.parse(file.text); + if (isRecord(parsed)) { + return { kind: "ok", value: parsed }; + } + } catch { + // Closed diagnostic below; parser output is never surfaced. + } + return { kind: "unreadable" }; +} + +async function readClaudeEnabled(path: string): Promise { + const file = await readJsonRecord(path); + if (file.kind === "absent") { + return { kind: "unset" }; + } + if (file.kind === "unreadable") { + return { kind: "unreadable" }; + } + const plugins = file.value.enabledPlugins; + if (plugins === undefined) { + return { kind: "unset" }; + } + if (!isRecord(plugins)) { + return { kind: "unreadable" }; + } + const value = plugins[HACK_PLUGIN_ID]; + if (value === undefined) { + return { kind: "unset" }; + } + return { kind: "set", enabled: value === true }; +} + +async function checkClaudeInstalledCopy(opts: { + readonly home: string; +}): Promise { + const registry = await readJsonRecord( + join(opts.home, ".claude", "plugins", "installed_plugins.json") + ); + if (registry.kind === "absent") { + return "not-installed"; + } + if (registry.kind === "unreadable") { + return "unreadable"; + } + const plugins = registry.value.plugins; + if (!isRecord(plugins)) { + return "unreadable"; + } + const raw = plugins[HACK_PLUGIN_ID]; + if (raw === undefined) { + return "not-installed"; + } + const installs = Array.isArray(raw) ? raw : [raw]; + const paths = installs + .map((install) => (isRecord(install) ? install.installPath : undefined)) + .filter( + (path): path is string => typeof path === "string" && isAbsolute(path) + ); + if (paths.length === 0) { + return "unreadable"; + } + for (const installPath of paths) { + if (await carriesReplacementSkills({ installPath })) { + return "verified"; + } + } + return "incomplete"; +} + +async function carriesReplacementSkills(opts: { + readonly installPath: string; +}): Promise { + for (const skill of REQUIRED_PLUGIN_SKILLS) { + const file = await readOptionalTextFile( + join(opts.installPath, "skills", skill, "SKILL.md") + ); + if (file.kind !== "text" || !file.text.includes(`name: ${skill}`)) { + return false; + } + } + return true; +} + +/** + * Codex: a project `.codex/config.toml` overrides the user one. The plugin + * counts only with `[plugins."hack@hack-dance"] enabled = true`. + */ +async function checkCodexPlugin(opts: { + readonly scope: SetupSyncScope; + readonly root: string; + readonly home: string; +}): Promise { + const configs = [ + ...(opts.scope === "project" + ? [join(opts.root, ".codex", "config.toml")] + : []), + join(opts.home, ".codex", "config.toml"), + ]; + return await firstSetting({ paths: configs, read: readCodexEnabled }); +} + +async function readCodexEnabled(path: string): Promise { + const file = await readOptionalTextFile(path); + if (file.kind === "absent") { + return { kind: "unset" }; + } + if (file.kind === "unreadable") { + return { kind: "unreadable" }; + } + let parsed: unknown; + try { + parsed = Bun.TOML.parse(file.text); + } catch { + return { kind: "unreadable" }; + } + if (!isRecord(parsed)) { + return { kind: "unreadable" }; + } + const plugins = parsed.plugins; + if (plugins === undefined) { + return { kind: "unset" }; + } + if (!isRecord(plugins)) { + return { kind: "unreadable" }; + } + const entry = plugins[HACK_PLUGIN_ID]; + if (entry === undefined) { + return { kind: "unset" }; + } + if (!isRecord(entry)) { + return { kind: "unreadable" }; + } + return { kind: "set", enabled: entry.enabled === true }; +} diff --git a/src/agents/setup-sync.ts b/src/agents/setup-sync.ts index 5de2430bb..3eb988541 100644 --- a/src/agents/setup-sync.ts +++ b/src/agents/setup-sync.ts @@ -108,7 +108,8 @@ export type SetupSyncTargetResult = { type Outcome = Omit; -type OwnedFileTarget = { +/** A standalone Hack-owned file target and the render that owns it. */ +export type OwnedFileTarget = { readonly target: SetupSyncTarget; readonly surface: IntegrationSurface; readonly relativePath: string; @@ -158,6 +159,17 @@ const GLOBAL_OWNED_FILES: readonly OwnedFileTarget[] = [ }, ]; +/** + * Standalone Hack-owned file targets for one scope, relative to the scope + * root (project root or home). Shared with the plugin migration engine so + * both act on the same paths and ownership surfaces. + */ +export function listOwnedFileTargets( + scope: SetupSyncScope +): readonly OwnedFileTarget[] { + return scope === "global" ? GLOBAL_OWNED_FILES : PROJECT_OWNED_FILES; +} + const CLAUDE_SETTINGS_PATHS: Readonly> = { project: ".claude/settings.local.json", global: ".claude/settings.json", @@ -214,8 +226,7 @@ export async function runSetupSyncScope(opts: { installMissing: opts.action === "sync" ? opts.installMissing : false, planInstall: opts.action === "check" && opts.installMissing, }; - const ownedFiles = - opts.scope === "global" ? GLOBAL_OWNED_FILES : PROJECT_OWNED_FILES; + const ownedFiles = listOwnedFileTargets(opts.scope); const results: SetupSyncTargetResult[] = []; for (const file of ownedFiles) { diff --git a/src/commands/setup-migrate.ts b/src/commands/setup-migrate.ts new file mode 100644 index 000000000..383ed8eb7 --- /dev/null +++ b/src/commands/setup-migrate.ts @@ -0,0 +1,524 @@ +import { join, resolve } from "node:path"; + +import { + applyPluginMigration, + type MigrationApplyEntry, + type MigrationPlanEntry, + type MigrationRollbackEntry, + type MigrationScopeRoot, + type PluginCheck, + type PluginMigrationApplyResult, + type PluginMigrationPlan, + type PluginMigrationRollbackResult, + planPluginMigration, + rollbackPluginMigration, +} from "../agents/plugin-migration.ts"; +import { + MIGRATION_CLIENTS, + type MigrationClient, +} from "../agents/plugin-replacement.ts"; +import type { SetupSyncTarget } from "../agents/setup-sync.ts"; +import type { CliContext, CommandArgs } from "../cli/command.ts"; +import { CliUsageError, defineCommand, defineOption } from "../cli/command.ts"; +import { optJson, optPath } from "../cli/options.ts"; +import { emitCliResult, okResult } from "../lib/cli-result.ts"; +import { resolveGlobalHackDir } from "../lib/config-paths.ts"; +import { findRepoRootForInit } from "../lib/project.ts"; +import { type DisplayStatusItem, display } from "../ui/display.ts"; +import { logger } from "../ui/logger.ts"; + +/** + * `hack setup migrate`: preview (default, read-only), `--apply`, or + * `--rollback ` for moving from standalone Hack skill and rule + * files to the skills-only `hack` plugin. See `plugin-migration.ts` for the + * ownership, verification and backup contract. + */ + +const optApply = defineOption({ + name: "apply", + type: "boolean", + long: "--apply", + description: + "Remove the previewed Hack-owned standalone files after backing them up (default: read-only preview)", +} as const); + +const optRollback = defineOption({ + name: "rollback", + type: "string", + long: "--rollback", + valueHint: "", + description: + "Restore the exact files recorded in a migration backup manifest", +} as const); + +const optSkipPluginCheck = defineOption({ + name: "skipPluginCheck", + type: "string", + long: "--skip-plugin-check", + valueHint: "", + description: + "Treat the plugin as verified for these comma-separated clients (claude, codex, cursor, or all) after checking it loads yourself", +} as const); + +const optMigrateGlobal = defineOption({ + name: "global", + type: "boolean", + long: "--global", + description: "Migrate user-scope files under HOME instead of the project", +} as const); + +const optMigrateAllScopes = defineOption({ + name: "allScopes", + type: "boolean", + long: "--all-scopes", + description: "Migrate both project and user-scope files", +} as const); + +const optMigrateClaude = defineOption({ + name: "claude", + type: "boolean", + long: "--claude", + description: "Limit to Claude Code files (default: all clients)", +} as const); + +const optMigrateCodex = defineOption({ + name: "codex", + type: "boolean", + long: "--codex", + description: "Limit to Codex files (default: all clients)", +} as const); + +const optMigrateCursor = defineOption({ + name: "cursor", + type: "boolean", + long: "--cursor", + description: "Limit to Cursor files (default: all clients)", +} as const); + +const setupMigrateOptions = [ + optPath, + optMigrateGlobal, + optMigrateAllScopes, + optMigrateClaude, + optMigrateCodex, + optMigrateCursor, + optSkipPluginCheck, + optApply, + optRollback, + optJson, +] as const; + +type SetupMigrateArgs = CommandArgs; + +export const setupMigrateSpec = defineCommand({ + name: "migrate", + summary: + "Preview, apply or roll back moving standalone agent files to the hack plugin", + group: "Agents", + options: setupMigrateOptions, + positionals: [], + subcommands: [], +} as const); + +const TARGET_LABELS: Readonly>> = { + "cursor-rules": "Cursor rules", + "claude-hack-init-skill": "Claude hack-init skill", + "codex-hack-cli-skill": "Codex hack-cli skill", + "codex-hack-init-skill": "Codex hack-init skill", + "shared-hack-cli-skill": "Shared Hack skill", +}; + +const CLIENT_LABELS: Readonly> = { + claude: "Claude Code", + codex: "Codex", + cursor: "Cursor", +}; + +/** Directory that holds migration backup manifests. */ +export function resolveMigrationBackupDir(): string { + return join(resolveGlobalHackDir(), "agent-migrations"); +} + +/** Parse `--skip-plugin-check`; unknown client names are a usage error. */ +export function parseSkipPluginCheck( + value: string | undefined +): readonly MigrationClient[] { + if (value === undefined) { + return []; + } + const names = value + .split(",") + .map((name) => name.trim()) + .filter((name) => name.length > 0); + if (names.length === 0) { + throw new CliUsageError( + "--skip-plugin-check needs claude, codex, cursor or all." + ); + } + const out = new Set(); + for (const name of names) { + if (name === "all") { + for (const client of MIGRATION_CLIENTS) { + out.add(client); + } + continue; + } + const client = MIGRATION_CLIENTS.find((candidate) => candidate === name); + if (!client) { + throw new CliUsageError( + "--skip-plugin-check accepts only claude, codex, cursor or all." + ); + } + out.add(client); + } + return [...out]; +} + +function resolveClients(opts: { + readonly claude: boolean; + readonly codex: boolean; + readonly cursor: boolean; +}): readonly MigrationClient[] { + const selected = MIGRATION_CLIENTS.filter((client) => opts[client]); + return selected.length > 0 ? selected : MIGRATION_CLIENTS; +} + +export async function handleSetupMigrate({ + ctx, + args, +}: { + readonly ctx: CliContext; + readonly args: SetupMigrateArgs; +}): Promise { + const options = args.options; + const rollback = options.rollback?.trim(); + if (options.rollback !== undefined) { + if (!rollback) { + throw new CliUsageError("--rollback needs a manifest path."); + } + if ( + options.apply || + options.global || + options.allScopes || + options.claude || + options.codex || + options.cursor || + options.skipPluginCheck !== undefined || + options.path !== undefined + ) { + throw new CliUsageError( + "--rollback takes only the manifest path (and --json); the manifest records the scopes and files." + ); + } + const result = await rollbackPluginMigration({ + manifestPath: resolve(ctx.cwd, rollback), + }); + return await reportRollback({ result, json: options.json === true }); + } + + const home = (process.env.HOME ?? "").trim(); + const scopes: MigrationScopeRoot[] = []; + if (!options.global || options.allScopes) { + const startDir = resolve(ctx.cwd, options.path ?? "."); + scopes.push({ + scope: "project", + root: await findRepoRootForInit(startDir), + }); + } + if (options.global || options.allScopes) { + scopes.push({ scope: "global", root: home }); + } + const input = { + scopes, + home, + clients: resolveClients({ + claude: options.claude === true, + codex: options.codex === true, + cursor: options.cursor === true, + }), + skipPluginCheck: parseSkipPluginCheck(options.skipPluginCheck), + }; + + if (!options.apply) { + const plan = await planPluginMigration(input); + return await reportPreview({ plan, json: options.json === true }); + } + const result = await applyPluginMigration({ + ...input, + backupDir: resolveMigrationBackupDir(), + }); + return await reportApply({ result, json: options.json === true }); +} + +// --------------------------------------------------------------------------- +// Exit codes and JSON payloads +// --------------------------------------------------------------------------- + +/** Preview is read-only; it fails only when a scope could not be resolved. */ +export function migratePreviewExitCode(plan: PluginMigrationPlan): number { + return plan.entries.some((entry) => entry.action === "error") ? 1 : 0; +} + +/** Apply fails when refused or when any file could not be handled. */ +export function migrateApplyExitCode( + result: PluginMigrationApplyResult +): number { + if (result.status === "refused") { + return 1; + } + if (result.status === "nothing-to-migrate") { + return 0; + } + return result.results.some( + (entry) => entry.status === "error" || entry.status === "changed" + ) + ? 1 + : 0; +} + +/** Rollback fails on an invalid manifest, a conflict or an error. */ +export function migrateRollbackExitCode( + result: PluginMigrationRollbackResult +): number { + if (result.status === "invalid-manifest") { + return 1; + } + return result.results.some( + (entry) => entry.status === "conflict" || entry.status === "error" + ) + ? 1 + : 0; +} + +function countBy( + values: readonly T[] +): Partial> { + const counts: Partial> = {}; + for (const value of values) { + counts[value] = (counts[value] ?? 0) + 1; + } + return counts; +} + +// --------------------------------------------------------------------------- +// Rendering +// --------------------------------------------------------------------------- + +function label(target: SetupSyncTarget): string { + return TARGET_LABELS[target] ?? target; +} + +function scopeLabel(scope: "project" | "global"): string { + return scope === "project" ? "Project" : "Global"; +} + +function describePlanEntry(entry: MigrationPlanEntry): string { + const at = entry.path ? ` at ${entry.path}` : ""; + switch (entry.action) { + case "remove": + return `${label(entry.target)}: would remove${at} (replaced by the hack plugin)`; + case "blocked": + return `${label(entry.target)}: blocked${at}. ${entry.message ?? ""}`.trim(); + case "preserve": + return `${label(entry.target)}: preserved${at}. ${entry.message ?? ""}`.trim(); + case "error": + return `${label(entry.target)}: ${entry.message ?? "error"}`; + default: + return `${label(entry.target)}: absent${at}`; + } +} + +function describeCheck(check: PluginCheck): string { + return `${CLIENT_LABELS[check.client]} plugin: ${check.status}. ${check.message}`; +} + +function planItems(plan: PluginMigrationPlan): DisplayStatusItem[] { + const scopes = [...new Set(plan.entries.map((entry) => entry.scope))]; + return scopes.map((scope) => { + const entries = plan.entries.filter((entry) => entry.scope === scope); + const present = entries.filter((entry) => entry.action !== "absent"); + // Human output names only the checks that decide a present file. + const relevant = new Set( + present.flatMap((entry) => + entry.replacement === "shared" + ? ["claude", "codex"] + : [entry.replacement] + ) + ); + const checks = plan.checks.filter( + (check) => check.scope === scope && relevant.has(check.client) + ); + const counts = countBy(entries.map((entry) => entry.action)); + let status: DisplayStatusItem["status"] = "ok"; + if ((counts.error ?? 0) > 0) { + status = "error"; + } else if ((counts.blocked ?? 0) > 0 || (counts.preserve ?? 0) > 0) { + status = "warn"; + } + const lines = [ + ...checks.map(describeCheck), + ...present.map(describePlanEntry), + ]; + return { + label: scopeLabel(scope), + status, + meta: `${counts.remove ?? 0} to remove, ${counts.blocked ?? 0} blocked, ${counts.preserve ?? 0} preserved`, + detail: lines.length > 0 ? lines.join("\n") : undefined, + }; + }); +} + +async function reportPreview(opts: { + readonly plan: PluginMigrationPlan; + readonly json: boolean; +}): Promise { + const exitCode = migratePreviewExitCode(opts.plan); + if (opts.json) { + emitCliResult({ + result: okResult({ + data: { + action: "preview", + dryRun: true, + counts: countBy(opts.plan.entries.map((entry) => entry.action)), + plugins: opts.plan.checks, + targets: opts.plan.entries, + }, + }), + }); + return exitCode; + } + const blocked = opts.plan.entries.some((entry) => entry.action === "blocked"); + await display.statusList({ + title: "Plugin migration preview (nothing changed)", + items: planItems(opts.plan), + }); + logger.info({ + message: blocked + ? "Install the hack plugin for the blocked clients and verify it loads in a fresh session (or pass --skip-plugin-check after checking it yourself), then rerun with --apply." + : "Rerun with --apply to back up and remove the files marked 'would remove'.", + }); + return exitCode; +} + +function describeApplyEntry(entry: MigrationApplyEntry): string { + const message = entry.message ? ` ${entry.message}` : ""; + return `${scopeLabel(entry.scope)} ${label(entry.target)}: ${entry.status} at ${entry.path}.${message}`; +} + +const REFUSAL_MESSAGES: Readonly< + Record< + Extract["reason"], + string + > +> = { + blocked: + "Migration refused; nothing was changed. The replacement plugin is not verified for at least one client with Hack-owned files.", + "scope-error": + "Migration refused; nothing was changed. A scope root could not be resolved.", + "backup-failed": + "Migration refused; nothing was changed. A file could not be read for backup.", +}; + +async function reportApply(opts: { + readonly result: PluginMigrationApplyResult; + readonly json: boolean; +}): Promise { + const { result } = opts; + const exitCode = migrateApplyExitCode(result); + if (opts.json) { + emitCliResult({ + result: okResult({ + data: { + action: "apply", + dryRun: false, + status: result.status, + ...(result.status === "refused" ? { reason: result.reason } : {}), + manifestPath: + result.status === "applied" ? result.manifestPath : null, + plugins: result.plan.checks, + targets: result.plan.entries, + results: result.status === "applied" ? result.results : [], + }, + }), + }); + return exitCode; + } + if (result.status === "refused") { + await display.statusList({ + title: "Plugin migration refused", + items: planItems(result.plan), + }); + logger.error({ message: REFUSAL_MESSAGES[result.reason] }); + return exitCode; + } + if (result.status === "nothing-to-migrate") { + await display.statusList({ + title: "Plugin migration: nothing to remove", + items: planItems(result.plan), + }); + return exitCode; + } + const counts = countBy(result.results.map((entry) => entry.status)); + await display.statusList({ + title: "Plugin migration applied", + items: [ + { + label: "Removed", + status: exitCode === 0 ? "ok" : "error", + meta: `${counts.removed ?? 0} removed`, + detail: result.results.map(describeApplyEntry).join("\n"), + }, + ], + }); + logger.info({ message: `Backup manifest: ${result.manifestPath}` }); + logger.info({ + message: `Undo with: hack setup migrate --rollback ${result.manifestPath}`, + }); + return exitCode; +} + +function describeRollbackEntry(entry: MigrationRollbackEntry): string { + const message = entry.message ? ` ${entry.message}` : ""; + return `${scopeLabel(entry.scope)} ${label(entry.target)}: ${entry.status} at ${entry.path}.${message}`; +} + +async function reportRollback(opts: { + readonly result: PluginMigrationRollbackResult; + readonly json: boolean; +}): Promise { + const { result } = opts; + const exitCode = migrateRollbackExitCode(result); + if (opts.json) { + emitCliResult({ + result: okResult({ + data: { + action: "rollback", + dryRun: false, + status: result.status, + ...(result.status === "invalid-manifest" + ? { message: result.message } + : {}), + results: result.status === "rolled-back" ? result.results : [], + }, + }), + }); + return exitCode; + } + if (result.status === "invalid-manifest") { + logger.error({ message: result.message }); + return exitCode; + } + const counts = countBy(result.results.map((entry) => entry.status)); + await display.statusList({ + title: "Plugin migration rolled back", + items: [ + { + label: "Restored", + status: exitCode === 0 ? "ok" : "error", + meta: `${counts.restored ?? 0} restored, ${counts["already-present"] ?? 0} already present, ${counts.conflict ?? 0} conflicts`, + detail: result.results.map(describeRollbackEntry).join("\n"), + }, + ], + }); + return exitCode; +} diff --git a/src/commands/setup.ts b/src/commands/setup.ts index 2711f94f5..62a67b92e 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -57,6 +57,7 @@ import { } from "../mcp/install.ts"; import { type DisplayStatusItem, display } from "../ui/display.ts"; import { logger } from "../ui/logger.ts"; +import { handleSetupMigrate, setupMigrateSpec } from "./setup-migrate.ts"; const optCheck = defineOption({ name: "check", @@ -253,6 +254,7 @@ export const setupCommand = defineCommand({ withHandler(codexSpec, handleSetupCodex), withHandler(agentsSpec, handleSetupAgents), withHandler(syncSpec, handleSetupSync), + withHandler(setupMigrateSpec, handleSetupMigrate), withHandler(mcpSpec, handleSetupMcp), ], } as const); diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 6ae45edea..8d5360a55 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -494,9 +494,12 @@ sentinel client config (`~/.claude/settings.json`, `~/.claude.json`, `.claude/settings.local.json` and `.mcp.json` into two fixtures. It then runs `init --auto`, `init --auto --with codex`, `agent prime`, `agent prime --check`, `agent onboard`, `agent patterns`, `doctor --json`, -`doctor --fix --no-interactive`, `setup sync --check`, and refresh-only -`setup sync --all-scopes` twice each (`setup sync --check` exits 1 because the -customized files are reported as preserved). After +`doctor --fix --no-interactive`, `setup sync --check`, refresh-only +`setup sync --all-scopes`, the read-only `setup migrate --all-scopes` preview, +and `setup migrate --all-scopes --apply --skip-plugin-check all` twice each +(`setup sync --check` exits 1 because the customized files are reported as +preserved; the migration apply exits 0 and removes nothing because no seeded +file is Hack-owned). After every run it requires byte-identical hashes for those trees, no new entries under them, and `git status` changes only under `.hack/`. Each command must also exit with its expected code, so a usage error cannot pass as "no write". diff --git a/tests/e2e/scenarios/agent-surface-freeze.ts b/tests/e2e/scenarios/agent-surface-freeze.ts index 06057ed51..179cffaef 100644 --- a/tests/e2e/scenarios/agent-surface-freeze.ts +++ b/tests/e2e/scenarios/agent-surface-freeze.ts @@ -45,7 +45,9 @@ type FreezeCommand = { * because the customized guidance is reported as preserved; it must not * repair it. Refresh-only `setup sync --all-scopes` (HACK-1132) exits 0: every * seeded file is customized or lacks a Hack section, so nothing is refreshed - * and nothing missing is created. + * and nothing missing is created. The plugin migration preview is read-only, + * and an explicit apply with every replacement check skipped still removes + * nothing, because no seeded file is Hack-owned (HACK-1132). */ const FREEZE_COMMANDS: readonly FreezeCommand[] = [ { @@ -114,6 +116,25 @@ const FREEZE_COMMANDS: readonly FreezeCommand[] = [ args: ["setup", "sync", "--all-scopes"], codes: [[0], [0]], }, + { + label: "setup migrate --all-scopes", + fixture: "project", + args: ["setup", "migrate", "--all-scopes"], + codes: [[0], [0]], + }, + { + label: "setup migrate --all-scopes --apply --skip-plugin-check all", + fixture: "project", + args: [ + "setup", + "migrate", + "--all-scopes", + "--apply", + "--skip-plugin-check", + "all", + ], + codes: [[0], [0]], + }, ]; type FreezeState = { diff --git a/tests/plugin-migration.test.ts b/tests/plugin-migration.test.ts new file mode 100644 index 000000000..bfd487423 --- /dev/null +++ b/tests/plugin-migration.test.ts @@ -0,0 +1,557 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + chmod, + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + rm, + stat, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { renderCodexSkill } from "../src/agents/codex-skill.ts"; +import { renderCursorRules } from "../src/agents/cursor.ts"; +import { renderHackInitSkill } from "../src/agents/hack-init-skill.ts"; +import { + applyPluginMigration, + type PluginMigrationInput, + planPluginMigration, + rollbackPluginMigration, +} from "../src/agents/plugin-migration.ts"; +import { + HACK_PLUGIN_ID, + type MigrationClient, +} from "../src/agents/plugin-replacement.ts"; + +/** + * Plugin migration engine (HACK-1132). Every path is under a per-test temp + * root: a fixture project and a fixture HOME passed explicitly. The process + * HOME is never read or written. + */ + +let root = ""; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "hack-plugin-migration-")); + await mkdir(join(root, "home"), { recursive: true }); + await mkdir(join(root, "repo"), { recursive: true }); +}); + +afterEach(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + root = ""; + } +}); + +const repo = (): string => join(root, "repo"); +const home = (): string => join(root, "home"); +const backupDir = (): string => join(root, "hack-home", "agent-migrations"); + +const CUSTOM_SKILL = + "---\nname: hack-init\n---\nTeam-edited hack-init skill. Keep me.\n"; +const UNKNOWN_RULE = "Our own cursor rule without any Hack marker.\n"; + +async function put(path: string, content: string, mode = 0o644) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content); + await chmod(path, mode); +} + +/** Hack-owned current renders plus customized and unknown neighbours. */ +async function seedProject(): Promise { + await put( + join(repo(), ".codex/skills/hack-cli/SKILL.md"), + renderCodexSkill(), + 0o640 + ); + await put(join(repo(), ".codex/skills/hack-init/SKILL.md"), CUSTOM_SKILL); + await put( + join(repo(), ".claude/skills/hack-init/SKILL.md"), + renderHackInitSkill() + ); + await put(join(repo(), ".cursor/rules/hack.mdc"), UNKNOWN_RULE); +} + +async function enableClaude(opts?: { + readonly skills?: readonly string[]; + readonly settingsRoot?: string; + readonly settingsFile?: string; + readonly enabled?: boolean; +}): Promise { + const installPath = join(root, "claude-plugin-copy"); + for (const skill of opts?.skills ?? ["hack-cli", "hack-init"]) { + await put( + join(installPath, "skills", skill, "SKILL.md"), + `---\nname: ${skill}\n---\n` + ); + } + await put( + join(home(), ".claude/plugins/installed_plugins.json"), + JSON.stringify({ + version: 2, + plugins: { [HACK_PLUGIN_ID]: [{ scope: "user", installPath }] }, + }) + ); + await put( + join( + opts?.settingsRoot ?? home(), + ".claude", + opts?.settingsFile ?? "settings.json" + ), + JSON.stringify({ + enabledPlugins: { [HACK_PLUGIN_ID]: opts?.enabled ?? true }, + }) + ); +} + +async function enableCodex(opts?: { + readonly configRoot?: string; + readonly body?: string; +}): Promise { + await put( + join(opts?.configRoot ?? home(), ".codex/config.toml"), + opts?.body ?? + `model = "x"\n\n[plugins."${HACK_PLUGIN_ID}"]\nenabled = true\n` + ); +} + +function input( + overrides: Partial = {} +): PluginMigrationInput { + return { + scopes: [{ scope: "project", root: repo() }], + home: home(), + clients: ["claude", "codex", "cursor"], + skipPluginCheck: [], + ...overrides, + }; +} + +/** Path to sha/mode for every file and symlink under the fixture root. */ +async function snapshotTree(): Promise> { + const out = new Map(); + async function visit(path: string): Promise { + const info = await lstat(path); + if (info.isDirectory()) { + for (const name of await readdir(path)) { + await visit(join(path, name)); + } + return; + } + const body = info.isSymbolicLink() + ? "symlink" + : new Bun.CryptoHasher("sha256") + .update(await readFile(path)) + .digest("hex"); + out.set(path, `${body}:${(info.mode & 0o777).toString(8)}`); + } + await visit(root); + return out; +} + +function actions( + plan: Awaited> +): Record { + return Object.fromEntries( + plan.entries.map((entry) => [ + `${entry.scope}:${entry.target}`, + entry.reason ? `${entry.action}:${entry.reason}` : entry.action, + ]) + ); +} + +function checks( + plan: Awaited> +): Record { + return Object.fromEntries( + plan.checks.map((check) => [`${check.scope}:${check.client}`, check.status]) + ); +} + +test("preview without a plugin blocks owned files, preserves custom ones and writes nothing", async () => { + await seedProject(); + const before = await snapshotTree(); + + const plan = await planPluginMigration(input()); + + expect(checks(plan)).toEqual({ + "project:claude": "not-installed", + "project:codex": "not-installed", + "project:cursor": "not-inspectable", + }); + expect(actions(plan)).toEqual({ + "project:cursor-rules": "preserve:missing-ownership-marker", + "project:claude-hack-init-skill": "blocked", + "project:codex-hack-cli-skill": "blocked", + "project:codex-hack-init-skill": "preserve:customized-content", + }); + for (const entry of plan.entries) { + expect(entry.message ?? "").not.toContain("Team-edited"); + } + expect(await snapshotTree()).toEqual(before); +}); + +test("apply refuses without writing when any owned file lacks a verified replacement", async () => { + await seedProject(); + await enableClaude(); + const before = await snapshotTree(); + + const result = await applyPluginMigration({ + ...input(), + backupDir: backupDir(), + }); + + expect(result.status).toBe("refused"); + expect(result.status === "refused" && result.reason).toBe("blocked"); + expect(actions(result.plan)["project:codex-hack-cli-skill"]).toBe("blocked"); + expect(actions(result.plan)["project:claude-hack-init-skill"]).toBe("remove"); + expect(await snapshotTree()).toEqual(before); +}); + +test("verified apply backs up and removes only owned files; rollback restores exact bytes and mode", async () => { + await seedProject(); + await enableClaude(); + await enableCodex(); + const before = await snapshotTree(); + const codexSkill = join(repo(), ".codex/skills/hack-cli/SKILL.md"); + const claudeSkill = join(repo(), ".claude/skills/hack-init/SKILL.md"); + + const result = await applyPluginMigration({ + ...input(), + backupDir: backupDir(), + hooks: { id: "fixture" }, + }); + + expect(result.status).toBe("applied"); + if (result.status !== "applied") { + return; + } + expect( + result.results.map((entry) => `${entry.target}:${entry.status}`).sort() + ).toEqual(["claude-hack-init-skill:removed", "codex-hack-cli-skill:removed"]); + // The removed skill directories were left empty, so they are gone too. + await expect(stat(dirname(codexSkill))).rejects.toThrow(); + await expect(stat(dirname(claudeSkill))).rejects.toThrow(); + // Customized and unknown neighbours are byte-identical. + for (const kept of [ + join(repo(), ".codex/skills/hack-init/SKILL.md"), + join(repo(), ".cursor/rules/hack.mdc"), + ]) { + expect((await snapshotTree()).get(kept)).toBe(before.get(kept)); + } + expect(result.manifestPath).toBe(join(backupDir(), "fixture.json")); + expect((await stat(result.manifestPath)).mode & 0o777).toBe(0o600); + const manifest = JSON.parse(await readFile(result.manifestPath, "utf8")); + expect(manifest.state).toBe("applied"); + expect(manifest.entries).toHaveLength(2); + + const rollback = await rollbackPluginMigration({ + manifestPath: result.manifestPath, + }); + expect(rollback.status).toBe("rolled-back"); + if (rollback.status !== "rolled-back") { + return; + } + expect(rollback.results.map((entry) => entry.status)).toEqual([ + "restored", + "restored", + ]); + const after = await snapshotTree(); + after.delete(result.manifestPath); + expect(after).toEqual(before); + + const again = await rollbackPluginMigration({ + manifestPath: result.manifestPath, + }); + expect( + again.status === "rolled-back" && + again.results.every((entry) => entry.status === "already-present") + ).toBe(true); +}); + +test("an installed Claude copy without the replacement skills is not a verified replacement", async () => { + await seedProject(); + await enableClaude({ skills: ["hack-cli"] }); + + const plan = await planPluginMigration(input({ clients: ["claude"] })); + + expect(checks(plan)["project:claude"]).toBe("incomplete"); + expect(actions(plan)["project:claude-hack-init-skill"]).toBe("blocked"); +}); + +test("nearest Claude settings decide: a project-local disable overrides a user enable", async () => { + await seedProject(); + await enableClaude(); + await put( + join(repo(), ".claude/settings.local.json"), + JSON.stringify({ enabledPlugins: { [HACK_PLUGIN_ID]: false } }) + ); + + const plan = await planPluginMigration(input({ clients: ["claude"] })); + + expect(checks(plan)["project:claude"]).toBe("disabled"); + expect(actions(plan)["project:claude-hack-init-skill"]).toBe("blocked"); +}); + +test("Codex must enable the plugin explicitly; disabled and malformed configs block", async () => { + await seedProject(); + const codex = { clients: ["codex"] as MigrationClient[] }; + + await enableCodex({ + body: `[plugins."${HACK_PLUGIN_ID}"]\nenabled = false\n`, + }); + expect(checks(await planPluginMigration(input(codex)))["project:codex"]).toBe( + "disabled" + ); + + await enableCodex({ body: "[plugins\nnot toml" }); + const malformed = await planPluginMigration(input(codex)); + expect(checks(malformed)["project:codex"]).toBe("unreadable"); + expect(actions(malformed)["project:codex-hack-cli-skill"]).toBe("blocked"); + + await enableCodex({ body: '[plugins."other@market"]\nenabled = true\n' }); + expect(checks(await planPluginMigration(input(codex)))["project:codex"]).toBe( + "not-installed" + ); +}); + +test("project-scope plugin config does not verify user-scope duplicates", async () => { + await seedProject(); + await put( + join(home(), ".codex/skills/hack-cli/SKILL.md"), + renderCodexSkill() + ); + await put(join(home(), ".ai/skills/hack-cli/SKILL.md"), renderCodexSkill()); + await enableCodex({ configRoot: repo() }); + const scopes = [ + { scope: "project", root: repo() }, + { scope: "global", root: home() }, + ] as const; + + const plan = await planPluginMigration(input({ scopes, clients: ["codex"] })); + + expect(checks(plan)).toEqual({ + "project:codex": "verified", + "global:codex": "not-installed", + }); + expect(actions(plan)["project:codex-hack-cli-skill"]).toBe("remove"); + expect(actions(plan)["global:codex-hack-cli-skill"]).toBe("blocked"); + expect(actions(plan)["global:shared-hack-cli-skill"]).toBe("blocked"); + + const before = await snapshotTree(); + const result = await applyPluginMigration({ + ...input({ scopes, clients: ["codex"] }), + backupDir: backupDir(), + }); + expect(result.status).toBe("refused"); + expect(await snapshotTree()).toEqual(before); +}); + +test("an explicit skip satisfies Cursor, which Hack cannot inspect", async () => { + await put(join(repo(), ".cursor/rules/hack.mdc"), renderCursorRules()); + + const unskipped = await planPluginMigration(input({ clients: ["cursor"] })); + expect(actions(unskipped)).toEqual({ "project:cursor-rules": "blocked" }); + + const skipped = await planPluginMigration( + input({ clients: ["cursor"], skipPluginCheck: ["cursor"] }) + ); + expect(checks(skipped)).toEqual({ "project:cursor": "skipped" }); + expect(actions(skipped)).toEqual({ "project:cursor-rules": "remove" }); +}); + +test("client selection limits targets; shared skill follows Claude or Codex", async () => { + const plan = await planPluginMigration( + input({ scopes: [{ scope: "global", root: home() }], clients: ["cursor"] }) + ); + expect(Object.keys(actions(plan))).toEqual(["global:cursor-rules"]); + + const codexOnly = await planPluginMigration( + input({ scopes: [{ scope: "global", root: home() }], clients: ["codex"] }) + ); + expect(Object.keys(actions(codexOnly)).sort()).toEqual([ + "global:codex-hack-cli-skill", + "global:codex-hack-init-skill", + "global:shared-hack-cli-skill", + ]); +}); + +test("an owned file reached through a symlinked directory is preserved", async () => { + const real = join(root, "elsewhere", "hack-cli"); + await put(join(real, "SKILL.md"), renderCodexSkill()); + await mkdir(join(repo(), ".codex/skills"), { recursive: true }); + await symlink(real, join(repo(), ".codex/skills/hack-cli")); + await enableCodex(); + const before = await snapshotTree(); + + const result = await applyPluginMigration({ + ...input({ clients: ["codex"] }), + backupDir: backupDir(), + }); + + expect(actions(result.plan)["project:codex-hack-cli-skill"]).toBe( + "preserve:symlinked-path" + ); + expect(result.status).toBe("nothing-to-migrate"); + expect(await snapshotTree()).toEqual(before); +}); + +test("an interrupted apply leaves a manifest that rollback recovers", async () => { + await seedProject(); + await enableClaude(); + await enableCodex(); + const before = await snapshotTree(); + + await expect( + applyPluginMigration({ + ...input(), + backupDir: backupDir(), + hooks: { + id: "interrupted", + beforeRemove: ({ index }) => { + if (index === 1) { + return Promise.reject(new Error("simulated interruption")); + } + return Promise.resolve(); + }, + }, + }) + ).rejects.toThrow("simulated interruption"); + + const manifestPath = join(backupDir(), "interrupted.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + expect(manifest.state).toBe("applying"); + expect(manifest.entries).toHaveLength(2); + + const rollback = await rollbackPluginMigration({ manifestPath }); + expect( + rollback.status === "rolled-back" && + rollback.results.map((entry) => entry.status).sort() + ).toEqual(["already-present", "restored"]); + const after = await snapshotTree(); + after.delete(manifestPath); + expect(after).toEqual(before); +}); + +test("a file edited after backup is left in place and reported as changed", async () => { + await seedProject(); + await enableCodex(); + const target = join(repo(), ".codex/skills/hack-cli/SKILL.md"); + + const result = await applyPluginMigration({ + ...input({ clients: ["codex"] }), + backupDir: backupDir(), + hooks: { + beforeRemove: async () => { + await writeFile(target, `${renderCodexSkill()}\nlocal edit\n`); + }, + }, + }); + + expect(result.status === "applied" && result.results).toEqual([ + expect.objectContaining({ + target: "codex-hack-cli-skill", + status: "changed", + }), + ]); + expect(await readFile(target, "utf8")).toContain("local edit"); +}); + +test("rollback never overwrites a file that reappeared at a restore path", async () => { + await seedProject(); + await enableClaude(); + await enableCodex(); + const result = await applyPluginMigration({ + ...input(), + backupDir: backupDir(), + }); + if (result.status !== "applied") { + throw new Error("apply did not run"); + } + const target = join(repo(), ".codex/skills/hack-cli/SKILL.md"); + await put(target, "user replacement\n"); + + const rollback = await rollbackPluginMigration({ + manifestPath: result.manifestPath, + }); + + expect( + rollback.status === "rolled-back" && + Object.fromEntries( + rollback.results.map((entry) => [entry.target, entry.status]) + ) + ).toEqual({ + "claude-hack-init-skill": "restored", + "codex-hack-cli-skill": "conflict", + }); + expect(await readFile(target, "utf8")).toBe("user replacement\n"); +}); + +test("rollback refuses a tampered manifest and restores nothing", async () => { + await seedProject(); + await enableClaude(); + await enableCodex(); + const result = await applyPluginMigration({ + ...input(), + backupDir: backupDir(), + }); + if (result.status !== "applied") { + throw new Error("apply did not run"); + } + const original = JSON.parse(await readFile(result.manifestPath, "utf8")); + let count = 0; + const tamper = async (mutate: (entry: Record) => void) => { + const copy = structuredClone(original); + mutate(copy.entries[0]); + count += 1; + const path = join(root, `tampered-${count}.json`); + await writeFile(path, JSON.stringify(copy)); + return path; + }; + const cases = [ + await tamper((entry) => { + entry.path = join(root, "outside", "SKILL.md"); + }), + await tamper((entry) => { + entry.contentBase64 = Buffer.from("other bytes").toString("base64"); + }), + await tamper((entry) => { + entry.target = "agents-md"; + }), + // Hack-owned bytes whose recorded digest does not match. + await tamper((entry) => { + entry.sha256 = "0".repeat(64); + }), + // Consistent digest, but the bytes are not a Hack render. + await tamper((entry) => { + const bytes = Buffer.from("---\nname: hack-cli\n---\nnot Hack's\n"); + entry.contentBase64 = bytes.toString("base64"); + entry.sha256 = new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); + }), + ]; + const before = await snapshotTree(); + + for (const manifestPath of cases) { + const rollback = await rollbackPluginMigration({ manifestPath }); + expect(rollback.status).toBe("invalid-manifest"); + } + expect( + (await rollbackPluginMigration({ manifestPath: join(root, "absent.json") })) + .status + ).toBe("invalid-manifest"); + expect(await snapshotTree()).toEqual(before); +}); + +test("an unresolved scope root refuses apply", async () => { + const result = await applyPluginMigration({ + ...input({ scopes: [{ scope: "global", root: " " }] }), + backupDir: backupDir(), + }); + expect(result.status === "refused" && result.reason).toBe("scope-error"); +}); diff --git a/tests/setup-migrate-command.test.ts b/tests/setup-migrate-command.test.ts new file mode 100644 index 000000000..0de7dfc58 --- /dev/null +++ b/tests/setup-migrate-command.test.ts @@ -0,0 +1,195 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; + +import { renderCodexSkill } from "../src/agents/codex-skill.ts"; +import { renderCursorRules } from "../src/agents/cursor.ts"; +import { CliUsageError } from "../src/cli/command.ts"; +import { parseSkipPluginCheck } from "../src/commands/setup-migrate.ts"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; + +/** + * `hack setup migrate` through the source CLI. HOME and HACK_HOME are private + * fixture directories; the contributor's HOME is never read or written. + */ + +const CLI = resolve(import.meta.dir, "..", "index.ts"); +/** Each test spawns the source CLI; a cold start under suite load exceeds 5 s. */ +const CLI_TEST_TIMEOUT_MS = 60_000; + +let root = ""; + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "hack-setup-migrate-")); + await mkdir(join(root, "repo", ".git"), { recursive: true }); +}); + +afterEach(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + root = ""; + } +}); + +const repo = (): string => join(root, "repo"); +const hackHome = (): string => join(root, "hack-home"); + +async function put(path: string, content: string): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, content); +} + +async function runCli(args: readonly string[]): Promise<{ + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +}> { + const child = Bun.spawn( + [process.execPath, CLI, "setup", "migrate", ...args], + { + cwd: repo(), + env: privateCliEnv({ + root, + env: { HACK_HOME: hackHome(), NO_COLOR: "1" }, + }), + stdout: "pipe", + stderr: "pipe", + } + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { exitCode, stdout, stderr }; +} + +async function snapshotRepo(): Promise> { + const out = new Map(); + async function visit(path: string): Promise { + const info = await lstat(path); + if (info.isDirectory()) { + for (const name of await readdir(path)) { + await visit(join(path, name)); + } + return; + } + out.set(path, await readFile(path, "base64")); + } + await visit(repo()); + return out; +} + +async function seed(): Promise<{ + readonly owned: string; + readonly rule: string; +}> { + const owned = join(repo(), ".codex/skills/hack-cli/SKILL.md"); + const rule = join(repo(), ".cursor/rules/hack.mdc"); + await put(owned, renderCodexSkill()); + await put(rule, `${renderCursorRules()}\nteam addition\n`); + return { owned, rule }; +} + +function data(stdout: string): Record { + const parsed: unknown = JSON.parse(stdout); + if ( + typeof parsed !== "object" || + parsed === null || + !("ok" in parsed) || + parsed.ok !== true || + !("data" in parsed) || + typeof parsed.data !== "object" || + parsed.data === null + ) { + throw new Error("unexpected envelope"); + } + return parsed.data as Record; +} + +test( + "preview is read-only, apply refuses unverified replacements, and rollback restores", + async () => { + const { owned, rule } = await seed(); + const before = await snapshotRepo(); + + const preview = await runCli(["--json"]); + expect(preview.exitCode).toBe(0); + const plan = data(preview.stdout); + expect(plan.action).toBe("preview"); + expect(plan.dryRun).toBe(true); + expect(plan.counts).toEqual({ preserve: 1, blocked: 1, absent: 2 }); + expect(await snapshotRepo()).toEqual(before); + + const refused = await runCli(["--apply"]); + expect(refused.exitCode).toBe(1); + expect(`${refused.stdout}${refused.stderr}`).toContain( + "Migration refused; nothing was changed." + ); + expect(await snapshotRepo()).toEqual(before); + await expect(lstat(join(hackHome(), "agent-migrations"))).rejects.toThrow(); + + const applied = await runCli([ + "--apply", + "--skip-plugin-check", + "codex,cursor", + "--json", + ]); + expect(applied.exitCode).toBe(0); + const result = data(applied.stdout); + expect(result.status).toBe("applied"); + const manifestPath = String(result.manifestPath); + expect(dirname(manifestPath)).toBe(join(hackHome(), "agent-migrations")); + await expect(lstat(owned)).rejects.toThrow(); + expect(await readFile(rule, "utf8")).toContain("team addition"); + + const rollback = await runCli(["--rollback", manifestPath]); + expect(rollback.exitCode).toBe(0); + expect(rollback.stdout).toContain("1 restored"); + expect(await snapshotRepo()).toEqual(before); + }, + CLI_TEST_TIMEOUT_MS +); + +test( + "a nonexistent manifest is refused without output of file content", + async () => { + const result = await runCli(["--rollback", join(root, "missing.json")]); + expect(result.exitCode).toBe(1); + expect(`${result.stdout}${result.stderr}`).toContain( + "nothing was restored" + ); + }, + CLI_TEST_TIMEOUT_MS +); + +test( + "rollback rejects scope, client and apply flags", + async () => { + for (const extra of [["--apply"], ["--global"], ["--codex"]]) { + const result = await runCli(["--rollback", "m.json", ...extra]); + expect(result.exitCode).not.toBe(0); + expect(`${result.stdout}${result.stderr}`).toContain( + "--rollback takes only the manifest path" + ); + } + }, + CLI_TEST_TIMEOUT_MS +); + +test("--skip-plugin-check accepts only known clients", () => { + expect(parseSkipPluginCheck(undefined)).toEqual([]); + expect(parseSkipPluginCheck("all")).toEqual(["claude", "codex", "cursor"]); + expect(parseSkipPluginCheck(" codex , cursor ")).toEqual(["codex", "cursor"]); + expect(() => parseSkipPluginCheck("vscode")).toThrow(CliUsageError); + expect(() => parseSkipPluginCheck(",")).toThrow(CliUsageError); +}); From 119fa6ac98989b1b3e23f355fe3675037980d5cf Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:49:32 -0400 Subject: [PATCH 06/21] fix(runtime): normalize image references at plan admission Docker Hub shorthand such as alpine:3.21 was refused as registry_image, and only after native foreground startup had already booted the VM. - Add a pure shared normalizer (provider::image_reference): alpine -> docker.io/library/alpine:latest, user/img -> docker.io/user/img:latest, index.docker.io -> docker.io; tags, digests, registries and ports are kept. Uppercase repositories, uppercase/non-sha256 digests, bad ports, empty and over-long references refuse with closed value-free reasons. - Compose plan admission validates with the same normalizer and keeps the authored image text, so existing plan identities are unchanged. - ensure-image, resolve-image and fetch-image accept shorthand. - New pure `runtime admit-image --reference REF --json`; native foreground up admits every authored service image before hooks, VM or provider start. Refs HACK-1116. Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 28 +- packages/runtime-core/src/main.rs | 13 +- .../src/project/compose/fields.rs | 116 +---- .../runtime-core/src/provider/image_ensure.rs | 14 +- .../src/provider/image_reference.rs | 441 ++++++++++++++++++ packages/runtime-core/src/provider/mod.rs | 1 + .../src/provider/registry_image.rs | 184 +++++--- packages/runtime-core/tests/local_contract.rs | 37 ++ .../runtime-core/tests/project_contract.rs | 35 ++ .../native-project-image-admission.ts | 82 ++++ src/backends/native-project-start.ts | 10 + tests/native-project-start.test.ts | 117 +++++ 12 files changed, 907 insertions(+), 171 deletions(-) create mode 100644 packages/runtime-core/src/provider/image_reference.rs create mode 100644 src/backends/native-project-image-admission.ts diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index b5f309b4c..231ec282b 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -517,7 +517,7 @@ OrbStack, registry credentials, or a running guest: ```sh ./hack-native --candidate-root /absolute/private/candidate-home runtime fetch-image \ - --reference 'namespace/repository@sha256:REVIEWED_MANIFEST_DIGEST' \ + --reference '[namespace/]repository@sha256:REVIEWED_MANIFEST_DIGEST' \ --archive /absolute/private/new-image.tar ``` @@ -525,9 +525,30 @@ For an existing mutable image declaration, resolve it explicitly first: ```sh ./hack-native --candidate-root /absolute/private/candidate-home runtime resolve-image \ - --reference 'namespace/repository:tag' --json + --reference '[namespace/]repository:tag' --json ``` +Image references are normalized as the Docker CLI does before any request: +`alpine` is `docker.io/library/alpine:latest`, `alpine:3.21` is +`docker.io/library/alpine:3.21`, and `user/img` is `docker.io/user/img:latest`. +`index.docker.io` becomes `docker.io`. Tags, digests, other registries and +ports are kept as written; only public Docker Hub `namespace/repository` +references are acquired. Repository names must be lowercase, and a normalized +name is limited to 255 bytes. Uppercase digests and malformed registry ports are +refused. Check a reference without network, credentials or candidate state: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home runtime admit-image \ + --reference 'alpine:3.21' --json +``` + +It returns the canonical `reference` and a `kind` of `docker_hub` or +`image_id`; any other registry refuses with `registry_image`. Native foreground +`hack up` runs this admission for every authored service image before startup +hooks, the VM or any provider starts. Compose plan review applies the same +syntax rules, keeps the authored image text in the reviewed plan, and does not +change existing plan identities. + Omitting the tag selects `latest`. Resolution returns `pinned_reference`, `source_digest`, `manifest_digest`, `image_id` and `platform`; it downloads only bounded metadata and verifies one unambiguous Linux ARM64 image configuration. @@ -777,7 +798,8 @@ restored one-off remain open. An uncertain shutdown retains its mapping and evid for recovery; do not delete managed state to retry. After starting an owned native pool, `runtime ensure-image --reference REF --json` -resolves a public Docker Hub tag or accepts an explicit digest, reuses the verified +resolves a public Docker Hub tag (including shorthand such as `alpine:3.21`) or +accepts an explicit digest, reuses the verified archive cache, and imports through the existing content-ID-checked loader. Tags are resolved on each call; a warm explicit digest needs no registry request. The cache is limited to 16 archives and 2 GiB; an incomplete or conflicting entry diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 76dbdb89b..14ecef1a8 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -97,9 +97,10 @@ Usage: hack-local runtime prepared-base verify --base-id [--store ] [--json] hack-local runtime prepared-base status --profile research|development [--store ] [--json] hack-local runtime prepared-base remove --base-id [--store ] [--json] - hack-local runtime ensure-image --reference --json - hack-local runtime resolve-image --reference --json - hack-local runtime fetch-image --reference --archive [--json] + hack-local runtime admit-image --reference <[namespace/]repository[:tag][@sha256:digest]|sha256:image-id> --json + hack-local runtime ensure-image --reference <[namespace/]repository[:tag]|[namespace/]repository@sha256:digest> --json + hack-local runtime resolve-image --reference <[namespace/]repository[:tag]> --json + hack-local runtime fetch-image --reference <[namespace/]repository@sha256:digest> --archive [--json] hack-local runtime load-image --archive --sha256 --image-id hack-local runtime network internet --json hack-local runtime up --profile development --internet --json @@ -937,6 +938,12 @@ fn run() -> Result<(), CandidateError> { reference, )?)?; } + ["runtime", "admit-image", "--reference", reference, "--json"] => { + // Pure: no candidate discovery, network, credentials or runtime effects. + print_json(&hack_runtime_core::provider::registry_image::admit( + reference, + )?)?; + } [ "runtime", "resolve-image", diff --git a/packages/runtime-core/src/project/compose/fields.rs b/packages/runtime-core/src/project/compose/fields.rs index 7bd67f4b3..032e7f6e1 100644 --- a/packages/runtime-core/src/project/compose/fields.rs +++ b/packages/runtime-core/src/project/compose/fields.rs @@ -315,118 +315,14 @@ pub(super) fn image( diagnostics.push(Diagnostic::error("structural_environment_reference", field, "Image interpolation requires an explicit future value-binding step; ambient environment is not read.")); return Ok(None); } - if value.len() > 512 - || value.is_empty() - || !value - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"._-/:@".contains(&b)) - || value.contains("://") - { - return Err(problem( - "invalid_image", - "Image must be a literal OCI-style reference without URLs, credentials or whitespace.", - )); - } - if let Some((_, digest)) = value.split_once('@') { - let Some(digest) = digest.strip_prefix("sha256:") else { - return Err(problem( - "invalid_image", - "Only sha256 image digests are supported.", - )); - }; - if digest.len() != 64 || !digest.bytes().all(|b| b.is_ascii_hexdigit()) { - return Err(problem("invalid_image", "Invalid image digest.")); - } - } - let reference = value.split('@').next().unwrap_or(""); - if !valid_image_name(reference) { - return Err(problem("invalid_image", "Invalid image reference.")); - } + // Plan admission uses the same normalizer as acquisition, so a reference the + // provider would refuse as malformed never reaches a VM start. The authored + // text is kept in the plan: normalizing it here would re-identify every + // existing reviewed plan and enrollment receipt. + crate::provider::image_reference::normalize(&value) + .map_err(|reason| problem("invalid_image", reason.message()))?; Ok(Some(value)) } -fn valid_image_name(reference: &str) -> bool { - let last = reference.rsplit('/').next().unwrap_or(""); - let repository = if let Some((_, tag)) = last.rsplit_once(':') { - if tag.is_empty() - || tag.len() > 128 - || !tag - .bytes() - .next() - .is_some_and(|b| b.is_ascii_alphanumeric() || b == b'_') - || !tag - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) - { - return false; - } - &reference[..reference.len() - tag.len() - 1] - } else { - reference - }; - let parts: Vec<_> = repository.split('/').collect(); - for (index, part) in parts.iter().enumerate() { - if index == 0 - && parts.len() > 1 - && (part.contains('.') || part.contains(':') || *part == "localhost") - { - let host = if let Some((host, port)) = part.split_once(':') { - if port.parse::().ok().filter(|p| *p > 0).is_none() { - return false; - } - host - } else { - part - }; - if host.is_empty() - || !host - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b".-".contains(&b)) - { - return false; - } - continue; - } - let bytes = part.as_bytes(); - let mut cursor = 0; - while cursor < bytes.len() { - let start = cursor; - while bytes - .get(cursor) - .is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) - { - cursor += 1; - } - if start == cursor { - return false; - } - if cursor == bytes.len() { - break; - } - match bytes[cursor] { - b'.' => cursor += 1, - b'_' => { - cursor += 1; - if bytes.get(cursor) == Some(&b'_') { - cursor += 1; - } - } - b'-' => { - while bytes.get(cursor) == Some(&b'-') { - cursor += 1; - } - } - _ => return false, - } - if cursor == bytes.len() { - return false; - } - } - if bytes.is_empty() { - return false; - } - } - true -} pub(super) fn number(value: &Value) -> Result { let number = match value { Value::String(s) => s.parse().ok(), diff --git a/packages/runtime-core/src/provider/image_ensure.rs b/packages/runtime-core/src/provider/image_ensure.rs index 84bc4713b..fe2c06af1 100644 --- a/packages/runtime-core/src/provider/image_ensure.rs +++ b/packages/runtime-core/src/provider/image_ensure.rs @@ -229,7 +229,7 @@ fn ensure_pinned( } }; let (_, pin) = registry_image::reference(&reference)?; - if acquired.source_digest != pin { + if acquired.source_digest != pin.as_str() { return Err(refused( "Acquired image does not match the requested immutable reference.", )); @@ -303,7 +303,7 @@ mod tests { fs::write(path, b"fixture").unwrap(); fs::set_permissions(path, fs::Permissions::from_mode(0o600)).unwrap(); Ok(registry_image::AcquiredImage { - source_digest: registry_image::reference(reference)?.1.into(), + source_digest: registry_image::reference(reference)?.1, manifest_digest: format!("sha256:{}", "b".repeat(64)), image_id: format!("sha256:{}", "c".repeat(64)), archive_sha256: key("fixture"), @@ -461,6 +461,16 @@ mod tests { assert!(ensure(&f.1, "untrusted/thing:bad tag").is_err()); assert!(ensure(&f.1, &pin(1)).is_err()); assert!(ensure(&f.1, "fixture/image:latest").is_err()); + // Docker Hub shorthand passes reference admission and stops at the absent + // runtime; an unacquirable registry refuses before any runtime check. + assert_ne!( + ensure(&f.1, "alpine:3.21").unwrap_err().code, + "registry_image" + ); + assert_eq!( + ensure(&f.1, "ghcr.io/fixture/image:1").unwrap_err().code, + "registry_image" + ); assert!(!f.1.state_root.join("artifacts/images").exists()); } #[test] diff --git a/packages/runtime-core/src/provider/image_reference.rs b/packages/runtime-core/src/provider/image_reference.rs new file mode 100644 index 000000000..c3b9392ba --- /dev/null +++ b/packages/runtime-core/src/provider/image_reference.rs @@ -0,0 +1,441 @@ +//! Pure OCI image reference normalization shared by plan admission and acquisition. +//! +//! Docker shorthand is expanded exactly as the Docker CLI does: an unqualified name +//! belongs to `docker.io`, a single Docker Hub path component gains `library/`, +//! the legacy `index.docker.io` domain becomes `docker.io`, and a reference with +//! neither tag nor digest gains `:latest`. Registries, ports, tags and digests are +//! otherwise kept exactly. No network, filesystem, credential or runtime state is read. +//! +//! Errors are a closed, value-free vocabulary: callers attach their own error code +//! and never echo the rejected input. + +const MAX_INPUT: usize = 512; +/// Docker distribution's bound on a complete normalized repository name. +const MAX_NAME: usize = 255; +const MAX_TAG: usize = 128; +const DOCKER_HUB: &str = "docker.io"; + +/// A closed rejection reason. Messages never include the rejected reference. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ReferenceError { + Empty, + TooLong, + InvalidCharacters, + InvalidDigest, + InvalidImageId, + InvalidDomain, + UppercaseRepository, + InvalidRepository, + InvalidTag, + NameTooLong, +} + +impl ReferenceError { + pub fn message(self) -> &'static str { + match self { + Self::Empty => "Image reference is empty.", + Self::TooLong => "Image reference exceeds 512 bytes.", + Self::InvalidCharacters => { + "Image must be a literal OCI-style reference without URLs, credentials or whitespace." + } + Self::InvalidDigest => "Only lowercase sha256 image digests are supported.", + Self::InvalidImageId => "Image IDs must be sha256 followed by 64 lowercase hex digits.", + Self::InvalidDomain => "Image registry must be a lowercase host with an optional port.", + Self::UppercaseRepository => "Image repository names must be lowercase.", + Self::InvalidRepository => "Invalid image repository path.", + Self::InvalidTag => { + "Image tags must be 1 to 128 word characters, '.' or '-', not starting with '.' or '-'." + } + Self::NameTooLong => "Normalized image name exceeds 255 bytes.", + } + } +} + +/// A fully qualified registry reference. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct NamedReference { + /// Registry host with optional port, for example `docker.io` or `localhost:5000`. + pub domain: String, + /// Repository path below the registry, for example `library/alpine`. + pub path: String, + /// Present unless the reference is pinned by digest alone. + pub tag: Option, + /// `sha256:<64 lowercase hex>` when pinned. + pub digest: Option, +} + +impl NamedReference { + /// The canonical `domain/path[:tag][@digest]` form. + pub fn canonical(&self) -> String { + let mut value = format!("{}/{}", self.domain, self.path); + if let Some(tag) = &self.tag { + value.push(':'); + value.push_str(tag); + } + if let Some(digest) = &self.digest { + value.push('@'); + value.push_str(digest); + } + value + } + + /// The `namespace/repository` path when this is a public Docker Hub reference + /// the anonymous acquisition path supports, otherwise `None`. + pub fn docker_hub_repository(&self) -> Option<&str> { + (self.domain == DOCKER_HUB && self.path.len() <= 200 && self.path.split('/').count() == 2) + .then_some(self.path.as_str()) + } +} + +/// A normalized image reference: a registry name or an already-local immutable image ID. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum ImageReference { + /// `sha256:<64 lowercase hex>` content ID that needs no registry resolution. + ImageId(String), + Named(NamedReference), +} + +impl ImageReference { + pub fn canonical(&self) -> String { + match self { + Self::ImageId(id) => id.clone(), + Self::Named(named) => named.canonical(), + } + } +} + +fn lowercase_hex(value: &str, length: usize) -> bool { + value.len() == length + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +fn sha256(value: &str) -> bool { + value + .strip_prefix("sha256:") + .is_some_and(|hex| lowercase_hex(hex, 64)) +} + +/// `[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*`, distribution's path component grammar. +fn path_component(value: &str) -> Result<(), ReferenceError> { + if value.bytes().any(|b| b.is_ascii_uppercase()) { + return Err(ReferenceError::UppercaseRepository); + } + let bytes = value.as_bytes(); + let alphanumeric = + |b: Option<&u8>| b.is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()); + let mut cursor = 0; + loop { + let start = cursor; + while alphanumeric(bytes.get(cursor)) { + cursor += 1; + } + if start == cursor { + return Err(ReferenceError::InvalidRepository); + } + if cursor == bytes.len() { + return Ok(()); + } + match bytes[cursor] { + b'.' => cursor += 1, + b'_' => { + cursor += 1; + if bytes.get(cursor) == Some(&b'_') { + cursor += 1; + } + } + b'-' => { + while bytes.get(cursor) == Some(&b'-') { + cursor += 1; + } + } + _ => return Err(ReferenceError::InvalidRepository), + } + } +} + +/// Lowercase DNS-style host labels, optionally followed by a decimal port in 1..=65535. +fn domain(value: &str) -> Result<(), ReferenceError> { + let (host, port) = match value.split_once(':') { + Some((host, port)) => (host, Some(port)), + None => (value, None), + }; + if let Some(port) = port { + if port.is_empty() + || !port.bytes().all(|b| b.is_ascii_digit()) + || port.parse::().ok().filter(|p| *p > 0).is_none() + { + return Err(ReferenceError::InvalidDomain); + } + } + let label = |label: &str| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') + && !label.starts_with('-') + && !label.ends_with('-') + }; + if host.is_empty() || !host.split('.').all(label) { + return Err(ReferenceError::InvalidDomain); + } + Ok(()) +} + +fn tag(value: &str) -> Result<(), ReferenceError> { + let word = |b: u8| b.is_ascii_alphanumeric() || b == b'_'; + if value.is_empty() + || value.len() > MAX_TAG + || !value.bytes().next().is_some_and(word) + || !value.bytes().all(|b| word(b) || b == b'.' || b == b'-') + { + return Err(ReferenceError::InvalidTag); + } + Ok(()) +} + +/// Normalize one authored image reference. Pure and deterministic; the result's +/// canonical form normalizes to itself. +pub fn normalize(value: &str) -> Result { + if value.is_empty() { + return Err(ReferenceError::Empty); + } + if value.len() > MAX_INPUT { + return Err(ReferenceError::TooLong); + } + if value.contains("://") + || !value + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"._-/:@".contains(&b)) + { + return Err(ReferenceError::InvalidCharacters); + } + // A bare content ID is not Docker Hub shorthand for a repository named sha256. + if let Some(id) = value.strip_prefix("sha256:") { + if value.contains('/') || value.contains('@') || !lowercase_hex(id, 64) { + return Err(ReferenceError::InvalidImageId); + } + return Ok(ImageReference::ImageId(value.into())); + } + let (name, digest) = match value.split_once('@') { + Some((name, digest)) => { + if !sha256(digest) { + return Err(ReferenceError::InvalidDigest); + } + (name, Some(digest.to_owned())) + } + None => (value, None), + }; + // Docker treats the first component as a registry only when it looks like a + // host (contains '.' or ':', or is localhost) and more components follow. + let (registry, remainder) = match name.split_once('/') { + Some((first, rest)) + if first.contains('.') + || first.contains(':') + || first == "localhost" + || first.bytes().any(|b| b.is_ascii_uppercase()) => + { + domain(first)?; + (first, rest) + } + _ => (DOCKER_HUB, name), + }; + let (path, tag_value) = match remainder.rsplit_once(':') { + Some((path, value)) => { + tag(value)?; + (path, Some(value)) + } + None => (remainder, None), + }; + if path.is_empty() { + return Err(ReferenceError::InvalidRepository); + } + for component in path.split('/') { + path_component(component)?; + } + let registry = if registry == "index.docker.io" { + DOCKER_HUB + } else { + registry + }; + let path = if registry == DOCKER_HUB && !path.contains('/') { + format!("library/{path}") + } else { + path.to_owned() + }; + if registry.len() + 1 + path.len() > MAX_NAME { + return Err(ReferenceError::NameTooLong); + } + let tag = match (tag_value, &digest) { + (Some(value), _) => Some(value.to_owned()), + (None, None) => Some("latest".to_owned()), + (None, Some(_)) => None, + }; + Ok(ImageReference::Named(NamedReference { + domain: registry.to_owned(), + path, + tag, + digest, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn canonical(value: &str) -> String { + normalize(value).unwrap().canonical() + } + + #[test] + fn docker_hub_shorthand_expands_like_the_docker_cli() { + let pin = format!("sha256:{}", "a".repeat(64)); + for (input, expected) in [ + ("alpine", "docker.io/library/alpine:latest".to_owned()), + ("alpine:3.21", "docker.io/library/alpine:3.21".to_owned()), + ("user/img", "docker.io/user/img:latest".to_owned()), + ( + "user/img:1.0-rc_1", + "docker.io/user/img:1.0-rc_1".to_owned(), + ), + ( + "docker.io/alpine", + "docker.io/library/alpine:latest".to_owned(), + ), + ( + "docker.io/library/alpine:3.21", + "docker.io/library/alpine:3.21".to_owned(), + ), + ( + "index.docker.io/user/img", + "docker.io/user/img:latest".to_owned(), + ), + ( + "postgres:16-alpine", + "docker.io/library/postgres:16-alpine".to_owned(), + ), + ( + "a__b-c/e---f.g:TAG_upper", + "docker.io/a__b-c/e---f.g:TAG_upper".to_owned(), + ), + // Without a slash, host-like text is a Docker Hub repository and tag. + ( + "registry.test:5000", + "docker.io/library/registry.test:5000".to_owned(), + ), + // Digests are kept and do not gain an implicit tag. + ( + &*format!("alpine@{pin}"), + format!("docker.io/library/alpine@{pin}"), + ), + ( + &*format!("alpine:3.21@{pin}"), + format!("docker.io/library/alpine:3.21@{pin}"), + ), + ( + &*format!("user/img@{pin}"), + format!("docker.io/user/img@{pin}"), + ), + // Registries, ports and nested paths are preserved verbatim. + ("ghcr.io/org/app:v1", "ghcr.io/org/app:v1".to_owned()), + ("localhost/app", "localhost/app:latest".to_owned()), + ("localhost:5000/app", "localhost:5000/app:latest".to_owned()), + ( + "registry.test:5000/team/sub/app:1", + "registry.test:5000/team/sub/app:1".to_owned(), + ), + ( + "registry.test:5000/app", + "registry.test:5000/app:latest".to_owned(), + ), + ( + &*format!("registry.test:5000/app@{pin}"), + format!("registry.test:5000/app@{pin}"), + ), + ( + &*format!("registry.test:5000/app:v2@{pin}"), + format!("registry.test:5000/app:v2@{pin}"), + ), + ] { + assert_eq!(canonical(input), expected, "{input}"); + assert_eq!(canonical(&expected), expected, "idempotent {expected}"); + } + let id = format!("sha256:{}", "e".repeat(64)); + assert_eq!(normalize(&id).unwrap(), ImageReference::ImageId(id.clone())); + } + + #[test] + fn invalid_references_refuse_with_closed_value_free_reasons() { + use ReferenceError::*; + let long_path = format!("{}/{}", "a".repeat(200), "b".repeat(60)); + let long_tag = format!("alpine:{}", "t".repeat(129)); + let cases: Vec<(String, ReferenceError)> = vec![ + ("".into(), Empty), + ("a".repeat(513), TooLong), + (long_path, NameTooLong), + (long_tag, InvalidTag), + ("Alpine".into(), UppercaseRepository), + ("user/Img:1".into(), UppercaseRepository), + ("Registry.Test/app".into(), InvalidDomain), + ("REGISTRY/app".into(), InvalidDomain), + ("registry.test:0/app".into(), InvalidDomain), + ("registry.test:65536/app".into(), InvalidDomain), + ("registry.test:/app".into(), InvalidDomain), + ("registry.test:50a/app".into(), InvalidDomain), + ("-bad.test/app".into(), InvalidDomain), + ("https://docker.io/a/b".into(), InvalidCharacters), + ("a/b:PRIVATE CANARY".into(), InvalidCharacters), + ("a/b:foo?bar".into(), InvalidCharacters), + ("user:secret@docker.io/a/b".into(), InvalidDigest), + ("alpine@sha256:bad".into(), InvalidDigest), + (format!("alpine@sha256:{}", "A".repeat(64)), InvalidDigest), + (format!("alpine@sha512:{}", "a".repeat(128)), InvalidDigest), + ("sha256:abc".into(), InvalidImageId), + (format!("sha256:{}", "E".repeat(64)), InvalidImageId), + ("a/b:".into(), InvalidTag), + ("a/b:-bad".into(), InvalidTag), + ("a/b:.bad".into(), InvalidTag), + ("a/..:tag".into(), InvalidRepository), + ("../bad".into(), InvalidDomain), + ("a//b".into(), InvalidRepository), + ("a/b/".into(), InvalidRepository), + ("bad-/x".into(), InvalidRepository), + ("a_-b".into(), InvalidRepository), + ("a___b".into(), InvalidRepository), + ("registry.test:5000/".into(), InvalidRepository), + ("registry.test/".into(), InvalidRepository), + ]; + for (input, reason) in cases { + assert_eq!(normalize(&input), Err(reason), "{input}"); + assert!(!reason.message().contains(&input) || input.is_empty()); + } + } + + #[test] + fn only_two_component_docker_hub_paths_are_acquirable() { + let named = |value: &str| match normalize(value).unwrap() { + ImageReference::Named(named) => named, + ImageReference::ImageId(_) => panic!("named"), + }; + assert_eq!( + named("alpine").docker_hub_repository(), + Some("library/alpine") + ); + assert_eq!( + named("user/img:1").docker_hub_repository(), + Some("user/img") + ); + for value in [ + "ghcr.io/org/app", + "localhost:5000/app", + "a/b/c", + "docker.io/a/b/c", + ] { + assert_eq!(named(value).docker_hub_repository(), None, "{value}"); + } + let long = format!("docker.io/{}/b", "a".repeat(199)); + assert_eq!(named(&long).docker_hub_repository(), None); + } +} diff --git a/packages/runtime-core/src/provider/mod.rs b/packages/runtime-core/src/provider/mod.rs index 7e09b85c0..410efb0cf 100644 --- a/packages/runtime-core/src/provider/mod.rs +++ b/packages/runtime-core/src/provider/mod.rs @@ -35,6 +35,7 @@ pub mod https_recovery; mod identity; pub mod image_ensure; mod image_load; +pub mod image_reference; pub mod managed_environment; #[cfg(feature = "native-config-plan")] mod native_environment; diff --git a/packages/runtime-core/src/provider/registry_image.rs b/packages/runtime-core/src/provider/registry_image.rs index c5a3a118b..19c02c89f 100644 --- a/packages/runtime-core/src/provider/registry_image.rs +++ b/packages/runtime-core/src/provider/registry_image.rs @@ -1,5 +1,6 @@ //! Anonymous, digest-pinned Docker Hub acquisition. No daemon, credential store or VM effects. use super::image_load; +use super::image_reference::{ImageReference, NamedReference, normalize}; use crate::CandidateError; use reqwest::{Url, blocking::Client, header::ACCEPT, redirect::Policy}; use serde::{Deserialize, Serialize}; @@ -30,27 +31,25 @@ fn valid_digest(value: &str) -> bool { .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) }) } -pub(super) fn reference(value: &str) -> Result<(&str, &str), CandidateError> { - let value = value.strip_prefix("docker.io/").unwrap_or(value); - let (name, hash) = value - .split_once('@') - .ok_or_else(|| error("A public Docker Hub reference pinned with @sha256 is required."))?; - let repo = name.split_once(':').map_or(name, |(repo, _)| repo); - if !valid_digest(hash) - || repo.len() > 200 - || repo.split('/').count() != 2 - || repo.split('/').any(|s| { - s.is_empty() - || !s - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b"._-".contains(&b)) - }) - { - return Err(error( - "Only namespace/repository@sha256 Docker Hub references are supported.", - )); +/// Normalize to a Docker Hub reference; other registries and image IDs refuse. +fn docker_hub(value: &str) -> Result { + match normalize(value) { + Ok(ImageReference::Named(named)) if named.docker_hub_repository().is_some() => Ok(named), + _ => Err(error( + "Only public Docker Hub [namespace/]repository[:tag][@sha256] references are supported.", + )), } - Ok((repo, hash)) +} +/// Normalized Docker Hub `namespace/repository` and its required sha256 pin. +/// Docker shorthand such as `alpine@sha256:...` expands to `library/alpine`. +pub(super) fn reference(value: &str) -> Result<(String, String), CandidateError> { + let pinned = || error("A public Docker Hub reference pinned with @sha256 is required."); + if !value.contains('@') { + return Err(pinned()); + } + let named = docker_hub(value)?; + let pin = named.digest.ok_or_else(pinned)?; + Ok((named.path, pin)) } #[derive(Clone, Deserialize)] struct Descriptor { @@ -278,41 +277,49 @@ fn authenticated_fetch(repo: &str) -> Result { Ok(fetch) } -pub(super) fn tagged_reference(value: &str) -> Result<(&str, &str), CandidateError> { - let value = value.strip_prefix("docker.io/").unwrap_or(value); +/// Normalized Docker Hub `namespace/repository` and tag for mutable resolution. +/// Shorthand expands as Docker does: `alpine` is `library/alpine:latest`. +pub(super) fn tagged_reference(value: &str) -> Result<(String, String), CandidateError> { if value.contains('@') { return Err(error( "Tag resolution requires a repository and optional tag, not a digest.", )); } - let (repo, tag) = value.split_once(':').unwrap_or((value, "latest")); - let tag_valid = !tag.is_empty() - && tag.len() <= 128 - && tag.bytes().enumerate().all(|(i, b)| { - b.is_ascii_alphanumeric() || b == b'_' || (i > 0 && matches!(b, b'.' | b'-')) - }); - let repo_valid = repo.len() <= 200 - && repo.split('/').count() == 2 - && repo.split('/').all(|part| { - !part.is_empty() - && part - .bytes() - .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b"._-".contains(&b)) - && part - .bytes() - .next() - .is_some_and(|b| b.is_ascii_alphanumeric()) - && part - .bytes() - .last() - .is_some_and(|b| b.is_ascii_alphanumeric()) - }); - if !tag_valid || !repo_valid { - return Err(error( - "Only public Docker Hub namespace/repository[:tag] references are supported.", - )); + let named = docker_hub(value)?; + let tag = named.tag.unwrap_or_else(|| "latest".into()); + Ok((named.path, tag)) +} + +/// The admitted form of one authored image reference. +#[derive(Debug, Serialize)] +pub struct AdmittedImage { + /// Canonical normalized reference, for example `docker.io/library/alpine:3.21`. + pub reference: String, + /// `image_id` for an already-local content ID, `docker_hub` for an acquirable reference. + pub kind: &'static str, +} + +/// Pure admission for one authored image reference. It normalizes Docker shorthand +/// and refuses anything the provider could not later acquire. No network, credential, +/// candidate state or runtime effect occurs, so a caller can refuse before any VM or +/// provider starts. Acquisition re-normalizes with the same function. +pub fn admit(value: &str) -> Result { + match normalize(value) { + Ok(ImageReference::ImageId(id)) => Ok(AdmittedImage { + reference: id, + kind: "image_id", + }), + Ok(ImageReference::Named(named)) if named.docker_hub_repository().is_some() => { + Ok(AdmittedImage { + reference: named.canonical(), + kind: "docker_hub", + }) + } + Ok(ImageReference::Named(_)) => Err(error( + "Only public Docker Hub references or sha256 image IDs are supported; other registries are not acquired.", + )), + Err(reason) => Err(error(reason.message())), } - Ok((repo, tag)) } #[derive(Debug, Serialize)] @@ -329,6 +336,7 @@ pub struct ResolvedImage { /// fetch must use pinned_reference: reusing the original tag would discard this review. pub fn resolve(reference_text: &str) -> Result { let (repo, tag) = tagged_reference(reference_text)?; + let (repo, tag) = (repo.as_str(), tag.as_str()); let fetch = authenticated_fetch(repo)?; let url = Url::parse(&format!( "https://registry-1.docker.io/v2/{repo}/manifests/{tag}" @@ -380,6 +388,7 @@ fn verify_platform(config_bytes: &[u8]) -> Result<(), CandidateError> { /// SIGINT/process cancellation stops acquisition; no child transfer outlives the caller. pub fn acquire(reference_text: &str, output: &Path) -> Result { let (repo, pin) = reference(reference_text)?; + let (repo, pin) = (repo.as_str(), pin.as_str()); if output.exists() { return Err(error("Image archive output already exists.")); } @@ -524,16 +533,30 @@ mod tests { } #[test] fn mutable_resolution_is_explicit_and_rejects_non_repository_inputs() { + let owned = |repo: &str, tag: &str| (repo.to_owned(), tag.to_owned()); assert_eq!( tagged_reference("bitnami/redis").unwrap(), - ("bitnami/redis", "latest") + owned("bitnami/redis", "latest") ); assert_eq!( tagged_reference("docker.io/chromedp/headless-shell:stable").unwrap(), - ("chromedp/headless-shell", "stable") + owned("chromedp/headless-shell", "stable") + ); + // Ordinary Docker Hub shorthand expands to the official library namespace. + assert_eq!( + tagged_reference("alpine:3.21").unwrap(), + owned("library/alpine", "3.21") + ); + assert_eq!( + tagged_reference("redis").unwrap(), + owned("library/redis", "latest") ); for value in [ - "redis", + "Redis", + "ghcr.io/a/b:tag", + "localhost:5000/a:tag", + "a/b/c:tag", + format!("sha256:{}", "e".repeat(64)).as_str(), "host.test/a/b:tag", "https://docker.io/a/b:tag", "a/b@sha256:bad", @@ -583,10 +606,19 @@ mod tests { ] { assert_eq!( reference(&format!("{prefix}@{pin}")).unwrap(), - ("imbios/bun-node", pin.as_str()) + ("imbios/bun-node".to_owned(), pin.clone()) + ); + } + for prefix in ["alpine", "alpine:3.21", "docker.io/alpine"] { + assert_eq!( + reference(&format!("{prefix}@{pin}")).unwrap(), + ("library/alpine".to_owned(), pin.clone()) ); } for value in [ + format!("ghcr.io/a/b@{pin}").as_str(), + format!("localhost:5000/a@{pin}").as_str(), + format!("Imbios/bun-node@{pin}").as_str(), "imbios/bun-node:latest", "https://docker.io/a/b", "user:secret@docker.io/a/b", @@ -597,6 +629,52 @@ mod tests { } } #[test] + fn admission_normalizes_hub_shorthand_and_refuses_unacquirable_references() { + let pin = format!("sha256:{}", "a".repeat(64)); + let id = format!("sha256:{}", "e".repeat(64)); + for (input, reference, kind) in [ + ( + "alpine", + "docker.io/library/alpine:latest".to_owned(), + "docker_hub", + ), + ( + "alpine:3.21", + "docker.io/library/alpine:3.21".to_owned(), + "docker_hub", + ), + ( + "user/img", + "docker.io/user/img:latest".to_owned(), + "docker_hub", + ), + ( + &*format!("alpine@{pin}"), + format!("docker.io/library/alpine@{pin}"), + "docker_hub", + ), + (id.as_str(), id.clone(), "image_id"), + ] { + let admitted = admit(input).unwrap(); + assert_eq!( + (admitted.reference.as_str(), admitted.kind), + (reference.as_str(), kind) + ); + } + for input in [ + "", + "PRIVATE_CANARY", + "ghcr.io/private-canary/app:v1", + "localhost:5000/private-canary", + "private-canary/nested/app", + "private-canary@sha256:bad", + ] { + let failure = admit(input).unwrap_err(); + assert_eq!(failure.code, "registry_image"); + assert!(!failure.message.to_ascii_lowercase().contains("canary")); + } + } + #[test] fn redirects_cannot_downgrade_or_reach_unrelated_or_local_origins() { for url in [ "https://registry-1.docker.io/v2/", diff --git a/packages/runtime-core/tests/local_contract.rs b/packages/runtime-core/tests/local_contract.rs index b84d0295c..aaccb03e0 100644 --- a/packages/runtime-core/tests/local_contract.rs +++ b/packages/runtime-core/tests/local_contract.rs @@ -28,6 +28,43 @@ fn json_image_fetch_still_refuses_unpinned_input_before_network_or_output() { assert!(!archive.exists()); } +#[test] +fn image_admission_normalizes_shorthand_without_candidate_state_or_network() { + let fixture = Fixture::new(); + let admit = |reference: &str| { + invoke( + &fixture, + &["runtime", "admit-image", "--reference", reference, "--json"], + ) + }; + let output = admit("alpine:3.21"); + assert_eq!(output.status.code(), Some(0)); + let admitted: Value = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(admitted["reference"], "docker.io/library/alpine:3.21"); + assert_eq!(admitted["kind"], "docker_hub"); + for reference in ["Private-Canary", "ghcr.io/private-canary/app:1"] { + let output = admit(reference); + assert_eq!(output.status.code(), Some(2)); + let failure: Value = serde_json::from_slice(&output.stderr).unwrap(); + assert_eq!(failure["code"], "registry_image"); + assert!(!String::from_utf8_lossy(&output.stderr).contains("anary")); + } + // Admission never creates candidate or home state. + for directory in [fixture.0.clone(), fixture.directory("fake-home")] { + let mut names: Vec<_> = std::fs::read_dir(&directory) + .unwrap() + .map(|entry| entry.unwrap().file_name().into_string().unwrap()) + .collect(); + names.sort(); + let expected: &[&str] = if directory == fixture.0 { + &["empty-path", "fake-home"] + } else { + &[] + }; + assert_eq!(names, expected); + } +} + struct Fixture(PathBuf); impl Fixture { diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index e48c9bdd5..e59942107 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -1070,6 +1070,38 @@ fn a_fifo_compose_input_is_rejected_without_waiting_for_a_writer() { assert_eq!(error.code, "invalid_compose_file"); } +#[test] +fn plan_admits_normalizable_image_references_and_keeps_authored_identity_text() { + let pin = format!("sha256:{}", "a".repeat(64)); + let long_tag = format!("alpine:{}", "t".repeat(129)); + let refused = [ + format!("alpine@sha256:{}", "A".repeat(64)), + format!("sha256:{}", "E".repeat(64)), + format!("{}/{}", "a".repeat(200), "b".repeat(60)), + long_tag, + ]; + for image in refused { + let fixture = Fixture::new(&format!("services:\n web:\n image: '{image}'\n")); + let error = project::plan(&fixture.candidate, fixture.options()).unwrap_err(); + assert_eq!(error.code, "invalid_image", "{image}"); + assert!(!error.message.contains(&image)); + } + for image in [ + "alpine".to_owned(), + "user/img".to_owned(), + format!("alpine:3.21@{pin}"), + format!("localhost:5000/team/app@{pin}"), + format!("sha256:{}", "e".repeat(64)), + ] { + let fixture = Fixture::new(&format!("services:\n web:\n image: '{image}'\n")); + let report = project::plan(&fixture.candidate, fixture.options()).unwrap(); + assert_eq!( + report.plan.services["web"].image.as_deref(), + Some(image.as_str()) + ); + } +} + #[test] fn malformed_images_and_control_characters_in_source_names_are_refused() { for image in [ @@ -1079,6 +1111,9 @@ fn malformed_images_and_control_characters_in_source_names_are_refused() { "registry:bad/repo:tag", "https://registry/repo", "user:password@registry/repo", + "Alpine:3.21", + "registry.test:0/repo", + "sha256:abc", ] { let fixture = Fixture::new(&format!("services:\n web:\n image: '{image}'\n")); assert_eq!( diff --git a/src/backends/native-project-image-admission.ts b/src/backends/native-project-image-admission.ts new file mode 100644 index 000000000..970ae42df --- /dev/null +++ b/src/backends/native-project-image-admission.ts @@ -0,0 +1,82 @@ +import { isRecord } from "../lib/guards.ts"; +import { + invokeNativeRuntime, + NativeRuntimeRequestError, + type NativeRuntimeSelection, +} from "./native-runtime-client.ts"; + +const IMAGE_ID = /^sha256:[a-f0-9]{64}$/; +const KINDS = new Set(["docker_hub", "image_id"]); + +/** Admission is pure in the runtime, so each request is short and bounded. */ +const ADMISSION_TIMEOUT_MS = 10_000; + +function refused(): Error { + return new Error( + "Native project image reference refused before runtime startup: use a public Docker Hub reference (shorthand such as alpine:3.21 is accepted) or a sha256 image ID; no VM, provider or image acquisition was started." + ); +} + +/** + * Admit every authored service image before any VM, provider or image effect. + * + * The native runtime owns reference normalization (`runtime admit-image`), so + * this adapter only collects distinct references and checks the closed response. + * Content IDs need no registry and are not sent. Admission does not pin or + * rewrite images: acquisition later re-normalizes with the same runtime rule. + * A refusal or malformed response throws a value-free error; other runtime + * request failures propagate unchanged. + */ +export async function admitNativeProjectImages(opts: { + readonly runtime: NativeRuntimeSelection; + readonly projectRoot: string; + readonly specs: Readonly>>>; + readonly signal?: AbortSignal; + readonly invoke?: typeof invokeNativeRuntime; +}): Promise { + const invoke = opts.invoke ?? invokeNativeRuntime; + const references = new Set(); + for (const spec of Object.values(opts.specs)) { + if (typeof spec.image !== "string") { + throw refused(); + } + if (!IMAGE_ID.test(spec.image)) { + references.add(spec.image); + } + } + for (const reference of references) { + if (opts.signal?.aborted) { + throw refused(); + } + let admitted: unknown; + try { + admitted = await invoke({ + runtime: opts.runtime, + cwd: opts.projectRoot, + args: ["runtime", "admit-image", "--reference", reference, "--json"], + timeoutMs: ADMISSION_TIMEOUT_MS, + signal: opts.signal, + }); + } catch (error) { + // A reference refusal gets actionable guidance; any other runtime failure + // (missing binary, timeout, cancellation) keeps its own closed diagnostic. + if ( + error instanceof NativeRuntimeRequestError && + error.nativeCode === "registry_image" + ) { + throw refused(); + } + throw error; + } + if ( + !( + isRecord(admitted) && + typeof admitted.reference === "string" && + typeof admitted.kind === "string" && + KINDS.has(admitted.kind) + ) + ) { + throw refused(); + } + } +} diff --git a/src/backends/native-project-start.ts b/src/backends/native-project-start.ts index 177abc1c1..d5fdfa1c2 100644 --- a/src/backends/native-project-start.ts +++ b/src/backends/native-project-start.ts @@ -34,6 +34,7 @@ import { waitNativeProjectFinalization, } from "./native-project-finalization.ts"; import { isNativeHttpsProbePath } from "./native-project-https.ts"; +import { admitNativeProjectImages } from "./native-project-image-admission.ts"; import { type NativeProjectInput, prepareNativeProjectInput, @@ -964,6 +965,15 @@ export async function startNativeProject(opts: { input, opts.dependencyFile !== undefined ); + // Refuse malformed or unacquirable references before hooks, VM or provider start. + await admitNativeProjectImages({ + runtime: opts.runtime, + projectRoot: opts.scope.projectRoot, + specs, + signal: opts.signal, + invoke: deps.invoke, + }); + requireActiveStartup(opts.signal); const bridgeCapacity = nativeBridgeCapacity(specs); const artifact = join(dirname(opts.runtime.binary), "hack-relay-guest"); const artifactFile = Bun.file(artifact); diff --git a/tests/native-project-start.test.ts b/tests/native-project-start.test.ts index 4a8721f2d..c365d1f46 100644 --- a/tests/native-project-start.test.ts +++ b/tests/native-project-start.test.ts @@ -100,6 +100,9 @@ async function fixture(withEnvironment = true) { if (opts.args[1] === "check-project-share") { return { source_admitted: true, pool_initialized: false }; } + if (opts.args[1] === "admit-image") { + return { reference: opts.args[3], kind: "docker_hub" }; + } events.push(opts.args.slice(0, 2).join(" ")); if (opts.args[0] === "runtime" && opts.args[1] === "status") { return { @@ -2670,6 +2673,120 @@ test("retained startup selects legacy labels only after runtime admission and be } }); +function withImages( + opts: Awaited>["opts"], + services: Record +) { + const prepare = opts.dependencies.prepare!; + opts.dependencies.prepare = async (request) => ({ + ...(await prepare(request)), + normalizedComposeJson: JSON.stringify({ + services: Object.fromEntries( + Object.entries(services).map(([name, image]) => [name, { image }]) + ), + }), + serviceNames: Object.keys(services), + }); +} + +test("an invalid image reference is refused before hooks, VM, provider or image effects", async () => { + const { opts, events } = await fixture(false); + withImages(opts, { web: "Private-Canary:3.21" }); + const invoke = opts.dependencies.invoke!; + const admissions: string[] = []; + opts.dependencies.invoke = async (request) => { + if (request.args[1] === "admit-image") { + admissions.push(String(request.args[3])); + throw new NativeRuntimeRequestError({ + message: "Image repository names must be lowercase.", + nativeCode: "registry_image", + }); + } + return await invoke(request); + }; + let served = false; + opts.dependencies.serve = async () => { + served = true; + return 0; + }; + let failure: unknown; + try { + await startNativeProject(opts); + } catch (error) { + failure = error; + } + expect(String(failure)).toContain( + "image reference refused before runtime startup" + ); + expect(String(failure)).not.toContain("Canary"); + expect(admissions).toEqual(["Private-Canary:3.21"]); + // No startup hook, runtime/VM start, image acquisition or graph effect occurred. + expect(events).toEqual([]); + expect(served).toBe(false); +}); + +test("Docker Hub shorthand is admitted once per distinct reference before the runtime starts", async () => { + const { opts, events } = await fixture(false); + const id = `sha256:${"e".repeat(64)}`; + withImages(opts, { + web: "alpine:3.21", + worker: "alpine:3.21", + cache: "redis", + pinned: id, + }); + opts.dependencies.review = async (request) => + request.run({ + planId: "a".repeat(64), + namespace: "b".repeat(64), + projectArgs: ["--project", request.projectRoot], + report: { plan: { enrollment_compatible: true, services: {} } }, + }); + const invoke = opts.dependencies.invoke!; + const order: string[] = []; + opts.dependencies.invoke = async (request) => { + if (request.args[1] === "admit-image") { + order.push(`admit ${request.args[3]}`); + expect(request.timeoutMs).toBeLessThanOrEqual(10_000); + } + if (request.args[0] === "runtime" && request.args[1] === "up") { + order.push("runtime up"); + } + if (request.args[1] === "ensure-image") { + order.push(`ensure ${request.args[3]}`); + return { image_id: id }; + } + return await invoke(request); + }; + expect(await startNativeProject(opts)).toBe(0); + expect(order).toEqual([ + "admit alpine:3.21", + "admit redis", + "runtime up", + "ensure alpine:3.21", + "ensure alpine:3.21", + "ensure redis", + ]); + expect(events.indexOf("before")).toBeLessThan(events.indexOf("runtime up")); +}); + +test("a runtime failure during image admission keeps its own diagnostic and starts nothing", async () => { + const { opts, events } = await fixture(false); + withImages(opts, { web: "alpine:3.21" }); + const invoke = opts.dependencies.invoke!; + opts.dependencies.invoke = async (request) => { + if (request.args[1] === "admit-image") { + throw new NativeRuntimeRequestError({ + message: "Native runtime request timed out.", + }); + } + return await invoke(request); + }; + await expect(startNativeProject(opts)).rejects.toThrow( + "Native runtime request timed out." + ); + expect(events).toEqual([]); +}); + test("retained startup reviews the saved content ID instead of a newly resolved mutable tag", async () => { const { opts, events } = await fixture(false); const saved = { From 7fd488732dad1b9b689008a694b7bf5ef79bef24 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:51:07 -0400 Subject: [PATCH 07/21] feat(lifecycle): ownership-safe shared host helper contract and fixtures Add a library contract for long-lived helpers shared across branches on one loopback port (HACK-1142, WU29). Starters are serialized by the projects-registry lock protocol; a listener is adopted only after its identity endpoint names the expected helper and branch upstream; owned helpers are recorded as PID plus process birth time and spawned behind an exec gate that only opens once the claim is durable, so a lost starter never leaves an unclaimed child. Readiness is the identity answer, bounded by backoff and one deadline, with cancellation stopping the owned child. Release drops one holder and stops an owned helper's birth-verified tree only for the last holder; adopted, foreign and stale-PID processes are never signalled. Failures are a closed, value-free set. Portable process fixtures cover adoption, wrong upstream, foreign listeners, concurrent independent starters, stale/reused PIDs, child restart, cancellation during retry, placeholder liveness, lost parent before and after the gate, descendants, impostor identity, invalid claims and an unavailable census. Co-Authored-By: Claude Opus 5.5 --- docs/lifecycle.md | 5 + docs/reference.md | 1 + docs/reference/shared-host-lifecycle.md | 184 +++++ src/lib/shared-host-lifecycle.ts | 1002 +++++++++++++++++++++++ tests/fixtures/shared-host-helper.ts | 71 ++ tests/fixtures/shared-host-starter.ts | 27 + tests/shared-host-lifecycle.test.ts | 916 +++++++++++++++++++++ 7 files changed, 2206 insertions(+) create mode 100644 docs/reference/shared-host-lifecycle.md create mode 100644 src/lib/shared-host-lifecycle.ts create mode 100644 tests/fixtures/shared-host-helper.ts create mode 100644 tests/fixtures/shared-host-starter.ts create mode 100644 tests/shared-host-lifecycle.test.ts diff --git a/docs/lifecycle.md b/docs/lifecycle.md index 4d0bc15d4..e9761669a 100644 --- a/docs/lifecycle.md +++ b/docs/lifecycle.md @@ -46,6 +46,11 @@ still be reconnecting. When the wrong listener would matter, pair `onConflict: " `lifecycle.up.before` hook that verifies the listener's identity (for example, a request to a known endpoint) and fails when it does not match. +The [shared host helper lifecycle contract](./reference/shared-host-lifecycle.md) defines the +identity-verified alternative: adoption only after the listener names the expected helper and upstream, +exact PID-plus-start-time claims, serialized starters, and release that never signals a foreign +process. It is a library contract today; `singleton` does not use it yet. + ## Config ```json diff --git a/docs/reference.md b/docs/reference.md index ee490977b..ae65b4fcf 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -12,6 +12,7 @@ - [Environment model](./env.md) - [Lifecycle & startup](./lifecycle.md) +- [Shared host helper lifecycle](./reference/shared-host-lifecycle.md) — identity-verified adoption, exact process claims and release rules for helpers shared across branches - [Sessions](./sessions.md) - [Agent-first setup](./guides/agent-first-setup.md) diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md new file mode 100644 index 000000000..0b77dfa82 --- /dev/null +++ b/docs/reference/shared-host-lifecycle.md @@ -0,0 +1,184 @@ +# Shared host helper lifecycle contract + +A shared host helper is a long-lived local process that several branches or clients +use through one fixed loopback port: an issuer bridge, a local API proxy, a database +port-forward or an opt-in tunnel. This contract defines how Hack starts, adopts, +reuses and releases such a helper without killing a process it does not own. + +Source: [`src/lib/shared-host-lifecycle.ts`](../../src/lib/shared-host-lifecycle.ts). +Fixtures: [`tests/shared-host-lifecycle.test.ts`](../../tests/shared-host-lifecycle.test.ts). + +Status: this is a library contract with portable process fixtures. No configuration +key or CLI command uses it yet. Consumers (WU18 shared helpers, bounded external +claims and opt-in tunnels) adopt it in their own changes. The existing +`lifecycle.singleton` port check in [Lifecycle & startup](../lifecycle.md) is +unchanged. It adopts any listener on the configured ports; this contract adopts +only a listener that proves its identity. + +## States + +| Axis | Values | Meaning | +| --- | --- | --- | +| Ownership | `owned` | Hack started the helper and holds an exact process claim. | +| | `adopted` | Someone else started it; its identity endpoint matched. Hack never signals it. | +| | `disabled` | The helper is off. Hack takes no lock, writes no claim and starts no child. | +| Phase | `starting` | Owned child spawned and claimed; readiness not yet proven. | +| | `ready` | The identity endpoint answered for the expected helper and upstream. | +| | `failed` | One closed failure reason (below). Terminal for this call. | + +A disabled public tunnel is the `disabled` case: it never spawns a child. + +## Identity and readiness + +The helper serves a JSON identity endpoint on `127.0.0.1:`: + +```json +{ "helper": "issuer-bridge", "upstream": "https://issuer.branch-a.test", "pid": 4242 } +``` + +- `helper` must equal the expected helper kind. Otherwise the listener is foreign. +- `upstream` must equal the upstream this branch requires. Callers derive it per + branch. A shared helper serving another upstream is refused, never retargeted. +- `pid` is optional. When present for an owned helper, it must equal the claimed PID. + A different process answering for the port is foreign. + +Readiness is this identity answer. A live process, an open port or a sleeping +placeholder is never readiness. Each probe has its own timeout and reads at most 4 KiB. + +## Persisted claim + +The claim is `/.claim.json`, owner-only (0600), replaced atomically, +and parsed strictly (exact keys, version 1): + +```json +{"version":1,"helper":"issuer-bridge","port":4100,"ownership":"owned","phase":"ready","pid":4242,"birth":"Mon Oct 19 10:00:00 2026","holders":["branch-a","branch-b"]} +{"version":1,"helper":"issuer-bridge","port":4100,"ownership":"adopted","holders":["branch-a"]} +``` + +- Ownership is the pair `pid` + `birth` (process start time from `ps lstart`). The + claim is live only when the census shows that exact pair. A missing PID, or the + same PID with a different birth time (PID reuse), makes the claim stale. A stale + claim grants nothing. Hack discards it and never signals that PID. +- `holders` names the branches or clients that use the helper. Holders carry over + when a stale owned helper is restarted. +- A malformed, unreadable, symlinked or group/world-accessible claim, or one for a + different port, fails closed with `claim-invalid`. Hack does not start, adopt or + stop anything until an operator inspects it. +- A census that did not observe the calling process (for example `ps` failed) fails + closed with `census-unavailable`. An empty census never reads as "every claimed + process is gone". + +## Start (`ensureSharedHostHelper`) + +1. `enabled: false` returns `disabled` immediately. +2. Take the helper's exclusive lock (`/.lock`, the projects-registry + lock protocol: hard-link publication, dead-owner reclamation by `ESRCH` only). + Competing starters wait in turn, bounded by the deadline. +3. A live owned claim is reused. The starter waits for readiness and adds its holder. + It does not spawn a second helper. If readiness fails, it leaves the helper for + the other holders. +4. Otherwise probe the port: + - identity matches: write an `adopted` claim and return `ready/adopted`; + - wrong upstream: `wrong-upstream`; another helper, a silent or non-identity + listener: `foreign-listener`. Either way the listener keeps running and no + child is spawned; + - connection refused: confirm with a loopback bind probe, then spawn. +5. Spawn through an exec gate in a new session: + `/bin/sh -c 'IFS= read -r hack_gate || exit 0; [ "$hack_gate" = open ] || exit 0; exec "$@" > = { + "foreign-listener": + "The shared helper port is held by a process that is not the expected helper; it was left running.", + "wrong-upstream": + "The shared helper on this port serves a different upstream; it was left running.", + "claim-invalid": + "The persisted shared helper claim is unreadable or belongs to another configuration; inspect and release it before retrying.", + "census-unavailable": + "The process census is unavailable, so helper ownership cannot be verified.", + "lock-unavailable": + "Another starter holds the shared helper lock beyond the deadline.", + "port-probe-failed": "The shared helper port could not be inspected.", + "spawn-failed": "The shared helper could not be started.", + "child-exited": "The shared helper exited before it became ready.", + deadline: "The shared helper did not become ready before the deadline.", + cancelled: "Shared helper startup was cancelled.", +}; + +/** Fixed, value-free message for a closed failure reason. */ +export function sharedHostFailureMessage(reason: SharedHostFailure): string { + return FAILURE_MESSAGES[reason]; +} + +/** Identity a helper must report at its identity endpoint. */ +export type SharedHostIdentity = { + /** Stable helper kind, for example `issuer-bridge`. */ + readonly helper: string; + /** The upstream this branch requires the helper to serve. */ + readonly upstream: string; +}; + +export type SharedHostHelperSpec = { + /** Claim and lock name; `[A-Za-z0-9][A-Za-z0-9_.-]{0,127}`. */ + readonly name: string; + /** Disabled helpers take no lock, write no claim and start no child. */ + readonly enabled: boolean; + /** Caller-owned private directory for the claim and lock. */ + readonly stateDir: string; + /** Loopback port the helper serves. */ + readonly port: number; + /** Absolute path of the JSON identity endpoint, for example `/__hack/identity`. */ + readonly identityPath: string; + readonly expected: SharedHostIdentity; + /** Helper argv; executed directly, never through a shell string. */ + readonly command: readonly string[]; + readonly cwd: string; + /** Complete child environment. The runner environment is never inherited. */ + readonly env: Readonly>; + /** One bound for lock wait, spawn and readiness. */ + readonly deadlineMs: number; + readonly backoff: { readonly initialMs: number; readonly maxMs: number }; + /** Per-request bound for one identity probe. */ + readonly probeTimeoutMs: number; +}; + +export type SharedHostEnsureResult = + | { readonly state: "disabled"; readonly ownership: "disabled" } + | { + readonly state: "ready"; + readonly ownership: "owned" | "adopted"; + readonly endpoint: string; + /** Present for owned helpers only. */ + readonly pid?: number; + readonly spawned: boolean; + } + | { + readonly state: "failed"; + readonly reason: SharedHostFailure; + readonly message: string; + }; + +/** Closed progress events. The observer is awaited (a test and UI seam). */ +export type SharedHostEvent = + | "lock-acquired" + | "claim-stale" + | "claim-reused" + | "adopted" + | "spawned" + | "claimed" + | "gate-opened" + | "probe-retry" + | "ready" + | "stopped-after-failure"; + +export type SharedHostReleaseResult = + | { + readonly state: + | "absent" + | "retained" + | "released-adopted" + | "stale-cleared" + | "stopped" + | "stop-unconfirmed"; + } + | { + readonly state: "failed"; + readonly reason: Extract< + SharedHostFailure, + | "claim-invalid" + | "census-unavailable" + | "lock-unavailable" + | "cancelled" + >; + readonly message: string; + }; + +type OwnedClaim = { + readonly version: 1; + readonly helper: string; + readonly port: number; + readonly ownership: "owned"; + readonly phase: "starting" | "ready"; + readonly pid: number; + readonly birth: string; + readonly holders: readonly string[]; +}; +type AdoptedClaim = { + readonly version: 1; + readonly helper: string; + readonly port: number; + readonly ownership: "adopted"; + readonly holders: readonly string[]; +}; +type Claim = OwnedClaim | AdoptedClaim; + +type ProbeObservation = + | { readonly kind: "absent" } + | { readonly kind: "unidentified" } + | { readonly kind: "foreign" } + | { readonly kind: "wrong-upstream" } + | { readonly kind: "match"; readonly pid: number | null } + | { readonly kind: "cancelled" }; + +const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const MAX_CLAIM_BYTES = 8192; +const MAX_IDENTITY_BYTES = 4096; +const MAX_HOLDERS = 256; +const STOP_POLL_MS = 50; +/** + * Exec gate: the helper argv runs only after the starter writes `open`. EOF + * (starter lost before its claim was durable) exits without running it. `exec` + * keeps the PID and birth time recorded in the claim. + */ +const GATE_SCRIPT = + 'IFS= read -r hack_gate || exit 0; [ "$hack_gate" = open ] || exit 0; exec "$@" +): SharedHostReleaseResult { + return { state: "failed", reason, message: FAILURE_MESSAGES[reason] }; +} + +function assertSpec(spec: SharedHostHelperSpec): void { + const valid = + NAME.test(spec.name) && + Number.isInteger(spec.port) && + spec.port > 0 && + spec.port <= 65_535 && + spec.identityPath.startsWith("/") && + spec.command.length > 0 && + spec.deadlineMs > 0 && + spec.probeTimeoutMs > 0 && + spec.backoff.initialMs > 0 && + spec.backoff.maxMs >= spec.backoff.initialMs; + if (!valid) { + throw new Error("Invalid shared host helper specification"); + } +} + +function assertHolder(holder: string): void { + if (!NAME.test(holder)) { + throw new Error("Invalid shared host helper holder"); + } +} + +function claimPath(spec: Pick) { + return join(spec.stateDir, `${spec.name}.claim.json`); +} + +function lockPath(spec: Pick) { + return join(spec.stateDir, `${spec.name}.lock`); +} + +function endpointFor(port: number): string { + return `http://127.0.0.1:${port}`; +} + +function exactKeys(value: Record, keys: readonly string[]) { + const actual = Object.keys(value).sort(); + const wanted = [...keys].sort(); + return ( + actual.length === wanted.length && + actual.every((key, index) => key === wanted[index]) + ); +} + +function parseHolders(value: unknown): readonly string[] | null { + if ( + !Array.isArray(value) || + value.length > MAX_HOLDERS || + !value.every((entry) => typeof entry === "string" && NAME.test(entry)) || + new Set(value).size !== value.length + ) { + return null; + } + return value as string[]; +} + +/** Strict v1 claim parser; any other shape is an uncertain receipt. */ +export function parseSharedHostClaim(text: string): Claim | null { + let value: unknown; + try { + value = JSON.parse(text); + } catch { + return null; + } + if (!isRecord(value) || value.version !== 1) { + return null; + } + const holders = parseHolders(value.holders); + const port = value.port; + if ( + holders === null || + typeof value.helper !== "string" || + !NAME.test(value.helper) || + typeof port !== "number" || + !Number.isInteger(port) || + port <= 0 || + port > 65_535 + ) { + return null; + } + if (value.ownership === "adopted") { + return exactKeys(value, [ + "version", + "helper", + "port", + "ownership", + "holders", + ]) + ? { + version: 1, + helper: value.helper, + port, + ownership: "adopted", + holders, + } + : null; + } + const pid = value.pid; + if ( + value.ownership !== "owned" || + !exactKeys(value, [ + "version", + "helper", + "port", + "ownership", + "phase", + "pid", + "birth", + "holders", + ]) || + (value.phase !== "starting" && value.phase !== "ready") || + typeof pid !== "number" || + !Number.isSafeInteger(pid) || + pid <= 1 || + typeof value.birth !== "string" || + value.birth.length === 0 || + value.birth.length > 64 + ) { + return null; + } + return { + version: 1, + helper: value.helper, + port, + ownership: "owned", + phase: value.phase, + pid, + birth: value.birth, + holders, + }; +} + +type ClaimRead = + | { readonly kind: "absent" } + | { readonly kind: "invalid" } + | { readonly kind: "claim"; readonly claim: Claim }; + +async function readClaim( + spec: Pick +): Promise { + let file: Awaited>; + try { + file = await readReplaceablePrivateFile({ + path: claimPath(spec), + directories: [spec.stateDir], + refusal: () => new ClaimRefusal(), + }); + } catch { + return { kind: "invalid" }; + } + if (file === null) { + return { kind: "absent" }; + } + if (file.text.length > MAX_CLAIM_BYTES || (file.mode & 0o077) !== 0) { + return { kind: "invalid" }; + } + const claim = parseSharedHostClaim(file.text); + if (claim === null || claim.helper !== spec.name) { + return { kind: "invalid" }; + } + return { kind: "claim", claim }; +} + +async function writeClaim( + spec: Pick, + claim: Claim +): Promise { + await replacePrivateFileAtomic({ + path: claimPath(spec), + directories: [spec.stateDir], + refusal: () => new ClaimRefusal(), + text: `${JSON.stringify(claim)}\n`, + }); +} + +async function removeClaim( + spec: Pick +): Promise { + try { + await unlink(claimPath(spec)); + } catch (error: unknown) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + } +} + +/** + * A census is usable only when it observed this process; an empty or failed + * `ps` must never be read as "every claimed process is gone". + */ +async function readCensus(): Promise { + const rows = await readProcessIdentities(); + return rows.some((row) => row.pid === process.pid) ? rows : null; +} + +function isLive( + census: readonly ProcessIdentityRow[], + claim: { readonly pid: number; readonly birth: string } +): boolean { + return census.some( + (row) => row.pid === claim.pid && row.birth === claim.birth + ); +} + +async function readBounded(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) { + return ""; + } + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + size += value.byteLength; + if (size > MAX_IDENTITY_BYTES) { + await reader.cancel().catch(() => undefined); + return null; + } + chunks.push(value); + } + } catch { + return null; + } + try { + return new TextDecoder("utf-8", { fatal: true }).decode( + Buffer.concat(chunks) + ); + } catch { + return null; + } +} + +/** Classify the listener on the helper port by its identity endpoint. */ +export async function probeSharedHostIdentity(opts: { + readonly port: number; + readonly identityPath: string; + readonly expected: SharedHostIdentity; + readonly timeoutMs: number; + readonly signal?: AbortSignal; +}): Promise { + const signals = [AbortSignal.timeout(opts.timeoutMs)]; + if (opts.signal) { + signals.push(opts.signal); + } + let response: Response; + try { + response = await fetch(`${endpointFor(opts.port)}${opts.identityPath}`, { + redirect: "manual", + signal: AbortSignal.any(signals), + }); + } catch (error: unknown) { + if (opts.signal?.aborted) { + return { kind: "cancelled" }; + } + // Only a refused connection proves no listener; a timeout or reset means + // something holds the port without answering as the helper. + return isRecord(error) && error.code === "ECONNREFUSED" + ? { kind: "absent" } + : { kind: "unidentified" }; + } + const text = await readBounded(response); + if (opts.signal?.aborted) { + return { kind: "cancelled" }; + } + if (response.status !== 200 || text === null) { + return { kind: "unidentified" }; + } + let body: unknown; + try { + body = JSON.parse(text); + } catch { + return { kind: "unidentified" }; + } + if ( + !isRecord(body) || + typeof body.helper !== "string" || + typeof body.upstream !== "string" + ) { + return { kind: "unidentified" }; + } + if (body.helper !== opts.expected.helper) { + return { kind: "foreign" }; + } + if (body.upstream !== opts.expected.upstream) { + return { kind: "wrong-upstream" }; + } + const pid = + typeof body.pid === "number" && Number.isSafeInteger(body.pid) + ? body.pid + : null; + return { kind: "match", pid }; +} + +/** + * Stop a birth-verified process tree: TERM, bounded grace, then KILL for the + * members that still carry their captured birth time. Nothing is signalled + * when the root no longer matches its claim. + */ +export async function stopVerifiedSharedHostProcess(opts: { + readonly pid: number; + readonly birth: string; + readonly graceMs: number; +}): Promise<"stopped" | "absent" | "unconfirmed" | "census-unavailable"> { + const census = await readCensus(); + if (census === null) { + return "census-unavailable"; + } + if (!isLive(census, opts)) { + return "absent"; + } + const tree = selectProcessTree(census, opts.pid); + const remaining = async (): Promise => { + const live = await readCensus(); + if (live === null) { + return null; + } + return tree.filter((row) => isLive(live, row)); + }; + await signalVerifiedProcessTree(tree, "SIGTERM"); + const termDeadline = performance.now() + opts.graceMs; + let left = await remaining(); + while (left !== null && left.length > 0 && performance.now() < termDeadline) { + await delay(STOP_POLL_MS); + left = await remaining(); + } + if (left !== null && left.length === 0) { + return "stopped"; + } + await signalVerifiedProcessTree(left ?? tree, "SIGKILL"); + const killDeadline = performance.now() + 1000; + left = await remaining(); + while (left !== null && left.length > 0 && performance.now() < killDeadline) { + await delay(STOP_POLL_MS); + left = await remaining(); + } + return left !== null && left.length === 0 ? "stopped" : "unconfirmed"; +} + +function withHolder(holders: readonly string[], holder: string): string[] { + return holders.includes(holder) ? [...holders] : [...holders, holder]; +} + +type EnsureContext = { + readonly spec: SharedHostHelperSpec; + readonly holder: string; + readonly signal?: AbortSignal; + readonly observe: (event: SharedHostEvent) => Promise; + readonly deadline: number; +}; + +function remainingMs(context: EnsureContext): number { + return context.deadline - performance.now(); +} + +type ReadinessOutcome = + | { readonly kind: "ready" } + | { + readonly kind: "failed"; + readonly reason: Extract< + SharedHostFailure, + | "foreign-listener" + | "wrong-upstream" + | "child-exited" + | "deadline" + | "cancelled" + | "census-unavailable" + >; + }; + +/** + * Probe until the identity endpoint answers for the expected helper and, when + * it reports a PID, for the owned process. Liveness alone is never readiness. + */ +async function awaitReadiness( + context: EnsureContext, + owned: { readonly pid: number; readonly birth: string } +): Promise { + const { spec } = context; + let wait = spec.backoff.initialMs; + while (true) { + if (context.signal?.aborted) { + return { kind: "failed", reason: "cancelled" }; + } + const census = await readCensus(); + if (census === null) { + return { kind: "failed", reason: "census-unavailable" }; + } + if (!isLive(census, owned)) { + return { kind: "failed", reason: "child-exited" }; + } + const left = remainingMs(context); + if (left <= 0) { + return { kind: "failed", reason: "deadline" }; + } + const probe = await probeSharedHostIdentity({ + port: spec.port, + identityPath: spec.identityPath, + expected: spec.expected, + timeoutMs: Math.min(spec.probeTimeoutMs, left), + signal: context.signal, + }); + switch (probe.kind) { + case "cancelled": + return { kind: "failed", reason: "cancelled" }; + case "wrong-upstream": + return { kind: "failed", reason: "wrong-upstream" }; + case "foreign": + return { kind: "failed", reason: "foreign-listener" }; + case "match": + if (probe.pid === null || probe.pid === owned.pid) { + return { kind: "ready" }; + } + return { kind: "failed", reason: "foreign-listener" }; + default: + break; + } + const sleep = Math.min(wait, remainingMs(context)); + if (sleep <= 0) { + return { kind: "failed", reason: "deadline" }; + } + await context.observe("probe-retry"); + try { + await delay(sleep, undefined, { signal: context.signal }); + } catch { + return { kind: "failed", reason: "cancelled" }; + } + wait = Math.min(wait * 2, spec.backoff.maxMs); + } +} + +async function ensureLocked( + context: EnsureContext +): Promise { + const { spec, holder } = context; + await context.observe("lock-acquired"); + const census = await readCensus(); + if (census === null) { + return failed("census-unavailable"); + } + const read = await readClaim(spec); + if (read.kind === "invalid") { + return failed("claim-invalid"); + } + let carried: readonly string[] = []; + if (read.kind === "claim") { + const claim = read.claim; + if (claim.port !== spec.port) { + return failed("claim-invalid"); + } + if (claim.ownership === "owned" && isLive(census, claim)) { + await context.observe("claim-reused"); + const outcome = await awaitReadiness(context, claim); + if (outcome.kind === "failed") { + // Other holders may depend on it: a reused helper is never stopped here. + return failed(outcome.reason); + } + await writeClaim(spec, { + ...claim, + phase: "ready", + holders: withHolder(claim.holders, holder), + }); + await context.observe("ready"); + return { + state: "ready", + ownership: "owned", + endpoint: endpointFor(spec.port), + pid: claim.pid, + spawned: false, + }; + } + if (claim.ownership === "owned") { + // Dead or reused PID: the claim no longer proves ownership of anything. + await context.observe("claim-stale"); + } + carried = claim.holders; + } + + const existing = await probeSharedHostIdentity({ + port: spec.port, + identityPath: spec.identityPath, + expected: spec.expected, + timeoutMs: Math.min(spec.probeTimeoutMs, Math.max(1, remainingMs(context))), + signal: context.signal, + }); + switch (existing.kind) { + case "cancelled": + return failed("cancelled"); + case "foreign": + case "unidentified": + return failed("foreign-listener"); + case "wrong-upstream": + return failed("wrong-upstream"); + case "match": + await writeClaim(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "adopted", + holders: withHolder(carried, holder), + }); + await context.observe("adopted"); + return { + state: "ready", + ownership: "adopted", + endpoint: endpointFor(spec.port), + spawned: false, + }; + default: + break; + } + try { + const occupied = await inspectListeningTcpPorts({ ports: [spec.port] }); + if (occupied.length > 0) { + return failed("foreign-listener"); + } + } catch { + return failed("port-probe-failed"); + } + if (read.kind === "claim") { + await removeClaim(spec); + } + return await spawnOwned(context, carried); +} + +async function spawnOwned( + context: EnsureContext, + carried: readonly string[] +): Promise { + const { spec, holder } = context; + let child: Bun.Subprocess<"pipe", "ignore", "ignore">; + try { + child = Bun.spawn( + ["/bin/sh", "-c", GATE_SCRIPT, "hack-shared-host-gate", ...spec.command], + { + cwd: spec.cwd, + env: { ...spec.env }, + stdin: "pipe", + stdout: "ignore", + stderr: "ignore", + // A new session keeps terminal signals for the starter away from a + // helper that other branches share. + detached: true, + } + ); + } catch { + return failed("spawn-failed"); + } + child.unref(); + await context.observe("spawned"); + const closeGate = () => { + try { + child.stdin.end(); + } catch { + // Already closed; the gate treats EOF as "do not run". + } + }; + const census = await readCensus(); + const row = census?.find((entry) => entry.pid === child.pid); + if (!row) { + closeGate(); + return failed(census === null ? "census-unavailable" : "spawn-failed"); + } + const owned = { pid: row.pid, birth: row.birth }; + const holders = withHolder(carried, holder); + try { + await writeClaim(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "starting", + pid: owned.pid, + birth: owned.birth, + holders, + }); + } catch { + closeGate(); + return failed("claim-invalid"); + } + await context.observe("claimed"); + if (context.signal?.aborted) { + closeGate(); + await removeClaim(spec); + return failed("cancelled"); + } + try { + child.stdin.write("open\n"); + await child.stdin.flush(); + } finally { + closeGate(); + } + await context.observe("gate-opened"); + const outcome = await awaitReadiness(context, owned); + if (outcome.kind === "failed") { + const stopped = await stopVerifiedSharedHostProcess({ + ...owned, + graceMs: 1000, + }); + // Keep the claim when the stop is unproven, so the process stays recorded. + if (stopped === "stopped" || stopped === "absent") { + await removeClaim(spec); + } + await context.observe("stopped-after-failure"); + return failed(outcome.reason); + } + await writeClaim(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: owned.pid, + birth: owned.birth, + holders, + }); + await context.observe("ready"); + return { + state: "ready", + ownership: "owned", + endpoint: endpointFor(spec.port), + pid: owned.pid, + spawned: true, + }; +} + +async function prepareStateDir(stateDir: string): Promise { + await mkdir(stateDir, { recursive: true, mode: 0o700 }); +} + +function isAbort(error: unknown, signal?: AbortSignal): boolean { + return ( + signal?.aborted === true || + (error instanceof Error && error.name === "AbortError") + ); +} + +/** + * Make the shared helper ready for `holder`, adopting an identity-verified + * helper, reusing the owned one, or starting it under an exact claim. + */ +export async function ensureSharedHostHelper(opts: { + readonly spec: SharedHostHelperSpec; + readonly holder: string; + readonly signal?: AbortSignal; + readonly observe?: (event: SharedHostEvent) => void | Promise; +}): Promise { + assertSpec(opts.spec); + assertHolder(opts.holder); + if (!opts.spec.enabled) { + return { state: "disabled", ownership: "disabled" }; + } + if (opts.signal?.aborted) { + return failed("cancelled"); + } + const observer = opts.observe; + const context: EnsureContext = { + spec: opts.spec, + holder: opts.holder, + signal: opts.signal, + observe: async (event) => { + await observer?.(event); + }, + deadline: performance.now() + opts.spec.deadlineMs, + }; + try { + await prepareStateDir(opts.spec.stateDir); + } catch { + return failed("claim-invalid"); + } + try { + return await withProjectsRegistryLock({ + lockPath: lockPath(opts.spec), + label: "shared host helper", + timeoutMs: opts.spec.deadlineMs, + signal: opts.signal, + run: () => ensureLocked(context), + }); + } catch (error: unknown) { + if (isAbort(error, opts.signal)) { + return failed("cancelled"); + } + if (error instanceof ClaimRefusal) { + return failed("claim-invalid"); + } + return failed("lock-unavailable"); + } +} + +/** + * Drop `holder`. The last holder of an owned helper stops it through its + * birth-verified tree; adopted helpers and stale claims are never signalled. + */ +export async function releaseSharedHostHelper(opts: { + readonly spec: Pick; + readonly holder: string; + readonly graceMs?: number; + readonly timeoutMs?: number; + readonly signal?: AbortSignal; +}): Promise { + if (!NAME.test(opts.spec.name)) { + throw new Error("Invalid shared host helper specification"); + } + assertHolder(opts.holder); + try { + await prepareStateDir(opts.spec.stateDir); + } catch { + return releaseFailed("claim-invalid"); + } + try { + return await withProjectsRegistryLock({ + lockPath: lockPath(opts.spec), + label: "shared host helper", + timeoutMs: opts.timeoutMs ?? 10_000, + signal: opts.signal, + run: () => releaseLocked(opts), + }); + } catch (error: unknown) { + if (isAbort(error, opts.signal)) { + return releaseFailed("cancelled"); + } + if (error instanceof ClaimRefusal) { + return releaseFailed("claim-invalid"); + } + return releaseFailed("lock-unavailable"); + } +} + +async function releaseLocked(opts: { + readonly spec: Pick; + readonly holder: string; + readonly graceMs?: number; +}): Promise { + const read = await readClaim(opts.spec); + if (read.kind === "absent") { + return { state: "absent" }; + } + if (read.kind === "invalid") { + return releaseFailed("claim-invalid"); + } + const claim = read.claim; + const holders = claim.holders.filter((entry) => entry !== opts.holder); + if (holders.length > 0) { + await writeClaim(opts.spec, { ...claim, holders }); + return { state: "retained" }; + } + if (claim.ownership === "adopted") { + await removeClaim(opts.spec); + return { state: "released-adopted" }; + } + const stopped = await stopVerifiedSharedHostProcess({ + pid: claim.pid, + birth: claim.birth, + graceMs: opts.graceMs ?? 2000, + }); + switch (stopped) { + case "census-unavailable": + return releaseFailed("census-unavailable"); + case "unconfirmed": + await writeClaim(opts.spec, { ...claim, holders }); + return { state: "stop-unconfirmed" }; + case "absent": + await removeClaim(opts.spec); + return { state: "stale-cleared" }; + default: + await removeClaim(opts.spec); + return { state: "stopped" }; + } +} diff --git a/tests/fixtures/shared-host-helper.ts b/tests/fixtures/shared-host-helper.ts new file mode 100644 index 000000000..54f73cd74 --- /dev/null +++ b/tests/fixtures/shared-host-helper.ts @@ -0,0 +1,71 @@ +/** + * Portable shared-host-helper fixture process for + * tests/shared-host-lifecycle.test.ts. + * + * argv: + * - `identity`: HTTP on 127.0.0.1:; `/__hack/identity` reports + * {helper, upstream, pid}; `/` answers `served:`. + * - `tcp`: a plain TCP listener that never answers (a foreign port owner). + * - `placeholder`: stays alive and never listens (a sleeping placeholder). + * - `parent`: spawns a `placeholder` grandchild, prints its PID, then serves + * like `identity` (a helper with its own child process). + * + * Every mode exits once its private HOME (the test's scratch root) is gone, + * so no fixture outlives its test even when a failing run skips cleanup. + */ +import { existsSync } from "node:fs"; + +const [mode, portText, helper, upstream] = process.argv.slice(2); +const port = Number(portText); +const home = process.env.HOME; +setInterval(() => { + if (!(home && existsSync(home))) { + process.exit(0); + } +}, 250); + +function serveIdentity(): void { + Bun.serve({ + hostname: "127.0.0.1", + port, + fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/__hack/identity") { + return Response.json({ helper, upstream, pid: process.pid }); + } + return new Response(`served:${upstream}`); + }, + }); +} + +switch (mode) { + case "identity": + serveIdentity(); + break; + case "tcp": + Bun.listen({ + hostname: "127.0.0.1", + port, + socket: { + data() { + // Accepts connections and never answers. + }, + }, + }); + break; + case "placeholder": + setInterval(() => undefined, 60_000); + break; + case "parent": { + const child = Bun.spawn( + [process.execPath, import.meta.path, "placeholder", "0", "-", "-"], + { stdin: "ignore", stdout: "ignore", stderr: "ignore" } + ); + console.log(`grandchild ${child.pid}`); + serveIdentity(); + break; + } + default: + process.exit(2); +} +console.log("listening"); diff --git a/tests/fixtures/shared-host-starter.ts b/tests/fixtures/shared-host-starter.ts new file mode 100644 index 000000000..077a9e348 --- /dev/null +++ b/tests/fixtures/shared-host-starter.ts @@ -0,0 +1,27 @@ +/** + * Independent starter client for tests/shared-host-lifecycle.test.ts. + * + * argv: [pause-event] + * Prints `event ` for each lifecycle event and `result ` at the + * end. With a pause event it stops forever after printing that event, so the + * test can kill this parent at a precise point. + */ +import { + ensureSharedHostHelper, + type SharedHostHelperSpec, +} from "../../src/lib/shared-host-lifecycle.ts"; + +const [specText, holder, pauseAt] = process.argv.slice(2); +const spec = JSON.parse(specText ?? "{}") as SharedHostHelperSpec; +const result = await ensureSharedHostHelper({ + spec, + holder: holder ?? "", + observe: async (event) => { + console.log(`event ${event}`); + if (event === pauseAt) { + // An interval keeps the event loop alive until the test kills us. + await new Promise(() => setInterval(() => undefined, 60_000)); + } + }, +}); +console.log(`result ${JSON.stringify(result)}`); diff --git a/tests/shared-host-lifecycle.test.ts b/tests/shared-host-lifecycle.test.ts new file mode 100644 index 000000000..5d6896cd7 --- /dev/null +++ b/tests/shared-host-lifecycle.test.ts @@ -0,0 +1,916 @@ +import { afterEach, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { + readProcessIdentities, + selectProcessTree, + signalVerifiedProcessTree, +} from "../src/lib/process-tree.ts"; +import { + ensureSharedHostHelper, + parseSharedHostClaim, + releaseSharedHostHelper, + type SharedHostEvent, + type SharedHostHelperSpec, +} from "../src/lib/shared-host-lifecycle.ts"; +import { restoreEnv } from "./helpers/env.ts"; + +/** + * Portable process fixtures for the shared host helper contract + * (docs/reference/shared-host-lifecycle.md). Every fixture is a local Bun + * process on a loopback port: no Docker, no real HOME, no privileged effect. + * Foreign processes are started and stopped by the test through their own + * handles; the library under test may only signal the helpers it claimed. + */ + +const HELPER = resolve(import.meta.dir, "fixtures/shared-host-helper.ts"); +const STARTER = resolve(import.meta.dir, "fixtures/shared-host-starter.ts"); +const TIMEOUT = 30_000; + +type Foreign = Bun.Subprocess<"ignore", "pipe", "ignore">; +const foreign: Foreign[] = []; +const starters: Bun.Subprocess<"ignore", "pipe", "ignore">[] = []; +const claimedHelpers: { pid: number; birth: string }[] = []; +const roots: string[] = []; + +afterEach(async () => { + for (const proc of [...starters.splice(0), ...foreign.splice(0)]) { + proc.kill("SIGKILL"); + await proc.exited; + } + // Reap every helper the library claimed, through the shared birth-verified + // tree primitives rather than the code under test, so a failing or mutated + // run cannot leak fixture processes onto a shared host. + const owned = [...claimedHelpers.splice(0)]; + for (const root of roots) { + const text = await readFile( + join(root, "state", "issuer-bridge.claim.json"), + "utf8" + ).catch(() => null); + const claim = text === null ? null : parseSharedHostClaim(text); + if (claim?.ownership === "owned") { + owned.push({ pid: claim.pid, birth: claim.birth }); + } + } + const census = await readProcessIdentities(); + for (const identity of owned) { + if ( + census.some( + (row) => row.pid === identity.pid && row.birth === identity.birth + ) + ) { + await signalVerifiedProcessTree( + selectProcessTree(census, identity.pid), + "SIGKILL" + ); + } + } + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); + +function childEnv(home: string): Record { + return { PATH: process.env.PATH ?? "/usr/bin:/bin", HOME: home }; +} + +async function scratch(): Promise { + const root = await mkdtemp(join(tmpdir(), "hack-shared-host-")); + roots.push(root); + return root; +} + +function freePort(): number { + const probe = Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { + data() { + // Bind probe only. + }, + }, + }); + const port = probe.port; + probe.stop(true); + return port; +} + +function specFor(opts: { + readonly root: string; + readonly port: number; + readonly upstream?: string; + readonly mode?: "identity" | "placeholder" | "parent"; + readonly deadlineMs?: number; +}): SharedHostHelperSpec { + const upstream = opts.upstream ?? "https://issuer.branch-a.test"; + return { + name: "issuer-bridge", + enabled: true, + stateDir: join(opts.root, "state"), + port: opts.port, + identityPath: "/__hack/identity", + expected: { helper: "issuer-bridge", upstream }, + command: [ + process.execPath, + HELPER, + opts.mode ?? "identity", + String(opts.port), + "issuer-bridge", + upstream, + ], + cwd: opts.root, + env: childEnv(opts.root), + deadlineMs: opts.deadlineMs ?? 10_000, + backoff: { initialMs: 25, maxMs: 200 }, + probeTimeoutMs: 500, + }; +} + +function lineReader(stream: ReadableStream) { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + const lines: string[] = []; + let buffer = ""; + return { + lines, + async waitFor(match: (line: string) => boolean): Promise { + const deadline = performance.now() + 15_000; + while (performance.now() < deadline) { + const found = lines.find(match); + if (found !== undefined) { + return found; + } + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const parts = buffer.split("\n"); + buffer = parts.pop() ?? ""; + lines.push(...parts); + } + const found = lines.find(match); + if (found === undefined) { + throw new Error(`fixture line not observed; saw ${lines.join(" | ")}`); + } + return found; + }, + }; +} + +async function startForeign(opts: { + readonly mode: "identity" | "tcp" | "placeholder"; + readonly port: number; + readonly helper?: string; + readonly upstream?: string; + readonly home: string; +}): Promise { + const proc = Bun.spawn( + [ + process.execPath, + HELPER, + opts.mode, + String(opts.port), + opts.helper ?? "issuer-bridge", + opts.upstream ?? "https://issuer.branch-a.test", + ], + { + env: childEnv(opts.home), + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + } + ); + foreign.push(proc); + await lineReader(proc.stdout).waitFor((line) => line === "listening"); + return proc; +} + +function startStarter(opts: { + readonly spec: SharedHostHelperSpec; + readonly holder: string; + readonly pauseAt?: SharedHostEvent; +}) { + const proc = Bun.spawn( + [ + process.execPath, + STARTER, + JSON.stringify(opts.spec), + opts.holder, + ...(opts.pauseAt ? [opts.pauseAt] : []), + ], + { + env: childEnv(opts.spec.cwd), + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + } + ); + starters.push(proc); + return { proc, output: lineReader(proc.stdout) }; +} + +async function starterResult(starter: ReturnType) { + const line = await starter.output.waitFor((entry) => + entry.startsWith("result ") + ); + return JSON.parse(line.slice("result ".length)) as Record; +} + +async function identityOf(pid: number) { + const row = (await readProcessIdentities()).find( + (entry) => entry.pid === pid + ); + return row ? { pid: row.pid, birth: row.birth } : null; +} + +async function isAlive(identity: { pid: number; birth: string }) { + return (await readProcessIdentities()).some( + (row) => row.pid === identity.pid && row.birth === identity.birth + ); +} + +async function waitGone(identity: { pid: number; birth: string }) { + const deadline = performance.now() + 5000; + while (performance.now() < deadline) { + if (!(await isAlive(identity))) { + return true; + } + await Bun.sleep(50); + } + return false; +} + +async function readClaim(spec: SharedHostHelperSpec) { + const text = await readFile( + join(spec.stateDir, `${spec.name}.claim.json`), + "utf8" + ).catch(() => null); + return text === null ? null : parseSharedHostClaim(text); +} + +async function trackClaimedHelper(spec: SharedHostHelperSpec) { + const claim = await readClaim(spec); + if (claim?.ownership !== "owned") { + throw new Error("expected an owned claim"); + } + const identity = { pid: claim.pid, birth: claim.birth }; + claimedHelpers.push(identity); + return identity; +} + +function recorder() { + const events: SharedHostEvent[] = []; + return { + events, + observe: (event: SharedHostEvent) => { + events.push(event); + }, + }; +} + +test( + "a disabled helper takes no lock, writes no claim and starts no child", + async () => { + const root = await scratch(); + const spec = { ...specFor({ root, port: freePort() }), enabled: false }; + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + expect(result).toEqual({ state: "disabled", ownership: "disabled" }); + expect(seen.events).toEqual([]); + expect(await readdir(root)).toEqual([]); + }, + TIMEOUT +); + +test( + "an owned helper becomes ready by identity, serves its branch upstream and is stopped by its last holder", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + const owned = await trackClaimedHelper(spec); + expect(result).toMatchObject({ + state: "ready", + ownership: "owned", + spawned: true, + pid: owned.pid, + endpoint: `http://127.0.0.1:${spec.port}`, + }); + expect(seen.events).toContain("claimed"); + expect(seen.events.indexOf("claimed")).toBeLessThan( + seen.events.indexOf("gate-opened") + ); + expect(await (await fetch(`http://127.0.0.1:${spec.port}/`)).text()).toBe( + "served:https://issuer.branch-a.test" + ); + expect(await readClaim(spec)).toMatchObject({ + phase: "ready", + holders: ["branch-a"], + }); + + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "stopped" } + ); + expect(await isAlive(owned)).toBe(false); + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); + +test( + "a healthy shared helper started elsewhere is adopted and never signalled on release", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const existing = await startForeign({ + mode: "identity", + port: spec.port, + home: root, + }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + expect(result).toEqual({ + state: "ready", + ownership: "adopted", + endpoint: `http://127.0.0.1:${spec.port}`, + spawned: false, + }); + expect(seen.events).not.toContain("spawned"); + expect(await (await fetch(`http://127.0.0.1:${spec.port}/`)).text()).toBe( + "served:https://issuer.branch-a.test" + ); + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "released-adopted" } + ); + await Bun.sleep(200); + expect(existing.exitCode).toBeNull(); + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); + +test( + "a helper serving the wrong upstream is refused and left running", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const existing = await startForeign({ + mode: "identity", + port: spec.port, + upstream: "https://issuer.production.example", + home: root, + }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + expect(result).toMatchObject({ state: "failed", reason: "wrong-upstream" }); + expect(JSON.stringify(result)).not.toContain("production"); + expect(seen.events).not.toContain("spawned"); + expect(await readClaim(spec)).toBeNull(); + await Bun.sleep(200); + expect(existing.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "foreign listeners on the helper port are refused and preserved", + async () => { + for (const fixture of [ + { mode: "tcp" as const }, + { mode: "identity" as const, helper: "unrelated-dev-server" }, + ]) { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), deadlineMs: 3000 }); + const existing = await startForeign({ + ...fixture, + port: spec.port, + home: root, + }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + expect(result).toMatchObject({ + state: "failed", + reason: "foreign-listener", + }); + expect(seen.events).not.toContain("spawned"); + await Bun.sleep(200); + expect(existing.exitCode).toBeNull(); + } + }, + TIMEOUT +); + +test( + "independent concurrent starters are serialized onto one owned helper; sibling holders and branches are preserved", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const first = startStarter({ spec, holder: "branch-a" }); + const second = startStarter({ spec, holder: "branch-b" }); + const results = await Promise.all([ + starterResult(first), + starterResult(second), + ]); + const owned = await trackClaimedHelper(spec); + expect(results.map((entry) => entry.state)).toEqual(["ready", "ready"]); + expect(results.map((entry) => entry.ownership)).toEqual(["owned", "owned"]); + expect(results.map((entry) => entry.pid)).toEqual([owned.pid, owned.pid]); + expect(results.filter((entry) => entry.spawned === true)).toHaveLength(1); + const spawnedEvents = [first, second].filter((starter) => + starter.output.lines.includes("event spawned") + ); + expect(spawnedEvents).toHaveLength(1); + expect([...((await readClaim(spec))?.holders ?? [])].sort()).toEqual([ + "branch-a", + "branch-b", + ]); + + // A sibling branch with a different upstream cannot take over the helper. + const sibling = await ensureSharedHostHelper({ + spec: specFor({ + root, + port: spec.port, + upstream: "https://issuer.branch-c.test", + }), + holder: "branch-c", + }); + expect(sibling).toMatchObject({ + state: "failed", + reason: "wrong-upstream", + }); + + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "retained" } + ); + expect(await isAlive(owned)).toBe(true); + expect(await (await fetch(`http://127.0.0.1:${spec.port}/`)).text()).toBe( + "served:https://issuer.branch-a.test" + ); + expect(await releaseSharedHostHelper({ spec, holder: "branch-b" })).toEqual( + { state: "stopped" } + ); + expect(await isAlive(owned)).toBe(false); + }, + TIMEOUT +); + +test( + "a stale claim whose PID was reused is discarded without signalling the live process", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const bystander = await startForeign({ + mode: "placeholder", + port: 0, + home: root, + }); + const reused = await identityOf(bystander.pid); + expect(reused).not.toBeNull(); + const staleClaim = { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: bystander.pid, + birth: "Thu Jan 1 00:00:00 1970", + holders: ["branch-old"], + }; + await mkdir(spec.stateDir, { recursive: true, mode: 0o700 }); + const claimFile = join(spec.stateDir, `${spec.name}.claim.json`); + await writeFile(claimFile, JSON.stringify(staleClaim), { mode: 0o600 }); + + // Release by the last holder must not signal the reused PID. + expect( + await releaseSharedHostHelper({ spec, holder: "branch-old" }) + ).toEqual({ state: "stale-cleared" }); + await Bun.sleep(200); + expect(bystander.exitCode).toBeNull(); + + // A starter treats the same stale claim as no ownership and starts fresh. + await writeFile(claimFile, JSON.stringify(staleClaim), { mode: 0o600 }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + const owned = await trackClaimedHelper(spec); + expect(seen.events).toContain("claim-stale"); + expect(result).toMatchObject({ state: "ready", ownership: "owned" }); + expect(owned.pid).not.toBe(bystander.pid); + expect((await readClaim(spec))?.holders).toEqual([ + "branch-old", + "branch-a", + ]); + await Bun.sleep(200); + expect(bystander.exitCode).toBeNull(); + expect(await isAlive(reused ?? owned)).toBe(true); + }, + TIMEOUT +); + +test( + "a crashed owned child is restarted under a fresh claim by the next starter", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const first = await ensureSharedHostHelper({ spec, holder: "branch-a" }); + const crashed = await trackClaimedHelper(spec); + expect(first).toMatchObject({ state: "ready", spawned: true }); + process.kill(crashed.pid, "SIGKILL"); + expect(await waitGone(crashed)).toBe(true); + + const seen = recorder(); + const second = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + const restarted = await trackClaimedHelper(spec); + expect(seen.events).toContain("claim-stale"); + expect(second).toMatchObject({ + state: "ready", + ownership: "owned", + spawned: true, + pid: restarted.pid, + }); + expect(restarted).not.toEqual(crashed); + expect((await readClaim(spec))?.holders).toEqual(["branch-a", "branch-b"]); + expect(await (await fetch(`http://127.0.0.1:${spec.port}/`)).text()).toBe( + "served:https://issuer.branch-a.test" + ); + }, + TIMEOUT +); + +test( + "cancellation during bounded readiness retries stops the owned placeholder and keeps foreign processes", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), mode: "placeholder" }); + const bystander = await startForeign({ + mode: "tcp", + port: freePort(), + home: root, + }); + const controller = new AbortController(); + let owned: { pid: number; birth: string } | null = null; + let retries = 0; + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + signal: controller.signal, + observe: async (event) => { + if (event === "claimed") { + owned = await trackClaimedHelper(spec); + } + if (event === "probe-retry") { + retries += 1; + if (retries === 3) { + controller.abort(); + } + } + }, + }); + expect(result).toMatchObject({ state: "failed", reason: "cancelled" }); + expect(retries).toBe(3); + expect(owned).not.toBeNull(); + expect(await isAlive(owned ?? { pid: 0, birth: "" })).toBe(false); + expect(await readClaim(spec)).toBeNull(); + expect(bystander.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a live placeholder is never readiness: the deadline bounds retries and stops the owned child", + async () => { + const root = await scratch(); + const spec = specFor({ + root, + port: freePort(), + mode: "placeholder", + deadlineMs: 800, + }); + let owned: { pid: number; birth: string } | null = null; + const started = performance.now(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: async (event) => { + if (event === "claimed") { + owned = await trackClaimedHelper(spec); + } + }, + }); + expect(result).toMatchObject({ state: "failed", reason: "deadline" }); + expect(performance.now() - started).toBeLessThan(5000); + expect(await isAlive(owned ?? { pid: 0, birth: "" })).toBe(false); + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); + +test( + "a reused owned claim is not ready when a different process answers its identity", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), deadlineMs: 2000 }); + const claimedProcess = await startForeign({ + mode: "placeholder", + port: 0, + home: root, + }); + const impostor = await startForeign({ + mode: "identity", + port: spec.port, + home: root, + }); + const identity = await identityOf(claimedProcess.pid); + await mkdir(spec.stateDir, { recursive: true, mode: 0o700 }); + await writeFile( + join(spec.stateDir, `${spec.name}.claim.json`), + JSON.stringify({ + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "starting", + pid: claimedProcess.pid, + birth: identity?.birth ?? "", + holders: ["branch-a"], + }), + { mode: 0o600 } + ); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + expect(seen.events).toContain("claim-reused"); + expect(result).toMatchObject({ + state: "failed", + reason: "foreign-listener", + }); + expect((await readClaim(spec))?.holders).toEqual(["branch-a"]); + await Bun.sleep(200); + expect(claimedProcess.exitCode).toBeNull(); + expect(impostor.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a starter lost before its claim opens the gate leaves no helper running", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const starter = startStarter({ + spec, + holder: "branch-a", + pauseAt: "claimed", + }); + await starter.output.waitFor((line) => line === "event claimed"); + const gated = await trackClaimedHelper(spec); + starter.proc.kill("SIGKILL"); + await starter.proc.exited; + expect(await waitGone(gated)).toBe(true); + const port = await fetch(`http://127.0.0.1:${spec.port}/`).catch( + () => null + ); + expect(port).toBeNull(); + + const seen = recorder(); + const next = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + await trackClaimedHelper(spec); + expect(seen.events).toContain("claim-stale"); + expect(next).toMatchObject({ state: "ready", spawned: true }); + }, + TIMEOUT +); + +test( + "a starter lost after readiness leaves a claimed helper that another client reuses and releases", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const starter = startStarter({ + spec, + holder: "branch-a", + pauseAt: "ready", + }); + await starter.output.waitFor((line) => line === "event ready"); + const owned = await trackClaimedHelper(spec); + starter.proc.kill("SIGKILL"); + await starter.proc.exited; + await Bun.sleep(200); + expect(await isAlive(owned)).toBe(true); + + const seen = recorder(); + const reused = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + expect(seen.events).toContain("claim-reused"); + expect(reused).toMatchObject({ + state: "ready", + ownership: "owned", + spawned: false, + pid: owned.pid, + }); + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "retained" } + ); + expect(await releaseSharedHostHelper({ spec, holder: "branch-b" })).toEqual( + { state: "stopped" } + ); + expect(await isAlive(owned)).toBe(false); + }, + TIMEOUT +); + +test( + "release stops the owned helper's own descendants and nothing else", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), mode: "parent" }); + const bystander = await startForeign({ + mode: "placeholder", + port: 0, + home: root, + }); + const result = await ensureSharedHostHelper({ spec, holder: "branch-a" }); + expect(result).toMatchObject({ state: "ready", ownership: "owned" }); + const owned = await trackClaimedHelper(spec); + const grandchildRow = (await readProcessIdentities()).find( + (row) => row.parentPid === owned.pid + ); + expect(grandchildRow).toBeDefined(); + const grandchild = { + pid: grandchildRow?.pid ?? 0, + birth: grandchildRow?.birth ?? "", + }; + claimedHelpers.push(grandchild); + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "stopped" } + ); + expect(await isAlive(owned)).toBe(false); + expect(await waitGone(grandchild)).toBe(true); + expect(bystander.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "an unreadable claim fails closed for start and release", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + await mkdir(spec.stateDir, { recursive: true, mode: 0o700 }); + await writeFile( + join(spec.stateDir, `${spec.name}.claim.json`), + '{"version":1,"helper":"issuer-bridge","ownership":"owned"}', + { mode: 0o600 } + ); + const seen = recorder(); + expect( + await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }) + ).toMatchObject({ state: "failed", reason: "claim-invalid" }); + expect(seen.events).not.toContain("spawned"); + expect( + await releaseSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "failed", reason: "claim-invalid" }); + }, + TIMEOUT +); + +test( + "an unavailable process census fails closed: no spawn, no signal, claim kept", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const bystander = await startForeign({ + mode: "placeholder", + port: 0, + home: root, + }); + const identity = await identityOf(bystander.pid); + await mkdir(spec.stateDir, { recursive: true, mode: 0o700 }); + const claimFile = join(spec.stateDir, `${spec.name}.claim.json`); + const claim = JSON.stringify({ + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: bystander.pid, + birth: identity?.birth ?? "", + holders: ["branch-a"], + }); + await writeFile(claimFile, claim, { mode: 0o600 }); + const bin = join(root, "bin"); + await mkdir(bin); + await writeFile(join(bin, "ps"), "#!/bin/sh\nexit 1\n", { mode: 0o755 }); + const savedPath = process.env.PATH; + process.env.PATH = `${bin}:${savedPath ?? "/usr/bin:/bin"}`; + try { + const seen = recorder(); + expect( + await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }) + ).toMatchObject({ state: "failed", reason: "census-unavailable" }); + expect(seen.events).not.toContain("spawned"); + expect( + await releaseSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "failed", reason: "census-unavailable" }); + } finally { + restoreEnv("PATH", savedPath); + } + expect(await readFile(claimFile, "utf8")).toBe(claim); + await Bun.sleep(200); + expect(bystander.exitCode).toBeNull(); + }, + TIMEOUT +); + +test("claim parser accepts only the exact v1 shapes", () => { + const owned = { + version: 1, + helper: "issuer-bridge", + port: 4100, + ownership: "owned", + phase: "ready", + pid: 4242, + birth: "Mon Oct 19 10:00:00 2026", + holders: ["branch-a"], + }; + const adopted = { + version: 1, + helper: "issuer-bridge", + port: 4100, + ownership: "adopted", + holders: ["branch-a"], + }; + expect(parseSharedHostClaim(JSON.stringify(owned))).toEqual(owned as never); + expect(parseSharedHostClaim(JSON.stringify(adopted))).toEqual( + adopted as never + ); + for (const invalid of [ + { ...owned, pid: 1 }, + { ...owned, birth: "" }, + { ...owned, phase: "failed" }, + { ...owned, extra: true }, + { ...owned, holders: ["branch-a", "branch-a"] }, + { ...owned, holders: ["../escape"] }, + { ...adopted, pid: 4242 }, + { ...adopted, version: 2 }, + { ...adopted, port: 70_000 }, + ]) { + expect(parseSharedHostClaim(JSON.stringify(invalid))).toBeNull(); + } + expect(parseSharedHostClaim("not json")).toBeNull(); +}); From 38ea93b6548516e0f34d8d43bff599ecf11e9647 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 11:59:42 -0400 Subject: [PATCH 08/21] feat(runtime-core): compact retired environment leases after confirmed cleanup Ordinary graph cleanup retired every bound environment slot but left each immutable intent in run/environment-leases, so admission, cleanup and restore scanned every historical generation of a repeatedly started graph. After `graph cleanup` commits, the same engine mutation lease now compacts the run's records. Each must match a receipt container that is absent and a guest slot that is absent, or nothing is written. Exact intent digests move into one value-free per-run tombstone bound to the VM incarnation, and only then are the active records unlinked. Prior boots fold into one sealed summary each; the current boot keeps per-slot reservations. A retired slot, or any slot of a sealed boot, is never recorded, delivered, retired as live or counted again. Pending, live, changed, misbound or foreign-incarnation records refuse the whole compaction before its first write; other runs' and standalone records are untouched. Exact residue of an interrupted compaction reads as retired and is completed by the next cleanup or by archive's resume-only pass; changed residue refuses. Recovery flows that carry an exact environment inventory keep using the fenced path without compaction. Refs HACK-1193 Co-Authored-By: Claude Opus 5.5 --- packages/runtime-core/README.md | 11 + .../src/provider/environment_recovery.rs | 66 +- .../environment_recovery/compaction.rs | 455 ++++++++++++ .../environment_recovery/compaction/tests.rs | 668 ++++++++++++++++++ .../environment_recovery/retention.rs | 2 +- .../src/provider/graph/environment.rs | 24 + .../runtime-core/src/provider/graph/mod.rs | 6 +- 7 files changed, 1213 insertions(+), 19 deletions(-) create mode 100644 packages/runtime-core/src/provider/environment_recovery/compaction.rs create mode 100644 packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index eb308245d..3342e1d79 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -916,6 +916,17 @@ Graph cleanup uses that same history capacity, counting active and archived records together, so repeated restores cannot exceed a smaller cleanup-only ceiling after allocation succeeds. Matching ownership, container bindings and unambiguous records are still required. +After ordinary `graph cleanup` commits, the same engine lease compacts that run's +records: each must match a receipt container that is absent and a guest slot that is +absent, or nothing is written. Their exact intent digests move into one value-free +per-run tombstone in `run/environment-lease-tombstones`, bound to the VM incarnation, +and only then are the active records unlinked. Prior boots fold into one sealed +summary each; the current boot keeps per-slot reservations (bounded at 16384). A +retired slot or any slot of a sealed boot is never recorded, delivered, retired as +live or counted again, so history no longer consumes the 4096 active-record budget. +Pending, live, changed, misbound or foreign-incarnation records refuse compaction, +and other runs' records are untouched. An interruption is resumed by the next +cleanup or archive; exact residue reads as retired, changed residue refuses. No retained evidence is automatically deleted to admit new work. This environment budget does not increase the runtime's separate container, CPU or memory limits and does not establish 32-branch qualification. diff --git a/packages/runtime-core/src/provider/environment_recovery.rs b/packages/runtime-core/src/provider/environment_recovery.rs index 6ec1ea917..a1e4cb831 100644 --- a/packages/runtime-core/src/provider/environment_recovery.rs +++ b/packages/runtime-core/src/provider/environment_recovery.rs @@ -1,7 +1,9 @@ //! Immutable, value-free allocation intent. Recovery grants cleanup, never renewed delivery. +mod compaction; mod retention; use super::{environment::EnvironmentLease, lifecycle::OwnedGuest, state}; use crate::{Candidate, CandidateError}; +pub(super) use compaction::{compact_graph, resume_graph}; pub use retention::{RetiredExport, export_retired}; use serde::{Deserialize, Serialize}; use std::{fs, path::PathBuf}; @@ -148,29 +150,39 @@ pub(super) fn record( if lease.native.is_some() { return Err(error()); } - if retention::reserved(candidate, &lease.slot)? { + record_intent( + candidate, + &Intent { + version: 1, + graph: lease.graph.clone(), + service: lease.service.clone(), + uid: lease.uid, + gid: lease.gid, + slot: lease.slot.clone(), + incarnation: lease.incarnation.clone(), + boot: lease.boot.clone(), + }, + ) +} +/// A slot reserved by standalone export or by its graph's compaction tombstone is never reused. +fn record_intent(candidate: &Candidate, intent: &Intent) -> Result<(), CandidateError> { + if retention::reserved(candidate, &intent.slot)? + || intent.graph.as_ref().map_or(Ok(false), |binding| { + compaction::reserved(candidate, &binding.run, &intent.incarnation, &intent.slot) + })? + { return Err(error()); } let directory = root(candidate); state::private_directory(&directory)?; preflight_records(candidate, 1)?; - let intent = Intent { - version: 1, - graph: lease.graph.clone(), - service: lease.service.clone(), - uid: lease.uid, - gid: lease.gid, - slot: lease.slot.clone(), - incarnation: lease.incarnation.clone(), - boot: lease.boot.clone(), - }; - validate(&intent, &lease.slot)?; - let path = directory.join(format!("{}.json", lease.slot)); + validate(intent, &intent.slot)?; + let path = directory.join(format!("{}.json", intent.slot)); match fs::symlink_metadata(&path) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} _ => return Err(error()), } - state::write(&path, &intent) + state::write(&path, intent) } fn read( candidate: &Candidate, @@ -287,11 +299,20 @@ pub(super) fn graph_slots( guest: &OwnedGuest<'_>, run: &str, ) -> Result, CandidateError> { + graph_slots_for(candidate, guest.incarnation(), run) +} +/// Exact residue of a durable compaction is retired history, not an allocation to retire again. +fn graph_slots_for( + candidate: &Candidate, + incarnation: &str, + run: &str, +) -> Result, CandidateError> { + let tombstone = compaction::load(candidate, run, incarnation)?; let mut bindings = Vec::new(); for slot in recorded_slots(candidate)? { - let intent = read_mode(candidate, &slot, guest.incarnation(), None, false)?; + let intent = read_mode(candidate, &slot, incarnation, None, false)?; if let Some(binding) = intent.graph { - if binding.run == run { + if binding.run == run && !compaction::residue(candidate, tombstone.as_ref(), &slot)? { bindings.push((slot, intent.service, binding)); } } @@ -347,6 +368,7 @@ fn active_exec_slot_for( || binding.run != run || binding.container != container || retention::reserved(candidate, slot)? + || compaction::reserved(candidate, run, incarnation, slot)? { return Err(error()); } @@ -386,6 +408,7 @@ pub(super) fn graph_inventory( if !hex(run, 32) || !hex(incarnation, 32) { return Err(error()); } + let tombstone = compaction::load(candidate, run, incarnation)?; let mut selected = std::collections::BTreeMap::new(); let mut insert = |intent: Intent| -> Result<(), CandidateError> { validate(&intent, &intent.slot)?; @@ -394,6 +417,7 @@ pub(super) fn graph_inventory( g.run == run && containers.get(&intent.service) == Some(&g.container) }) || retention::reserved(candidate, &intent.slot)? + || compaction::reserves(tombstone.as_ref(), &intent.slot) || selected.len() >= MAX_GRAPH_INTENT_ENTRIES || selected.insert(intent.slot.clone(), intent).is_some() { @@ -403,7 +427,9 @@ pub(super) fn graph_inventory( }; for slot in recorded_slots(candidate)? { let intent = read_mode(candidate, &slot, incarnation, None, false)?; - if intent.graph.as_ref().is_some_and(|g| g.run == run) { + if intent.graph.as_ref().is_some_and(|g| g.run == run) + && !compaction::residue(candidate, tombstone.as_ref(), &slot)? + { insert(intent)?; } } @@ -568,6 +594,12 @@ pub(super) fn retire( if retention::reserved(candidate, slot)? { return retention::verify_retired(candidate, guest, slot, lease); } + // A compacted graph slot is retained history; never treat it as a live allocation. + if let Some(binding) = read_mode(candidate, slot, guest.incarnation(), lease, false)?.graph { + if compaction::reserved(candidate, &binding.run, guest.incarnation(), slot)? { + return Err(error()); + } + } let intent = read(candidate, slot, guest.incarnation(), lease)?; retire_intent(guest, &intent) } diff --git a/packages/runtime-core/src/provider/environment_recovery/compaction.rs b/packages/runtime-core/src/provider/environment_recovery/compaction.rs new file mode 100644 index 000000000..d3ac05792 --- /dev/null +++ b/packages/runtime-core/src/provider/environment_recovery/compaction.rs @@ -0,0 +1,455 @@ +//! Compaction of retired graph-bound allocation intents. +//! +//! Ordinary graph cleanup retires every bound slot under the provider mutation lease, but each +//! immutable intent used to stay in `run/environment-leases` indefinitely. Admission, cleanup and +//! restore then scanned every historical generation of a repeatedly started graph. After the +//! caller confirms each recorded container and guest slot absent, compaction retains the exact +//! intent digest in one per-run tombstone and unlinks the active record. +//! +//! Commit and recovery points, in order: +//! 1. The tombstone gains per-slot `retired` digests (atomic rename plus directory sync). No +//! active record is unlinked before this is durable, so a leftover tombstone temporary file +//! is an uncommitted write with no authority and is discarded. +//! 2. Active records whose exact bytes match a retired digest are unlinked; the directory is +//! synced. Until then such a record is residue: read-only inventory treats it as retired and +//! the next compaction completes the unlink. Changed bytes are refused, never removed. +//! 3. Retired slots of prior guest boots are folded into one sealed summary per boot. A prior +//! boot cannot allocate again, so the whole boot stays reserved without per-slot entries. +//! +//! Reservation: a retired slot, or any slot of a sealed boot, can never be recorded again, +//! selected for delivery, retired as live, or counted as a live allocation. Records are never +//! deleted to make room: an uncertain (pending), live, changed, misbound or foreign-owned record +//! refuses the whole compaction before its first write. Records of other runs, standalone +//! records and other incarnations' state are never modified. +use super::*; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +const VERSION: u32 = 1; +/// Per-slot reservations retained for boots that may still allocate (normally the current one). +const MAX_RETIRED_SLOTS: usize = 16 * 1024; +const MAX_SEALED_BOOTS: usize = 4096; +/// Covers both maxima in pretty-printed JSON with margin; larger state is refused, not truncated. +const TOMBSTONE_LIMIT: u64 = 4 * 1024 * 1024; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Sealed { + slots: u32, + sha256: String, +} + +/// Value-free retained evidence for one graph run. Bound to the owning VM incarnation. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Tombstone { + version: u32, + run: String, + incarnation: String, + sealed: BTreeMap, + retired: BTreeMap, +} + +#[derive(Debug, PartialEq, Eq)] +enum Reservation<'a> { + Free, + Retired(&'a str), + Sealed, +} + +/// Value-free outcome counts for one compaction. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] +pub(crate) struct Compaction { + /// Newly confirmed records moved into the tombstone. + pub compacted: usize, + /// Residue of an interrupted compaction whose unlink was completed. + pub completed: usize, + /// Prior boots folded into sealed reservations. + pub sealed_boots: usize, +} + +/// `Resume` only completes an interrupted compaction: it selects no new record and seals nothing, +/// so it is safe where live allocations of the run may remain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Mode { + Compact, + Resume, +} + +/// Owner identity and receipt bindings for one compaction. +#[derive(Clone, Copy)] +pub(super) struct Request<'a> { + pub mode: Mode, + /// VM incarnation that owns every record; others refuse. + pub incarnation: &'a str, + /// Current guest boot; it may still allocate, so it is never sealed. + pub boot: &'a str, + pub run: &'a str, + /// Receipt service-to-container bindings each selected record must match. + pub containers: &'a BTreeMap, +} + +/// Fault boundaries observed by in-process interruption tests. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Step { + Recorded, + Unlinked, + Synced, + Sealed, +} + +fn directory(candidate: &Candidate) -> PathBuf { + candidate + .state_root + .join("run/environment-lease-tombstones") +} + +/// Boot UUID of a `hack-env-lease--` slot. Callers validate the slot first; an +/// invalid identity yields an empty boot that matches no reservation. +fn boot_of(slot: &str) -> &str { + let start = "hack-env-lease-".len(); + slot.get(start..start + 36).unwrap_or("") +} + +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +fn present(path: &Path) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(state::io(e)), + } +} + +fn sync(directory: &Path) -> Result<(), CandidateError> { + fs::File::open(directory) + .and_then(|f| f.sync_all()) + .map_err(state::io) +} + +impl Tombstone { + fn new(run: &str, incarnation: &str) -> Self { + Self { + version: VERSION, + run: run.into(), + incarnation: incarnation.into(), + sealed: BTreeMap::new(), + retired: BTreeMap::new(), + } + } + + fn validate(&self, run: &str, incarnation: &str) -> Result<(), CandidateError> { + if self.version != VERSION + || self.run != run + || self.incarnation != incarnation + || !hex(run, 32) + || !hex(incarnation, 32) + || self.retired.len() > MAX_RETIRED_SLOTS + || self.sealed.len() > MAX_SEALED_BOOTS + || self + .sealed + .iter() + .any(|(boot, sealed)| !uuid(boot) || sealed.slots == 0 || !hex(&sealed.sha256, 64)) + || self.retired.iter().any(|(slot, sha)| { + !valid_slot(slot) || !hex(sha, 64) || self.sealed.contains_key(boot_of(slot)) + }) + { + return Err(error()); + } + Ok(()) + } + + fn reservation(&self, slot: &str) -> Reservation<'_> { + if !valid_slot(slot) || self.sealed.contains_key(boot_of(slot)) { + return Reservation::Sealed; + } + match self.retired.get(slot) { + Some(sha) => Reservation::Retired(sha), + None => Reservation::Free, + } + } +} + +fn load_at(path: &Path, run: &str, incarnation: &str) -> Result, CandidateError> { + if !present(path)? { + return Ok(None); + } + let tombstone: Tombstone = state::read_bounded(path, TOMBSTONE_LIMIT)?; + tombstone.validate(run, incarnation)?; + Ok(Some(tombstone)) +} + +/// Read-only. A run that cannot name a valid binding has no tombstone to consult. +pub(super) fn load( + candidate: &Candidate, + run: &str, + incarnation: &str, +) -> Result, CandidateError> { + if !hex(run, 32) { + return Ok(None); + } + let parent = directory(candidate); + crate::reject_aliased_state(&parent)?; + if !parent.try_exists().map_err(state::io)? { + return Ok(None); + } + state::check_private_directory(&parent)?; + load_at(&parent.join(format!("{run}.json")), run, incarnation) +} + +/// Whether an already loaded tombstone reserves the slot. +pub(super) fn reserves(tombstone: Option<&Tombstone>, slot: &str) -> bool { + tombstone.is_some_and(|tombstone| tombstone.reservation(slot) != Reservation::Free) +} + +/// Whether the slot is reserved by retained evidence, whether or not residue remains active. +pub(super) fn reserved( + candidate: &Candidate, + run: &str, + incarnation: &str, + slot: &str, +) -> Result { + Ok(reserves(load(candidate, run, incarnation)?.as_ref(), slot)) +} + +/// Read-only classification for inventory. `true` means the committed active record is exact +/// residue of a durable compaction: retired, never live, and removed by the next compaction. +/// A sealed-boot replay, changed bytes or an uncertain pending record refuses. +pub(super) fn residue( + candidate: &Candidate, + tombstone: Option<&Tombstone>, + slot: &str, +) -> Result { + let Some(tombstone) = tombstone else { + return Ok(false); + }; + match tombstone.reservation(slot) { + Reservation::Free => Ok(false), + Reservation::Sealed => Err(error()), + Reservation::Retired(sha) => { + let committed = root(candidate).join(format!("{slot}.json")); + if present(&committed.with_extension("pending"))? + || digest(&retention::bytes(&committed)?) != sha + { + return Err(error()); + } + Ok(true) + } + } +} + +/// A leftover temporary file means the rename never committed (see module commit point 1). +fn discard_uncommitted(path: &Path) -> Result<(), CandidateError> { + let metadata = match fs::symlink_metadata(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(state::io(e)), + Ok(metadata) => metadata, + }; + // SAFETY: geteuid has no preconditions. + if !metadata.file_type().is_file() + || metadata.nlink() != 1 + || metadata.uid() != unsafe { libc::geteuid() } + || metadata.len() > TOMBSTONE_LIMIT + { + return Err(error()); + } + fs::remove_file(path).map_err(state::io)?; + sync(path.parent().ok_or_else(error)?) +} + +/// Commit point: atomic rename plus parent sync. Only the uncommitted temporary is replaced. +fn commit(parent: &Path, path: &Path, tombstone: &Tombstone) -> Result<(), CandidateError> { + state::private_directory(parent)?; + discard_uncommitted(&path.with_extension("pending"))?; + state::write(path, tombstone) +} + +/// Compact every confirmed-retired record bound to `run`. The caller holds the provider mutation +/// lease, has completed graph cleanup and passes the receipt's service-to-container bindings. +pub(crate) fn compact_graph( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + run: &str, + containers: &BTreeMap, +) -> Result { + compact_with( + candidate, + Request { + mode: Mode::Compact, + incarnation: guest.incarnation(), + boot: guest.boot_id(), + run, + containers, + }, + |intent| { + if let Some(binding) = &intent.graph { + super::super::engine::require_container_absent(guest, &binding.container)?; + } + retention::absent(guest, &intent.slot) + }, + |_| Ok(()), + ) +} + +/// Complete only an interrupted compaction of `run`, under the caller's mutation lease. Exact +/// residue is unlinked; every other record is left untouched. No tombstone means no effect. +pub(crate) fn resume_graph( + candidate: &Candidate, + guest: &OwnedGuest<'_>, + run: &str, +) -> Result { + if load(candidate, run, guest.incarnation())?.is_none() { + return Ok(Compaction::default()); + } + compact_with( + candidate, + Request { + mode: Mode::Resume, + incarnation: guest.incarnation(), + boot: guest.boot_id(), + run, + containers: &BTreeMap::new(), + }, + |_| Err(error()), + |_| Ok(()), + ) +} + +/// Filesystem state machine behind [`compact_graph`]. `confirm` proves one selected allocation +/// absent; `fault` lets tests stop at each commit boundary as an interruption would. +pub(super) fn compact_with( + candidate: &Candidate, + request: Request<'_>, + confirm: impl Fn(&Intent) -> Result<(), CandidateError>, + mut fault: impl FnMut(Step) -> Result<(), CandidateError>, +) -> Result { + let Request { + mode, + incarnation, + boot, + run, + containers, + } = request; + if !hex(run, 32) || !hex(incarnation, 32) || !uuid(boot) { + return Err(error()); + } + let parent = directory(candidate); + let path = parent.join(format!("{run}.json")); + let mut tombstone = + load(candidate, run, incarnation)?.unwrap_or_else(|| Tombstone::new(run, incarnation)); + if tombstone.sealed.contains_key(boot) { + return Err(error()); + } + + // Inventory and confirm every record bound to this run before the first write. + let active = root(candidate); + let mut residue = Vec::new(); + let mut selected = BTreeMap::new(); + for slot in recorded_slots(candidate)? { + let intent = read_mode(candidate, &slot, incarnation, None, false)?; + let Some(binding) = intent.graph.as_ref().filter(|binding| binding.run == run) else { + continue; + }; + let reservation = tombstone.reservation(&slot); + if mode == Mode::Resume && reservation == Reservation::Free { + continue; + } + let committed = active.join(format!("{slot}.json")); + if present(&committed.with_extension("pending"))? { + return Err(error()); + } + let sha = digest(&retention::bytes(&committed)?); + match reservation { + Reservation::Sealed => return Err(error()), + Reservation::Retired(expected) if expected == sha => residue.push(slot), + Reservation::Retired(_) => return Err(error()), + Reservation::Free => { + if containers.get(&intent.service) != Some(&binding.container) + || retention::reserved(candidate, &slot)? + { + return Err(error()); + } + confirm(&intent)?; + selected.insert(slot, sha); + } + } + } + let outcome = Compaction { + compacted: selected.len(), + completed: residue.len(), + sealed_boots: 0, + }; + + if !selected.is_empty() { + if tombstone.retired.len() + selected.len() > MAX_RETIRED_SLOTS { + return Err(error()); + } + tombstone.retired.extend(selected.clone()); + commit(&parent, &path, &tombstone)?; + fault(Step::Recorded)?; + } + for slot in residue.iter().chain(selected.keys()) { + let committed = active.join(format!("{slot}.json")); + if tombstone.retired.get(slot) != Some(&digest(&retention::bytes(&committed)?)) { + return Err(error()); + } + fs::remove_file(&committed).map_err(state::io)?; + fault(Step::Unlinked)?; + } + if outcome.compacted + outcome.completed > 0 { + sync(&active)?; + fault(Step::Synced)?; + } + + // Resume may leave unselected records of this run active, so it never seals a boot. + if mode == Mode::Resume { + return Ok(outcome); + } + // Every record of this run was unlinked above or refused the compaction, so no active + // allocation of a prior boot remains. Its whole boot is then reserved by one summary. + let prior: BTreeSet = tombstone + .retired + .keys() + .map(|slot| boot_of(slot).to_owned()) + .filter(|retired| retired != boot) + .collect(); + if prior.is_empty() { + return Ok(outcome); + } + if tombstone.sealed.len() + prior.len() > MAX_SEALED_BOOTS { + return Err(error()); + } + for retired in &prior { + let slots: Vec<(String, String)> = tombstone + .retired + .iter() + .filter(|(slot, _)| boot_of(slot) == retired) + .map(|(slot, sha)| (slot.clone(), sha.clone())) + .collect(); + let mut summary = Sha256::new(); + for (slot, sha) in &slots { + summary.update(format!("{slot} {sha}\n")); + tombstone.retired.remove(slot); + } + tombstone.sealed.insert( + retired.clone(), + Sealed { + slots: u32::try_from(slots.len()).map_err(|_| error())?, + sha256: format!("{:x}", summary.finalize()), + }, + ); + } + commit(&parent, &path, &tombstone)?; + fault(Step::Sealed)?; + Ok(Compaction { + sealed_boots: prior.len(), + ..outcome + }) +} + +#[cfg(test)] +mod tests; diff --git a/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs b/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs new file mode 100644 index 000000000..f4a9de77a --- /dev/null +++ b/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs @@ -0,0 +1,668 @@ +//! Filesystem-only tests. Guest absence is injected through `confirm`; no VM, engine, +//! real HOME or process interruption is involved. Faults stop the state machine in-process. +use super::super::{active_exec_slot_for, graph_slots_for, preflight_records, record_intent}; +use super::*; +use std::{ + cell::Cell, + io::Read, + os::unix::fs::{MetadataExt, OpenOptionsExt}, +}; + +const RUN: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OTHER_RUN: &str = "cccccccccccccccccccccccccccccccc"; +const INCARNATION: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const FOREIGN_INCARNATION: &str = "dddddddddddddddddddddddddddddddd"; +const PRIOR: &str = "11111111-1111-4111-8111-111111111111"; +const CURRENT: &str = "22222222-2222-4222-8222-222222222222"; +const SERVICES: usize = 14; + +/// Bytes, inode and modification time of each directory entry. +type Snapshot = BTreeMap, u64, i64, i64)>; + +struct Fixture { + candidate: Candidate, + next: Cell, +} + +impl Fixture { + fn new() -> Self { + let mut bytes = [0; 16]; + fs::File::open("/dev/urandom") + .unwrap() + .read_exact(&mut bytes) + .unwrap(); + let token: String = bytes.iter().map(|b| format!("{b:02x}")).collect(); + let path = std::env::temp_dir() + .canonicalize() + .unwrap() + .join(format!("hack-env-compaction-{token}")); + state::private_directory(&path).unwrap(); + let candidate = Candidate::discover(&path).unwrap(); + state::private_directory(&root(&candidate)).unwrap(); + Self { + candidate, + next: Cell::new(0), + } + } + + fn intent(&self, run: Option<&str>, boot: &str, service: usize) -> Intent { + let token = self.next.get(); + self.next.set(token + 1); + Intent { + version: 1, + graph: run.map(|run| GraphBinding { + run: run.into(), + container: format!("hkg-{run}-container-{service}"), + }), + service: format!("svc{service}"), + uid: 0, + gid: 0, + slot: format!("hack-env-lease-{boot}-{token:032x}"), + incarnation: INCARNATION.into(), + boot: boot.into(), + } + } + + /// One start of every service: a generation of committed records. + fn generation(&self, run: &str, boot: &str) -> Vec { + (0..SERVICES) + .map(|service| { + let intent = self.intent(Some(run), boot, service); + self.commit(&intent); + intent + }) + .collect() + } + + fn commit(&self, intent: &Intent) { + state::write(&self.active(&intent.slot), intent).unwrap(); + } + + fn active(&self, slot: &str) -> PathBuf { + root(&self.candidate).join(format!("{slot}.json")) + } + + fn tombstone_path(&self, run: &str) -> PathBuf { + directory(&self.candidate).join(format!("{run}.json")) + } + + fn compact( + &self, + mode: Mode, + incarnation: &str, + boot: &str, + run: &str, + fault: impl FnMut(Step) -> Result<(), CandidateError>, + ) -> Result { + compact_with( + &self.candidate, + Request { + mode, + incarnation, + boot, + run, + containers: &containers(run), + }, + |_| Ok(()), + fault, + ) + } + + fn compact_now(&self, run: &str) -> Result { + self.compact(Mode::Compact, INCARNATION, CURRENT, run, |_| Ok(())) + } + + fn slots(&self, run: &str) -> Result, CandidateError> { + Ok(graph_slots_for(&self.candidate, INCARNATION, run)? + .into_iter() + .map(|(slot, _, _)| slot) + .collect()) + } + + /// Bytes plus inode and modification time of every entry, to prove "untouched". + fn snapshot(&self, directory: &Path) -> Snapshot { + let Ok(entries) = fs::read_dir(directory) else { + return BTreeMap::new(); + }; + entries + .map(|entry| { + let entry = entry.unwrap(); + let metadata = fs::symlink_metadata(entry.path()).unwrap(); + let bytes = if metadata.file_type().is_symlink() { + fs::read_link(entry.path()) + .unwrap() + .into_os_string() + .into_encoded_bytes() + } else { + fs::read(entry.path()).unwrap() + }; + ( + entry.file_name().into_string().unwrap(), + ( + bytes, + metadata.ino(), + metadata.mtime(), + metadata.mtime_nsec(), + ), + ) + }) + .collect() + } + + fn state(&self) -> (Snapshot, Snapshot) { + ( + self.snapshot(&root(&self.candidate)), + self.snapshot(&directory(&self.candidate)), + ) + } + + /// Records of another run, a standalone record and a pending record of another run. + fn foreign(&self) -> Vec { + let other = self.intent(Some(OTHER_RUN), CURRENT, 0); + self.commit(&other); + let standalone = self.intent(None, CURRENT, 1); + self.commit(&standalone); + let uncertain = self.intent(Some(OTHER_RUN), CURRENT, 2); + let pending = self.active(&uncertain.slot).with_extension("pending"); + write_raw(&pending, &serde_json::to_vec(&uncertain).unwrap()); + vec![ + self.active(&other.slot), + self.active(&standalone.slot), + pending, + ] + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + fs::remove_dir_all(&self.candidate.checkout).unwrap(); + } +} + +fn containers(run: &str) -> BTreeMap { + (0..SERVICES) + .map(|service| { + ( + format!("svc{service}"), + format!("hkg-{run}-container-{service}"), + ) + }) + .collect() +} + +fn write_raw(path: &Path, bytes: &[u8]) { + std::io::Write::write_all( + &mut fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .unwrap(), + bytes, + ) + .unwrap(); +} + +fn stop_at(target: usize) -> impl FnMut(Step) -> Result<(), CandidateError> { + let mut seen = 0; + move |_| { + seen += 1; + if seen == target + 1 { + return Err(CandidateError::new("test_interrupted", "Interrupted.")); + } + Ok(()) + } +} + +#[test] +fn compaction_retains_digests_seals_prior_boots_and_frees_active_budget() { + let fixture = Fixture::new(); + let mut retired = fixture.generation(RUN, PRIOR); + retired.extend(fixture.generation(RUN, PRIOR)); + let current = fixture.generation(RUN, CURRENT); + retired.extend(current.clone()); + let originals: BTreeMap> = retired + .iter() + .map(|intent| { + ( + intent.slot.clone(), + fs::read(fixture.active(&intent.slot)).unwrap(), + ) + }) + .collect(); + let foreign = fixture.foreign(); + let untouched = fixture.snapshot(&root(&fixture.candidate)); + let budget = MAX_INTENT_ENTRIES - foreign.len(); + assert!(preflight_records(&fixture.candidate, budget).is_err()); + + let confirmed = Cell::new(0); + let outcome = compact_with( + &fixture.candidate, + Request { + mode: Mode::Compact, + incarnation: INCARNATION, + boot: CURRENT, + run: RUN, + containers: &containers(RUN), + }, + |intent| { + assert_eq!(intent.graph.as_ref().unwrap().run, RUN); + confirmed.set(confirmed.get() + 1); + Ok(()) + }, + |_| Ok(()), + ) + .unwrap(); + assert_eq!( + outcome, + Compaction { + compacted: 3 * SERVICES, + completed: 0, + sealed_boots: 1, + } + ); + assert_eq!(confirmed.get(), 3 * SERVICES, "every record proven absent"); + + let remaining = fixture.snapshot(&root(&fixture.candidate)); + assert_eq!(remaining.len(), foreign.len()); + for (name, entry) in &remaining { + assert_eq!(untouched.get(name), Some(entry), "foreign record changed"); + } + assert!(preflight_records(&fixture.candidate, budget).is_ok()); + + let tombstone = load(&fixture.candidate, RUN, INCARNATION).unwrap().unwrap(); + assert_eq!(tombstone.sealed.len(), 1); + assert_eq!(tombstone.sealed[PRIOR].slots, 2 * SERVICES as u32); + assert_eq!(tombstone.retired.len(), SERVICES); + for intent in ¤t { + assert_eq!( + tombstone.retired[&intent.slot], + digest(&originals[&intent.slot]) + ); + } + for slot in originals.keys() { + assert!(reserved(&fixture.candidate, RUN, INCARNATION, slot).unwrap()); + } + assert!(fixture.slots(RUN).unwrap().is_empty()); + assert_eq!( + fixture.slots(OTHER_RUN).unwrap().len(), + 2, + "committed and pending" + ); + + // A fresh allocation in the current boot stays admissible. + let fresh = fixture.intent(Some(RUN), CURRENT, 0); + assert!(!reserved(&fixture.candidate, RUN, INCARNATION, &fresh.slot).unwrap()); + record_intent(&fixture.candidate, &fresh).unwrap(); + assert_eq!(fixture.slots(RUN).unwrap(), vec![fresh.slot.clone()]); + fs::remove_file(fixture.active(&fresh.slot)).unwrap(); + + let bytes = fs::read(fixture.tombstone_path(RUN)).unwrap(); + assert_eq!(fixture.compact_now(RUN).unwrap(), Compaction::default()); + assert_eq!( + fs::read(fixture.tombstone_path(RUN)).unwrap(), + bytes, + "re-run is idempotent" + ); +} + +#[test] +fn interruption_at_each_compaction_step_recovers_without_loss() { + let build = || { + let fixture = Fixture::new(); + let mut intents = fixture.generation(RUN, PRIOR); + intents.extend(fixture.generation(RUN, CURRENT)); + let originals: BTreeMap> = intents + .iter() + .map(|intent| { + ( + intent.slot.clone(), + fs::read(fixture.active(&intent.slot)).unwrap(), + ) + }) + .collect(); + let foreign = fixture.foreign(); + let untouched = fixture.snapshot(&root(&fixture.candidate)); + (fixture, originals, foreign, untouched) + }; + let (reference, _, _, _) = build(); + let steps = Cell::new(0); + reference + .compact(Mode::Compact, INCARNATION, CURRENT, RUN, |_| { + steps.set(steps.get() + 1); + Ok(()) + }) + .unwrap(); + // Recorded, one unlink per record, directory sync, prior-boot seal. + assert_eq!(steps.get(), 2 * SERVICES + 3); + let expected = load(&reference.candidate, RUN, INCARNATION) + .unwrap() + .unwrap(); + + for target in 0..steps.get() { + let (fixture, originals, foreign, untouched) = build(); + assert!( + fixture + .compact(Mode::Compact, INCARNATION, CURRENT, RUN, stop_at(target)) + .is_err() + ); + // No loss: every record is still active with its exact bytes, or durably retained. + let tombstone = load(&fixture.candidate, RUN, INCARNATION).unwrap().unwrap(); + for (slot, bytes) in &originals { + let active = fixture.active(slot); + if active.exists() { + assert_eq!(&fs::read(&active).unwrap(), bytes); + assert_eq!(tombstone.retired.get(slot), Some(&digest(bytes))); + } else { + assert!( + reserves(Some(&tombstone), slot), + "step {target} lost {slot}" + ); + } + } + // Read-only inventory treats durable residue as retired, never as live. + assert!(fixture.slots(RUN).unwrap().is_empty(), "step {target}"); + #[cfg(target_os = "macos")] + assert!( + graph_inventory( + &fixture.candidate, + INCARNATION, + RUN, + &containers(RUN), + &fixture + .candidate + .state_root + .join("graph-environment-archive"), + ) + .unwrap() + .is_empty() + ); + let now = fixture.snapshot(&root(&fixture.candidate)); + for path in &foreign { + let name = path.file_name().unwrap().to_str().unwrap(); + assert_eq!(untouched.get(name), now.get(name), "step {target} foreign"); + } + // Alternate recovery entry points: archive's resume-only pass, or the next cleanup. + if target % 2 == 0 { + let resumed = fixture + .compact(Mode::Resume, INCARNATION, CURRENT, RUN, |_| Ok(())) + .unwrap(); + assert_eq!(resumed.compacted, 0); + assert_eq!(resumed.sealed_boots, 0); + } + fixture.compact_now(RUN).unwrap(); + assert_eq!( + load(&fixture.candidate, RUN, INCARNATION).unwrap().unwrap(), + expected, + "step {target} converges" + ); + for slot in originals.keys() { + assert!(!fixture.active(slot).exists()); + } + assert_eq!(fixture.compact_now(RUN).unwrap(), Compaction::default()); + } +} + +#[test] +fn uncommitted_tombstone_write_has_no_authority_and_is_replaced() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + let parent = directory(&fixture.candidate); + state::private_directory(&parent).unwrap(); + let torn = fixture.tombstone_path(RUN).with_extension("pending"); + write_raw(&torn, b"{\"version\":1,\"run\":"); + // Before commit point 1, every record is still live inventory. + assert_eq!(fixture.slots(RUN).unwrap().len(), SERVICES); + assert!(!reserved(&fixture.candidate, RUN, INCARNATION, &intents[0].slot).unwrap()); + assert_eq!(fixture.compact_now(RUN).unwrap().compacted, SERVICES); + assert!(!torn.exists()); + assert!(fixture.slots(RUN).unwrap().is_empty()); + // A temporary that is not a private regular file refuses rather than being removed. + std::os::unix::fs::symlink("elsewhere", &torn).unwrap(); + fixture.generation(RUN, CURRENT); + let before = fixture.state(); + assert!(fixture.compact_now(RUN).is_err()); + assert_eq!(fixture.state(), before); +} + +#[test] +fn wrong_owner_stale_boot_live_misbound_and_uncertain_records_refuse_without_effects() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + fixture.foreign(); + let before = fixture.state(); + let unchanged = |label: &str| assert_eq!(fixture.state(), before, "{label}"); + + assert!( + fixture + .compact(Mode::Compact, FOREIGN_INCARNATION, CURRENT, RUN, |_| Ok(())) + .is_err() + ); + unchanged("wrong owner"); + assert!(!directory(&fixture.candidate).exists()); + + let live = intents[3].slot.clone(); + assert!( + compact_with( + &fixture.candidate, + Request { + mode: Mode::Compact, + incarnation: INCARNATION, + boot: CURRENT, + run: RUN, + containers: &containers(RUN), + }, + |intent| if intent.slot == live { + Err(error()) + } else { + Ok(()) + }, + |_| Ok(()), + ) + .is_err() + ); + unchanged("live slot"); + + let mut misbound = containers(RUN); + misbound.insert("svc5".into(), format!("hkg-{RUN}-container-6")); + assert!( + compact_with( + &fixture.candidate, + Request { + mode: Mode::Compact, + incarnation: INCARNATION, + boot: CURRENT, + run: RUN, + containers: &misbound, + }, + |_| Ok(()), + |_| Ok(()), + ) + .is_err() + ); + unchanged("misbound container"); + + let uncertain = fixture.intent(Some(RUN), CURRENT, 7); + let pending = fixture.active(&uncertain.slot).with_extension("pending"); + write_raw(&pending, &serde_json::to_vec(&uncertain).unwrap()); + let with_pending = fixture.state(); + assert!(fixture.compact_now(RUN).is_err()); + assert_eq!(fixture.state(), with_pending, "uncertain record"); + // Resume never selects, so it also leaves the uncertain record alone. + assert_eq!( + fixture + .compact(Mode::Resume, INCARNATION, CURRENT, RUN, |_| Ok(())) + .unwrap(), + Compaction::default() + ); + assert_eq!(fixture.state(), with_pending); + fs::remove_file(&pending).unwrap(); + + // Seal CURRENT by compacting from a later boot; the stale boot may not compact again. + let later = "33333333-3333-4333-8333-333333333333"; + assert_eq!( + fixture + .compact(Mode::Compact, INCARNATION, later, RUN, |_| Ok(())) + .unwrap() + .sealed_boots, + 1 + ); + let sealed = fixture.state(); + assert!( + fixture + .compact(Mode::Compact, INCARNATION, CURRENT, RUN, |_| Ok(())) + .is_err() + ); + assert_eq!(fixture.state(), sealed, "stale boot"); +} + +#[test] +fn replayed_changed_and_copied_state_never_resurrects_a_retired_lease() { + let fixture = Fixture::new(); + let prior = fixture.generation(RUN, PRIOR); + let current = fixture.generation(RUN, CURRENT); + let replay = |intent: &Intent| { + let path = fixture.active(&intent.slot); + let bytes = serde_json::to_vec_pretty(intent).unwrap(); + write_raw(&path, &bytes); + (path, bytes) + }; + let prior_bytes = fs::read(fixture.active(&prior[0].slot)).unwrap(); + let current_bytes = fs::read(fixture.active(¤t[0].slot)).unwrap(); + fixture.compact_now(RUN).unwrap(); + + // Exact current-boot replay: retired history, never delivery or a new allocation. + let (path, bytes) = replay(¤t[0]); + assert_eq!(bytes, current_bytes, "exact copy of the retired record"); + assert!(fixture.slots(RUN).unwrap().is_empty()); + assert!(record_intent(&fixture.candidate, ¤t[0]).is_err()); + let exec = |slot: &str| { + active_exec_slot_for( + &fixture.candidate, + INCARNATION, + CURRENT, + RUN, + "svc0", + &format!("hkg-{RUN}-container-0"), + Some(slot), + ) + }; + assert!(exec(¤t[0].slot).is_err()); + assert_eq!(fixture.compact_now(RUN).unwrap().completed, 1); + assert!(!path.exists()); + // With no residue left, the retired identity still cannot be recorded again. + assert!(record_intent(&fixture.candidate, ¤t[2]).is_err()); + assert!(!fixture.active(¤t[2].slot).exists()); + + // Stale-boot replay of a sealed boot refuses everywhere and is retained untouched. + let (path, _) = replay(&prior[0]); + assert_eq!(fs::read(&path).unwrap(), prior_bytes); + let before = fixture.state(); + assert!(fixture.slots(RUN).is_err()); + assert!(fixture.compact_now(RUN).is_err()); + assert!(record_intent(&fixture.candidate, &prior[0]).is_err()); + #[cfg(target_os = "macos")] + assert!( + graph_inventory( + &fixture.candidate, + INCARNATION, + RUN, + &containers(RUN), + &fixture + .candidate + .state_root + .join("graph-environment-archive"), + ) + .is_err() + ); + assert_eq!(fixture.state(), before); + fs::remove_file(&path).unwrap(); + + // A changed record under a retired slot identity is refused, never removed, and + // blocks compaction of an otherwise eligible record before any tombstone write. + let mut changed = current[1].clone(); + changed.uid = 1000; + changed.gid = 1000; + let (path, bytes) = replay(&changed); + let eligible = fixture.intent(Some(RUN), CURRENT, 4); + fixture.commit(&eligible); + let before = fixture.state(); + assert!(fixture.slots(RUN).is_err()); + assert!(fixture.compact_now(RUN).is_err()); + assert!(exec(&changed.slot).is_err()); + assert_eq!(fixture.state(), before); + assert_eq!(fs::read(&path).unwrap(), bytes); + fs::remove_file(&path).unwrap(); + assert_eq!(fixture.compact_now(RUN).unwrap().compacted, 1); + + // Copied stores: another run's name, another incarnation, or a linked copy refuse. + let tombstone = fixture.tombstone_path(RUN); + let copied = fixture.tombstone_path(OTHER_RUN); + fs::copy(&tombstone, &copied).unwrap(); + assert!(load(&fixture.candidate, OTHER_RUN, INCARNATION).is_err()); + assert!(fixture.slots(OTHER_RUN).is_err()); + fs::remove_file(&copied).unwrap(); + assert!(load(&fixture.candidate, RUN, FOREIGN_INCARNATION).is_err()); + assert!(graph_slots_for(&fixture.candidate, FOREIGN_INCARNATION, RUN).is_err()); + fs::hard_link(&tombstone, &copied).unwrap(); + assert!(load(&fixture.candidate, RUN, INCARNATION).is_err()); + assert!(fixture.compact_now(RUN).is_err()); + fs::remove_file(&copied).unwrap(); + assert!( + load(&fixture.candidate, RUN, INCARNATION) + .unwrap() + .is_some() + ); +} + +#[test] +fn malformed_tombstones_refuse() { + let fixture = Fixture::new(); + fixture.generation(RUN, PRIOR); + fixture.compact_now(RUN).unwrap(); + let path = fixture.tombstone_path(RUN); + let original: serde_json::Value = state::read(&path).unwrap(); + let mut cases = Vec::new(); + let mut overlap = original.clone(); + overlap["retired"][format!("hack-env-lease-{PRIOR}-{}", "f".repeat(32))] = + serde_json::Value::String("0".repeat(64)); + cases.push(overlap); + let mut empty_seal = original.clone(); + empty_seal["sealed"][PRIOR]["slots"] = 0.into(); + cases.push(empty_seal); + let mut unknown = original.clone(); + unknown["values"] = serde_json::json!({"TOKEN": "x"}); + cases.push(unknown); + let mut version = original; + version["version"] = 2.into(); + cases.push(version); + for case in cases { + fs::remove_file(&path).unwrap(); + state::write(&path, &case).unwrap(); + assert!(load(&fixture.candidate, RUN, INCARNATION).is_err()); + assert!(fixture.compact_now(RUN).is_err()); + } +} + +#[test] +fn record_changed_after_tombstone_commit_is_retained_not_unlinked() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + let target = fixture.active(&intents[0].slot); + let mut changed = intents[0].clone(); + changed.uid = 1000; + let replacement = serde_json::to_vec_pretty(&changed).unwrap(); + // An interruption after commit point 1 during which the record is replaced. + let outcome = fixture.compact(Mode::Compact, INCARNATION, CURRENT, RUN, |step| { + if step == Step::Recorded { + fs::remove_file(&target).unwrap(); + write_raw(&target, &replacement); + } + Ok(()) + }); + assert!(outcome.is_err()); + assert_eq!(fs::read(&target).unwrap(), replacement); + assert!(fixture.slots(RUN).is_err()); + assert!(fixture.compact_now(RUN).is_err()); + assert_eq!(fs::read(&target).unwrap(), replacement); +} diff --git a/packages/runtime-core/src/provider/environment_recovery/retention.rs b/packages/runtime-core/src/provider/environment_recovery/retention.rs index 6812d07f1..5d3b2272c 100644 --- a/packages/runtime-core/src/provider/environment_recovery/retention.rs +++ b/packages/runtime-core/src/provider/environment_recovery/retention.rs @@ -39,7 +39,7 @@ fn present(path: &Path) -> Result { Err(e) => Err(state::io(e)), } } -fn bytes(path: &Path) -> Result, CandidateError> { +pub(super) fn bytes(path: &Path) -> Result, CandidateError> { let mut file = fs::OpenOptions::new() .read(true) .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) diff --git a/packages/runtime-core/src/provider/graph/environment.rs b/packages/runtime-core/src/provider/graph/environment.rs index d965f8b91..bd470fa8f 100644 --- a/packages/runtime-core/src/provider/graph/environment.rs +++ b/packages/runtime-core/src/provider/graph/environment.rs @@ -134,6 +134,27 @@ pub(super) fn verify_cleanup( environment_recovery::verify_graph_retired(engine.guest(), expected) } +/// Compact this graph's confirmed-retired allocation records after ordinary cleanup. The caller +/// holds the engine's mutation lease and has committed the cleaned receipt; every record must +/// match a receipt container that is absent, and its guest slot must be absent. +pub(super) fn compact_retired( + candidate: &Candidate, + engine: &Engine<'_>, + receipt: &Receipt, +) -> Result<(), CandidateError> { + if !receipt.environment_attached { + return Ok(()); + } + let containers = receipt + .resources + .values() + .filter(|r| r.kind == Kind::Container) + .map(|r| (r.key.clone(), r.name.clone())) + .collect(); + environment_recovery::compact_graph(candidate, engine.guest(), &receipt.run, &containers)?; + Ok(()) +} + pub(super) fn require_replay_supported(receipt: &Receipt) -> Result<(), CandidateError> { if receipt.environment_attached { return Err(error( @@ -156,6 +177,9 @@ pub(super) fn archive_slots( if !receipt.environment_attached { return Ok(()); } + // Finish an interrupted compaction so exact residue is not stranded in the active + // directory after the graph leaves cleanup. No new record is compacted here. + environment_recovery::resume_graph(candidate, engine.guest(), &receipt.run)?; let containers = receipt .resources .values() diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index e277f8981..f94177fab 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1624,7 +1624,11 @@ pub fn cleanup( let engine = Engine::connect_cleanup(candidate)?; let (receipt, root) = load(candidate, &engine, run)?; cleanup_enrollment::ordinary_mutation(&root, &receipt)?; - cleanup_owned(candidate, &engine, receipt, &root, remove_data) + let receipt = cleanup_owned(candidate, &engine, receipt, &root, remove_data)?; + // Only ordinary cleanup compacts. Recovery flows retain an exact environment inventory + // across their confirmation and use the fenced path without this effect. + environment::compact_retired(candidate, &engine, &receipt)?; + Ok(receipt) } // The caller retains the VM mutation lease through this entire effect and any From c4b7521fb6e14490d4e10e6198aa9e0a70f3dc3b Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:00:48 -0400 Subject: [PATCH 09/21] feat(runtime): quarantine a corrupted disposable build cache without touching app data Add an opt-in, single-target quarantine for a regenerable cache directory inside an ordinary owned volume of a stopped, data-retaining graph (HACK-1198). - Compose volumes may declare `x-hack-disposable-cache` (at most eight distinct, non-nested, volume-relative directories). Shared dependency cache volumes and paths that are or contain a mounted subpath are refused. The declaration is recorded on the admitted volume resource; absent declarations keep plan and receipt bytes unchanged. - `graph inspect-cache-quarantine` measures the exact target under the provider lease and returns a selection digest binding receipt bytes, volume incarnation, root and target directory identities and measured bytes. - `graph quarantine-cache --expect-selection` renames the directory on the same filesystem into a Hack-owned `hack-quarantine` sibling of the volume's `_data` root. Nothing is deleted; reclaimed bytes are reported as zero. - A durable private record is committed before the guest effect and completed after the guest syncs the move; replay resumes only the identical selection. - Refuses active graphs, incomplete journals, undeclared or traversing paths, foreign labels, changed volume incarnation, any container consumer, symlinked or replaced targets, ambiguous destinations and torn records. Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 81 +++ packages/runtime-core/src/graph_cli.rs | 50 ++ packages/runtime-core/src/main.rs | 2 + .../runtime-core/src/project/compose/mod.rs | 63 +- .../src/project/compose/service.rs | 7 + packages/runtime-core/src/project/mod.rs | 5 + .../src/provider/graph/cache_provenance.rs | 2 + .../src/provider/graph/cache_quarantine.rs | 568 +++++++++++++++++ .../provider/graph/cache_quarantine/tests.rs | 583 ++++++++++++++++++ .../runtime-core/src/provider/graph/config.rs | 15 + .../src/provider/graph/environment/tests.rs | 1 + .../src/provider/graph/host_relay.rs | 1 + .../src/provider/graph/hostname_change.rs | 1 + .../runtime-core/src/provider/graph/mod.rs | 6 + .../src/provider/graph/native/mod.rs | 2 + .../provider/graph/restore_history/tests.rs | 1 + .../src/provider/graph/shutdown.rs | 1 + .../runtime-core/src/provider/graph/source.rs | 1 + .../src/provider/graph/storage_inventory.rs | 1 + .../runtime-core/src/provider/graph/tests.rs | 48 ++ .../src/provider/graph/volume_subpaths.rs | 2 +- .../tests/cache_quarantine_cli_contract.rs | 105 ++++ .../runtime-core/tests/project_contract.rs | 89 +++ 23 files changed, 1632 insertions(+), 3 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/cache_quarantine.rs create mode 100644 packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs create mode 100644 packages/runtime-core/tests/cache_quarantine_cli_contract.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index b5f309b4c..9c5c76a9d 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -1516,6 +1516,87 @@ the dead predecessor publication. That selection belongs to the interrupted operation; replacing a selection from an older restored generation still requires independent restore-history proof. +### Quarantine a corrupted disposable build cache + +Some frameworks keep a regenerable build cache inside a named volume next to other +generated output. When such a cache is corrupted (for example a Turbopack checksum +mismatch reported as a misleading `Module not found`), you can move exactly that +cache directory aside. Nothing is deleted, and other volumes and directories are +unchanged. + +First declare the regenerable directories on the volume in the Compose file. Paths +are relative to the volume root. List at most eight distinct directories; one entry +cannot contain another: + +```yaml +volumes: + www_next: + x-hack-disposable-cache: + - dev/cache/turbopack +``` + +The declaration is part of the reviewed plan. It is recorded in the graph receipt +when the graph is admitted, and only a receipt-recorded entry can be selected later. +A volume that is also a shared dependency cache cannot declare disposable +directories. Declaring a path grants no automatic cleanup. + +Stop the graph with an ordinary retaining `down` (phase `stopped-data-retained`). +With the owned runtime running, inspect the single target. This step is read-only: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph inspect-cache-quarantine \ + --run-id --volume www_next --path dev/cache/turbopack --json +``` + +The response reports the target's `target_bytes` (regular-file bytes), +`allocated_bytes`, `entries` and a `selection_sha256`. The selection binds the exact +receipt bytes, the volume name, creation time and root directory identity, the +target directory identity and the measured bytes. Review it, then quarantine: + +```sh +./hack-native --candidate-root /absolute/private/candidate-home graph quarantine-cache \ + --run-id --volume www_next --path dev/cache/turbopack \ + --expect-selection --json +``` + +Both commands hold the provider mutation lease. They refuse when any of these is +true: + +- the graph is active, its journal is incomplete, or interrupted cleanup is pending +- the path is not exactly a declared entry, or is absolute, traversing or empty +- the volume is missing, carries foreign ownership labels, or has changed + incarnation +- any container mounts the volume, including stopped containers +- a component of the path, or the target itself, is a symbolic link or not a + directory +- the target directory was replaced after inspection + +The guest renames the directory under a lock on the volume root into +`/var/lib/docker/volumes//hack-quarantine/`. That directory is beside +the volume's `_data` root on the same filesystem, so the move never copies, and no +container can mount it. The result reports `quarantined_bytes` and +`reclaimed_bytes: 0`. The bytes remain for diagnosis until the owning volume +itself is explicitly removed (for example `graph cleanup --remove-data`). Restart, +restore and ordinary cleanup never collect them. Then use a normal retaining +restart. The framework regenerates its cache. + +A private record under the candidate home's `run/cache-quarantine/` is written +before the guest effect and completed after the guest syncs the move. If the +command is interrupted, repeat it with the same selection: `inspect-cache-quarantine` +reports `resume_required` with the recorded digest. Resumption accepts the recorded +directory either still at its source or already at its destination with the same +inode. It refuses a regenerated directory, a different destination or both +present, and never moves new bytes. A torn record write refuses automatic replay. +Resume before restarting the graph. A restart changes the receipt, so the pending +record can no longer be resumed and further quarantines of that volume are +refused. The bytes stay where the interruption left them. +Records are retained (at most 256) and are not collected automatically. + +This operation does not classify the cause of the corruption. One checksum error +does not show an interrupted write or a storage integrity fault. Container health +also does not prove that application routes work; check the public route +separately after restart. + ### Explicit cleanup after a dead foreground owner For a fully ready graph whose foreground owner died while its pool remains on the diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index dd9ced0c8..5f8b8bf85 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -335,6 +335,56 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result (run, volume, path), + _ => return Err(invalid()), + }; + return graph::inspect_cache_quarantine( + candidate, + graph::CacheQuarantineTarget { run, volume, path }, + ); + } + if *action == "quarantine-cache" { + let (run, volume, path, expected) = match *args { + [ + "--run-id", + run, + "--volume", + volume, + "--path", + path, + "--expect-selection", + expected, + ] + | [ + "--run-id", + run, + "--volume", + volume, + "--path", + path, + "--expect-selection", + expected, + "--json", + ] => (run, volume, path, expected), + _ => return Err(invalid()), + }; + return graph::quarantine_cache( + candidate, + graph::CacheQuarantineTarget { run, volume, path }, + expected, + ); + } if *action == "inspect-source-device-rebind" { let run = match *args { ["--run-id", run] | ["--run-id", run, "--json"] => run, diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 76dbdb89b..6e1d9a41c 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -40,6 +40,8 @@ Usage: hack-local graph inspect-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file [--json] hack-local graph recover-absent-publication-cleanup --run-id <32-hex> --original-owner-file --host-inspection-file --expect-selection <64-hex> --retain-data --accept-unpinned-post-reboot [--json] hack-local graph inspect-source-device-rebind --run-id <32-hex> [--json] + hack-local graph inspect-cache-quarantine --run-id <32-hex> --volume --path [--json] + hack-local graph quarantine-cache --run-id <32-hex> --volume --path --expect-selection <64-hex> [--json] hack-local graph recover-source-device-rebind --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] diff --git a/packages/runtime-core/src/project/compose/mod.rs b/packages/runtime-core/src/project/compose/mod.rs index c236abe2a..ca8d52f57 100644 --- a/packages/runtime-core/src/project/compose/mod.rs +++ b/packages/runtime-core/src/project/compose/mod.rs @@ -36,6 +36,38 @@ pub(super) fn original_environment_files( Ok(result) } +/// Maximum declared disposable cache directories per volume. +const MAX_DISPOSABLE_CACHES: usize = 8; + +/// Parse an explicit list of volume-relative regenerable cache directories. +/// Entries are exact, distinct and non-nested, so a later quarantine can select +/// exactly one declared directory and never an ancestor that holds other data. +fn disposable_caches(value: Option<&Value>) -> Result, CandidateError> { + let invalid = || { + problem( + "invalid_disposable_cache", + "x-hack-disposable-cache must list at most eight distinct, non-nested, volume-relative directories.", + ) + }; + let values = strings(value).map_err(|_| invalid())?; + if values.len() > MAX_DISPOSABLE_CACHES + || values + .iter() + .any(|value| !super::valid_volume_subpath(value)) + { + return Err(invalid()); + } + for (index, value) in values.iter().enumerate() { + for other in &values[index + 1..] { + let nested = |a: &str, b: &str| a == b || b.starts_with(&format!("{a}/")); + if nested(value, other) || nested(other, value) { + return Err(invalid()); + } + } + } + Ok(values) +} + pub(super) fn compile( candidate: &Candidate, project: &Path, @@ -78,7 +110,13 @@ pub(super) fn compile( } else { object(value)? }; - keys(v, &["driver", "external"], &field, &mut diagnostics); + keys( + v, + &["driver", "external", "x-hack-disposable-cache"], + &field, + &mut diagnostics, + ); + let disposable_caches = disposable_caches(v.get("x-hack-disposable-cache"))?; if boolean(v.get("external"), false)? { diagnostics.push(Diagnostic::error( "external_volume", @@ -94,7 +132,7 @@ pub(super) fn compile( "Only owned local volumes are modeled.", )); } - volumes.insert(name.clone(),VolumePlan {driver:if driver=="local" {"local".into()} else {"[unsupported]".into()},ownership:"new candidate namespace; retained on ordinary stop; no active v4 volume adoption".into()}); + volumes.insert(name.clone(),VolumePlan {driver:if driver=="local" {"local".into()} else {"[unsupported]".into()},ownership:"new candidate namespace; retained on ordinary stop; no active v4 volume adoption".into(),disposable_caches}); } } let mut networks = BTreeMap::new(); @@ -167,6 +205,27 @@ pub(super) fn compile( &volumes, &mut diagnostics, )?; + // Quarantining a directory that is, or contains, a mounted volume subpath + // would remove a mount source that retained startup never recreates. + for service in services.values() { + if service.mounts.iter().any(|mount| { + mount.kind == "volume" + && mount.subpath.as_deref().is_some_and(|subpath| { + volumes + .get(&mount.source) + .is_some_and(|volume: &VolumePlan| { + volume.disposable_caches.iter().any(|cache| { + subpath == cache || subpath.starts_with(&format!("{cache}/")) + }) + }) + }) + }) { + return Err(problem( + "invalid_disposable_cache", + "A disposable cache directory cannot be or contain a mounted volume subpath.", + )); + } + } graph::validate(&services, &networks, profiles, &mut diagnostics); Ok(PlanData {schema_version:1,kind:"compose-enrollment-review-only".into(),candidate_root:candidate.checkout.clone(),source:project.into(),namespace:String::new(),branch:None,compose_file:file.into(),compose_sha256:format!("{:x}",Sha256::digest(input)),original_compose_sha256:None,hostname_change_sha256:None,active_profiles:profiles.to_vec(),services,networks,volumes,registry:None,generated_files:super::generated::parse(m.get("x-hack-generated-files"))?, original_environment_files:std::collections::BTreeSet::new(),source_selection:SourceSelection {policy:String::new(),identity_kind:String::new(),metadata_sha256:String::new(),ignore_files:BTreeMap::new(),entries:vec![],excluded_paths:vec![],exclusion_rules:vec![]},diagnostics,enrollment_compatible:false,runtime_execution_supported:false, diff --git a/packages/runtime-core/src/project/compose/service.rs b/packages/runtime-core/src/project/compose/service.rs index 3c7d72b2c..875c562a6 100644 --- a/packages/runtime-core/src/project/compose/service.rs +++ b/packages/runtime-core/src/project/compose/service.rs @@ -794,6 +794,13 @@ pub(super) fn compile( "Dependency cache volume must be declared; values omitted.", )); } + // A shared dependency cache outlives and is consumed beyond one graph. + if !volumes[&cache.volume].disposable_caches.is_empty() { + return Err(problem( + "invalid_disposable_cache", + "A shared dependency cache volume cannot declare disposable cache directories.", + )); + } diagnostics.push(Diagnostic::warning("dependency_cache_binding_required", &format!("{field}.labels"), "Dependency cache execution requires a verified source revision, pinned installer image and shared cache binding; declaration alone is not a completed install.")); } if routing_invalid diff --git a/packages/runtime-core/src/project/mod.rs b/packages/runtime-core/src/project/mod.rs index fc9a01f03..043076259 100644 --- a/packages/runtime-core/src/project/mod.rs +++ b/packages/runtime-core/src/project/mod.rs @@ -290,6 +290,11 @@ pub struct NetworkPlan { pub struct VolumePlan { pub driver: String, pub ownership: String, + /// Volume-relative directories the project explicitly declared regenerable + /// (`x-hack-disposable-cache`). Declaration grants no automatic effect; it is + /// the only path set an explicit, single-target quarantine may select. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub disposable_caches: Vec, } #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(deny_unknown_fields)] diff --git a/packages/runtime-core/src/provider/graph/cache_provenance.rs b/packages/runtime-core/src/provider/graph/cache_provenance.rs index 1f8af3da4..fa81b076f 100644 --- a/packages/runtime-core/src/provider/graph/cache_provenance.rs +++ b/packages/runtime-core/src/provider/graph/cache_provenance.rs @@ -279,6 +279,7 @@ mod tests { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: "deps".into(), name: "owned".into(), @@ -370,6 +371,7 @@ mod tests { outbound: false, cache: Some(cache.clone()), cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Volume, key: "data".into(), name: cache.name(), diff --git a/packages/runtime-core/src/provider/graph/cache_quarantine.rs b/packages/runtime-core/src/provider/graph/cache_quarantine.rs new file mode 100644 index 000000000..c8dc69b54 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/cache_quarantine.rs @@ -0,0 +1,568 @@ +//! Explicit, single-target quarantine of one admitted disposable cache directory +//! inside an ordinary owned volume of a stopped, data-retaining graph. +//! +//! Authority is the conjunction of: the receipt's admitted +//! `x-hack-disposable-cache` declaration for that exact path, an operator-reviewed +//! selection digest (receipt bytes, volume incarnation, directory identities and +//! measured bytes), the provider mutation lease, owned volume labels, and the +//! absence of every container consumer. Nothing is deleted. The directory is +//! renamed on the same filesystem into a Hack-owned `hack-quarantine` sibling of +//! the volume's `_data` root, which no container mounts. Quarantined bytes stay +//! until the owning volume itself is explicitly removed; ordinary cleanup, +//! restart and restore never collect them, so reclaimed bytes are always zero. +//! +//! Recovery: a durable host record is committed before the guest rename and is +//! marked complete only after the guest reports a synchronized move. A pending +//! record admits only its identical selection. The guest accepts the target at +//! its original path or already at its destination with the recorded inode, and +//! refuses every other layout. A torn record write refuses automatic replay. +use super::*; +use sha2::{Digest, Sha256}; +use std::os::unix::fs::DirBuilderExt; +use std::path::Path; + +const RECORD_LIMIT: u64 = 16 * 1024; +/// Records are retained as diagnosis evidence; there is no automatic collection. +const MAX_RECORDS: usize = 256; +const QUARANTINE_DIRECTORY: &str = "hack-quarantine"; + +fn refused() -> CandidateError { + error( + "graph_cache_quarantine", + "The disposable cache selection is missing, unsafe, changed or in use; no cache bytes were moved.", + ) +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} + +/// One explicit target. `volume` is the logical Compose volume name and `path` +/// must equal one admitted disposable directory of that volume. +#[derive(Clone, Copy, Debug)] +pub struct CacheQuarantineTarget<'a> { + pub run: &'a str, + pub volume: &'a str, + pub path: &'a str, +} + +/// Exact reviewed state. Its digest is the only accepted `--expect-selection`. +/// Boot identity is deliberately excluded so an interrupted move can resume +/// after a guest restart when every recorded identity still matches. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Selection { + version: u32, + run: String, + owner: String, + receipt_sha256: String, + volume: String, + volume_name: String, + volume_created: String, + root_identity: String, + path: String, + target_identity: String, + target_bytes: u64, + allocated_bytes: u64, + entries: u64, +} +impl Selection { + fn sha256(&self) -> Result { + Ok(digest(&serde_json::to_vec(self).map_err(|_| refused())?)) + } + /// The destination name; distinct directory identities never share one. + fn quarantine_id(digest: &str) -> &str { + &digest[..32] + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Record { + version: u32, + selection_sha256: String, + selection: Selection, + completed: bool, +} +impl Record { + fn valid(&self) -> bool { + self.version == 1 + && self.selection.version == 1 + && hex(&self.selection_sha256, 64) + && self + .selection + .sha256() + .is_ok_and(|value| value == self.selection_sha256) + } +} + +/// Pure admission over the loaded receipt. Only an admitted ordinary volume of a +/// stopped retained graph with no incomplete journal can select an exact path. +fn authorize<'r>( + receipt: &'r Receipt, + journal_incomplete: bool, + target: &CacheQuarantineTarget<'_>, +) -> Result<&'r Resource, CandidateError> { + if journal_incomplete + || receipt.phase != "stopped-data-retained" + || !project::valid_volume_subpath(target.path) + { + return Err(refused()); + } + let resource = receipt + .resources + .get(&format!("volume:{}", target.volume)) + .ok_or_else(refused)?; + if resource.kind != Kind::Volume + || resource.key != target.volume + || resource.phase != "created" + || resource.cache.is_some() + || !resource.disposable_caches.iter().any(|p| p == target.path) + { + return Err(refused()); + } + Ok(resource) +} + +/// Parse the read-only guest observation. `None` means the target is absent. +fn parse_observation(output: &str) -> Result, CandidateError> { + if output == "absent\n" { + return Ok(None); + } + let line = output + .strip_prefix("present ") + .and_then(|v| v.strip_suffix('\n')) + .filter(|v| v.len() <= 128 && !v.contains('\n')) + .ok_or_else(refused)?; + let mut fields = line.split(' '); + let identity = fields + .next() + .filter(|v| volume_subpaths::valid_identity(v)) + .ok_or_else(refused)?; + let mut number = || -> Result { + fields + .next() + .filter(|v| !v.is_empty() && v.len() <= 20 && v.bytes().all(|b| b.is_ascii_digit())) + .and_then(|v| v.parse().ok()) + .ok_or_else(refused) + }; + let (bytes, allocated, entries) = (number()?, number()?, number()?); + if fields.next().is_some() || entries == 0 { + return Err(refused()); + } + Ok(Some((identity.to_owned(), bytes, allocated, entries))) +} + +fn same_device(a: &str, b: &str) -> bool { + a.split_once(':').map(|v| v.0) == b.split_once(':').map(|v| v.0) +} + +/// Retained private records under `state_root/run/cache-quarantine`. +struct Records { + directory: PathBuf, + records: Vec, +} +impl Records { + fn load(candidate: &Candidate) -> Result { + let run = candidate.state_root.join("run"); + state::check_private_directory(&run).map_err(|_| refused())?; + let directory = run.join("cache-quarantine"); + let mut records = Vec::new(); + match fs::symlink_metadata(&directory) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err(refused()), + Ok(metadata) => { + if !metadata.is_dir() { + return Err(refused()); + } + state::check_private_directory(&directory).map_err(|_| refused())?; + for entry in fs::read_dir(&directory).map_err(|_| refused())? { + let entry = entry.map_err(|_| refused())?; + // A torn `.pending` write or any foreign entry refuses replay. + let name = entry.file_name(); + let name = name.to_str().ok_or_else(refused)?; + let id = name.strip_suffix(".json").ok_or_else(refused)?; + if records.len() >= MAX_RECORDS + || !hex(id, 32) + || !entry.file_type().map_err(|_| refused())?.is_file() + { + return Err(refused()); + } + let record: Record = + state::read_bounded(&entry.path(), RECORD_LIMIT).map_err(|_| refused())?; + if !record.valid() || Selection::quarantine_id(&record.selection_sha256) != id { + return Err(refused()); + } + records.push(record); + } + } + } + Ok(Self { directory, records }) + } + /// At most one incomplete operation per volume; it must be resumed first. + fn pending(&self, volume_name: &str) -> Result, CandidateError> { + let mut pending = self + .records + .iter() + .filter(|r| !r.completed && r.selection.volume_name == volume_name); + let first = pending.next(); + if pending.next().is_some() { + return Err(refused()); + } + Ok(first) + } + fn write(&self, record: &Record) -> Result<(), CandidateError> { + if !record.valid() { + return Err(refused()); + } + let bytes = serde_json::to_vec_pretty(record).map_err(|_| refused())?; + if bytes.len() as u64 > RECORD_LIMIT { + return Err(refused()); + } + match fs::symlink_metadata(&self.directory) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + fs::DirBuilder::new() + .mode(0o700) + .create(&self.directory) + .map_err(|_| refused())?; + fs::File::open(self.directory.parent().ok_or_else(refused)?) + .and_then(|f| f.sync_all()) + .map_err(|_| refused())?; + } + Err(_) => return Err(refused()), + Ok(_) => {} + } + state::check_private_directory(&self.directory).map_err(|_| refused())?; + let id = Selection::quarantine_id(&record.selection_sha256); + state::write(&self.directory.join(format!("{id}.json")), record).map_err(|_| refused()) + } +} + +fn journal_incomplete(root: &Path, receipt: &Receipt) -> Result { + let pending = root.join("state.pending"); + #[cfg(target_os = "macos")] + let interrupted = interrupted_start_cleanup::incomplete(root, &receipt.run)?; + #[cfg(not(target_os = "macos"))] + let interrupted = false; + Ok(pending.exists() + || pending.is_symlink() + || interrupted + || startup::require_dependency_rebind_complete(root, receipt).is_err()) +} + +/// Evidence observed under the provider lease for one admitted target. +struct Admitted { + receipt: Receipt, + resource: Resource, + inspected: Value, + mountpoint: String, + created: String, + root_identity: String, + receipt_sha256: String, +} +fn admit( + candidate: &Candidate, + engine: &Engine<'_>, + target: &CacheQuarantineTarget<'_>, +) -> Result { + let (receipt, root) = load(candidate, engine, target.run)?; + let incomplete = journal_incomplete(&root, &receipt)?; + let resource = authorize(&receipt, incomplete, target)?.clone(); + // Owner/graph/plan labels are verified here; a foreign volume that reuses + // the recorded name fails with its own ownership diagnostic. + let inspected = inspect_resource(engine, &receipt, &resource)?.ok_or_else(refused)?; + let mountpoint = format!("/var/lib/docker/volumes/{}/_data", resource.name); + let created = inspected["CreatedAt"] + .as_str() + .filter(|v| !v.is_empty() && v.len() <= 128) + .ok_or_else(refused)? + .to_owned(); + if inspected["Name"] != resource.name.as_str() + || inspected["Mountpoint"] != mountpoint.as_str() + || inspected["Driver"] != "local" + { + return Err(refused()); + } + // Stopped containers count: any consumer could start and observe the move. + volume_subpaths::no_consumers( + &engine.request(Method::GET, "/v1.53/containers/json?all=true", None)?, + &resource.name, + )?; + let root_identity = volume_subpaths::directory_identity(engine, &resource, &inspected)?; + let receipt_sha256 = digest(&serde_json::to_vec_pretty(&receipt).map_err(|_| refused())?); + Ok(Admitted { + receipt, + resource, + inspected, + mountpoint, + created, + root_identity, + receipt_sha256, + }) +} +impl Admitted { + fn observe( + &self, + engine: &Engine<'_>, + target: &CacheQuarantineTarget<'_>, + ) -> Result, CandidateError> { + let output = engine.guest().execute( + OBSERVE, + &[&self.mountpoint, &self.root_identity, target.path], + None, + )?; + let Some((target_identity, target_bytes, allocated_bytes, entries)) = + parse_observation(&output)? + else { + return Ok(None); + }; + if !same_device(&target_identity, &self.root_identity) { + return Err(refused()); + } + Ok(Some(Selection { + version: 1, + run: self.receipt.run.clone(), + owner: self.receipt.owner.clone(), + receipt_sha256: self.receipt_sha256.clone(), + volume: self.resource.key.clone(), + volume_name: self.resource.name.clone(), + volume_created: self.created.clone(), + root_identity: self.root_identity.clone(), + path: target.path.to_owned(), + target_identity, + target_bytes, + allocated_bytes, + entries, + })) + } + /// A pending record resumes only for the identical receipt and volume incarnation. + fn resumes(&self, selection: &Selection, target: &CacheQuarantineTarget<'_>) -> bool { + selection.run == self.receipt.run + && selection.owner == self.receipt.owner + && selection.receipt_sha256 == self.receipt_sha256 + && selection.volume == self.resource.key + && selection.volume_name == self.resource.name + && selection.volume_created == self.created + && selection.root_identity == self.root_identity + && selection.path == target.path + && same_device(&selection.target_identity, &self.root_identity) + } +} + +fn report(selection: &Selection, sha: &str, phase: &str, mutation: bool) -> Value { + json!({ + "run":selection.run, + "volume":selection.volume, + "path":selection.path, + "phase":phase, + "selection_sha256":sha, + "receipt_sha256":selection.receipt_sha256, + "quarantine":Selection::quarantine_id(sha), + "target_bytes":selection.target_bytes, + "allocated_bytes":selection.allocated_bytes, + "entries":selection.entries, + "quarantined_bytes":if mutation {selection.target_bytes} else {0}, + "reclaimed_bytes":0, + "data_retained":true, + "quarantine_retained_until":"explicit_volume_removal", + "mutation":mutation + }) +} + +/// Read-only selection for `quarantine_cache`. It measures the exact admitted +/// target under the provider lease and returns the digest to review; a pending +/// interrupted operation for the volume is reported for resumption instead. +pub fn inspect_cache_quarantine( + candidate: &Candidate, + target: CacheQuarantineTarget<'_>, +) -> Result { + let engine = Engine::connect_cleanup(candidate)?; + let admitted = admit(candidate, &engine, &target)?; + let records = Records::load(candidate)?; + if let Some(record) = records.pending(&admitted.resource.name)? { + if !admitted.resumes(&record.selection, &target) { + return Err(refused()); + } + return Ok(report( + &record.selection, + &record.selection_sha256, + "resume_required", + false, + )); + } + let Some(selection) = admitted.observe(&engine, &target)? else { + return Ok( + json!({"run":target.run,"volume":target.volume,"path":target.path,"phase":"target_absent","reclaimed_bytes":0,"mutation":false}), + ); + }; + let sha = selection.sha256()?; + Ok(report(&selection, &sha, "selectable", false)) +} + +/// Move one admitted disposable cache directory into the volume's owned +/// quarantine. Requires the exact digest from `inspect_cache_quarantine`; +/// repeating it after interruption resumes the same recorded operation. +pub fn quarantine_cache( + candidate: &Candidate, + target: CacheQuarantineTarget<'_>, + expected_selection: &str, +) -> Result { + if !hex(expected_selection, 64) { + return Err(refused()); + } + let engine = Engine::connect_cleanup(candidate)?; + let admitted = admit(candidate, &engine, &target)?; + let records = Records::load(candidate)?; + let (mut record, resumed) = match records.pending(&admitted.resource.name)? { + Some(record) => { + if record.selection_sha256 != expected_selection + || !admitted.resumes(&record.selection, &target) + { + return Err(refused()); + } + (record.clone(), true) + } + None => { + let selection = admitted.observe(&engine, &target)?.ok_or_else(refused)?; + let sha = selection.sha256()?; + // An identical completed selection can never be replayed. + if sha != expected_selection + || records.records.len() >= MAX_RECORDS + || records + .records + .iter() + .any(|r| Selection::quarantine_id(&r.selection_sha256) == &sha[..32]) + { + return Err(refused()); + } + let record = Record { + version: 1, + selection_sha256: sha, + selection, + completed: false, + }; + // Commit point 1: intent is durable before any guest effect. + records.write(&record)?; + (record, false) + } + }; + let id = Selection::quarantine_id(&record.selection_sha256).to_owned(); + let quarantine = format!( + "/var/lib/docker/volumes/{}/{QUARANTINE_DIRECTORY}", + admitted.resource.name + ); + let output = engine.guest().execute( + MOVE, + &[ + &admitted.mountpoint, + &quarantine, + &id, + target.path, + &admitted.root_identity, + &record.selection.target_identity, + ], + None, + )?; + let moved = match output.as_str() { + "cache-quarantine-v1 moved\n" => true, + "cache-quarantine-v1 resumed\n" => false, + _ => return Err(refused()), + }; + if inspect_resource(&engine, &admitted.receipt, &admitted.resource)?.as_ref() + != Some(&admitted.inspected) + { + return Err(refused()); + } + // Commit point 2: completion follows the guest's synchronized rename. + record.completed = true; + records.write(&record)?; + let mut value = report( + &record.selection, + &record.selection_sha256, + "quarantined", + true, + ); + value["resumed"] = json!(resumed); + value["moved_now"] = json!(moved); + Ok(value) +} + +// Read-only. Rejects symlinked or non-directory components and reports the +// target's identity plus apparent file bytes, allocated bytes and entry count +// on the volume filesystem. Symlinks inside the target are counted, not followed. +const OBSERVE: &str = r#"set -efu +root=$1; root_identity=$2; relative=$3 +parent=$root +while test "$parent" != /; do test ! -L "$parent"; test -d "$parent"; parent=${parent%/*}; test -n "$parent" || parent=/; done +test "$(stat -c %d:%i "$root")" = "$root_identity" +path=$root +previous_ifs=$IFS; IFS=/; set -- $relative; IFS=$previous_ifs +test "$#" -ge 1 +for part in "$@"; do + case "$part" in ''|.|..) exit 65;; esac + path=$path/$part + if test ! -e "$path" && test ! -L "$path"; then printf 'absent\n'; exit 0; fi + test ! -L "$path" + test -d "$path" +done +identity=$(stat -c %d:%i "$path") +test "${identity%%:*}" = "${root_identity%%:*}" +bytes=$({ find "$path" -xdev -type f -exec stat -c %s {} + || echo failed; } | awk '$0=="failed"{bad=1;next} {s+=$1} END{if(bad)exit 1; printf "%.0f\n", s}') +allocated=$({ find "$path" -xdev -exec stat -c '%b %B' {} + || echo failed; } | awk '$0=="failed"{bad=1;next} {s+=$1*$2;n+=1} END{if(bad)exit 1; printf "%.0f %.0f\n", s, n}') +test "$(stat -c %d:%i "$root")" = "$root_identity" +printf 'present %s %s %s\n' "$identity" "$bytes" "$allocated" +"#; + +// The only mutation: one same-filesystem rename(2) under the volume root lock. +// `mv` cannot fall back to copying because both ends are checked to share the +// root's device. Accepts the exact target at its source or already at its +// destination; any other layout refuses without effects. +const MOVE: &str = r#"set -efu +root=$1; quarantine=$2; id=$3; relative=$4; root_identity=$5; target_identity=$6 +case "$id" in *[!0-9a-f]*|'') exit 64;; esac +test "${#id}" -eq 32 +test "$quarantine" = "${root%/*}/hack-quarantine" +parent=$root +while test "$parent" != /; do test ! -L "$parent"; test -d "$parent"; parent=${parent%/*}; test -n "$parent" || parent=/; done +test "$(stat -c %d:%i "$root")" = "$root_identity" +test "${target_identity%%:*}" = "${root_identity%%:*}" +exec 9<"$root" +flock -w 5 9 +test "$(stat -Lc %d:%i /proc/self/fd/9)" = "$root_identity" +if test ! -e "$quarantine" && test ! -L "$quarantine"; then mkdir -m 700 "$quarantine"; fi +test ! -L "$quarantine" +test -d "$quarantine" +test "$(stat -c %u:%a "$quarantine")" = "$(id -u):700" +test "$(stat -c %d "$quarantine")" = "${root_identity%%:*}" +path=$root +previous_ifs=$IFS; IFS=/; set -- $relative; IFS=$previous_ifs +test "$#" -ge 1 +count=$#; index=0 +for part in "$@"; do + case "$part" in ''|.|..) exit 65;; esac + index=$((index + 1)) + path=$path/$part + if test "$index" -lt "$count"; then test ! -L "$path"; test -d "$path"; fi +done +destination=$quarantine/$id +if test -e "$path" || test -L "$path"; then + test ! -L "$path" + test -d "$path" + test "$(stat -c %d:%i "$path")" = "$target_identity" + test ! -e "$destination" + test ! -L "$destination" + mv "$path" "$destination" + state=moved +else + test ! -L "$destination" + test -d "$destination" + state=resumed +fi +test "$(stat -c %d:%i "$destination")" = "$target_identity" +test "$(stat -c %d:%i "$root")" = "$root_identity" +sync -f "$root" +printf 'cache-quarantine-v1 %s\n' "$state" +"#; + +#[cfg(test)] +#[path = "cache_quarantine/tests.rs"] +mod tests; diff --git a/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs b/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs new file mode 100644 index 000000000..9d1f60ad2 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs @@ -0,0 +1,583 @@ +use super::*; +use std::os::unix::fs::{MetadataExt, PermissionsExt}; +use std::process::{Command, Output}; + +const RUN: &str = "0123456789abcdef0123456789abcdef"; + +fn receipt(phase: &str, volume: Value) -> Receipt { + serde_json::from_value(json!({ + "version":1,"run":RUN,"owner":"owner","namespace":"a".repeat(64), + "plan_id":"b".repeat(64),"phase":phase,"readiness":{}, + "resources":{ + "volume:www_next":volume, + "volume:redis":{"kind":"volume","key":"redis","name":"hkg-redis","id":null,"image":null,"phase":"created"} + } + })) + .unwrap() +} +fn cache_volume() -> Value { + json!({"disposable_caches":["dev/cache/turbopack","dev/cache/images"],"kind":"volume","key":"www_next","name":"hkg-www","id":null,"image":null,"phase":"created"}) +} +fn target<'a>(volume: &'a str, path: &'a str) -> CacheQuarantineTarget<'a> { + CacheQuarantineTarget { + run: RUN, + volume, + path, + } +} + +#[test] +fn only_an_admitted_declared_path_of_a_stopped_owned_volume_is_selectable() { + let stopped = receipt("stopped-data-retained", cache_volume()); + assert_eq!( + authorize(&stopped, false, &target("www_next", "dev/cache/turbopack")) + .unwrap() + .name, + "hkg-www" + ); + // Application data: undeclared volumes, ancestors, siblings and children. + for (volume, path) in [ + ("redis", "dump"), + ("redis", "dev/cache/turbopack"), + ("www_next", "dev/cache"), + ("www_next", "dev"), + ("www_next", "dev/cache/turbopack/sst"), + ("www_next", "server"), + ("missing", "dev/cache/turbopack"), + ] { + assert!( + authorize(&stopped, false, &target(volume, path)).is_err(), + "{volume}:{path}" + ); + } + // Traversal and absolute forms are refused before any lookup. + for path in [ + "", + "/var/lib/docker", + "../hkg-redis/_data", + "dev/cache/../../../escape", + "dev//cache/turbopack", + "dev/./cache/turbopack", + "dev/cache/turbopack/", + "$HOME", + "dev\\cache", + "dev/cache/\nturbopack", + ] { + assert!( + authorize(&stopped, false, &target("www_next", path)).is_err(), + "{path:?}" + ); + } + // Active, incomplete and non-retained graphs never admit a move. + for phase in [ + "ready-observed", + "preparing", + "restoring", + "failed-retained", + "removed", + ] { + let active = receipt(phase, cache_volume()); + assert!(authorize(&active, false, &target("www_next", "dev/cache/turbopack")).is_err()); + } + assert!(authorize(&stopped, true, &target("www_next", "dev/cache/turbopack")).is_err()); + let mut missing = cache_volume(); + missing["phase"] = json!("absent"); + assert!( + authorize( + &receipt("stopped-data-retained", missing), + false, + &target("www_next", "dev/cache/turbopack") + ) + .is_err() + ); + // A shared dependency cache has consumers beyond this graph. + let mut shared = cache_volume(); + shared["cache"] = json!({"scope":"c".repeat(64),"fingerprint":"d".repeat(64),"image":format!("sha256:{}","e".repeat(64))}); + assert!( + authorize( + &receipt("stopped-data-retained", shared), + false, + &target("www_next", "dev/cache/turbopack") + ) + .is_err() + ); + // A tampered receipt cannot smuggle a traversal through its declaration. + let mut forged = cache_volume(); + forged["disposable_caches"] = json!(["../hkg-redis/_data", "/var/lib/docker", ""]); + let forged = receipt("stopped-data-retained", forged); + for path in ["../hkg-redis/_data", "/var/lib/docker", ""] { + assert!( + authorize(&forged, false, &target("www_next", path)).is_err(), + "{path:?}" + ); + } +} + +#[test] +fn active_or_stopped_consumers_refuse_before_any_observation() { + for state in ["running", "exited", "created", "paused"] { + let containers = json!([ + {"State":"running","Mounts":[{"Type":"volume","Name":"hkg-redis"}]}, + {"State":state,"Mounts":[{"Type":"bind","Name":""},{"Type":"volume","Name":"hkg-www"}]} + ]); + assert!(volume_subpaths::no_consumers(&containers, "hkg-www").is_err()); + } + assert!( + volume_subpaths::no_consumers( + &json!([{"State":"running","Mounts":[{"Type":"volume","Name":"hkg-redis"}]}]), + "hkg-www" + ) + .is_ok() + ); +} + +#[test] +fn observation_output_is_closed_and_bounded() { + assert_eq!(parse_observation("absent\n").unwrap(), None); + assert_eq!( + parse_observation("present 12:34 100 4096 3\n").unwrap(), + Some(("12:34".into(), 100, 4096, 3)) + ); + for malformed in [ + "", + "absent", + "present 12:34 100 4096 3", + "present 12:34 100 4096 0\n", + "present 12 100 4096 3\n", + "present 12:34 -1 4096 3\n", + "present 12:34 100 4096 3 extra\n", + "present 12:34 100 4096\n", + "present 12:34 99999999999999999999999 1 1\n", + "present 12:34 1 1 1\nabsent\n", + ] { + assert!(parse_observation(malformed).is_err(), "{malformed:?}"); + } +} + +fn selection(target_identity: &str) -> Selection { + Selection { + version: 1, + run: RUN.into(), + owner: "owner".into(), + receipt_sha256: "f".repeat(64), + volume: "www_next".into(), + volume_name: "hkg-www".into(), + volume_created: "2026-09-28T19:10:13Z".into(), + root_identity: "7:100".into(), + path: "dev/cache/turbopack".into(), + target_identity: target_identity.into(), + target_bytes: 647, + allocated_bytes: 8192, + entries: 3, + } +} +fn record(target_identity: &str, completed: bool) -> Record { + let selection = selection(target_identity); + Record { + version: 1, + selection_sha256: selection.sha256().unwrap(), + selection, + completed, + } +} + +#[test] +fn durable_records_are_private_exact_bounded_and_refuse_torn_or_forged_state() { + let fixture = super::super::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + state::private_directory(&candidate.state_root.join("run")).unwrap(); + let records = Records::load(&candidate).unwrap(); + assert!(records.pending("hkg-www").unwrap().is_none()); + let pending = record("7:200", false); + records.write(&pending).unwrap(); + let loaded = Records::load(&candidate).unwrap(); + assert_eq!( + loaded.pending("hkg-www").unwrap().unwrap().selection, + pending.selection + ); + assert!(loaded.pending("hkg-redis").unwrap().is_none()); + let directory = candidate.state_root.join("run/cache-quarantine"); + let path = directory.join(format!("{}.json", &pending.selection_sha256[..32])); + // A record whose selection no longer matches its digest is never resumed. + let mut forged: Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); + forged["selection"]["path"] = json!("server"); + fs::write(&path, serde_json::to_vec_pretty(&forged).unwrap()).unwrap(); + assert!(Records::load(&candidate).is_err()); + fs::remove_file(&path).unwrap(); + records.write(&pending).unwrap(); + // Torn write evidence and foreign entries refuse automatic replay. + fs::write(path.with_extension("pending"), b"torn").unwrap(); + assert!(Records::load(&candidate).is_err()); + fs::remove_file(path.with_extension("pending")).unwrap(); + fs::write(directory.join("foreign"), b"").unwrap(); + assert!(Records::load(&candidate).is_err()); + fs::remove_file(directory.join("foreign")).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(Records::load(&candidate).is_err()); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + // Two unresolved operations for one volume are ambiguous. + let second = record("7:300", false); + records.write(&second).unwrap(); + assert!( + Records::load(&candidate) + .unwrap() + .pending("hkg-www") + .is_err() + ); + // Completion is retained as diagnosis evidence, not collected. + records + .write(&Record { + completed: true, + ..second.clone() + }) + .unwrap(); + records + .write(&Record { + completed: true, + ..pending.clone() + }) + .unwrap(); + let done = Records::load(&candidate).unwrap(); + assert!(done.pending("hkg-www").unwrap().is_none()); + assert_eq!(done.records.len(), 2); + let mut invalid = pending.clone(); + invalid.selection.target_bytes += 1; + assert!(records.write(&invalid).is_err()); +} + +/// Run a production guest script on this host. Linux runs it unchanged; macOS +/// receives PATH shims translating GNU `stat -c`, util-linux `flock`, and +/// `sync -f` to the same syscalls (lstat, fstat of fd 9, flock, fsync). +fn script(fixture: &Path, program: &str, args: &[&str]) -> Output { + let mut command = Command::new("sh"); + command.args(["-c", program, "sh"]).args(args); + if cfg!(target_os = "macos") { + let bin = fixture.join("shim"); + if !bin.exists() { + fs::create_dir(&bin).unwrap(); + for (name, body) in [ + ( + "stat", + r#"#!/bin/sh +if test "$1" = -Lc; then exec python3 -c 'import os; s=os.fstat(9); print(str(s.st_dev)+":"+str(s.st_ino))'; fi +test "$1" = -c || exit 70 +format=$2; shift 2 +case "$format" in + %d:%i) exec /usr/bin/stat -f '%d:%i' "$@";; + %d) exec /usr/bin/stat -f '%d' "$@";; + %u:%a) exec /usr/bin/stat -f '%u:%Lp' "$@";; + %s) exec /usr/bin/stat -f '%z' "$@";; + '%b %B') exec /usr/bin/stat -f '%b 512' "$@";; +esac +exit 70 +"#, + ), + ( + "flock", + r#"#!/bin/sh +test "$1" = -w && test "$3" = 9 || exit 70 +exec python3 -c 'import fcntl,time +end=time.monotonic()+float(__import__("sys").argv[1]) +while True: + try: + fcntl.flock(9, fcntl.LOCK_EX|fcntl.LOCK_NB) + break + except BlockingIOError: + if time.monotonic() >= end: raise SystemExit(1) + time.sleep(0.01)' "$2" +"#, + ), + ( + "sync", + r#"#!/bin/sh +test "$1" = -f || exit 70 +exec python3 -c 'import os,sys; fd=os.open(sys.argv[1],os.O_RDONLY); os.fsync(fd); os.close(fd)' "$2" +"#, + ), + ] { + let path = bin.join(name); + fs::write(&path, body).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + } + } + command.env( + "PATH", + format!("{}:{}", bin.display(), std::env::var("PATH").unwrap()), + ); + } + command.output().unwrap() +} +fn identity(path: &Path) -> String { + let m = fs::symlink_metadata(path).unwrap(); + format!("{}:{}", m.dev(), m.ino()) +} +fn stdout(output: Output) -> String { + assert!( + output.status.success(), + "status={}; stderr={}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} + +/// A synthetic volume: a corrupt cache, a sibling persistent marker, and app data. +struct Volume { + fixture: super::super::tests::Fixture, + root: PathBuf, + quarantine: PathBuf, + target: PathBuf, +} +const PATH: &str = "dev/cache/turbopack"; +const ID: &str = "0123456789abcdef0123456789abcdef"; +impl Volume { + fn new() -> Self { + let fixture = super::super::tests::Fixture::new(); + let root = fixture.0.join("hkg-www/_data"); + let target = root.join(PATH); + fs::create_dir_all(target.join("sst")).unwrap(); + fs::write(target.join("sst/00001.sst"), vec![7; 1000]).unwrap(); + fs::write(target.join("CURRENT"), b"corrupt-checksum").unwrap(); + std::os::unix::fs::symlink("/etc/hosts", target.join("link")).unwrap(); + fs::write(root.join("dev/cache/marker"), b"persistent").unwrap(); + fs::write(root.join("BUILD_ID"), b"keep").unwrap(); + let quarantine = fixture.0.join("hkg-www/hack-quarantine"); + Self { + fixture, + root, + quarantine, + target, + } + } + fn observe(&self, path: &str) -> Output { + script( + &self.fixture.0, + OBSERVE, + &[self.root.to_str().unwrap(), &identity(&self.root), path], + ) + } + fn move_with( + &self, + quarantine: &Path, + id: &str, + root_identity: &str, + target_identity: &str, + ) -> Output { + script( + &self.fixture.0, + MOVE, + &[ + self.root.to_str().unwrap(), + quarantine.to_str().unwrap(), + id, + PATH, + root_identity, + target_identity, + ], + ) + } + fn move_target(&self, target_identity: &str) -> Output { + self.move_with(&self.quarantine, ID, &identity(&self.root), target_identity) + } + fn untouched_application_data(&self) { + assert_eq!( + fs::read(self.root.join("dev/cache/marker")).unwrap(), + b"persistent" + ); + assert_eq!(fs::read(self.root.join("BUILD_ID")).unwrap(), b"keep"); + } +} + +#[test] +fn observation_measures_exact_bytes_without_following_links() { + let volume = Volume::new(); + let output = stdout(volume.observe(PATH)); + let (observed, bytes, allocated, entries) = parse_observation(&output).unwrap().unwrap(); + assert_eq!(observed, identity(&volume.target)); + // Two regular files; the symlink target's size is never read. + assert_eq!(bytes, 1000 + 16); + assert!(allocated > 0); + // target, sst, sst/00001.sst, CURRENT, link + assert_eq!(entries, 5); + assert_eq!(stdout(volume.observe("dev/cache/absent")), "absent\n"); + assert!(!volume.observe("../escape").status.success()); + assert!(!volume.observe("dev/../BUILD_ID").status.success()); + assert!(!volume.observe("BUILD_ID").status.success()); + assert!( + !script( + &volume.fixture.0, + OBSERVE, + &[volume.root.to_str().unwrap(), "0:0", PATH] + ) + .status + .success() + ); + volume.untouched_application_data(); +} + +#[test] +fn move_is_a_retaining_rename_that_resumes_after_interruption() { + let volume = Volume::new(); + let target_identity = identity(&volume.target); + assert_eq!( + stdout(volume.move_target(&target_identity)), + "cache-quarantine-v1 moved\n" + ); + let destination = volume.quarantine.join(ID); + assert!(!volume.target.exists()); + assert_eq!(identity(&destination), target_identity); + assert_eq!( + fs::read(destination.join("sst/00001.sst")).unwrap(), + vec![7; 1000] + ); + assert_eq!( + fs::read(destination.join("CURRENT")).unwrap(), + b"corrupt-checksum" + ); + assert_eq!( + fs::metadata(&volume.quarantine) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o700 + ); + volume.untouched_application_data(); + // Host loss after the guest rename but before host completion: replay + // recognizes the exact inode at its destination and moves nothing else. + assert_eq!( + stdout(volume.move_target(&target_identity)), + "cache-quarantine-v1 resumed\n" + ); + // A regenerated cache at the source is a different directory; the pending + // operation refuses rather than moving the new bytes too. + fs::create_dir(&volume.target).unwrap(); + fs::write(volume.target.join("fresh"), b"new").unwrap(); + assert!(!volume.move_target(&target_identity).status.success()); + assert_eq!(fs::read(volume.target.join("fresh")).unwrap(), b"new"); + assert_eq!(identity(&destination), target_identity); +} + +#[test] +fn interrupted_before_rename_resumes_by_moving_the_recorded_directory() { + let volume = Volume::new(); + let target_identity = identity(&volume.target); + // Intent was durable and the quarantine root was created, then the host died. + fs::create_dir(&volume.quarantine).unwrap(); + fs::set_permissions(&volume.quarantine, fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!( + stdout(volume.move_target(&target_identity)), + "cache-quarantine-v1 moved\n" + ); + assert_eq!(identity(&volume.quarantine.join(ID)), target_identity); +} + +#[test] +fn replaced_symlinked_or_ambiguous_targets_refuse_without_effects() { + // Replaced: same path, different directory incarnation. + let volume = Volume::new(); + let original = identity(&volume.target); + fs::rename(&volume.target, volume.root.join("dev/cache/old")).unwrap(); + fs::create_dir(&volume.target).unwrap(); + assert!(!volume.move_target(&original).status.success()); + assert!(volume.target.exists() && !volume.quarantine.join(ID).exists()); + + // Symlinked target pointing at application data outside the cache. + let volume = Volume::new(); + let original = identity(&volume.target); + fs::remove_dir_all(&volume.target).unwrap(); + std::os::unix::fs::symlink(&volume.root, &volume.target).unwrap(); + assert!(!volume.move_target(&original).status.success()); + assert!(!volume.observe(PATH).status.success()); + assert!(fs::symlink_metadata(&volume.target).unwrap().is_symlink()); + volume.untouched_application_data(); + + // Symlinked ancestor redirecting the walk outside the volume. + let volume = Volume::new(); + let outside = volume.fixture.0.join("outside/cache/turbopack"); + fs::create_dir_all(&outside).unwrap(); + let original = identity(&outside); + fs::rename(volume.root.join("dev"), volume.fixture.0.join("dev-moved")).unwrap(); + std::os::unix::fs::symlink(volume.fixture.0.join("outside"), volume.root.join("dev")).unwrap(); + assert!(!volume.move_target(&original).status.success()); + assert!(!volume.observe(PATH).status.success()); + assert!(outside.exists()); + + // Resume never adopts a different directory found at the destination. + let volume = Volume::new(); + let original = identity(&volume.target); + fs::create_dir_all(volume.quarantine.join(ID)).unwrap(); + fs::set_permissions(&volume.quarantine, fs::Permissions::from_mode(0o700)).unwrap(); + fs::rename(&volume.target, volume.root.join("dev/cache/elsewhere")).unwrap(); + assert!(!volume.move_target(&original).status.success()); + + // Source and destination both present: never choose one. + let volume = Volume::new(); + let original = identity(&volume.target); + fs::create_dir_all(volume.quarantine.join(ID)).unwrap(); + fs::set_permissions(&volume.quarantine, fs::Permissions::from_mode(0o700)).unwrap(); + assert!(!volume.move_target(&original).status.success()); + assert!(volume.target.join("CURRENT").exists()); +} + +#[test] +fn foreign_quarantine_root_lock_and_identity_refuse_without_effects() { + let volume = Volume::new(); + let original = identity(&volume.target); + // Group/world-accessible quarantine root. + fs::create_dir(&volume.quarantine).unwrap(); + fs::set_permissions(&volume.quarantine, fs::Permissions::from_mode(0o755)).unwrap(); + assert!(!volume.move_target(&original).status.success()); + fs::remove_dir(&volume.quarantine).unwrap(); + // Symlinked quarantine root. + let elsewhere = volume.fixture.0.join("elsewhere"); + fs::create_dir(&elsewhere).unwrap(); + std::os::unix::fs::symlink(&elsewhere, &volume.quarantine).unwrap(); + assert!(!volume.move_target(&original).status.success()); + assert!(fs::read_dir(&elsewhere).unwrap().next().is_none()); + fs::remove_file(&volume.quarantine).unwrap(); + // Quarantine outside the volume's own directory (even a private one), a + // malformed id, or a replaced volume root. + fs::set_permissions(&elsewhere, fs::Permissions::from_mode(0o700)).unwrap(); + assert!( + !volume + .move_with(&elsewhere, ID, &identity(&volume.root), &original) + .status + .success() + ); + assert!( + !volume + .move_with( + &volume.quarantine, + "../escape", + &identity(&volume.root), + &original + ) + .status + .success() + ); + assert!( + !volume + .move_with(&volume.quarantine, ID, "0:0", &original) + .status + .success() + ); + assert!(volume.target.exists()); + // A live holder of the volume root lock excludes the move. + { + use std::os::fd::AsRawFd; + let directory = fs::File::open(&volume.root).unwrap(); + // SAFETY: the owned descriptor stays open for the duration of flock. + assert_eq!( + unsafe { libc::flock(directory.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + assert!(!volume.move_target(&original).status.success()); + assert!(volume.target.exists()); + } + assert_eq!( + stdout(volume.move_target(&original)), + "cache-quarantine-v1 moved\n" + ); + volume.untouched_application_data(); +} diff --git a/packages/runtime-core/src/provider/graph/config.rs b/packages/runtime-core/src/provider/graph/config.rs index 0765381a8..257424239 100644 --- a/packages/runtime-core/src/provider/graph/config.rs +++ b/packages/runtime-core/src/provider/graph/config.rs @@ -290,12 +290,26 @@ pub(super) fn prepare_delivery( for (kind, entries) in [(Kind::Network, networks), (Kind::Volume, volumes)] { for (index, key) in entries.into_iter().enumerate() { let cache = (kind == Kind::Volume).then(|| caches.get(&key)).flatten(); + let disposable_caches = if kind == Kind::Volume { + plan.volumes[&key].disposable_caches.clone() + } else { + Vec::new() + }; + // A shared dependency cache has consumers outside this graph; its + // directories can never be selected through one graph's declaration. + if cache.is_some() && !disposable_caches.is_empty() { + return Err(error( + "graph_disposable_cache", + "A shared dependency cache volume cannot declare disposable cache directories.", + )); + } let resource = Resource { routing: None, networks: None, outbound: kind == Kind::Network && !plan.networks[&key].internal, cache: cache.cloned(), cache_provenance: None, + disposable_caches, kind, key: key.clone(), name: cache.map_or_else( @@ -320,6 +334,7 @@ pub(super) fn prepare_delivery( outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: name.clone(), name: format!("hkg-{run}-container-{index}"), diff --git a/packages/runtime-core/src/provider/graph/environment/tests.rs b/packages/runtime-core/src/provider/graph/environment/tests.rs index 616afbbcb..c4ea92c03 100644 --- a/packages/runtime-core/src/provider/graph/environment/tests.rs +++ b/packages/runtime-core/src/provider/graph/environment/tests.rs @@ -49,6 +49,7 @@ fn container_attachment_records_ownership_before_create_and_omits_values_from_me outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: "job".into(), name: format!("hkg-{run}-container-0"), diff --git a/packages/runtime-core/src/provider/graph/host_relay.rs b/packages/runtime-core/src/provider/graph/host_relay.rs index 9c2af9652..fc1c221f6 100644 --- a/packages/runtime-core/src/provider/graph/host_relay.rs +++ b/packages/runtime-core/src/provider/graph/host_relay.rs @@ -1010,6 +1010,7 @@ mod tests { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: "web".into(), name: "owned-web".into(), diff --git a/packages/runtime-core/src/provider/graph/hostname_change.rs b/packages/runtime-core/src/provider/graph/hostname_change.rs index 45a96fcef..80a3382ab 100644 --- a/packages/runtime-core/src/provider/graph/hostname_change.rs +++ b/packages/runtime-core/src/provider/graph/hostname_change.rs @@ -228,6 +228,7 @@ pub(super) mod tests { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind, key: key.into(), name: format!("owned-{key}"), diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index e277f8981..65497ffa9 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -51,7 +51,9 @@ pub(in crate::provider) use source_device_rebind::https_devices; pub use source_device_rebind::{ inspect as inspect_source_device_rebind, recover as recover_source_device_rebind, }; +mod cache_quarantine; mod initializer_cache; +pub use cache_quarantine::{CacheQuarantineTarget, inspect_cache_quarantine, quarantine_cache}; mod volume_subpaths; pub use dependency_hosts::dependency_address; #[cfg(target_os = "macos")] @@ -220,6 +222,10 @@ pub struct Resource { pub cache: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub cache_provenance: Option, + /// Admitted `x-hack-disposable-cache` directories of an ordinary owned volume. + /// Fixed at admission; only these exact paths may be explicitly quarantined. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub disposable_caches: Vec, pub kind: Kind, pub key: String, pub name: String, diff --git a/packages/runtime-core/src/provider/graph/native/mod.rs b/packages/runtime-core/src/provider/graph/native/mod.rs index fbb51b810..741f3e072 100644 --- a/packages/runtime-core/src/provider/graph/native/mod.rs +++ b/packages/runtime-core/src/provider/graph/native/mod.rs @@ -155,6 +155,7 @@ fn configuration_with_source( outbound: !internal, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Network, key: key.clone(), name: format!("hkn-{}-network-{index}", review.scope().run), @@ -182,6 +183,7 @@ fn configuration_with_source( outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: name.clone(), name: format!("hkn-{}-container-{index}", review.scope().run), diff --git a/packages/runtime-core/src/provider/graph/restore_history/tests.rs b/packages/runtime-core/src/provider/graph/restore_history/tests.rs index f2a262736..c6a54503c 100644 --- a/packages/runtime-core/src/provider/graph/restore_history/tests.rs +++ b/packages/runtime-core/src/provider/graph/restore_history/tests.rs @@ -25,6 +25,7 @@ fn receipt(index: usize) -> Receipt { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: "web".into(), name: "owned-web".into(), diff --git a/packages/runtime-core/src/provider/graph/shutdown.rs b/packages/runtime-core/src/provider/graph/shutdown.rs index 915f7ae5c..011b2139b 100644 --- a/packages/runtime-core/src/provider/graph/shutdown.rs +++ b/packages/runtime-core/src/provider/graph/shutdown.rs @@ -409,6 +409,7 @@ mod tests { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Container, key: "service".into(), name: "owned-service".into(), diff --git a/packages/runtime-core/src/provider/graph/source.rs b/packages/runtime-core/src/provider/graph/source.rs index 9156fe81a..11aa9ed0c 100644 --- a/packages/runtime-core/src/provider/graph/source.rs +++ b/packages/runtime-core/src/provider/graph/source.rs @@ -890,6 +890,7 @@ mod tests { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), }, )]), }; diff --git a/packages/runtime-core/src/provider/graph/storage_inventory.rs b/packages/runtime-core/src/provider/graph/storage_inventory.rs index 95a9578b0..03c224c03 100644 --- a/packages/runtime-core/src/provider/graph/storage_inventory.rs +++ b/packages/runtime-core/src/provider/graph/storage_inventory.rs @@ -207,6 +207,7 @@ mod tests { outbound: false, cache: Some(cache.clone()), cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Volume, key: "deps".into(), name: cache.name(), diff --git a/packages/runtime-core/src/provider/graph/tests.rs b/packages/runtime-core/src/provider/graph/tests.rs index aa61c788c..b45dc5d7f 100644 --- a/packages/runtime-core/src/provider/graph/tests.rs +++ b/packages/runtime-core/src/provider/graph/tests.rs @@ -400,6 +400,7 @@ fn larger_graphs_fit_by_resources_and_receipts_keep_matching_bounds() { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind, ..template.clone() }, @@ -417,6 +418,7 @@ fn larger_graphs_fit_by_resources_and_receipts_keep_matching_bounds() { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind, ..template.clone() }, @@ -1880,6 +1882,7 @@ fn outbound_network_is_pinned_and_requires_explicit_pool_capability() { outbound: true, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Network, key: "default".into(), name: "owned-network".into(), @@ -1909,6 +1912,50 @@ fn outbound_network_is_pinned_and_requires_explicit_pool_capability() { ); } +#[test] +fn admitted_receipts_carry_only_declared_disposable_cache_directories() { + let fixture = Fixture::new(); + let home = Fixture::new(); + let candidate = Candidate::discover(&home.0).unwrap(); + let document = json!({"services":{"web":{"image":format!("sha256:{}","a".repeat(64)),"read_only":true,"network_mode":"none","volumes":[ + {"type":"volume","source":"www_next","target":"/app/.next"}, + {"type":"volume","source":"redis","target":"/data"} + ]}},"volumes":{"www_next":{"x-hack-disposable-cache":["dev/cache/turbopack"]},"redis":{}}}); + fs::write(fixture.0.join("compose.yaml"), document.to_string()).unwrap(); + let review = project::plan(&candidate, fixture.options()).unwrap(); + let inputs = project::inputs::compile( + &candidate, + fixture.options(), + &review.plan_id, + &BTreeMap::new(), + ) + .unwrap(); + let prepared = config::prepare( + inputs, + &BTreeMap::from([("web".into(), Condition::Started)]), + &"a".repeat(32), + &"b".repeat(32), + None, + ) + .unwrap(); + assert_eq!( + prepared.resources["volume:www_next"].disposable_caches, + ["dev/cache/turbopack"] + ); + assert!( + prepared.resources["volume:redis"] + .disposable_caches + .is_empty() + ); + let encoded = serde_json::to_value(&prepared.resources["volume:redis"]).unwrap(); + assert!(encoded.get("disposable_caches").is_none()); + let decoded: Resource = serde_json::from_value( + serde_json::to_value(&prepared.resources["volume:www_next"]).unwrap(), + ) + .unwrap(); + assert_eq!(decoded.disposable_caches, ["dev/cache/turbopack"]); +} + #[test] fn two_subpath_mounts_keep_one_volume_and_exact_docker_selection() { let fixture = Fixture::new(); @@ -1977,6 +2024,7 @@ fn endpoint_selects_service_primary_not_unrelated_first_graph_network() { outbound: false, cache: None, cache_provenance: None, + disposable_caches: Vec::new(), kind: Kind::Network, key: key.into(), name: name.into(), diff --git a/packages/runtime-core/src/provider/graph/volume_subpaths.rs b/packages/runtime-core/src/provider/graph/volume_subpaths.rs index 12cec8b74..08a8e360c 100644 --- a/packages/runtime-core/src/provider/graph/volume_subpaths.rs +++ b/packages/runtime-core/src/provider/graph/volume_subpaths.rs @@ -189,7 +189,7 @@ pub(super) fn forget_removed( &expected_labels(receipt, resource), ) } -fn no_consumers(value: &Value, volume: &str) -> Result<(), CandidateError> { +pub(super) fn no_consumers(value: &Value, volume: &str) -> Result<(), CandidateError> { let containers = value .as_array() .filter(|v| v.len() <= 4096) diff --git a/packages/runtime-core/tests/cache_quarantine_cli_contract.rs b/packages/runtime-core/tests/cache_quarantine_cli_contract.rs new file mode 100644 index 000000000..5ede4e0c3 --- /dev/null +++ b/packages/runtime-core/tests/cache_quarantine_cli_contract.rs @@ -0,0 +1,105 @@ +//! Cache quarantine is opt-in and single-target: its exact argument shape and +//! reviewed selection are checked before any provider access. +use serde_json::Value; +use std::{fs, path::Path, process::Command}; + +fn run(home: &Path, arguments: &[String]) -> (Option, Value) { + let checkout = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../..") + .canonicalize() + .unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_hack-runtime-candidate")) + .arg("--candidate-root") + .arg(&checkout) + .args(arguments) + .env_clear() + .env("PATH", "/nonexistent") + .env("HOME", home) + .output() + .unwrap(); + assert!(output.stdout.is_empty(), "{arguments:?}"); + ( + output.status.code(), + serde_json::from_slice(&output.stderr).unwrap(), + ) +} + +#[test] +fn cache_quarantine_requires_one_exact_target_and_reviewed_selection() { + let home = std::env::temp_dir().join(format!( + "hack-cache-quarantine-cli-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + fs::create_dir(&home).unwrap(); + let strings = |values: &[&str]| values.iter().map(|v| (*v).to_owned()).collect::>(); + let run_id = "a".repeat(32); + let full = strings(&[ + "graph", + "quarantine-cache", + "--run-id", + &run_id, + "--volume", + "www_next", + "--path", + "dev/cache/turbopack", + "--expect-selection", + &"b".repeat(64), + "--json", + ]); + let mut cases = Vec::new(); + for flag in ["--run-id", "--volume", "--path", "--expect-selection"] { + let mut missing = full.clone(); + let index = missing.iter().position(|value| value == flag).unwrap(); + missing.drain(index..index + 2); + cases.push(missing); + } + // A second target, repeated flag, reordered flags or unrelated mutation flag. + let mut second = full.clone(); + second.extend(strings(&["--path", "dev/cache/images"])); + cases.push(second); + let mut duplicate = full.clone(); + duplicate.push("--json".into()); + cases.push(duplicate); + let mut reordered = full.clone(); + reordered.swap(4, 6); + reordered.swap(5, 7); + cases.push(reordered); + let mut remove = full.clone(); + remove.insert(10, "--remove-data".into()); + cases.push(remove); + // Read-only selection takes no expected digest. + let mut inspect = full.clone(); + inspect[1] = "inspect-cache-quarantine".into(); + cases.push(inspect.clone()); + inspect.truncate(8); + inspect.push("--expect-selection".into()); + cases.push(inspect); + for arguments in cases { + let (status, failure) = run(&home, &arguments); + assert_eq!(status, Some(2), "{arguments:?}"); + assert_eq!(failure["code"], "graph_arguments", "{arguments:?}"); + assert_eq!(fs::read_dir(&home).unwrap().count(), 0); + } + // A malformed selection digest refuses before the provider is contacted. + for selection in [ + "B".repeat(64), + "b".repeat(63), + "b".repeat(65), + String::new(), + ] { + let mut arguments = full.clone(); + arguments[9] = selection; + let (status, failure) = run(&home, &arguments); + assert_eq!(status, Some(2), "{arguments:?}"); + assert_eq!(failure["code"], "graph_cache_quarantine", "{arguments:?}"); + // Closed, value-free diagnostic: no path or digest is echoed. + let message = failure["message"].as_str().unwrap(); + assert!(!message.contains("turbopack") && !message.contains("www_next")); + assert_eq!(fs::read_dir(&home).unwrap().count(), 0); + } + fs::remove_dir(&home).unwrap(); +} diff --git a/packages/runtime-core/tests/project_contract.rs b/packages/runtime-core/tests/project_contract.rs index e48c9bdd5..a254806e2 100644 --- a/packages/runtime-core/tests/project_contract.rs +++ b/packages/runtime-core/tests/project_contract.rs @@ -2289,3 +2289,92 @@ fn named_volume_subpaths_are_explicit_canonical_and_plan_bound() { assert!(project::plan(&fixture.candidate, fixture.options()).is_err()); } } + +#[test] +fn disposable_cache_declarations_are_exact_volume_relative_and_review_bound() { + let service = "services:\n web:\n image: alpine:3.21\n command: [echo, hello]\n volumes: ['www_next:/app/.next', 'redis:/data']\n"; + let plain = Fixture::new(&format!( + "{service}volumes:\n www_next: {{}}\n redis: {{}}\n" + )) + .plan(); + let fixture = Fixture::new(&format!( + "{service}volumes:\n www_next: {{x-hack-disposable-cache: [dev/cache/turbopack, dev/cache/images]}}\n redis: {{}}\n" + )); + let review = fixture.plan(); + assert!(review.plan.enrollment_compatible); + assert_eq!( + review.plan.volumes["www_next"].disposable_caches, + ["dev/cache/turbopack", "dev/cache/images"] + ); + assert!(review.plan.volumes["redis"].disposable_caches.is_empty()); + // The declaration is reviewed intent; absent declarations keep plan bytes. + assert_ne!(plain.plan_id, review.plan_id); + assert!( + serde_json::to_value(&review.plan.volumes["redis"]) + .unwrap() + .get("disposable_caches") + .is_none() + ); + let nine = (0..9) + .map(|n| format!("c{n}")) + .collect::>() + .join(", "); + for declaration in [ + "[../escape]".to_owned(), + "[/var/lib/docker]".into(), + "[dev/../redis]".into(), + "[dev//cache]".into(), + "[dev/cache/]".into(), + "['']".into(), + "['$HOME']".into(), + "[dev/cache, dev/cache]".into(), + "[dev/cache, dev/cache/turbopack]".into(), + "[dev/cache/turbopack, dev/cache]".into(), + format!("[{nine}]"), + "dev/cache".into(), + "[1]".into(), + ] { + let fixture = Fixture::new(&format!( + "{service}volumes:\n www_next: {{x-hack-disposable-cache: {declaration}}}\n redis: {{}}\n" + )); + assert_eq!( + project::plan(&fixture.candidate, fixture.options()) + .unwrap_err() + .code, + "invalid_disposable_cache", + "{declaration}" + ); + } + // A declared cache cannot be, or contain, a mounted volume subpath; a cache + // inside a mounted subpath remains selectable. + let subpath = |cache: &str| { + format!( + "services:\n web:\n image: alpine:3.21\n command: [echo, hello]\n volumes:\n - {{type: volume, source: www_next, target: /app/.next, volume: {{subpath: apps/www}}}}\nvolumes:\n www_next: {{x-hack-disposable-cache: [{cache}]}}\n" + ) + }; + for cache in ["apps/www", "apps"] { + let fixture = Fixture::new(&subpath(cache)); + assert_eq!( + project::plan(&fixture.candidate, fixture.options()) + .unwrap_err() + .code, + "invalid_disposable_cache", + "{cache}" + ); + } + let nested = Fixture::new(&subpath("apps/www/dev/cache/turbopack")).plan(); + assert_eq!( + nested.plan.volumes["www_next"].disposable_caches, + ["apps/www/dev/cache/turbopack"] + ); + // A shared dependency cache volume is never a per-graph disposable target. + let fixture = Fixture::new( + "services:\n web:\n image: alpine:3.21\n command: [echo, hello]\n labels: {hack.dependencies.cache-volume: deps, hack.dependencies.lockfiles: bun.lock}\n volumes: ['deps:/app/node_modules']\nvolumes:\n deps: {x-hack-disposable-cache: [.cache]}\n", + ); + assert_eq!( + project::plan(&fixture.candidate, fixture.options()) + .unwrap_err() + .code, + "invalid_disposable_cache" + ); +} From 0983d7a6852f6d81f4266ec095d9312f10bc2671 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:16:22 -0400 Subject: [PATCH 10/21] fix(secrets): converge concurrent key provisioning and add sentinel scan controls HACK-1127 (WU14) F9. - tests/env-secret-sentinel-scan.test.ts: CI synthetic-sentinel scan of a primary checkout and linked worktree after env add/materialize/exec, up (Compose generation), env apply, logs, list, explain and doctor plus a careless `git add -A` commit. Zero raw/hex/base64 hits required in output, child argv, tracked files, Git objects, HACK_HOME, HOME caches and runtime files; plaintext only in ignored, untracked, 0600 .hack/.env and compose.env.override.yml. - The modern compose.env.override.yml (decrypted values) is now written private (0600, atomic, symlink refusal). - encrypted_file first-use key bootstrap publishes the key exclusively (link(2) of a fsynced 0600 sibling) so concurrent first users adopt one key instead of replacing a key the store was already encrypted with. - Project key creation runs under a cooperative lock in the git common dir when no key exists, key files are published complete, and a read that overlaps a publication is retried briefly; primary and linked checkouts now converge. - Cross-context v1 controls: moves to another key are refused value-free; same-key moves are pinned as accepted and documented as residual risk. - docs/security/env-secrets*.md, docs/env.md, docs/architecture.md updated. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 3 +- docs/env.md | 6 +- docs/security/env-secrets-threat-model.md | 74 +- docs/security/env-secrets.md | 52 +- src/commands/project.ts | 13 +- src/lib/private-file.ts | 57 ++ src/lib/project-env-config.ts | 116 ++- src/lib/secret-store.ts | 77 +- tests/aes-gcm-v1-context-binding.test.ts | 331 +++++++++ tests/env-secret-sentinel-scan.test.ts | 668 ++++++++++++++++++ tests/helpers/secret-fixture.ts | 91 +++ ...ecret-key-provisioning-convergence.test.ts | 389 ++++++++++ 12 files changed, 1805 insertions(+), 72 deletions(-) create mode 100644 tests/env-secret-sentinel-scan.test.ts create mode 100644 tests/helpers/secret-fixture.ts create mode 100644 tests/secret-key-provisioning-convergence.test.ts diff --git a/docs/architecture.md b/docs/architecture.md index 2feaacffb..eedd166c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -113,7 +113,8 @@ Projects can declare shareable env overlays and safely inject secrets into compo At runtime, hack resolves the selected overlay and injects the effective env directly into compose and host-side command flows. For service-scoped values, hack still generates `.hack/.internal/compose.env.override.yml` so compose services get the right per-service environment -without requiring `.hack/.env` to exist. +without requiring `.hack/.env` to exist. That file holds resolved plaintext, so it is written like +`.hack/.env`: atomically, mode `0600`, and never through a symlink. See `docs/env.md` for the current env model and migration notes. diff --git a/docs/env.md b/docs/env.md index 3452d237e..e2c5a60a2 100644 --- a/docs/env.md +++ b/docs/env.md @@ -557,6 +557,7 @@ Write-path guarantees in a linked worktree (first secret added from any checkout - otherwise an existing shared key (git common dir) is reused - otherwise the primary checkout's key is adopted by copying it to the shared location, so sibling checkouts converge on one key - only then is a new key generated, and it is written to the shared location — never silently to the checkout +- concurrent first use from several checkouts converges on one key: when no key exists, creation runs under a cooperative lock at `/.hack.secret.key.lock`, and a key file is published only when complete, never empty or partial (Hack v4 writers do not take the lock) - if the shared location cannot be resolved (git unavailable) or written, Hack falls back to a checkout-local key and prints a loud divergence warning - `hack doctor` flags divergent `.hack.secret.key` contents across checkouts with the exact paths @@ -652,7 +653,10 @@ Some older surfaces are still compatibility-oriented: `.lock` so concurrent writers, including separate processes, do not lose updates. A store path that is a symlink is refused for writes. Explicit key provisioning also uses an atomic mode-`0600` replacement and refuses symlinked - key paths or immediate parent directories. This protects individual key writes; + key paths or immediate parent directories. A key generated on first use is + published only when no key file exists, so concurrent first users adopt the key + published first instead of replacing it under a store already encrypted with + it. This protects individual key writes; it does not make key replacement and store re-encryption a crash-safe rotation transaction. Keep the previous key until recovery has been verified. The v1 store envelope and legacy `v1:` values authenticate no associated data: they are not diff --git a/docs/security/env-secrets-threat-model.md b/docs/security/env-secrets-threat-model.md index dea4e2144..9e968dcde 100644 --- a/docs/security/env-secrets-threat-model.md +++ b/docs/security/env-secrets-threat-model.md @@ -76,6 +76,9 @@ refer to `hack-dance/hack`. | A symlinked `.hack/.env`, overlay file or `.hack` directory redirects a write, or a reader sees a partial file. | `src/lib/private-env-file.ts` (`readPrivateEnvFile`, `writePrivateEnvFileIfChanged`) and `replacePrivateFileAtomic` in `src/lib/private-file.ts`: a private sibling is written and fsynced, then renamed over the target; symlinks and non-regular entries are refused with one fixed message (#302). Used by `src/lib/hack-env.ts` and `materializeProjectEnv`. | | A world-readable env file or store. | `.hack/.env`, overlays and the `encrypted_file` store are written 0600, and a same-content file with group or other access is tightened (#302). The `hack doctor` env key check reports `env-key-weak-permissions` and `env-key-wrong-owner` (#309). | | Two writers of the `encrypted_file` store lose an update. | An in-process queue plus a cooperative `.lock` using the owner protocol from `src/lib/projects-registry-lock.ts`; a symlinked store path is refused for writes (`src/lib/secret-store.ts`, #302). | +| Concurrent first users of the `encrypted_file` store each generate a key; the last key file wins and the store, already encrypted under another key, can no longer be decrypted. | The generated key is published with `publishPrivateFileExclusive` (`src/lib/private-file.ts`): a fsynced 0600 sibling is linked into place only when no key file exists, so losers adopt the published key. A store found non-empty after the first key read re-reads the key file before the keychain fallback. Proven across eight worker processes by `tests/secret-key-provisioning-convergence.test.ts`. | +| Concurrent first secrets in the primary checkout and a linked worktree create two different project keys (checkout-local and shared), so values do not decrypt across checkouts; or a reader refuses a key file that another process is still publishing. | When no key exists, `ensureProjectEnvSecretKey` (`src/lib/project-env-config.ts`) creates it under a cooperative lock at `/.hack.secret.key.lock` and repeats the lookup under the lock. Key files are published complete with `publishPrivateFileExclusive`, and a key read that overlaps a publication is retried for about a quarter second while the entry is a regular file. Proven by `tests/secret-key-provisioning-convergence.test.ts`. Hack v4 writers do not take the lock. | +| Resolved plaintext for Compose lands in a world-readable file or is redirected through a symlink. | The modern projection `.hack/.internal/compose.env.override.yml` is written with `writePrivateEnvFileIfChanged` (0600, atomic, symlink refusal) in `resolveModernComposeEnvOverrides` (`src/commands/project.ts`). The legacy dotenv path writes `${KEY}` interpolations only, not values. | ### Malformed or hostile ciphertext @@ -83,6 +86,7 @@ refer to `hack-dance/hack`. | --- | --- | | Truncated tags, oversized ciphertext or non-canonical base64 cause a wrong decision or a crash. | `decryptProjectEnvValue` (`src/lib/project-env-config.ts`) and `parseEncryptedEnvelope`/`decryptStore` (`src/lib/secret-store.ts`) require canonical base64 (`decodeCanonicalBase64` in `src/lib/private-file.ts`), a 12-byte IV and a 16-byte tag, pin `authTagLength: 16`, bound the size before decoding, and fail with one fixed error (#288). | | Ciphertext is moved to another variable, scope or file and still decrypts. | Not prevented. The current v1 format authenticates no associated data. `tests/aes-gcm-v1-context-binding.test.ts` pins this behavior so that any change is deliberate (#302). See [open gaps](#open-gaps). | +| Ciphertext is moved to another key context: another project, a different `HACK_ENV_SECRET_KEY`, or a store with another key. | Refused with the fixed `Invalid secure env value.` or store error, which contains neither the value nor any key. Cross-context controls in `tests/aes-gcm-v1-context-binding.test.ts` cover each move. The key alone decides: every move that keeps the key is accepted (see [cross-context substitution](#cross-context-substitution-accepted-under-v1)). | ### Public-artifact and git leakage @@ -92,6 +96,7 @@ refer to `hack-dance/hack`. | A key copy ships in build output or a static site. | `hack doctor` reports `env-key-public-path` for copies under `public/`, `static/`, `dist/`, `build/`, `out/`, `www/`, `_site/`, `.next/` and `.output/` (#309). | | A weak, short or repetitive key. | `env-key-wrong-length` and `env-key-low-entropy` (#309). Generated keys are 32 random bytes in base64url. | | Materialized `.hack/.env` or local overrides are committed. | Hack owns a committed `.hack/.gitignore` block, and `hack doctor` flags tracked generated files (see [ignore rules](../env.md#ignore-rules-committed-hackgitignore)). | +| A secret leaks into a public artifact through a path nobody listed. | `tests/env-secret-sentinel-scan.test.ts` runs in the CI `test` job. It gives a primary checkout and a linked worktree random synthetic secrets through `env add --secret --value-stdin` (shared and `--local`), runs materialize, exec, up (Compose generation), env apply, logs, list, explain and doctor, then commits everything with `git add -A`. It requires zero occurrences of any value or the project key, raw or hex/base64/base64url encoded, in CLI output, recorded child argv, tracked files, every Git object, HACK_HOME, the private HOME and its caches, and generated Compose and runtime files. Plaintext may exist only in `.hack/.env` and `.hack/.internal/compose.env.override.yml`, and key text only in key files; each must be git-ignored, untracked and mode 0600. Values may reach a child's environment only for the exec'd command and `docker compose`. Positive controls prove delivery and that the scanner finds planted encodings in files and Git objects. Docker is a recording shim; no engine, keychain or installed Hack is reached. | | A dependency vulnerability ships in Hack. | `bun install --frozen-lockfile`, and the read-only `scripts/dependabot-inventory.ts` lists open alerts (#309). Hack does not sandbox project dependencies that run under `hack host exec`; they receive the injected env by design. | ### Agent and MCP exfiltration @@ -110,31 +115,33 @@ These gaps are known and not closed on `next`. Each one needs its own change. secret. A crash or a partial run leaves files encrypted under two different keys, and Hack has no journal or resume step for this. There is also no command that rotates the `encrypted_file` store key. -2. **Crypto decision record pending independent review.** The KDF (a single SHA-256 - of the key text), the missing AAD binding, and the v1 envelope format are - characterized in tests but have no written decision record. #288 was flagged for an - independent crypto review (WU14) that has not happened yet. Any format change must - keep v1 values readable. +2. **Independent crypto review.** The KDF (a single SHA-256 of the key text), the + missing AAD binding and the v1 envelope format are characterized in tests, and + [the bounded decision record](env-secrets.md) keeps the format unchanged. The + independent crypto review flagged in #288 (WU14) has not happened. Any format + change must keep v1 values readable. 3. **Live keychain and 1Password custody receipts.** No test or receipt proves, on a real macOS host, that values stored through the `keychain` backend or piped from 1Password stay only in the expected custody: the keychain item, the encrypted YAML or store, and the target process environment. The CLI tests disable the keychain fallback (`HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK=1`) and never call 1Password. -4. **Plaintext Compose override.** For the Compose runtime, `src/commands/project.ts` - writes resolved values, including decrypted secrets, to - `.hack/.internal/compose.env.override.yml` with `writeTextFileIfChanged` from - `src/lib/fs.ts`. That call uses `Bun.write`, so the file gets the process umask - mode (often 0644) and a symlink at that path is followed. The #302 rules do not - cover it. +4. **Plaintext materializations stay plaintext.** `.hack/.env` and + `.hack/.internal/compose.env.override.yml` are now both private (0600, atomic, + symlink refusal) and git-ignored, and the sentinel scan enforces that. They still + hold decrypted values by design, so backups, file sync and anything running as the + user can read them. Other generated files under `.hack/.internal/` and + `.hack/.branch/` are written with the process umask; the scan shows they hold no + values today. 5. **YAML env file writes.** `writeProjectEnvConfigFile` in `src/lib/project-env-config.ts` also uses `writeTextFileIfChanged`, so `.hack/hack.env.*.yaml` writes follow a symlink at the file. These files hold ciphertext and non-secret plaintext, but a hostile repository could redirect the write. -6. **`encrypted_file` key file hygiene.** `readEncryptedFileKey` and - `writeEncryptedFileKey` in `src/lib/secret-store.ts` read with `readTextFile` and - write with `writeFile` plus `chmod`. They do not use the `src/lib/private-file.ts` - helpers, so they follow symlinks and do not create the key exclusively. +6. **`encrypted_file` key file reads.** Key writes now use the + `src/lib/private-file.ts` helpers: explicit provisioning replaces the key + atomically, and first-use generation publishes it exclusively. `readEncryptedFileKey` + in `src/lib/secret-store.ts` still reads with `readTextFile`, which follows a + symlink at the key path. 7. **Directory TOCTOU.** The #302 directory checks are path-based `lstat` with identity rechecks, not `openat`/`renameat`. A swap of `.hack` between the recheck and `rename(2)` is not fully closed. The store lock is cooperative, and v4 writers @@ -150,6 +157,43 @@ These gaps are known and not closed on `next`. Each one needs its own change. tokens with `security add-generic-password ... -w `, which puts the value in argv for the life of that process. The node surface is unsupported; fix or remove it before that surface returns. +11. **Cooperative key-creation lock.** Concurrent project key creation converges only + among writers that take `/.hack.secret.key.lock`. Hack v4, a + manually copied key, or a git dir that cannot hold the lock (creation then runs + unserialized, as before) can still produce divergent keys; `hack doctor` reports + divergent key files. A crashed lock owner is reclaimed only when its process is + gone, by the protocol in `src/lib/projects-registry-lock.ts`. +12. **Limits of the sentinel scan.** The scan covers the files, argv and output that + the listed commands produce with a recording Docker shim. It does not cover a real + engine (`docker inspect`, container logs, image layers), application output, crash + reports, session archives, the keychain backend or 1Password, other CLI commands, + or the native runtime backend. A zero-hit result is evidence for those paths and + encodings only; it is not a complete-redaction claim. + +## Cross-context substitution accepted under v1 + +Under the v1 formats the key alone decides whether ciphertext opens. The tests in +`tests/aes-gcm-v1-context-binding.test.ts` refuse every move to another key and +pin these moves as accepted. An attacker who can write a repository's env files can +therefore swap or copy authentic ciphertext among them and change which value a +variable receives, without knowing any key: + +- between variables and scopes in one file +- between overlays of one project (for example `staging` to `default`) +- between the shared env file and the `--local` override +- from the primary checkout to a linked worktree, which share one key +- between unrelated projects that use the same `HACK_ENV_SECRET_KEY` (for example + one CI secret for several repositories) +- between `encrypted_file` stores with the same key, whatever their paths +- across formats when the same key text is used for the project key and the + `encrypted_file` store: a store envelope re-framed as a `v1:` value decrypts to the + whole store map (the reverse is refused because the plaintext is not a store map) + +Mitigations available today: review changes to `.hack/hack.env.*.yaml` like code, +use a different key per repository and per backend, and do not reuse a project key +as `HACK_SECRETS_FILE_KEY`. Binding ciphertext to its context needs a new envelope +version with associated data and a migration contract (see +[the decision record](env-secrets.md)). ## Related diff --git a/docs/security/env-secrets.md b/docs/security/env-secrets.md index 64ff7cf41..1ce7ddaa8 100644 --- a/docs/security/env-secrets.md +++ b/docs/security/env-secrets.md @@ -48,21 +48,36 @@ therefore remains part of the threat model even after rotation. The inline `v1:` and encrypted-store envelope authenticate no associated data. Inline ciphertext is not bound to its variable, project or path; the store scopes -entry names inside its encrypted map. Cross-context copying needs explicit controls -in any future format. Existing negative controls reject incorrect decryption -material, altered or truncated payloads, invalid IV/tag lengths and oversized -ciphertext. Passing those controls does not prove all backends, exports or endpoint -behavior safe. +entry names inside its encrypted map. Cross-context controls now pin the outcome: +ciphertext moved to another key (project, `HACK_ENV_SECRET_KEY` or store key) is +refused with a fixed, value-free error, and every move that keeps the key is +accepted. The [threat model](env-secrets-threat-model.md#cross-context-substitution-accepted-under-v1) +lists the accepted moves as residual risk; binding them needs a future envelope +version. Existing negative controls also reject altered or truncated payloads, +invalid IV/tag lengths and oversized ciphertext. Passing those controls does not +prove all backends, exports or endpoint behavior safe. + +A synthetic-sentinel scan (`tests/env-secret-sentinel-scan.test.ts`, CI `test` +job) enforces the artifact policy above for one primary checkout and one linked +worktree: after env set, materialize, exec, Compose generation, env apply, logs and +doctor and a careless `git add -A` commit, no value or key appears in tracked files, +Git objects, output, child argv, HACK_HOME or caches; plaintext exists only in the +private, ignored, mode-0600 `.hack/.env` and `.hack/.internal/compose.env.override.yml`. +Its limits are listed in the threat model. ## Key writes and rotation -Project-env key creation already uses exclusive private-file creation and -worktree selection rules. Encrypted-store provisioning now stages and syncs a -mode-0600 sibling and atomically replaces a regular key; key-path and immediate -parent symlinks refuse. Store set/delete have their own cooperative lock. These -are separate operations: neither the store lock nor atomic key replacement makes -key provisioning converge across concurrent processes or coordinates rotation of -a key with its ciphertext. +Project-env key creation publishes a complete mode-0600 key file only when the +path is free, and, when no key exists in a git checkout, runs under a cooperative +lock in the git common dir, so concurrent first users in primary and linked +checkouts converge on one key. Explicit encrypted-store provisioning stages and +syncs a mode-0600 sibling and atomically replaces a regular key; key-path and +immediate parent symlinks refuse. A store key generated on first use is published +only when no key file exists, so concurrent first users adopt one key instead of +replacing a key the store is already encrypted with. Store set/delete have their +own cooperative lock. `tests/secret-key-provisioning-convergence.test.ts` proves +both convergence cases across eight worker processes. None of this coordinates +rotation of a key with its ciphertext, and Hack v4 writers take no lock. Before rotating real data, preserve a private recoverable key/ciphertext snapshot, validate recovery, inventory every overlay, linked-worktree override and backend, @@ -82,11 +97,14 @@ ordering model and compatibility gates. It is not a production rotation command. ## Remaining acceptance -HACK-1127 stays open for a complete backend/export inventory, synthetic Git-history -and runtime leakage controls, concurrent key provisioning, cross-context controls, -nonce/usage limits, crash-safe paired rotation and interrupted recovery, primary -and linked-worktree isolation, and independent review where a new construction is -proposed. Compare maintained alternatives only against a concrete distribution +HACK-1127 stays open for a complete backend/export inventory, leakage controls +beyond the synthetic scan (a real engine, application output, crash reports, +archives, the keychain backend and the native runtime), context binding for the +accepted cross-context moves, nonce/usage limits, crash-safe paired rotation and +interrupted recovery, and independent review where a new construction is proposed. +Synthetic Git-history, output, argv and cache scans, concurrent key provisioning +and cross-context controls now exist as regression tests; they are evidence for +the paths they exercise, not complete qualification. Compare maintained alternatives only against a concrete distribution and migration design; this decision does not claim they have been benchmarked or qualified. Production rotation, private off-machine backup and publication require their actual authorized destination and scope. diff --git a/src/commands/project.ts b/src/commands/project.ts index 073ce8ee2..d3a703976 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -220,6 +220,7 @@ import { resolvePreferredOpenHost, } from "../lib/open-host.ts"; import { openUrl } from "../lib/os.ts"; +import { writePrivateEnvFileIfChanged } from "../lib/private-env-file.ts"; import { defaultProjectSlugFromPath, findProjectContext, @@ -1314,10 +1315,14 @@ export async function resolveModernComposeEnvOverrides(opts: { } await ensureHackDirGitignore({ projectDir: opts.project.projectDir }); - const overrideDir = resolve(opts.project.projectDir, ".internal"); - await ensureDir(overrideDir); - const overridePath = resolve(overrideDir, "compose.env.override.yml"); - await writeTextFileIfChanged(overridePath, text); + const overridePath = resolve( + opts.project.projectDir, + ".internal", + "compose.env.override.yml" + ); + // The modern projection holds resolved plaintext, including decrypted + // secrets: write it like .hack/.env (0600, atomic, symlink refusal). + await writePrivateEnvFileIfChanged({ envFile: overridePath, text }); return { composeFiles: [overridePath], env: modern.globalEnv, diff --git a/src/lib/private-file.ts b/src/lib/private-file.ts index 3776ef235..1cb9d1adc 100644 --- a/src/lib/private-file.ts +++ b/src/lib/private-file.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import { constants, type Stats } from "node:fs"; import { type FileHandle, + link, lstat, open, realpath, @@ -363,6 +364,62 @@ export async function replacePrivateFileAtomic( await syncDirectory({ path: dirname(opts.path) }); } +/** + * Publishes `text` at `path` only when no entry exists there, and reports + * whether this call published it. + * + * The complete contents are staged in a private (0600, fsynced) sibling and + * published with link(2), which fails with EEXIST instead of replacing an + * existing entry. Unlike {@link createPrivateFileExclusive}, readers never see + * an empty or partial file, so a concurrent loser can adopt the winner's file + * immediately. An existing regular file returns false untouched; a symlink + * (dangling or not), a non-regular entry, or a symlinked or replaced directory + * from `directories` throws `refusal()`. The staging file is always removed. + */ +export async function publishPrivateFileExclusive( + opts: ReplaceablePrivateFileOptions & { readonly text: string } +): Promise { + const directories = await captureRealDirectories(opts); + if (await observeReplaceableTarget(opts)) { + return false; + } + const staging = join( + dirname(opts.path), + `.${basename(opts.path)}.${randomUUID()}.tmp` + ); + const file = await open( + staging, + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW | + constants.O_WRONLY, + 0o600 + ); + let published = false; + try { + // The umask can only remove bits; restore owner read/write explicitly. + await file.chmod(0o600); + await file.writeFile(opts.text, "utf8"); + await file.sync(); + await recheckRealDirectories({ directories, refusal: opts.refusal }); + try { + await link(staging, opts.path); + published = true; + } catch (error: unknown) { + if (!(isRecord(error) && error.code === "EEXIST")) { + throw error; + } + // Lost the race: classify what is there now (regular file or refusal). + await observeReplaceableTarget(opts); + } + } finally { + await file.close(); + await unlink(staging).catch(() => undefined); + } + await syncDirectory({ path: dirname(opts.path) }); + return published; +} + /** Best effort: persist the directory entry where the platform allows it. */ async function syncDirectory(opts: { readonly path: string }): Promise { let directory: FileHandle | null = null; diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 3a18af37c..20537975a 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -56,8 +56,8 @@ import { } from "./native-project-inputs.ts"; import { writePrivateEnvFileIfChanged } from "./private-env-file.ts"; import { - createPrivateFileExclusive, decodeCanonicalBase64, + publishPrivateFileExclusive, readBoundedPrivateFile, } from "./private-file.ts"; import { readProjectDefaultEnvConfig } from "./project.ts"; @@ -66,6 +66,7 @@ import { assertLegacyProjectInputFamily, ProjectInputSelectionError, } from "./project-input-selection.ts"; +import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; import { isNativeManagedLocalTracked, resolvePrimaryLocalProjectDir, @@ -2865,26 +2866,40 @@ async function readNativeProjectEnvKeyFile(opts: { }); } +const PROJECT_ENV_KEY_READ_ATTEMPTS = 10; +const PROJECT_ENV_KEY_READ_RETRY_DELAY_MS = 25; + /** * Legacy key files are refused unless they are regular, non-symlink, bounded * UTF-8 files, so a committed `.hack.secret.key` symlink never selects another * file as the key. Directory components keep their legacy spelling (temporary * and home directories are often reached through symlinks); only the final * component is pinned. Absent and whitespace-only files return null. + * + * A read that overlaps a concurrent creator's publication can observe the + * file's metadata change (the publisher drops its staging link) and refuse. + * While the entry is still a regular file the read is retried for a bounded + * period (about a quarter second); a persistent refusal still throws. */ async function readLegacyProjectEnvKeyFile(opts: { readonly path: string; }): Promise { - try { - return await readBoundedPrivateFile({ - path: opts.path, - directories: [], - canonical: false, - maxBytes: NATIVE_CONFIG_INPUT_LIMIT, - refusal: projectEnvKeyFileError, - }); - } catch { - throw projectEnvKeyFileError(); + for (let attempt = 1; ; attempt += 1) { + try { + return await readBoundedPrivateFile({ + path: opts.path, + directories: [], + canonical: false, + maxBytes: NATIVE_CONFIG_INPUT_LIMIT, + refusal: projectEnvKeyFileError, + }); + } catch { + const entry = await lstat(opts.path).catch(() => null); + if (attempt >= PROJECT_ENV_KEY_READ_ATTEMPTS || !entry?.isFile()) { + throw projectEnvKeyFileError(); + } + await Bun.sleep(PROJECT_ENV_KEY_READ_RETRY_DELAY_MS); + } } } @@ -2952,9 +2967,61 @@ export type EnsureProjectEnvSecretKeyResult = { * symlinks, a git-tracked selected or adopted key is refused before any write, * and new key files are created exclusively with mode 0600 (an existing entry, * including a dangling symlink, is never replaced or followed). + * + * Concurrency: when no key exists, creation in a git checkout runs under a + * cooperative lock at `/.hack.secret.key.lock`, so concurrent + * first users across primary and linked checkouts converge on one key. Legacy + * v4 writers do not take this lock. */ export async function ensureProjectEnvSecretKey(opts: { readonly projectRoot: string; +}): Promise { + await assertLegacyProjectInputFamily({ projectRoot: opts.projectRoot }); + const sharedLocation = await resolveProjectEnvSharedKeyLocation({ + projectRoot: opts.projectRoot, + }); + if ( + sharedLocation === null || + (await pathExists(resolveProjectEnvKeyPath(opts))) || + (await pathExists(sharedLocation.path)) + ) { + return await ensureProjectEnvSecretKeyUnlocked(opts); + } + // No key yet. The primary checkout mints a checkout-local key and a linked + // worktree mints a shared one, so concurrent first users in different + // checkouts would each select a different key. One cooperative lock in the + // git common dir serializes creation; under it the full lookup runs again + // and adopts a key another checkout created meanwhile. + let started = false; + try { + return await withProjectsRegistryLock({ + lockPath: `${sharedLocation.path}.lock`, + label: "project env key", + run: async () => { + started = true; + return await ensureProjectEnvSecretKeyUnlocked(opts); + }, + }); + } catch (error: unknown) { + if (started || !isProjectEnvKeyLockUnavailable(error)) { + throw error; + } + // The git dir cannot hold a lock (for example it is read-only): create + // the key unserialized, as before; the writer's own fallbacks apply. + return await ensureProjectEnvSecretKeyUnlocked(opts); + } +} + +function isProjectEnvKeyLockUnavailable(error: unknown): boolean { + return ( + isRecord(error) && + typeof error.code === "string" && + ["EACCES", "ENOENT", "EPERM", "EROFS"].includes(error.code) + ); +} + +async function ensureProjectEnvSecretKeyUnlocked(opts: { + readonly projectRoot: string; }): Promise { await assertLegacyProjectInputFamily({ projectRoot: opts.projectRoot }); const keyPath = resolveProjectEnvKeyPath({ projectRoot: opts.projectRoot }); @@ -3099,10 +3166,11 @@ async function writeProjectEnvKeyWithFallback(opts: { } /** - * Creates the key file exclusively (O_CREAT|O_EXCL|O_NOFOLLOW, mode 0600, - * fsync) and returns the key text now stored at `path`. When the path is - * already occupied, a valid untracked regular key file (e.g. a concurrent - * writer's) is adopted; anything else, including a symlink, is refused. + * Publishes the key file only when the path is free (a fsynced mode-0600 + * sibling linked into place, so no reader ever sees an empty or partial key) + * and returns the key text now stored at `path`. When the path is already + * occupied, a valid untracked regular key file (e.g. a concurrent writer's) + * is adopted; anything else, including a symlink, is refused. */ async function writeProjectEnvKeyFile(opts: { readonly projectRoot: string; @@ -3112,16 +3180,16 @@ async function writeProjectEnvKeyFile(opts: { await assertLegacyProjectInputFamily({ projectRoot: opts.projectRoot }); await ensureDir(dirname(opts.path)); await assertLegacyProjectInputFamily({ projectRoot: opts.projectRoot }); - try { - await createPrivateFileExclusive({ - path: opts.path, - text: `${opts.keyText}\n`, - }); + const published = await publishPrivateFileExclusive({ + path: opts.path, + text: `${opts.keyText}\n`, + // Like an O_EXCL|O_NOFOLLOW create, only the final component is checked, + // so a checkout reached through a symlinked root stays usable. + directories: [], + refusal: projectEnvKeyFileError, + }); + if (published) { return opts.keyText; - } catch (error: unknown) { - if (!(isRecord(error) && error.code === "EEXIST")) { - throw error; - } } return await adoptExistingProjectEnvKeyFile({ path: opts.path }); } diff --git a/src/lib/secret-store.ts b/src/lib/secret-store.ts index 87f7070b1..31a177cc0 100644 --- a/src/lib/secret-store.ts +++ b/src/lib/secret-store.ts @@ -17,6 +17,9 @@ import { resolveGlobalHackDir } from "./config-paths.ts"; import { ensureDir, readTextFile } from "./fs.ts"; import { decodeCanonicalBase64, + publishPrivateFileExclusive, + type ReplaceablePrivateFileOptions, + readReplaceablePrivateFile, replacePrivateFileAtomic, } from "./private-file.ts"; import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; @@ -627,17 +630,33 @@ async function resolveEncryptedFileKeyMaterial(input?: { })); if (configuredKeyPath && shouldBootstrapFileKey) { const generated = randomBytes(32).toString("base64url"); - await writeEncryptedFileKey({ + const key = await createOrAdoptEncryptedFileKey({ keyPath: configuredKeyPath, key: generated, }); - cachedEncryptedFileKey = generated; + cachedEncryptedFileKey = key; encryptedFileKeyFailureCooldownUntilMs = 0; return { - key: generated, - source: "generated", + key, + source: key === generated ? "generated" : "file", }; } + if (configuredKeyPath) { + // A first user publishes its key file before it writes the store, so a + // store found non-empty after the first key read implies a key file that + // may have appeared since. Prefer it to the keychain. + const publishedKey = await readEncryptedFileKey({ + keyPath: configuredKeyPath, + }); + if (publishedKey) { + cachedEncryptedFileKey = publishedKey; + encryptedFileKeyFailureCooldownUntilMs = 0; + return { + key: publishedKey, + source: "file", + }; + } + } const nowMs = Date.now(); if (nowMs < encryptedFileKeyFailureCooldownUntilMs) { @@ -742,22 +761,60 @@ async function readEncryptedFileKey(input: { return key.length > 0 ? key : null; } -async function writeEncryptedFileKey(input: { +function encryptedFileKeyOptions(input: { readonly keyPath: string; - readonly key: string; -}): Promise { - await ensureDir(dirname(input.keyPath)); - await replacePrivateFileAtomic({ +}): ReplaceablePrivateFileOptions { + return { path: input.keyPath, - text: `${input.key}\n`, directories: [dirname(input.keyPath)], refusal: () => new Error( "Refused encrypted secret key write: the key path or its directory is aliased or not a regular file; values omitted." ), + }; +} + +/** Explicit provisioning: replaces the key file atomically (0600). */ +async function writeEncryptedFileKey(input: { + readonly keyPath: string; + readonly key: string; +}): Promise { + await ensureDir(dirname(input.keyPath)); + await replacePrivateFileAtomic({ + ...encryptedFileKeyOptions(input), + text: `${input.key}\n`, }); } +/** + * First-use bootstrap of a generated key; returns the key now stored. + * + * Concurrent first users must converge on one key: a key that is replaced + * after another process has encrypted the store with it leaves that store + * undecryptable. The generated key is therefore published only when no key + * file exists (complete contents, never a partial file); otherwise the key + * already published is adopted. An existing empty key file is replaced as + * before, since a published key is never empty. + */ +async function createOrAdoptEncryptedFileKey(input: { + readonly keyPath: string; + readonly key: string; +}): Promise { + await ensureDir(dirname(input.keyPath)); + const options = encryptedFileKeyOptions(input); + if ( + await publishPrivateFileExclusive({ ...options, text: `${input.key}\n` }) + ) { + return input.key; + } + const existing = (await readReplaceablePrivateFile(options))?.text.trim(); + if (existing) { + return existing; + } + await replacePrivateFileAtomic({ ...options, text: `${input.key}\n` }); + return input.key; +} + async function readEncryptedEntries(input: { readonly filePath: string; readonly encryptionKey: string; diff --git a/tests/aes-gcm-v1-context-binding.test.ts b/tests/aes-gcm-v1-context-binding.test.ts index 3f29993bb..753151bfe 100644 --- a/tests/aes-gcm-v1-context-binding.test.ts +++ b/tests/aes-gcm-v1-context-binding.test.ts @@ -39,6 +39,7 @@ import { } from "../src/lib/project-env-config.ts"; import { resolveSecretStore } from "../src/lib/secret-store.ts"; import { restoreEnv } from "./helpers/env.ts"; +import { createSecretWorktreeFixture } from "./helpers/secret-fixture.ts"; const STORE_KEY = "synthetic-v1-binding-store-key"; const ENV_KEY = "synthetic-v1-binding-env-key"; @@ -480,3 +481,333 @@ for (const [label, aad] of [ } }); } + +// --------------------------------------------------------------------------- +// Cross-context substitution controls (HACK-1127) +// +// Each case moves authentic v1 ciphertext into another context. The key alone +// decides the outcome: a different key refuses with the fixed error and never +// reveals the value or a key; the same key accepts, whatever the variable, +// scope, overlay, override file, checkout, project or backend. Accepted moves +// are residual risks in docs/security/env-secrets-threat-model.md. +// --------------------------------------------------------------------------- + +const OTHER_ENV_KEY = "synthetic-v1-binding-other-env-key"; + +type ProjectPaths = { + readonly projectRoot: string; + readonly projectDir: string; +}; + +async function createKeyedProject(opts: { + readonly name: string; + readonly keyText: string | null; +}): Promise { + const projectRoot = resolve(requireRoot(), opts.name); + const projectDir = resolve(projectRoot, ".hack"); + await mkdir(projectDir, { recursive: true }); + await writeFile( + resolve(projectDir, PROJECT_COMPOSE_FILENAME), + "services:\n api: {}\n" + ); + await writeFile( + resolve(projectDir, PROJECT_CONFIG_FILENAME), + `${JSON.stringify({ name: opts.name, dev_host: `${opts.name}.hack` })}\n` + ); + if (opts.keyText !== null) { + await writeFile( + resolve(projectRoot, PROJECT_ENV_KEY_FILENAME), + `${opts.keyText}\n` + ); + } + return { projectRoot, projectDir }; +} + +/** Encrypts a value through the real writer; returns its file and token. */ +async function sealThroughWriter(opts: { + readonly project: ProjectPaths; + readonly envName?: string | null; + readonly key?: string; + readonly value?: string; + readonly local?: boolean; +}): Promise<{ readonly filePath: string; readonly secure: string }> { + const key = opts.key ?? "TOKEN"; + const written = await setProjectEnvValue({ + projectRoot: opts.project.projectRoot, + projectDir: opts.project.projectDir, + envName: opts.envName ?? null, + scope: "global", + key, + value: opts.value ?? SENTINEL, + secret: true, + local: opts.local, + }); + const config = await readLegacyConfig(written.filePath); + return { + filePath: written.filePath, + secure: secureOf(config, "global", key), + }; +} + +/** Sets one secure entry in an env file, keeping the rest of the file. */ +async function plantSecure(opts: { + readonly filePath: string; + readonly scope: string; + readonly key: string; + readonly secure: string; +}): Promise { + const text = await readFile(opts.filePath, "utf8").catch(() => null); + const config: LegacyConfig = + text === null + ? { + version: 1, + environment: "default", + secretsprovider: "project_key", + values: {}, + } + : (YAML.parse(text) as LegacyConfig); + await writeFile( + opts.filePath, + JSON.stringify({ + ...config, + values: { + ...config.values, + [opts.scope]: { + ...(config.values[opts.scope] ?? {}), + [opts.key]: { secure: opts.secure }, + }, + }, + }) + ); +} + +async function resolveSelected(opts: { + readonly project: ProjectPaths; + readonly envName?: string | null; +}) { + const resolved = await resolveProjectEnvConfig({ + projectRoot: opts.project.projectRoot, + projectDir: opts.project.projectDir, + envName: opts.envName ?? null, + serviceNames: ["api"], + }); + if (!resolved) { + throw new Error("missing resolved config"); + } + return resolved; +} + +async function expectFixedEnvRefusal(pending: Promise): Promise { + const error = await pending.then( + () => null, + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + expect(error instanceof Error ? error.message : "").toBe( + "Invalid secure env value." + ); + for (const privateText of [SENTINEL, ENV_KEY, OTHER_ENV_KEY, STORE_KEY]) { + expect(String(error)).not.toContain(privateText); + expect(JSON.stringify(error)).not.toContain(privateText); + } +} + +async function caughtStoreRead(): Promise { + return await (await resolveSecretStore({ projectName: "app" })) + .get({ key: "TOKEN" }) + .then( + () => null, + (caught: unknown) => caught + ); +} + +test("cross-context: a v1 value moved to a project with another key is refused without disclosure", async () => { + const source = await createKeyedProject({ name: "source", keyText: ENV_KEY }); + const target = await createKeyedProject({ + name: "target", + keyText: OTHER_ENV_KEY, + }); + const { secure } = await sealThroughWriter({ project: source }); + await plantSecure({ + filePath: resolve(target.projectDir, "hack.env.default.yaml"), + scope: "global", + key: "TOKEN", + secure, + }); + await expectFixedEnvRefusal(resolveSelected({ project: target })); +}); + +test("cross-context: a HACK_ENV_SECRET_KEY that differs from the sealing key is refused", async () => { + const source = await createKeyedProject({ name: "source", keyText: ENV_KEY }); + const { secure } = await sealThroughWriter({ project: source }); + // A keyless checkout (for example CI) falls back to HACK_ENV_SECRET_KEY. + const keyless = await createKeyedProject({ name: "keyless", keyText: null }); + await plantSecure({ + filePath: resolve(keyless.projectDir, "hack.env.default.yaml"), + scope: "global", + key: "TOKEN", + secure, + }); + process.env.HACK_ENV_SECRET_KEY = OTHER_ENV_KEY; + await expectFixedEnvRefusal(resolveSelected({ project: keyless })); + // Control: the sealing key opens the same moved value. + process.env.HACK_ENV_SECRET_KEY = ENV_KEY; + expect((await resolveSelected({ project: keyless })).globalEnv.TOKEN).toBe( + SENTINEL + ); +}); + +test("cross-context (accepted): a v1 value moved between overlays of one project decrypts", async () => { + const project = await createKeyedProject({ + name: "overlays", + keyText: ENV_KEY, + }); + const sealed = await sealThroughWriter({ project, envName: "staging" }); + const defaultFile = resolve(project.projectDir, "hack.env.default.yaml"); + expect(sealed.filePath).not.toBe(defaultFile); + await plantSecure({ + filePath: defaultFile, + scope: "api", + key: "MOVED_FROM_STAGING", + secure: sealed.secure, + }); + expect( + (await resolveSelected({ project, envName: "default" })).serviceEnv.api + ?.MOVED_FROM_STAGING + ).toBe(SENTINEL); +}); + +test("cross-context (accepted): a v1 value moved from the shared file to the local override decrypts", async () => { + const project = await createKeyedProject({ name: "local", keyText: ENV_KEY }); + const shared = await sealThroughWriter({ project }); + const local = await sealThroughWriter({ + project, + key: "LOCAL_TOKEN", + value: OTHER_SENTINEL, + local: true, + }); + expect(local.filePath).not.toBe(shared.filePath); + await plantSecure({ + filePath: local.filePath, + scope: "global", + key: "LOCAL_TOKEN", + secure: shared.secure, + }); + expect((await resolveSelected({ project })).globalEnv.LOCAL_TOKEN).toBe( + SENTINEL + ); +}); + +test("cross-context (accepted): projects that share HACK_ENV_SECRET_KEY accept each other's values", async () => { + process.env.HACK_ENV_SECRET_KEY = ENV_KEY; + const first = await createKeyedProject({ name: "first", keyText: null }); + const second = await createKeyedProject({ name: "second", keyText: null }); + // Sealed under the shared environment key, as a CI-only checkout would be. + await writeLegacyValue({ + configPath: resolve(first.projectDir, "hack.env.default.yaml"), + secure: legacyValue(), + }); + expect((await resolveSelected({ project: first })).globalEnv.TOKEN).toBe( + SENTINEL + ); + await copyFile( + resolve(first.projectDir, "hack.env.default.yaml"), + resolve(second.projectDir, "hack.env.default.yaml") + ); + expect((await resolveSelected({ project: second })).globalEnv.TOKEN).toBe( + SENTINEL + ); +}); + +test("cross-context (accepted): a v1 value moved from the primary checkout to a linked worktree decrypts", async () => { + const fixture = await createSecretWorktreeFixture({ + parent: requireRoot(), + home: join(requireRoot(), "home"), + name: "worktree-binding", + }); + const primary = { + projectRoot: fixture.primaryRoot, + projectDir: join(fixture.primaryRoot, ".hack"), + }; + const linked = { + projectRoot: fixture.linkedRoot, + projectDir: join(fixture.linkedRoot, ".hack"), + }; + const { secure } = await sealThroughWriter({ project: primary }); + await plantSecure({ + filePath: resolve(linked.projectDir, "hack.env.default.yaml"), + scope: "api", + key: "FROM_PRIMARY", + secure, + }); + expect( + (await resolveSelected({ project: linked })).serviceEnv.api?.FROM_PRIMARY + ).toBe(SENTINEL); +}); + +test("cross-context: an encrypted_file envelope moved to a store with another key is refused", async () => { + const storePath = resolve(requireRoot(), "secrets.enc.json"); + await configureStore({ storePath }); + await (await resolveSecretStore({ projectName: "app" })).set({ + key: "TOKEN", + value: SENTINEL, + }); + const otherPath = resolve(requireRoot(), "other.enc.json"); + await copyFile(storePath, otherPath); + await configureStore({ storePath: otherPath }); + process.env.HACK_SECRETS_FILE_KEY = OTHER_ENV_KEY; + const error = await caughtStoreRead(); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("Cannot decrypt encrypted secret store at "); + for (const privateText of [SENTINEL, STORE_KEY, OTHER_ENV_KEY]) { + expect(String(error)).not.toContain(privateText); + } +}); + +test("cross-context (accepted, reused key text only): store ciphertext re-framed as a v1 env value decrypts to the whole store map", async () => { + const storePath = resolve(requireRoot(), "secrets.enc.json"); + await configureStore({ storePath }); + await (await resolveSecretStore({ projectName: "app" })).set({ + key: "TOKEN", + value: SENTINEL, + }); + const envelope = JSON.parse(await readFile(storePath, "utf8")) as Record< + string, + string + >; + const project = await createKeyedProject({ + name: "reused-key", + keyText: STORE_KEY, + }); + await plantSecure({ + filePath: resolve(project.projectDir, "hack.env.default.yaml"), + scope: "global", + key: "REFRAMED", + secure: ["v1", envelope.iv, envelope.tag, envelope.ciphertext].join(":"), + }); + expect((await resolveSelected({ project })).globalEnv.REFRAMED).toBe( + STORE_PLAINTEXT + ); +}); + +test("cross-context: a v1 env value re-framed as a store envelope under a reused key is refused", async () => { + const project = await createKeyedProject({ + name: "reframe-source", + keyText: STORE_KEY, + }); + const { secure } = await sealThroughWriter({ project }); + const [, iv, tag, ciphertext] = secure.split(":"); + const storePath = resolve(requireRoot(), "reframed.enc.json"); + await writeFile( + storePath, + `${JSON.stringify({ version: 1, algorithm: "aes-256-gcm", iv, tag, ciphertext })}\n` + ); + await configureStore({ storePath }); + // Authentication succeeds under the reused key; the plaintext is not a map. + const error = await caughtStoreRead(); + expect(error).toBeInstanceOf(Error); + expect(String(error)).toContain("Decrypted secret store payload"); + for (const privateText of [SENTINEL, STORE_KEY]) { + expect(String(error)).not.toContain(privateText); + } +}); diff --git a/tests/env-secret-sentinel-scan.test.ts b/tests/env-secret-sentinel-scan.test.ts new file mode 100644 index 000000000..3a3af311d --- /dev/null +++ b/tests/env-secret-sentinel-scan.test.ts @@ -0,0 +1,668 @@ +import { afterAll, beforeAll, expect, test } from "bun:test"; +import { createHash, randomBytes } from "node:crypto"; +import { + chmod, + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join, relative, resolve, sep } from "node:path"; + +import { privateCliEnv, privateCliHome } from "./helpers/private-cli-env.ts"; +import { + createSecretWorktreeFixture, + runFixtureGit, + type SecretWorktreeFixture, +} from "./helpers/secret-fixture.ts"; + +/** + * Synthetic-sentinel scan of public artifacts (HACK-1127). + * + * A primary checkout and a linked worktree get synthetic secrets through the + * source CLI (`env add --secret --value-stdin`, shared and `--local`), then + * run materialize, exec, up (Compose generation), env apply, logs, list, + * explain and doctor. Everything is then committed carelessly + * (`git add -A`), as a user might before pushing a public repository. + * + * Required result: zero occurrences of any sentinel value or of the project + * key, raw or hex/base64/base64url encoded, in CLI output, captured child + * argv, tracked files, any Git object of the fixture repository, or any other + * file on disk (HACK_HOME, the private HOME and its caches, generated Compose + * and runtime files, host-command records and logs). The only exceptions are + * the private-by-design plaintext materializations (`.hack/.env` and + * `.hack/.internal/compose.env.override.yml`) and the key files themselves, + * each of which must be git-ignored, untracked and mode 0600. + * + * Positive controls prove the scan is not vacuous: every sentinel reaches the + * exec'd command, the encrypted env file is committed, and each allowed + * plaintext artifact actually contains a sentinel. + * + * Isolation: private HOME/HACK_HOME/Docker client config (privateCliEnv), a + * `docker` shim first on PATH that only records its argv and fails, so no real + * engine, OrbStack, keychain or client configuration is reached. Sentinels are + * random per run and are never printed; failures report paths and forms only. + */ + +const REPO_ROOT = resolve(import.meta.dir, ".."); +const ENTRYPOINT = resolve(REPO_ROOT, "index.ts"); +const RECORDER = resolve(import.meta.dir, "fixtures/spawn-argv-recorder.ts"); +const PROJECT_KEY_FILENAME = ".hack.secret.key"; +const SCAN_TIMEOUT_MS = 240_000; + +type CliRun = { + readonly label: string; + readonly exitCode: number; + readonly stdout: string; + readonly stderr: string; +}; + +type SpawnRecord = { + readonly via: string; + readonly argv: unknown; + readonly env: unknown; +}; + +const runId = randomBytes(6).toString("hex"); +/** Synthetic secret values; never printed. */ +const SENTINELS = { + PRIMARY_GLOBAL: `hacksentinel-primary-global-${runId}-v4qz`, + PRIMARY_API: `hacksentinel-primary-api-${runId}-k9wd`, + LINKED_GLOBAL: `hacksentinel-linked-global-${runId}-p2mx`, + LINKED_LOCAL: `hacksentinel-linked-local-${runId}-t7ry`, +} as const; +type SentinelName = keyof typeof SENTINELS; + +let root = ""; +let fixture: SecretWorktreeFixture | null = null; +const runs: CliRun[] = []; +const execDigests = new Map>(); + +function requireFixture(): SecretWorktreeFixture { + if (!fixture) { + throw new Error("Missing fixture"); + } + return fixture; +} + +function home(): string { + return privateCliHome(root); +} + +function spawnLogPath(): string { + return join(root, "logs", "spawn.jsonl"); +} + +function sha256(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex"); +} + +async function cli(opts: { + readonly label: string; + readonly cwd: string; + readonly args: readonly string[]; + readonly stdin?: string; +}): Promise { + // Only the shim directory (docker shim, git link) and the base system + // directories: no installed hack, Docker client, brew or OrbStack tools. + // `hack` re-invocations (for example env apply -> restart) then fall back + // to this source CLI. + const env = privateCliEnv({ + root, + env: { + PATH: [join(root, "shim"), "/usr/bin", "/bin"].join(":"), + HACK_HOME: join(root, "hack-home"), + HACK_NO_INTERACTIVE: "1", + NO_COLOR: "1", + HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", + HACK_TEST_SPAWN_ARGV_LOG: spawnLogPath(), + }, + }); + const child = Bun.spawn( + [process.execPath, "--preload", RECORDER, ENTRYPOINT, ...opts.args], + { + cwd: opts.cwd, + env, + stdin: + opts.stdin === undefined + ? "ignore" + : new TextEncoder().encode(opts.stdin), + stdout: "pipe", + stderr: "pipe", + } + ); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + const run = { label: opts.label, exitCode, stdout, stderr }; + runs.push(run); + return run; +} + +/** Value-free description of a failed run (exit code only). */ +function expectExit(run: CliRun, code: number): void { + expect({ label: run.label, exitCode: run.exitCode }).toEqual({ + label: run.label, + exitCode: code, + }); +} + +/** Hashes (never values) of the named variables as seen by the exec'd child. */ +const EXEC_PROBE = [ + "const { createHash } = require('node:crypto');", + "const out = {};", + "for (const name of process.argv.slice(1)) {", + " const value = process.env[name];", + " out[name] = value === undefined ? null : createHash('sha256').update(value, 'utf8').digest('hex');", + "}", + "process.stdout.write(JSON.stringify(out));", +].join("\n"); + +async function addSecret(opts: { + readonly cwd: string; + readonly name: SentinelName; + readonly extra?: readonly string[]; +}): Promise { + const run = await cli({ + label: `env add ${opts.name}`, + cwd: opts.cwd, + args: [ + "env", + "add", + opts.name, + "--secret", + "--value-stdin", + ...(opts.extra ?? []), + ], + stdin: `${SENTINELS[opts.name]}\n`, + }); + expectExit(run, 0); +} + +async function exerciseCheckout(opts: { + readonly label: string; + readonly cwd: string; + readonly expected: readonly SentinelName[]; +}): Promise { + expectExit( + await cli({ + label: `${opts.label} materialize`, + cwd: opts.cwd, + args: ["env", "materialize"], + }), + 0 + ); + const exec = await cli({ + label: `${opts.label} exec`, + cwd: opts.cwd, + args: [ + "env", + "exec", + "--", + process.execPath, + "-e", + EXEC_PROBE, + ...opts.expected, + ], + }); + expectExit(exec, 0); + execDigests.set( + opts.label, + JSON.parse(exec.stdout) as Record + ); + // Compose generation runs before the (shimmed) engine call fails. + for (const [suffix, args] of [ + ["up", ["up", "--detach"]], + ["env apply", ["env", "apply", "--service", "api"]], + ["logs", ["logs", "--no-follow", "--compose"]], + ["env list", ["env", "list", "--json"]], + ["env explain", ["env", "explain", "PRIMARY_GLOBAL", "--json"]], + ["doctor", ["doctor", "--json"]], + ] as const) { + await cli({ label: `${opts.label} ${suffix}`, cwd: opts.cwd, args }); + } +} + +function commitEverything(cwd: string, message: string): void { + runFixtureGit({ cwd, home: home(), args: ["add", "-A"] }); + runFixtureGit({ + cwd, + home: home(), + args: ["commit", "-q", "--allow-empty", "-m", message], + }); +} + +beforeAll(async () => { + // Canonical path, so paths reported by git compare equal to walked paths. + root = await realpath(await mkdtemp(join(tmpdir(), "hack-sentinel-scan-"))); + await mkdir(join(root, "logs"), { recursive: true }); + await mkdir(join(root, "shim"), { recursive: true }); + const shim = join(root, "shim", "docker"); + await writeFile( + shim, + [ + "#!/bin/sh", + `for arg in "$@"; do printf '%s\\n' "$arg" >> '${join(root, "logs", "docker-argv.log")}'; done`, + "echo 'docker shim: no engine' >&2", + "exit 1", + "", + ].join("\n") + ); + await chmod(shim, 0o755); + await symlink( + await realpath(Bun.which("git") ?? "/usr/bin/git"), + join(root, "shim", "git") + ); + fixture = await createSecretWorktreeFixture({ + parent: join(root, "src"), + home: home(), + name: `sentinel-${runId}`, + }); + const { primaryRoot, linkedRoot } = fixture; + + await addSecret({ cwd: primaryRoot, name: "PRIMARY_GLOBAL" }); + await addSecret({ + cwd: primaryRoot, + name: "PRIMARY_API", + extra: ["--service", "api"], + }); + commitEverything(primaryRoot, "careless: primary secrets"); + runFixtureGit({ + cwd: linkedRoot, + home: home(), + args: ["merge", "-q", "--ff-only", "main"], + }); + await addSecret({ cwd: linkedRoot, name: "LINKED_GLOBAL" }); + await addSecret({ + cwd: linkedRoot, + name: "LINKED_LOCAL", + extra: ["--local"], + }); + + await exerciseCheckout({ + label: "primary", + cwd: primaryRoot, + expected: ["PRIMARY_GLOBAL"], + }); + await exerciseCheckout({ + label: "linked", + cwd: linkedRoot, + expected: ["PRIMARY_GLOBAL", "LINKED_GLOBAL", "LINKED_LOCAL"], + }); + commitEverything(primaryRoot, "careless: primary after runtime"); + commitEverything(linkedRoot, "careless: linked after runtime"); +}, SCAN_TIMEOUT_MS); + +afterAll(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + } +}); + +// --------------------------------------------------------------------------- +// Scanner +// --------------------------------------------------------------------------- + +type Needle = { readonly secret: string; readonly form: string }; + +/** + * Raw, hex, and the alignment-independent core of standard and URL-safe + * base64 for each byte offset modulo 3, so an encoded copy embedded at any + * position is still found. + */ +function encodedForms(text: string): string[] { + const bytes = Buffer.from(text, "utf8"); + const forms = new Set([text, bytes.toString("hex")]); + for (let shift = 0; shift < 3; shift += 1) { + const encoded = Buffer.concat([Buffer.alloc(shift), bytes]).toString( + "base64" + ); + const core = encoded.slice(4, -4); + forms.add(core); + forms.add(core.replaceAll("+", "-").replaceAll("/", "_")); + } + return [...forms].filter((form) => form.length >= 16); +} + +function needlesFor(secrets: Readonly>): Needle[] { + return Object.entries(secrets).flatMap(([secret, value]) => + encodedForms(value).map((form) => ({ secret, form })) + ); +} + +function findHits(text: string, needles: readonly Needle[]): string[] { + return [ + ...new Set( + needles + .filter((needle) => text.includes(needle.form)) + .map((needle) => needle.secret) + ), + ].sort(); +} + +async function walkFiles(dir: string, skip: (path: string) => boolean) { + const files: string[] = []; + const pending = [dir]; + while (pending.length > 0) { + const current = pending.pop() ?? ""; + for (const entry of await readdir(current, { withFileTypes: true })) { + const path = join(current, entry.name); + if (skip(path)) { + continue; + } + if (entry.isDirectory()) { + pending.push(path); + } else if (entry.isFile()) { + files.push(path); + } + } + } + return files.sort(); +} + +async function projectKeyFiles(): Promise { + const { primaryRoot } = requireFixture(); + const commonDir = runFixtureGit({ + cwd: primaryRoot, + home: home(), + args: ["rev-parse", "--path-format=absolute", "--git-common-dir"], + }).trim(); + const candidates = [ + join(primaryRoot, PROJECT_KEY_FILENAME), + join(requireFixture().linkedRoot, PROJECT_KEY_FILENAME), + join(commonDir, PROJECT_KEY_FILENAME), + ]; + const present: string[] = []; + for (const path of candidates) { + if ((await lstat(path).catch(() => null))?.isFile()) { + present.push(path); + } + } + return present; +} + +/** Every sentinel plus the project key text, keyed by a printable label. */ +async function allSecrets(): Promise> { + const secrets: Record = { ...SENTINELS }; + const keys = new Set(); + for (const path of await projectKeyFiles()) { + keys.add((await readFile(path, "utf8")).trim()); + } + let index = 0; + for (const key of keys) { + secrets[`PROJECT_KEY_${index}`] = key; + index += 1; + } + return secrets; +} + +/** Plaintext is allowed only in these documented private materializations. */ +const PRIVATE_PLAINTEXT_NAMES = new Set([".env", "compose.env.override.yml"]); + +function isPrivatePlaintextArtifact(path: string): boolean { + const { primaryRoot, linkedRoot } = requireFixture(); + return [primaryRoot, linkedRoot].some((checkout) => { + const inside = relative(join(checkout, ".hack"), path); + return ( + !inside.startsWith("..") && + PRIVATE_PLAINTEXT_NAMES.has(basename(path)) && + (inside === ".env" || inside.startsWith(`.internal${sep}`)) + ); + }); +} + +function readSpawnRecords(text: string): SpawnRecord[] { + return text + .split("\n") + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as SpawnRecord); +} + +// --------------------------------------------------------------------------- +// Positive controls +// --------------------------------------------------------------------------- + +test("positive control: the workflow delivered every sentinel to the exec'd command", () => { + expect(runs.length).toBeGreaterThan(10); + expect(execDigests.get("primary")).toEqual({ + PRIMARY_GLOBAL: sha256(SENTINELS.PRIMARY_GLOBAL), + }); + expect(execDigests.get("linked")).toEqual({ + PRIMARY_GLOBAL: sha256(SENTINELS.PRIMARY_GLOBAL), + LINKED_GLOBAL: sha256(SENTINELS.LINKED_GLOBAL), + LINKED_LOCAL: sha256(SENTINELS.LINKED_LOCAL), + }); +}); + +test("positive control: the scanner finds every planted encoding in files and Git objects", async () => { + const plantDir = join(root, "plant"); + await mkdir(plantDir, { recursive: true }); + const needles = needlesFor({ PLANTED: SENTINELS.PRIMARY_API }); + const encoded = [ + SENTINELS.PRIMARY_API, + Buffer.from(SENTINELS.PRIMARY_API).toString("hex"), + Buffer.from(`x${SENTINELS.PRIMARY_API}`).toString("base64"), + Buffer.from(`xy${SENTINELS.PRIMARY_API}`).toString("base64url"), + ]; + for (const [index, text] of encoded.entries()) { + expect(findHits(`prefix ${text} suffix`, needles)).toEqual(["PLANTED"]); + await writeFile(join(plantDir, `planted-${index}`), text); + } + const files = await walkFiles(plantDir, () => false); + expect(files.length).toBe(encoded.length); + for (const file of files) { + expect(findHits(await readFile(file, "latin1"), needles)).toEqual([ + "PLANTED", + ]); + } + // A blob written into a scratch repository is found by the object scan. + const repo = join(plantDir, "repo"); + await mkdir(repo); + runFixtureGit({ cwd: repo, home: home(), args: ["init", "-q"] }); + await writeFile(join(repo, "blob"), encoded[0] ?? ""); + runFixtureGit({ + cwd: repo, + home: home(), + args: ["hash-object", "-w", "blob"], + }); + await rm(join(repo, "blob")); + expect( + findHits( + runFixtureGit({ + cwd: repo, + home: home(), + args: ["cat-file", "--batch-all-objects", "--batch"], + }), + needles + ) + ).toEqual(["PLANTED"]); + await rm(plantDir, { recursive: true, force: true }); +}); + +// --------------------------------------------------------------------------- +// Zero-hit scans +// --------------------------------------------------------------------------- + +test("no sentinel or key appears in CLI output or captured child argv", async () => { + const needles = needlesFor(await allSecrets()); + const outputHits = runs.flatMap((run) => + findHits(`${run.stdout}\n${run.stderr}`, needles).map( + (secret) => `${run.label}: ${secret}` + ) + ); + expect(outputHits).toEqual([]); + + const records = readSpawnRecords(await readFile(spawnLogPath(), "utf8")); + expect(records.length).toBeGreaterThan(0); + const argvHits = records.flatMap((record) => + findHits(JSON.stringify(record.argv), needles).map( + (secret) => `${record.via}: ${secret}` + ) + ); + expect(argvHits).toEqual([]); + const dockerArgv = await readFile( + join(root, "logs", "docker-argv.log"), + "utf8" + ); + expect(dockerArgv).toContain("compose"); + expect(findHits(dockerArgv, needles)).toEqual([]); +}); + +test("no sentinel or key is tracked or stored in any Git object after a careless commit", async () => { + const { primaryRoot, linkedRoot } = requireFixture(); + const needles = needlesFor(await allSecrets()); + // Every object in the shared object store: reachable or not, both branches. + const objects = runFixtureGit({ + cwd: primaryRoot, + home: home(), + args: ["cat-file", "--batch-all-objects", "--batch"], + }); + expect(findHits(objects, needles)).toEqual([]); + + for (const checkout of [primaryRoot, linkedRoot]) { + const tracked = runFixtureGit({ + cwd: checkout, + home: home(), + args: ["ls-files", "-z"], + }) + .split("\0") + .filter((path) => path.length > 0); + // Not vacuous: the encrypted env file was committed. + expect(tracked).toContain(".hack/hack.env.default.yaml"); + for (const privatePath of [ + ".hack/.env", + ".hack/.internal/compose.env.override.yml", + ".hack/hack.env.local.yaml", + PROJECT_KEY_FILENAME, + ]) { + expect(tracked).not.toContain(privatePath); + } + const trackedHits: string[] = []; + for (const path of tracked) { + const text = await readFile(join(checkout, path), "latin1"); + for (const secret of findHits(text, needles)) { + trackedHits.push(`${path}: ${secret}`); + } + } + expect(trackedHits).toEqual([]); + } +}); + +test("plaintext exists on disk only in private, ignored, 0600 materializations", async () => { + const secrets = await allSecrets(); + const keyFiles = new Set(await projectKeyFiles()); + expect(keyFiles.size).toBeGreaterThan(0); + const needles = needlesFor(secrets); + const valueNeedles = needlesFor(SENTINELS); + const files = await walkFiles( + root, + (path) => + path.endsWith(`${sep}.git${sep}objects`) || path === spawnLogPath() + ); + // Not vacuous: HACK_HOME, the private HOME and generated runtime files. + expect(files.some((path) => path.includes(`${sep}hack-home${sep}`))).toBe( + true + ); + expect( + files.some((path) => path.endsWith("compose.runtime.override.yml")) + ).toBe(true); + + const unexpected: string[] = []; + const allowed: string[] = []; + for (const path of files) { + const hits = findHits(await readFile(path, "latin1"), needles); + if (hits.length === 0) { + continue; + } + const keyOnly = hits.every((secret) => secret.startsWith("PROJECT_KEY_")); + if (keyFiles.has(path) && keyOnly) { + continue; + } + const valueOnly = hits.every((secret) => secret in SENTINELS); + if (isPrivatePlaintextArtifact(path) && valueOnly) { + allowed.push(path); + continue; + } + unexpected.push(`${relative(root, path)}: ${hits.join(",")}`); + } + expect(unexpected).toEqual([]); + + // Positive control: both checkouts materialized both private artifacts. + const { primaryRoot, linkedRoot } = requireFixture(); + const expectedArtifacts = [primaryRoot, linkedRoot].flatMap((checkout) => [ + join(checkout, ".hack", ".env"), + join(checkout, ".hack", ".internal", "compose.env.override.yml"), + ]); + for (const path of expectedArtifacts) { + expect(allowed).toContain(path); + expect( + findHits(await readFile(path, "utf8"), valueNeedles).length + ).toBeGreaterThan(0); + } + + // Every plaintext artifact and key file is private and ignored. + for (const path of [...allowed, ...keyFiles]) { + expect({ + path: relative(root, path), + mode: (await lstat(path)).mode & 0o777, + }).toEqual({ path: relative(root, path), mode: 0o600 }); + } + for (const checkout of [primaryRoot, linkedRoot]) { + const inCheckout = [...allowed, ...keyFiles] + .filter((path) => !relative(checkout, path).startsWith("..")) + .map((path) => relative(checkout, path)); + const ignored = runFixtureGit({ + cwd: checkout, + home: home(), + args: ["check-ignore", "--no-index", ...inCheckout], + }) + .split("\n") + .filter((line) => line.length > 0); + expect(ignored.sort()).toEqual([...inCheckout].sort()); + } +}); + +test("secret values reach child environments only for runtime delivery", async () => { + const valueNeedles = needlesFor(SENTINELS); + const records = readSpawnRecords(await readFile(spawnLogPath(), "utf8")); + const shimDir = join(root, "shim"); + const unexpected: string[] = []; + let execDeliveries = 0; + for (const record of records) { + const argv = Array.isArray(record.argv) + ? record.argv.map((part) => String(part)) + : []; + const program = argv[0] ?? ""; + // No keychain access, no installed hack, and Docker only through the + // recording shim. + expect(["security", "hack"]).not.toContain(basename(program)); + if (["docker", "orbctl", "orb", "colima"].includes(basename(program))) { + expect(program === basename(program) || program.startsWith(shimDir)).toBe( + true + ); + } + if ( + findHits(JSON.stringify(record.env ?? null), valueNeedles).length === 0 + ) { + continue; + } + if (program === process.execPath && argv[2] === EXEC_PROBE) { + execDeliveries += 1; + continue; + } + if (basename(program) === "docker" && argv[1] === "compose") { + continue; + } + unexpected.push(`${record.via} ${basename(program)} ${argv[1] ?? ""}`); + } + expect(unexpected).toEqual([]); + expect(execDeliveries).toBe(2); +}); diff --git a/tests/helpers/secret-fixture.ts b/tests/helpers/secret-fixture.ts new file mode 100644 index 000000000..d8829be1b --- /dev/null +++ b/tests/helpers/secret-fixture.ts @@ -0,0 +1,91 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +/** + * Disposable Git fixtures for env-secret tests: a Hack project committed in a + * primary checkout plus one linked worktree, and a private Git runner. + * + * Git never reads the runner's HOME, global or system config: HOME is the + * fixture's private home, and global/system config are disabled. + */ + +export type SecretWorktreeFixture = { + readonly primaryRoot: string; + readonly linkedRoot: string; +}; + +/** Runs git with a private HOME and no global/system config; throws on failure. */ +export function runFixtureGit(opts: { + readonly cwd: string; + readonly home: string; + readonly args: readonly string[]; +}): string { + const result = Bun.spawnSync(["git", ...opts.args], { + cwd: opts.cwd, + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + HOME: opts.home, + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_AUTHOR_NAME: "Hack Test", + GIT_AUTHOR_EMAIL: "hack-test@example.invalid", + GIT_COMMITTER_NAME: "Hack Test", + GIT_COMMITTER_EMAIL: "hack-test@example.invalid", + }, + stdout: "pipe", + stderr: "pipe", + }); + if (result.exitCode !== 0) { + throw new Error( + `git ${opts.args[0] ?? ""} failed (exit ${result.exitCode}): ${result.stderr.toString()}` + ); + } + return result.stdout.toString(); +} + +/** + * Creates `/primary` (a committed Hack project with one `api` service + * and a plaintext-only env overlay) and `/linked`, a linked worktree on + * a new branch. Nothing secret-related is pre-seeded: no key, no secure value + * and no Hack ignore rules, so the CLI under test owns all of them. + */ +export async function createSecretWorktreeFixture(opts: { + readonly parent: string; + readonly home: string; + readonly name: string; +}): Promise { + const primaryRoot = resolve(opts.parent, "primary"); + const linkedRoot = resolve(opts.parent, "linked"); + const hackDir = join(primaryRoot, ".hack"); + await mkdir(hackDir, { recursive: true }); + await writeFile( + join(hackDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.20\n" + ); + await writeFile( + join(hackDir, "hack.config.json"), + `${JSON.stringify({ name: opts.name, dev_host: `${opts.name}.hack` }, null, 2)}\n` + ); + await writeFile( + join(hackDir, "hack.env.default.yaml"), + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global:\n PLAIN_FLAG: plain\n" + ); + await writeFile(join(primaryRoot, "README.md"), "# fixture\n"); + runFixtureGit({ + cwd: primaryRoot, + home: opts.home, + args: ["init", "-q", "-b", "main"], + }); + runFixtureGit({ cwd: primaryRoot, home: opts.home, args: ["add", "-A"] }); + runFixtureGit({ + cwd: primaryRoot, + home: opts.home, + args: ["commit", "-q", "-m", "fixture: init"], + }); + runFixtureGit({ + cwd: primaryRoot, + home: opts.home, + args: ["worktree", "add", "-q", "-b", "feature", linkedRoot], + }); + return { primaryRoot, linkedRoot }; +} diff --git a/tests/secret-key-provisioning-convergence.test.ts b/tests/secret-key-provisioning-convergence.test.ts new file mode 100644 index 000000000..845d9bd16 --- /dev/null +++ b/tests/secret-key-provisioning-convergence.test.ts @@ -0,0 +1,389 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { + lstat, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +import { PROJECT_ENV_KEY_FILENAME } from "../src/constants.ts"; +import { + resolveProjectEnvConfig, + resolveProjectEnvSharedKeyLocation, + setProjectEnvValue, +} from "../src/lib/project-env-config.ts"; +import { resolveSecretStore } from "../src/lib/secret-store.ts"; +import { restoreEnv } from "./helpers/env.ts"; +import { privateCliEnv, privateCliHome } from "./helpers/private-cli-env.ts"; +import { createSecretWorktreeFixture } from "./helpers/secret-fixture.ts"; + +/** + * Concurrent first-use key provisioning must converge on one key (HACK-1127). + * + * Each case starts several fresh worker processes (fresh module state, so no + * in-process key cache can hide a race), releases them together through a + * stdin barrier, and then proves that every worker ended up with the same key + * and that every value it wrote decrypts with the key left on disk. Workers + * report only a SHA-256 digest of the key, never the key. + * + * Isolation: private HOME, HACK_HOME and global config inside a temp root; + * the encrypted-store case configures an explicit key path and resolves every + * worker while the store is still empty, so the keychain fallback is never + * reached; the parent reads back with the keychain fallback disabled. + */ + +const SECRET_STORE_MODULE = resolve( + import.meta.dir, + "..", + "src", + "lib", + "secret-store.ts" +); +const PROJECT_ENV_MODULE = resolve( + import.meta.dir, + "..", + "src", + "lib", + "project-env-config.ts" +); +const WORKERS = 8; +const WRITES_PER_WORKER = 4; +const PARENT_ENV_KEYS = [ + "HOME", + "HACK_HOME", + "HACK_GLOBAL_CONFIG_PATH", + "HACK_SECRETS_FILE_KEY", + "HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK", + "HACK_ENV_SECRET_KEY", +] as const; + +let root: string | null = null; +const previousEnv = new Map(); + +beforeEach(async () => { + root = await mkdtemp(join(tmpdir(), "hack-key-convergence-")); + for (const key of PARENT_ENV_KEYS) { + previousEnv.set(key, process.env[key]); + } + process.env.HOME = privateCliHome(root); + process.env.HACK_HOME = join(root, "hack-home"); + process.env.HACK_GLOBAL_CONFIG_PATH = join(root, "hack.config.json"); + process.env.HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK = "1"; + Reflect.deleteProperty(process.env, "HACK_SECRETS_FILE_KEY"); + Reflect.deleteProperty(process.env, "HACK_ENV_SECRET_KEY"); +}); + +afterEach(async () => { + if (root) { + await rm(root, { recursive: true, force: true }); + root = null; + } + for (const key of PARENT_ENV_KEYS) { + restoreEnv(key, previousEnv.get(key)); + } + previousEnv.clear(); +}); + +function requireRoot(): string { + if (!root) { + throw new Error("Missing temp root"); + } + return root; +} + +function digest(text: string): string { + return createHash("sha256").update(text, "utf8").digest("hex"); +} + +type WorkerReport = { + readonly digest: string; + readonly keyPath?: string; + readonly created?: boolean; +}; + +type WorkerExit = { + readonly code: number; + readonly stderr: string; + readonly report: WorkerReport | null; +}; + +/** + * Runs `count` workers of `script`. Each worker prints `loaded`, waits for + * `start`, does its provisioning, prints `ready `, waits for `go`, does + * its writes and exits. The parent releases each phase only after every + * worker reached it, so provisioning overlaps and no write precedes the last + * provisioning. + */ +async function runWorkers(opts: { + readonly script: string; + readonly args: (index: number) => readonly string[]; + readonly count: number; +}): Promise { + const base = requireRoot(); + const env = privateCliEnv({ + root: base, + env: { + HACK_HOME: process.env.HACK_HOME, + HACK_GLOBAL_CONFIG_PATH: process.env.HACK_GLOBAL_CONFIG_PATH, + }, + }); + const workers = Array.from({ length: opts.count }, (_, index) => + Bun.spawn({ + cmd: [process.execPath, opts.script, ...opts.args(index)], + cwd: base, + env, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }) + ); + const readers = workers.map((worker) => lineReader(worker.stdout)); + const waitAll = async (prefix: string): Promise => + await Promise.all( + readers.map(async (next) => { + const line = await next(); + if (!line?.startsWith(prefix)) { + throw new Error(`worker did not reach ${prefix}`); + } + return line.slice(prefix.length).trim(); + }) + ); + const release = async (word: string): Promise => { + for (const worker of workers) { + worker.stdin.write(`${word}\n`); + await worker.stdin.flush(); + } + }; + let reports: string[] = []; + try { + await waitAll("loaded"); + await release("start"); + reports = await waitAll("ready"); + await release("go"); + } catch (error: unknown) { + for (const worker of workers) { + worker.kill(); + } + const stderr = await Promise.all( + workers.map(async (worker) => await new Response(worker.stderr).text()) + ); + throw new Error( + `${error instanceof Error ? error.message : String(error)}\n${stderr.join("\n")}` + ); + } finally { + for (const worker of workers) { + await worker.stdin.end(); + } + } + return await Promise.all( + workers.map(async (worker, index) => ({ + code: await worker.exited, + stderr: await new Response(worker.stderr).text(), + report: reports[index] + ? (JSON.parse(reports[index] ?? "") as WorkerReport) + : null, + })) + ); +} + +function lineReader( + stream: ReadableStream +): () => Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let buffered = ""; + return async () => { + while (!buffered.includes("\n")) { + const chunk = await reader.read(); + if (chunk.done) { + return null; + } + buffered += decoder.decode(chunk.value, { stream: true }); + } + const at = buffered.indexOf("\n"); + const line = buffered.slice(0, at); + buffered = buffered.slice(at + 1); + return line; + }; +} + +/** Worker preamble: phase barrier on stdin. */ +const BARRIER_PREAMBLE = [ + "const lines = (async function* () {", + " let buffered = '';", + " for await (const chunk of Bun.stdin.stream()) {", + " buffered += new TextDecoder().decode(chunk);", + " let at = buffered.indexOf('\\n');", + " while (at >= 0) { yield buffered.slice(0, at); buffered = buffered.slice(at + 1); at = buffered.indexOf('\\n'); }", + " }", + "})();", + "async function waitFor(word) {", + " const next = await lines.next();", + " if (next.done || next.value !== word) { process.exit(3); }", + "}", + "process.stdout.write('loaded\\n');", + "await waitFor('start');", +].join("\n"); + +async function writeGlobalStoreConfig(opts: { + readonly storePath: string; + readonly keyPath: string; +}): Promise { + await writeFile( + process.env.HACK_GLOBAL_CONFIG_PATH ?? "", + `${JSON.stringify({ + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { path: opts.storePath, keyPath: opts.keyPath }, + }, + }, + })}\n` + ); +} + +test("concurrent first use of an encrypted_file store converges on one key and loses no value", async () => { + const base = requireRoot(); + const storePath = join(base, "store", "secrets.enc.json"); + const keyPath = join(base, "keys", "secrets-file.key"); + await writeGlobalStoreConfig({ storePath, keyPath }); + const script = join(base, "store-worker.ts"); + await writeFile( + script, + [ + `import { resolveSecretStore } from ${JSON.stringify(SECRET_STORE_MODULE)};`, + BARRIER_PREAMBLE, + "const id = process.argv[2] ?? 'W';", + "const store = await resolveSecretStore({ projectName: 'app' });", + "process.stdout.write(`ready ${JSON.stringify({ digest: 'n/a' })}\\n`);", + "await waitFor('go');", + `for (let index = 0; index < ${WRITES_PER_WORKER}; index += 1) {`, + " await store.set({ key: `${id}_${index}`, value: `synthetic-${id}-${index}` });", + "}", + "", + ].join("\n") + ); + const exits = await runWorkers({ + script, + count: WORKERS, + args: (index) => [`W${index}`], + }); + for (const exit of exits) { + expect({ code: exit.code, stderr: exit.stderr }).toEqual({ + code: 0, + stderr: "", + }); + } + + // One complete key file, private, with no staging leftovers. + const keyInfo = await lstat(keyPath); + expect(keyInfo.isFile()).toBe(true); + expect(keyInfo.mode & 0o777).toBe(0o600); + expect(keyInfo.nlink).toBe(1); + expect((await readFile(keyPath, "utf8")).trim().length).toBeGreaterThan(0); + expect(await readdir(join(base, "keys"))).toEqual(["secrets-file.key"]); + + // Every worker's values decrypt with the key on disk (keychain disabled). + const store = await resolveSecretStore({ projectName: "app" }); + for (let worker = 0; worker < WORKERS; worker += 1) { + for (let index = 0; index < WRITES_PER_WORKER; index += 1) { + expect(await store.get({ key: `W${worker}_${index}` })).toBe( + `synthetic-W${worker}-${index}` + ); + } + } +}); + +test("concurrent project key creation from primary and linked checkouts converges", async () => { + const base = requireRoot(); + const fixture = await createSecretWorktreeFixture({ + parent: base, + home: privateCliHome(base), + name: "key-convergence", + }); + const script = join(base, "project-key-worker.ts"); + await writeFile( + script, + [ + `import { createHash } from "node:crypto";`, + `import { ensureProjectEnvSecretKey } from ${JSON.stringify(PROJECT_ENV_MODULE)};`, + BARRIER_PREAMBLE, + "const ensured = await ensureProjectEnvSecretKey({ projectRoot: process.argv[2] ?? '' });", + "const digest = createHash('sha256').update(ensured.keyText, 'utf8').digest('hex');", + "process.stdout.write(`ready ${JSON.stringify({ digest, keyPath: ensured.keyPath, created: ensured.created })}\\n`);", + "await waitFor('go');", + "", + ].join("\n") + ); + const exits = await runWorkers({ + script, + count: WORKERS, + args: (index) => [ + index % 2 === 0 ? fixture.primaryRoot : fixture.linkedRoot, + ], + }); + for (const exit of exits) { + expect({ code: exit.code, stderr: exit.stderr }).toEqual({ + code: 0, + stderr: "", + }); + } + const digests = new Set(exits.map((exit) => exit.report?.digest)); + expect(digests.size).toBe(1); + expect(exits.filter((exit) => exit.report?.created).length).toBe(1); + + // Every key file that exists holds that one key. + const shared = await resolveProjectEnvSharedKeyLocation({ + projectRoot: fixture.linkedRoot, + }); + const candidates = [ + resolve(fixture.primaryRoot, PROJECT_ENV_KEY_FILENAME), + resolve(fixture.linkedRoot, PROJECT_ENV_KEY_FILENAME), + shared?.path ?? "", + ]; + let present = 0; + for (const path of candidates) { + const text = await readFile(path, "utf8").catch(() => null); + if (text !== null) { + present += 1; + expect(digest(text.trim())).toBe([...digests][0] ?? ""); + expect((await lstat(path)).mode & 0o777).toBe(0o600); + } + } + expect(present).toBeGreaterThan(0); + + // A value encrypted in one checkout decrypts in the other. + for (const [writer, reader] of [ + [fixture.primaryRoot, fixture.linkedRoot], + [fixture.linkedRoot, fixture.primaryRoot], + ] as const) { + await setProjectEnvValue({ + projectRoot: writer, + projectDir: join(writer, ".hack"), + envName: null, + scope: "global", + key: "CROSS_CHECKOUT", + value: "synthetic-cross-checkout-value", + secret: true, + local: true, + }); + await writeFile( + join(reader, ".hack", "hack.env.local.yaml"), + await readFile(join(writer, ".hack", "hack.env.local.yaml"), "utf8") + ); + const resolved = await resolveProjectEnvConfig({ + projectRoot: reader, + projectDir: join(reader, ".hack"), + envName: null, + serviceNames: ["api"], + }); + expect(resolved?.globalEnv.CROSS_CHECKOUT).toBe( + "synthetic-cross-checkout-value" + ); + } +}); From a6c15bdaf54ad9be3c0c3d1c3d1355d7b71937d0 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:27:32 -0400 Subject: [PATCH 11/21] test(e2e): read only the P02 checkout probe markers The hosted run showed the legacy hack run path printing its internal DNS/TLS notices on stdout ahead of the probe output. Match the probe's own git= and checkout= lines instead of the whole stream. Co-Authored-By: Claude Opus 5.5 --- tests/e2e/scenarios/pattern-p02-portable-toolchain.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts b/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts index 54c483bea..5e8be4f38 100644 --- a/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts +++ b/tests/e2e/scenarios/pattern-p02-portable-toolchain.ts @@ -107,8 +107,16 @@ export const patternP02PortableToolchainScenario: Scenario = { }); const checkout = async (cwd: string, expected: string) => { const result = await passes(cwd, ["sh", "-c", CHECKOUT_PROBE]); + // Read only the probe's own marker lines. Without global DNS/TLS the + // legacy `hack run` path also prints its internal DNS/TLS notices on + // stdout (observed on the hosted runner); that is reported separately. + const probed = result.stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => /^(?:git|checkout)=/.test(line)) + .join("\n"); expect({ - that: result.stdout.trim() === expected, + that: probed === expected, message: "toolchain must mount the invoking checkout", result, }); From 5d0f337ff32895596ec84c436023811a76af4e9e Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:28:00 -0400 Subject: [PATCH 12/21] fix(lifecycle): keep shared helper ownership exact across time zones and lost starts Read process births in UTC so clients with different TZ values agree on a live owned helper; previously a second client saw it as stale, re-adopted it and the last release leaked it. A stale owned claim whose PID still answers as the helper is now kept (claim-invalid) instead of adopted. Report deadline (not foreign-listener) when lock contention or the deadline cut the port probe short. Stop a helper left in phase starting by a lost starter once it still cannot become ready. Rethrow observer errors after closing the gate or stopping the started helper, and report state errors raised under the lock as claim-invalid/state-unavailable rather than lock-unavailable. Bound the claim read by bytes before reading it. Add a lock barrier to the concurrent-starters fixture and regression tests for every documented check; register the suite in the Linux process job. Refs: HACK-1142 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- docs/reference/shared-host-lifecycle.md | 135 +++-- src/lib/private-file.ts | 15 +- src/lib/process-tree.ts | 12 +- src/lib/shared-host-lifecycle.ts | 452 ++++++++++++----- tests/fixtures/shared-host-helper.ts | 30 +- tests/fixtures/shared-host-starter.ts | 50 +- tests/shared-host-lifecycle.test.ts | 626 +++++++++++++++++++++++- 8 files changed, 1114 insertions(+), 208 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 987da5080..739a3c1d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -402,4 +402,4 @@ jobs: - name: Install dependencies run: bun install --frozen-lockfile - name: Run process and terminal regressions - run: bun test tests/daemon-command.test.ts tests/daemon-orphan.test.ts tests/host-exec-lifetime.test.ts tests/host-exec-tty.test.ts tests/shell-observation.test.ts + run: bun test tests/daemon-command.test.ts tests/daemon-orphan.test.ts tests/host-exec-lifetime.test.ts tests/host-exec-tty.test.ts tests/shell-observation.test.ts tests/shared-host-lifecycle.test.ts diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index 0b77dfa82..601bf98c4 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -43,27 +43,35 @@ The helper serves a JSON identity endpoint on `127.0.0.1:`: A different process answering for the port is foreign. Readiness is this identity answer. A live process, an open port or a sleeping -placeholder is never readiness. Each probe has its own timeout and reads at most 4 KiB. +placeholder is never readiness. Each probe has its own timeout and reads at most +4 KiB; a larger answer is not an identity. ## Persisted claim -The claim is `/.claim.json`, owner-only (0600), replaced atomically, -and parsed strictly (exact keys, version 1): +The claim is `/.claim.json`, owner-only (0600), at most 8 KiB, +replaced atomically, and parsed strictly (exact keys, version 1): ```json {"version":1,"helper":"issuer-bridge","port":4100,"ownership":"owned","phase":"ready","pid":4242,"birth":"Mon Oct 19 10:00:00 2026","holders":["branch-a","branch-b"]} {"version":1,"helper":"issuer-bridge","port":4100,"ownership":"adopted","holders":["branch-a"]} ``` -- Ownership is the pair `pid` + `birth` (process start time from `ps lstart`). The - claim is live only when the census shows that exact pair. A missing PID, or the - same PID with a different birth time (PID reuse), makes the claim stale. A stale - claim grants nothing. Hack discards it and never signals that PID. +- Ownership is the pair `pid` + `birth`. `birth` is the process start time from + `ps lstart`, always read with `TZ=UTC0` and `LC_ALL=C`. `ps` prints `lstart` in + local time, so without a fixed zone two clients with different `TZ` values, or a + host whose zone changes, would see different births for the same live process. +- The claim is live only when the census shows that exact pair. A missing PID, or + the same PID with a different birth time (PID reuse), makes the claim stale. A + stale claim grants nothing. Hack discards it and never signals that PID. +- One exception: if a stale owned claim's PID still answers the identity endpoint + as the expected helper, ownership is uncertain. Hack keeps the claim and fails + with `claim-invalid`. It neither adopts the helper (which would drop the owned + record and leak the helper on the last release) nor signals it. - `holders` names the branches or clients that use the helper. Holders carry over when a stale owned helper is restarted. -- A malformed, unreadable, symlinked or group/world-accessible claim, or one for a - different port, fails closed with `claim-invalid`. Hack does not start, adopt or - stop anything until an operator inspects it. +- A malformed, oversized, unreadable, symlinked or group/world-accessible claim, or + one for a different port, fails closed with `claim-invalid`. Hack does not start, + adopt or stop anything until an operator inspects it. - A census that did not observe the calling process (for example `ps` failed) fails closed with `census-unavailable`. An empty census never reads as "every claimed process is gone". @@ -73,26 +81,37 @@ and parsed strictly (exact keys, version 1): 1. `enabled: false` returns `disabled` immediately. 2. Take the helper's exclusive lock (`/.lock`, the projects-registry lock protocol: hard-link publication, dead-owner reclamation by `ESRCH` only). - Competing starters wait in turn, bounded by the deadline. + Competing starters wait in turn (`lock-waiting`, then `lock-acquired`). One + deadline bounds the lock wait, the port probe, the spawn and readiness. 3. A live owned claim is reused. The starter waits for readiness and adds its holder. - It does not spawn a second helper. If readiness fails, it leaves the helper for - the other holders. -4. Otherwise probe the port: + It does not spawn a second helper. + - A `ready` helper may serve other holders, so if readiness fails the starter + leaves it running and returns the failure. + - A `starting` claim seen under the lock has lost its starter, because a starter + holds the lock until its helper is ready or stopped. If that helper still + cannot become ready (`deadline`, `child-exited`, `foreign-listener`, + `wrong-upstream`), the starter stops it through its birth-verified tree and + removes the claim. `cancelled` and `census-unavailable` leave it recorded. +4. Otherwise, if the deadline has already passed, return `deadline`. Then probe the + port: - identity matches: write an `adopted` claim and return `ready/adopted`; - - wrong upstream: `wrong-upstream`; another helper, a silent or non-identity - listener: `foreign-listener`. Either way the listener keeps running and no - child is spawned; + - wrong upstream: `wrong-upstream`; another helper, a non-identity answer or a + listener that stays silent for the full probe timeout: `foreign-listener`. + Either way the listener keeps running and no child is spawned; + - a probe whose timeout was shortened by the deadline and then timed out proves + nothing about the listener: `deadline`; - connection refused: confirm with a loopback bind probe, then spawn. 5. Spawn through an exec gate in a new session: `/bin/sh -c 'IFS= read -r hack_gate || exit 0; [ "$hack_gate" = open ] || exit 0; exec "$@" `. - PID reuse within one `lstart` second. Birth times have one-second resolution, the same assumption as the existing lifecycle process-tree helpers. +- Linux `lstart` drift. Linux `ps` computes `lstart` from the boot time, which can + move when the wall clock is changed, so a live helper's birth can change. Start + then fails closed if the helper reports its PID in the identity answer, but a + release would read the claim as stale and clear it without stopping the helper. + macOS records the start time directly and is not affected. - Wiring into `hack.config.json`, `hack up`/`down` or the daemon, and migrating existing `lifecycle.singleton` configurations. - Native (macOS VM) and container-runtime effects. These fixtures qualify the diff --git a/src/lib/private-file.ts b/src/lib/private-file.ts index 3776ef235..6101e353b 100644 --- a/src/lib/private-file.ts +++ b/src/lib/private-file.ts @@ -278,7 +278,10 @@ async function observeReplaceableTarget(opts: { * unreadable file is never mistaken for an empty one and overwritten. */ export async function readReplaceablePrivateFile( - opts: ReplaceablePrivateFileOptions + opts: ReplaceablePrivateFileOptions & { + /** Largest accepted file size in bytes; a larger file throws `refusal()` before it is read. */ + readonly maxBytes?: number; + } ): Promise { const directories = await captureRealDirectories(opts); const observed = await observeReplaceableTarget(opts); @@ -303,11 +306,19 @@ export async function readReplaceablePrivateFile( if ( !opened.isFile() || opened.dev !== observed.dev || - opened.ino !== observed.ino + opened.ino !== observed.ino || + (opts.maxBytes !== undefined && opened.size > opts.maxBytes) ) { throw opts.refusal(); } const text = await file.readFile({ encoding: "utf8" }); + if ( + opts.maxBytes !== undefined && + Buffer.byteLength(text, "utf8") > opts.maxBytes + ) { + // The file grew in place after the size check. + throw opts.refusal(); + } await recheckRealDirectories({ directories, refusal: opts.refusal }); return { text, mode: opened.mode & 0o7777 }; } finally { diff --git a/src/lib/process-tree.ts b/src/lib/process-tree.ts index c354fc612..a6bb346b7 100644 --- a/src/lib/process-tree.ts +++ b/src/lib/process-tree.ts @@ -5,11 +5,19 @@ export type ProcessIdentityRow = { }; const IDENTITY_ROW = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/; -/** Capture only process identity and lineage; never command arguments or environment. */ +/** + * Capture only process identity and lineage; never command arguments or environment. + * + * `birth` is the `lstart` start time rendered in UTC with the C locale. `ps` + * prints `lstart` in local time, so without a fixed zone two clients with + * different `TZ` values, or one host whose zone changes, would see different + * births for the same live process. Births may be persisted and compared across + * processes, so the zone is pinned here (POSIX `UTC0` needs no zoneinfo). + */ export async function readProcessIdentities(): Promise { try { const proc = Bun.spawn(["ps", "-A", "-o", "pid=,ppid=,lstart="], { - env: { ...process.env, LC_ALL: "C" }, + env: { ...process.env, LC_ALL: "C", TZ: "UTC0" }, stdin: "ignore", stdout: "pipe", stderr: "ignore", diff --git a/src/lib/shared-host-lifecycle.ts b/src/lib/shared-host-lifecycle.ts index a50187e74..f436b6b49 100644 --- a/src/lib/shared-host-lifecycle.ts +++ b/src/lib/shared-host-lifecycle.ts @@ -29,15 +29,19 @@ import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; * - A listener is only ever adopted after its identity endpoint names the * expected helper and upstream. Anything else on the port is refused and * left running: this module never signals a process it cannot prove it owns. - * - Ownership is a persisted PID plus process birth time. A claim whose PID is + * - Ownership is a persisted PID plus process birth time (`ps lstart`, read in + * UTC so clients with different time zones agree). A claim whose PID is * gone or now belongs to a different process (PID reuse) is stale and is - * discarded without signalling anything. + * discarded without signalling anything. A stale claim whose PID still + * answers as the helper is uncertain: it is kept, never adopted. * - A child is spawned behind an exec gate: it waits on stdin until its claim * is durable. If the starter dies first, the gate reads EOF and exits, so a * spawned helper is never left running without a claim. * - Readiness is the identity endpoint answering for the expected helper, not - * process liveness. Retries back off and are bounded by one deadline; - * cancellation stops the owned child and removes its claim. + * process liveness. Retries back off and are bounded by one deadline that + * also covers the lock wait; cancellation stops the owned child and removes + * its claim. A `starting` claim found under the lock has lost its starter; + * if that helper still cannot become ready, it is stopped. * - Release removes one holder. Only when no holder remains is an owned helper * stopped, through its birth-verified process tree. Adopted helpers are * never signalled. @@ -54,6 +58,7 @@ export type SharedHostFailure = | "foreign-listener" | "wrong-upstream" | "claim-invalid" + | "state-unavailable" | "census-unavailable" | "lock-unavailable" | "port-probe-failed" @@ -68,7 +73,9 @@ const FAILURE_MESSAGES: Readonly> = { "wrong-upstream": "The shared helper on this port serves a different upstream; it was left running.", "claim-invalid": - "The persisted shared helper claim is unreadable or belongs to another configuration; inspect and release it before retrying.", + "The persisted shared helper claim is unreadable, uncertain or belongs to another configuration; inspect and release it before retrying.", + "state-unavailable": + "The shared helper state directory could not be read or written.", "census-unavailable": "The process census is unavailable, so helper ownership cannot be verified.", "lock-unavailable": @@ -135,6 +142,7 @@ export type SharedHostEnsureResult = /** Closed progress events. The observer is awaited (a test and UI seam). */ export type SharedHostEvent = + | "lock-waiting" | "lock-acquired" | "claim-stale" | "claim-reused" @@ -158,16 +166,19 @@ export type SharedHostReleaseResult = } | { readonly state: "failed"; - readonly reason: Extract< - SharedHostFailure, - | "claim-invalid" - | "census-unavailable" - | "lock-unavailable" - | "cancelled" - >; + readonly reason: ReleaseFailure; readonly message: string; }; +type ReleaseFailure = Extract< + SharedHostFailure, + | "claim-invalid" + | "state-unavailable" + | "census-unavailable" + | "lock-unavailable" + | "cancelled" +>; + type OwnedClaim = { readonly version: 1; readonly helper: string; @@ -190,6 +201,8 @@ type Claim = OwnedClaim | AdoptedClaim; type ProbeObservation = | { readonly kind: "absent" } | { readonly kind: "unidentified" } + /** The listener did not answer within this probe's own timeout. */ + | { readonly kind: "timeout" } | { readonly kind: "foreign" } | { readonly kind: "wrong-upstream" } | { readonly kind: "match"; readonly pid: number | null } @@ -214,16 +227,40 @@ class ClaimRefusal extends Error { } } +/** Carries an error thrown by the caller's observer; it is rethrown as is. */ +class ObserverError extends Error { + readonly original: unknown; + constructor(original: unknown) { + super("shared host observer failed"); + this.original = original; + } +} + +/** + * Wraps an error thrown while the helper lock is held, so it is never + * reported as a lock acquisition failure. + */ +class LockedRunError extends Error { + readonly original: unknown; + constructor(original: unknown) { + super("shared host locked operation failed"); + this.original = original; + } +} + +async function runLocked(run: () => Promise): Promise { + try { + return await run(); + } catch (error: unknown) { + throw new LockedRunError(error); + } +} + function failed(reason: SharedHostFailure): SharedHostEnsureResult { return { state: "failed", reason, message: FAILURE_MESSAGES[reason] }; } -function releaseFailed( - reason: Extract< - SharedHostFailure, - "claim-invalid" | "census-unavailable" | "lock-unavailable" | "cancelled" - > -): SharedHostReleaseResult { +function releaseFailed(reason: ReleaseFailure): SharedHostReleaseResult { return { state: "failed", reason, message: FAILURE_MESSAGES[reason] }; } @@ -373,6 +410,7 @@ async function readClaim( path: claimPath(spec), directories: [spec.stateDir], refusal: () => new ClaimRefusal(), + maxBytes: MAX_CLAIM_BYTES, }); } catch { return { kind: "invalid" }; @@ -380,7 +418,8 @@ async function readClaim( if (file === null) { return { kind: "absent" }; } - if (file.text.length > MAX_CLAIM_BYTES || (file.mode & 0o077) !== 0) { + // Only an owner-only receipt is trusted; group/world access makes it uncertain. + if ((file.mode & 0o077) !== 0) { return { kind: "invalid" }; } const claim = parseSharedHostClaim(file.text); @@ -472,7 +511,8 @@ export async function probeSharedHostIdentity(opts: { readonly timeoutMs: number; readonly signal?: AbortSignal; }): Promise { - const signals = [AbortSignal.timeout(opts.timeoutMs)]; + const timeout = AbortSignal.timeout(opts.timeoutMs); + const signals = [timeout]; if (opts.signal) { signals.push(opts.signal); } @@ -486,8 +526,11 @@ export async function probeSharedHostIdentity(opts: { if (opts.signal?.aborted) { return { kind: "cancelled" }; } - // Only a refused connection proves no listener; a timeout or reset means - // something holds the port without answering as the helper. + if (timeout.aborted) { + return { kind: "timeout" }; + } + // Only a refused connection proves no listener; a reset means something + // holds the port without answering as the helper. return isRecord(error) && error.code === "ECONNREFUSED" ? { kind: "absent" } : { kind: "unidentified" }; @@ -496,9 +539,19 @@ export async function probeSharedHostIdentity(opts: { if (opts.signal?.aborted) { return { kind: "cancelled" }; } + if (text === null && timeout.aborted) { + return { kind: "timeout" }; + } if (response.status !== 200 || text === null) { return { kind: "unidentified" }; } + return classifyIdentity(text, opts.expected); +} + +function classifyIdentity( + text: string, + expected: SharedHostIdentity +): ProbeObservation { let body: unknown; try { body = JSON.parse(text); @@ -512,10 +565,10 @@ export async function probeSharedHostIdentity(opts: { ) { return { kind: "unidentified" }; } - if (body.helper !== opts.expected.helper) { + if (body.helper !== expected.helper) { return { kind: "foreign" }; } - if (body.upstream !== opts.expected.upstream) { + if (body.upstream !== expected.upstream) { return { kind: "wrong-upstream" }; } const pid = @@ -662,10 +715,83 @@ async function awaitReadiness( } } +type OwnedIdentity = { readonly pid: number; readonly birth: string }; + +/** + * Readiness failures after which an owned helper that never reached `ready` is + * stopped. `cancelled` and `census-unavailable` leave it recorded instead: a + * cancelled caller or a blind census is not evidence that the start failed. + */ +const ABANDONED_START_FAILURES: ReadonlySet = new Set([ + "child-exited", + "deadline", + "foreign-listener", + "wrong-upstream", +]); + +/** + * Stop an owned helper whose start failed and forget its claim. When the stop + * cannot be confirmed the claim is kept, so the process stays recorded. + */ +async function stopFailedStart(opts: { + readonly context: EnsureContext; + readonly owned: OwnedIdentity; + readonly notify: boolean; +}): Promise { + const stopped = await stopVerifiedSharedHostProcess({ + ...opts.owned, + graceMs: 1000, + }); + if (stopped === "stopped" || stopped === "absent") { + await removeClaim(opts.context.spec); + } + if (opts.notify) { + await opts.context.observe("stopped-after-failure"); + } +} + +/** + * Reuse a live owned helper. A `ready` helper may serve other holders, so a + * failed readiness check never stops it. A `starting` claim seen under the + * lock has no live starter (a starter holds the lock until its helper is + * ready or stopped), so a helper that still cannot become ready is an + * abandoned start and is stopped through its birth-verified tree. + */ +async function reuseOwned( + context: EnsureContext, + claim: OwnedClaim +): Promise { + const { spec, holder } = context; + await context.observe("claim-reused"); + const outcome = await awaitReadiness(context, claim); + if (outcome.kind === "failed") { + if ( + claim.phase === "starting" && + ABANDONED_START_FAILURES.has(outcome.reason) + ) { + await stopFailedStart({ context, owned: claim, notify: true }); + } + return failed(outcome.reason); + } + await writeClaim(spec, { + ...claim, + phase: "ready", + holders: withHolder(claim.holders, holder), + }); + await context.observe("ready"); + return { + state: "ready", + ownership: "owned", + endpoint: endpointFor(spec.port), + pid: claim.pid, + spawned: false, + }; +} + async function ensureLocked( context: EnsureContext ): Promise { - const { spec, holder } = context; + const { spec } = context; await context.observe("lock-acquired"); const census = await readCensus(); if (census === null) { @@ -676,55 +802,87 @@ async function ensureLocked( return failed("claim-invalid"); } let carried: readonly string[] = []; + let staleOwned: OwnedClaim | null = null; if (read.kind === "claim") { const claim = read.claim; if (claim.port !== spec.port) { return failed("claim-invalid"); } if (claim.ownership === "owned" && isLive(census, claim)) { - await context.observe("claim-reused"); - const outcome = await awaitReadiness(context, claim); - if (outcome.kind === "failed") { - // Other holders may depend on it: a reused helper is never stopped here. - return failed(outcome.reason); - } - await writeClaim(spec, { - ...claim, - phase: "ready", - holders: withHolder(claim.holders, holder), - }); - await context.observe("ready"); - return { - state: "ready", - ownership: "owned", - endpoint: endpointFor(spec.port), - pid: claim.pid, - spawned: false, - }; + return await reuseOwned(context, claim); } if (claim.ownership === "owned") { // Dead or reused PID: the claim no longer proves ownership of anything. await context.observe("claim-stale"); + staleOwned = claim; } carried = claim.holders; } + const settled = await adoptOrRefuse(context, { carried, staleOwned }); + if (settled !== null) { + return settled; + } + try { + const occupied = await inspectListeningTcpPorts({ ports: [spec.port] }); + if (occupied.length > 0) { + return failed("foreign-listener"); + } + } catch { + return failed("port-probe-failed"); + } + if (read.kind === "claim") { + await removeClaim(spec); + } + return await spawnOwned(context, carried); +} +/** + * Probe the helper port when no live owned claim exists. Returns the final + * result (adopted, refused, deadline or cancelled), or null when nothing + * answers and the caller may spawn. + */ +async function adoptOrRefuse( + context: EnsureContext, + prior: { + readonly carried: readonly string[]; + readonly staleOwned: OwnedClaim | null; + } +): Promise { + const { spec, holder } = context; + const { carried, staleOwned } = prior; + // The lock wait shares the deadline; never start a probe after it passed. + const left = remainingMs(context); + if (left <= 0) { + return failed("deadline"); + } + const cutByDeadline = left < spec.probeTimeoutMs; const existing = await probeSharedHostIdentity({ port: spec.port, identityPath: spec.identityPath, expected: spec.expected, - timeoutMs: Math.min(spec.probeTimeoutMs, Math.max(1, remainingMs(context))), + timeoutMs: cutByDeadline ? left : spec.probeTimeoutMs, signal: context.signal, }); switch (existing.kind) { case "cancelled": return failed("cancelled"); - case "foreign": + case "timeout": + // A probe cut short by the deadline proves nothing about the listener; + // one that used its full timeout met a listener that does not answer. + return failed(cutByDeadline ? "deadline" : "foreign-listener"); case "unidentified": + case "foreign": return failed("foreign-listener"); case "wrong-upstream": return failed("wrong-upstream"); case "match": + if (staleOwned !== null && existing.pid === staleOwned.pid) { + // The claimed PID answers as the helper, but its birth no longer + // matches: ownership is uncertain. Adopting it would drop the owned + // record (and a later release would leak the helper); stopping it + // could signal a stranger. Keep the claim and fail closed. + return failed("claim-invalid"); + } await writeClaim(spec, { version: 1, helper: spec.name, @@ -740,20 +898,8 @@ async function ensureLocked( spawned: false, }; default: - break; - } - try { - const occupied = await inspectListeningTcpPorts({ ports: [spec.port] }); - if (occupied.length > 0) { - return failed("foreign-listener"); - } - } catch { - return failed("port-probe-failed"); - } - if (read.kind === "claim") { - await removeClaim(spec); + return null; } - return await spawnOwned(context, carried); } async function spawnOwned( @@ -780,7 +926,6 @@ async function spawnOwned( return failed("spawn-failed"); } child.unref(); - await context.observe("spawned"); const closeGate = () => { try { child.stdin.end(); @@ -788,73 +933,83 @@ async function spawnOwned( // Already closed; the gate treats EOF as "do not run". } }; - const census = await readCensus(); - const row = census?.find((entry) => entry.pid === child.pid); - if (!row) { - closeGate(); - return failed(census === null ? "census-unavailable" : "spawn-failed"); - } - const owned = { pid: row.pid, birth: row.birth }; - const holders = withHolder(carried, holder); + let owned: OwnedIdentity | null = null; + let gateOpened = false; try { + await context.observe("spawned"); + const census = await readCensus(); + const row = census?.find((entry) => entry.pid === child.pid); + if (!row) { + closeGate(); + return failed(census === null ? "census-unavailable" : "spawn-failed"); + } + const identity: OwnedIdentity = { pid: row.pid, birth: row.birth }; + const holders = withHolder(carried, holder); await writeClaim(spec, { version: 1, helper: spec.name, port: spec.port, ownership: "owned", phase: "starting", - pid: owned.pid, - birth: owned.birth, + pid: identity.pid, + birth: identity.birth, holders, }); - } catch { - closeGate(); - return failed("claim-invalid"); - } - await context.observe("claimed"); - if (context.signal?.aborted) { - closeGate(); - await removeClaim(spec); - return failed("cancelled"); - } - try { - child.stdin.write("open\n"); - await child.stdin.flush(); - } finally { + owned = identity; + await context.observe("claimed"); + if (context.signal?.aborted) { + closeGate(); + await removeClaim(spec); + return failed("cancelled"); + } + gateOpened = true; + try { + child.stdin.write("open\n"); + await child.stdin.flush(); + } catch { + // The gate died before it could run the helper. + closeGate(); + await stopFailedStart({ context, owned: identity, notify: false }); + return failed("spawn-failed"); + } closeGate(); - } - await context.observe("gate-opened"); - const outcome = await awaitReadiness(context, owned); - if (outcome.kind === "failed") { - const stopped = await stopVerifiedSharedHostProcess({ - ...owned, - graceMs: 1000, + await context.observe("gate-opened"); + const outcome = await awaitReadiness(context, identity); + if (outcome.kind === "failed") { + await stopFailedStart({ context, owned: identity, notify: true }); + return failed(outcome.reason); + } + await writeClaim(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: identity.pid, + birth: identity.birth, + holders, }); - // Keep the claim when the stop is unproven, so the process stays recorded. - if (stopped === "stopped" || stopped === "absent") { - await removeClaim(spec); + await context.observe("ready"); + return { + state: "ready", + ownership: "owned", + endpoint: endpointFor(spec.port), + pid: identity.pid, + spawned: true, + }; + } catch (error: unknown) { + // An observer or a state write failed. Never leave a gate waiting while + // this starter lives, nor a running helper the caller was not handed. + closeGate(); + if (owned !== null) { + const recorded = owned; + await (gateOpened + ? stopFailedStart({ context, owned: recorded, notify: false }) + : removeClaim(spec) + ).catch(() => undefined); } - await context.observe("stopped-after-failure"); - return failed(outcome.reason); + throw error; } - await writeClaim(spec, { - version: 1, - helper: spec.name, - port: spec.port, - ownership: "owned", - phase: "ready", - pid: owned.pid, - birth: owned.birth, - holders, - }); - await context.observe("ready"); - return { - state: "ready", - ownership: "owned", - endpoint: endpointFor(spec.port), - pid: owned.pid, - spawned: true, - }; } async function prepareStateDir(stateDir: string): Promise { @@ -868,9 +1023,45 @@ function isAbort(error: unknown, signal?: AbortSignal): boolean { ); } +function hasErrorCode(error: unknown): boolean { + return isRecord(error) && typeof error.code === "string"; +} + +/** + * Map an error to a closed reason. A caller's observer error is rethrown + * unchanged; errors raised while the lock is held are never `lock-unavailable`. + */ +function classifyFailure( + error: unknown, + signal: AbortSignal | undefined +): ReleaseFailure { + if (error instanceof ObserverError) { + throw error.original; + } + if (error instanceof LockedRunError) { + const inner = error.original; + if (inner instanceof ObserverError) { + throw inner.original; + } + if (inner instanceof ClaimRefusal) { + return "claim-invalid"; + } + return isAbort(inner, signal) ? "cancelled" : "state-unavailable"; + } + if (isAbort(error, signal)) { + return "cancelled"; + } + // The lock times out with a plain error; a filesystem error carries a code. + return hasErrorCode(error) ? "state-unavailable" : "lock-unavailable"; +} + /** * Make the shared helper ready for `holder`, adopting an identity-verified * helper, reusing the owned one, or starting it under an exact claim. + * + * An error thrown by `observe` propagates unchanged after cleanup: a gate that + * has not opened is closed and its claim removed, and a helper started by this + * call is stopped. A holder recorded before the error stays recorded. */ export async function ensureSharedHostHelper(opts: { readonly spec: SharedHostHelperSpec; @@ -892,37 +1083,38 @@ export async function ensureSharedHostHelper(opts: { holder: opts.holder, signal: opts.signal, observe: async (event) => { - await observer?.(event); + try { + await observer?.(event); + } catch (error: unknown) { + throw new ObserverError(error); + } }, deadline: performance.now() + opts.spec.deadlineMs, }; try { await prepareStateDir(opts.spec.stateDir); } catch { - return failed("claim-invalid"); + return failed("state-unavailable"); } try { + await context.observe("lock-waiting"); return await withProjectsRegistryLock({ lockPath: lockPath(opts.spec), label: "shared host helper", timeoutMs: opts.spec.deadlineMs, signal: opts.signal, - run: () => ensureLocked(context), + run: () => runLocked(() => ensureLocked(context)), }); } catch (error: unknown) { - if (isAbort(error, opts.signal)) { - return failed("cancelled"); - } - if (error instanceof ClaimRefusal) { - return failed("claim-invalid"); - } - return failed("lock-unavailable"); + return failed(classifyFailure(error, opts.signal)); } } /** * Drop `holder`. The last holder of an owned helper stops it through its * birth-verified tree; adopted helpers and stale claims are never signalled. + * Releasing a holder that is not recorded changes nothing and reports + * `retained` while other holders remain. */ export async function releaseSharedHostHelper(opts: { readonly spec: Pick; @@ -938,7 +1130,7 @@ export async function releaseSharedHostHelper(opts: { try { await prepareStateDir(opts.spec.stateDir); } catch { - return releaseFailed("claim-invalid"); + return releaseFailed("state-unavailable"); } try { return await withProjectsRegistryLock({ @@ -946,16 +1138,10 @@ export async function releaseSharedHostHelper(opts: { label: "shared host helper", timeoutMs: opts.timeoutMs ?? 10_000, signal: opts.signal, - run: () => releaseLocked(opts), + run: () => runLocked(() => releaseLocked(opts)), }); } catch (error: unknown) { - if (isAbort(error, opts.signal)) { - return releaseFailed("cancelled"); - } - if (error instanceof ClaimRefusal) { - return releaseFailed("claim-invalid"); - } - return releaseFailed("lock-unavailable"); + return releaseFailed(classifyFailure(error, opts.signal)); } } diff --git a/tests/fixtures/shared-host-helper.ts b/tests/fixtures/shared-host-helper.ts index 54f73cd74..b88f4cc17 100644 --- a/tests/fixtures/shared-host-helper.ts +++ b/tests/fixtures/shared-host-helper.ts @@ -4,7 +4,11 @@ * * argv: * - `identity`: HTTP on 127.0.0.1:; `/__hack/identity` reports - * {helper, upstream, pid}; `/` answers `served:`. + * {helper, upstream, pid}; `/` answers `served:`; + * `/__fixture/env` lists this process's environment variable names. + * - `stubborn`: like `identity`, but ignores SIGTERM (only SIGKILL stops it). + * - `oversized`: like `identity`, but pads the identity answer past 4 KiB. + * - `exit`: exits immediately (a helper that dies during startup). * - `tcp`: a plain TCP listener that never answers (a foreign port owner). * - `placeholder`: stays alive and never listens (a sleeping placeholder). * - `parent`: spawns a `placeholder` grandchild, prints its PID, then serves @@ -24,14 +28,22 @@ setInterval(() => { } }, 250); -function serveIdentity(): void { +function serveIdentity(opts: { readonly padding?: number } = {}): void { Bun.serve({ hostname: "127.0.0.1", port, fetch(request) { const url = new URL(request.url); if (url.pathname === "/__hack/identity") { - return Response.json({ helper, upstream, pid: process.pid }); + return Response.json({ + helper, + upstream, + pid: process.pid, + ...(opts.padding ? { padding: "x".repeat(opts.padding) } : {}), + }); + } + if (url.pathname === "/__fixture/env") { + return Response.json(Object.keys(process.env).sort()); } return new Response(`served:${upstream}`); }, @@ -42,6 +54,18 @@ switch (mode) { case "identity": serveIdentity(); break; + case "stubborn": + process.on("SIGTERM", () => { + // Ignored on purpose: the stop path must escalate to SIGKILL. + }); + serveIdentity(); + break; + case "oversized": + serveIdentity({ padding: 8192 }); + break; + case "exit": + process.exit(3); + break; case "tcp": Bun.listen({ hostname: "127.0.0.1", diff --git a/tests/fixtures/shared-host-starter.ts b/tests/fixtures/shared-host-starter.ts index 077a9e348..9b0ee3226 100644 --- a/tests/fixtures/shared-host-starter.ts +++ b/tests/fixtures/shared-host-starter.ts @@ -1,27 +1,63 @@ /** * Independent starter client for tests/shared-host-lifecycle.test.ts. * - * argv: [pause-event] + * argv: [options-json] + * options: { pauseAt?: event, resume?: "never" | "stdin", linger?: boolean } + * * Prints `event ` for each lifecycle event and `result ` at the - * end. With a pause event it stops forever after printing that event, so the - * test can kill this parent at a precise point. + * end. With `pauseAt` it stops after printing that event: forever by default + * (so the test can kill this parent at a precise point, possibly while it + * holds the helper lock), or until a line arrives on stdin with + * `resume: "stdin"`. With `linger` it stays alive after printing the result. */ import { ensureSharedHostHelper, + type SharedHostEvent, type SharedHostHelperSpec, } from "../../src/lib/shared-host-lifecycle.ts"; -const [specText, holder, pauseAt] = process.argv.slice(2); +type StarterOptions = { + readonly pauseAt?: SharedHostEvent; + readonly resume?: "never" | "stdin"; + readonly linger?: boolean; +}; + +const [specText, holder, optionsText] = process.argv.slice(2); const spec = JSON.parse(specText ?? "{}") as SharedHostHelperSpec; +const options = JSON.parse(optionsText ?? "{}") as StarterOptions; + +async function waitForStdinLine(): Promise { + const reader = Bun.stdin.stream().getReader(); + const decoder = new TextDecoder(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done || decoder.decode(value, { stream: true }).includes("\n")) { + return; + } + } + } finally { + reader.releaseLock(); + } +} + const result = await ensureSharedHostHelper({ spec, holder: holder ?? "", observe: async (event) => { console.log(`event ${event}`); - if (event === pauseAt) { - // An interval keeps the event loop alive until the test kills us. - await new Promise(() => setInterval(() => undefined, 60_000)); + if (event !== options.pauseAt) { + return; + } + if (options.resume === "stdin") { + await waitForStdinLine(); + return; } + // An interval keeps the event loop alive until the test kills us. + await new Promise(() => setInterval(() => undefined, 60_000)); }, }); console.log(`result ${JSON.stringify(result)}`); +if (options.linger) { + setInterval(() => undefined, 60_000); +} diff --git a/tests/shared-host-lifecycle.test.ts b/tests/shared-host-lifecycle.test.ts index 5d6896cd7..87c44a907 100644 --- a/tests/shared-host-lifecycle.test.ts +++ b/tests/shared-host-lifecycle.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; import { + chmod, mkdir, mkdtemp, readdir, @@ -14,6 +15,7 @@ import { selectProcessTree, signalVerifiedProcessTree, } from "../src/lib/process-tree.ts"; +import { withProjectsRegistryLock } from "../src/lib/projects-registry-lock.ts"; import { ensureSharedHostHelper, parseSharedHostClaim, @@ -37,7 +39,7 @@ const TIMEOUT = 30_000; type Foreign = Bun.Subprocess<"ignore", "pipe", "ignore">; const foreign: Foreign[] = []; -const starters: Bun.Subprocess<"ignore", "pipe", "ignore">[] = []; +const starters: Bun.Subprocess<"pipe", "pipe", "ignore">[] = []; const claimedHelpers: { pid: number; birth: string }[] = []; const roots: string[] = []; @@ -107,7 +109,15 @@ function specFor(opts: { readonly root: string; readonly port: number; readonly upstream?: string; - readonly mode?: "identity" | "placeholder" | "parent"; + readonly mode?: + | "identity" + | "placeholder" + | "parent" + | "stubborn" + | "oversized" + | "exit"; + readonly backoff?: SharedHostHelperSpec["backoff"]; + readonly probeTimeoutMs?: number; readonly deadlineMs?: number; }): SharedHostHelperSpec { const upstream = opts.upstream ?? "https://issuer.branch-a.test"; @@ -129,16 +139,39 @@ function specFor(opts: { cwd: opts.root, env: childEnv(opts.root), deadlineMs: opts.deadlineMs ?? 10_000, - backoff: { initialMs: 25, maxMs: 200 }, - probeTimeoutMs: 500, + backoff: opts.backoff ?? { initialMs: 25, maxMs: 200 }, + probeTimeoutMs: opts.probeTimeoutMs ?? 500, }; } +/** + * Reads fixture stdout continuously, so `lines` is current even while the + * test is not waiting (a negative check can inspect what was printed). + */ function lineReader(stream: ReadableStream) { const reader = stream.getReader(); const decoder = new TextDecoder(); const lines: string[] = []; let buffer = ""; + let closed = false; + const pump = (async () => { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + buffer += decoder.decode(value, { stream: true }); + const parts = buffer.split("\n"); + buffer = parts.pop() ?? ""; + lines.push(...parts); + } + } catch { + // The fixture was killed; keep what was read. + } finally { + closed = true; + } + })(); return { lines, async waitFor(match: (line: string) => boolean): Promise { @@ -148,14 +181,10 @@ function lineReader(stream: ReadableStream) { if (found !== undefined) { return found; } - const { done, value } = await reader.read(); - if (done) { + if (closed) { break; } - buffer += decoder.decode(value, { stream: true }); - const parts = buffer.split("\n"); - buffer = parts.pop() ?? ""; - lines.push(...parts); + await Promise.race([pump, Bun.sleep(10)]); } const found = lines.find(match); if (found === undefined) { @@ -167,7 +196,7 @@ function lineReader(stream: ReadableStream) { } async function startForeign(opts: { - readonly mode: "identity" | "tcp" | "placeholder"; + readonly mode: "identity" | "tcp" | "placeholder" | "oversized"; readonly port: number; readonly helper?: string; readonly upstream?: string; @@ -198,6 +227,12 @@ function startStarter(opts: { readonly spec: SharedHostHelperSpec; readonly holder: string; readonly pauseAt?: SharedHostEvent; + /** Resume a paused starter with {@link resumeStarter} instead of never. */ + readonly resumable?: boolean; + /** Keep the starter process alive after it prints its result. */ + readonly linger?: boolean; + /** The starter client's own time zone. */ + readonly tz?: string; }) { const proc = Bun.spawn( [ @@ -205,11 +240,18 @@ function startStarter(opts: { STARTER, JSON.stringify(opts.spec), opts.holder, - ...(opts.pauseAt ? [opts.pauseAt] : []), + JSON.stringify({ + pauseAt: opts.pauseAt, + resume: opts.resumable ? "stdin" : "never", + linger: opts.linger === true, + }), ], { - env: childEnv(opts.spec.cwd), - stdin: "ignore", + env: { + ...childEnv(opts.spec.cwd), + ...(opts.tz ? { TZ: opts.tz } : {}), + }, + stdin: "pipe", stdout: "pipe", stderr: "ignore", } @@ -218,6 +260,11 @@ function startStarter(opts: { return { proc, output: lineReader(proc.stdout) }; } +async function resumeStarter(starter: ReturnType) { + starter.proc.stdin.write("resume\n"); + await starter.proc.stdin.flush(); +} + async function starterResult(starter: ReturnType) { const line = await starter.output.waitFor((entry) => entry.startsWith("result ") @@ -435,8 +482,20 @@ test( async () => { const root = await scratch(); const spec = specFor({ root, port: freePort() }); - const first = startStarter({ spec, holder: "branch-a" }); + // Barrier: the first starter holds the lock until it is resumed, and the + // second is already contending for it, so both runs overlap for certain. + const first = startStarter({ + spec, + holder: "branch-a", + pauseAt: "lock-acquired", + resumable: true, + }); + await first.output.waitFor((line) => line === "event lock-acquired"); const second = startStarter({ spec, holder: "branch-b" }); + await second.output.waitFor((line) => line === "event lock-waiting"); + await Bun.sleep(300); + expect(second.output.lines).not.toContain("event lock-acquired"); + await resumeStarter(first); const results = await Promise.all([ starterResult(first), starterResult(second), @@ -643,7 +702,7 @@ test( ); test( - "a reused owned claim is not ready when a different process answers its identity", + "a reused ready claim is not ready when a different process answers its identity, and neither process is signalled", async () => { const root = await scratch(); const spec = specFor({ root, port: freePort(), deadlineMs: 2000 }); @@ -666,7 +725,7 @@ test( helper: spec.name, port: spec.port, ownership: "owned", - phase: "starting", + phase: "ready", pid: claimedProcess.pid, birth: identity?.birth ?? "", holders: ["branch-a"], @@ -914,3 +973,536 @@ test("claim parser accepts only the exact v1 shapes", () => { } expect(parseSharedHostClaim("not json")).toBeNull(); }); + +async function writeClaimFile( + spec: SharedHostHelperSpec, + value: unknown, + opts: { readonly mode?: number; readonly padding?: number } = {} +) { + await mkdir(spec.stateDir, { recursive: true, mode: 0o700 }); + const path = join(spec.stateDir, `${spec.name}.claim.json`); + const text = `${JSON.stringify(value)}${" ".repeat(opts.padding ?? 0)}`; + await writeFile(path, text, { mode: 0o600 }); + await chmod(path, opts.mode ?? 0o600); + return { path, text }; +} + +async function processGroupOf(pid: number): Promise { + const proc = Bun.spawn(["ps", "-o", "pgid=", "-p", String(pid)], { + env: { PATH: process.env.PATH ?? "/usr/bin:/bin", LC_ALL: "C" }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + const text = await new Response(proc.stdout).text(); + await proc.exited; + return Number(text.trim()); +} + +test( + "clients in different time zones agree on an owned helper's birth: it is reused, stays owned and is stopped by its last holder", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + // POSIX zone strings need no zoneinfo, so the zones differ on minimal hosts too. + const first = startStarter({ spec, holder: "branch-a", tz: "UTC0" }); + expect(await starterResult(first)).toMatchObject({ + state: "ready", + ownership: "owned", + spawned: true, + }); + const owned = await trackClaimedHelper(spec); + const second = startStarter({ spec, holder: "branch-b", tz: "HST10" }); + expect(await starterResult(second)).toMatchObject({ + state: "ready", + ownership: "owned", + spawned: false, + pid: owned.pid, + }); + expect(second.output.lines).toContain("event claim-reused"); + expect(second.output.lines).not.toContain("event claim-stale"); + expect(second.output.lines).not.toContain("event adopted"); + expect(await readClaim(spec)).toMatchObject({ + ownership: "owned", + pid: owned.pid, + birth: owned.birth, + holders: ["branch-a", "branch-b"], + }); + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "retained" } + ); + expect(await releaseSharedHostHelper({ spec, holder: "branch-b" })).toEqual( + { state: "stopped" } + ); + expect(await isAlive(owned)).toBe(false); + }, + TIMEOUT +); + +test( + "a starter lost after opening the gate leaves a starting helper that the next starter stops when it never becomes ready", + async () => { + const root = await scratch(); + const spec = specFor({ + root, + port: freePort(), + mode: "placeholder", + deadlineMs: 1500, + }); + const starter = startStarter({ + spec, + holder: "branch-a", + pauseAt: "gate-opened", + }); + await starter.output.waitFor((line) => line === "event gate-opened"); + const abandoned = await trackClaimedHelper(spec); + starter.proc.kill("SIGKILL"); + await starter.proc.exited; + expect(await readClaim(spec)).toMatchObject({ phase: "starting" }); + expect(await isAlive(abandoned)).toBe(true); + + const seen = recorder(); + const next = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + expect(next).toMatchObject({ state: "failed", reason: "deadline" }); + expect(seen.events).toContain("claim-reused"); + expect(seen.events).toContain("stopped-after-failure"); + expect(await waitGone(abandoned)).toBe(true); + expect(await readClaim(spec)).toBeNull(); + + // The helper port is free again for a fresh, working start. + const fresh = await ensureSharedHostHelper({ + spec: specFor({ root, port: spec.port }), + holder: "branch-c", + }); + await trackClaimedHelper(spec); + expect(fresh).toMatchObject({ state: "ready", spawned: true }); + }, + TIMEOUT +); + +test( + "a lock wait that uses up the deadline reports deadline, never a foreign listener, and adopts nothing", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), deadlineMs: 600 }); + const healthy = await startForeign({ + mode: "identity", + port: spec.port, + home: root, + }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: async (event) => { + seen.observe(event); + if (event === "lock-acquired") { + // Stands in for lock contention that consumed the whole deadline. + await Bun.sleep(700); + } + }, + }); + expect(result).toMatchObject({ state: "failed", reason: "deadline" }); + expect(seen.events).not.toContain("adopted"); + expect(await readClaim(spec)).toBeNull(); + expect(healthy.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a port probe cut short by the deadline reports deadline; a full-length silent probe is a foreign listener", + async () => { + const root = await scratch(); + const port = freePort(); + const silent = await startForeign({ mode: "tcp", port, home: root }); + const cut = await ensureSharedHostHelper({ + spec: specFor({ root, port, deadlineMs: 600, probeTimeoutMs: 5000 }), + holder: "branch-a", + }); + expect(cut).toMatchObject({ state: "failed", reason: "deadline" }); + const full = await ensureSharedHostHelper({ + spec: specFor({ root, port, deadlineMs: 5000, probeTimeoutMs: 300 }), + holder: "branch-a", + }); + expect(full).toMatchObject({ state: "failed", reason: "foreign-listener" }); + expect(silent.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a stale owned claim whose PID still answers as the helper is kept, not adopted, and nothing is signalled", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const answering = await startForeign({ + mode: "identity", + port: spec.port, + home: root, + }); + const { path, text } = await writeClaimFile(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: answering.pid, + birth: "Thu Jan 1 00:00:00 1970", + holders: ["branch-a"], + }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }); + expect(result).toMatchObject({ state: "failed", reason: "claim-invalid" }); + expect(seen.events).toContain("claim-stale"); + expect(seen.events).not.toContain("adopted"); + expect(await readFile(path, "utf8")).toBe(text); + await Bun.sleep(200); + expect(answering.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a claim for another port, a group-readable claim and an oversized claim fail closed", + async () => { + const root = await scratch(); + const bystander = await startForeign({ + mode: "placeholder", + port: 0, + home: root, + }); + const identity = await identityOf(bystander.pid); + const cases = [ + { label: "other port", portOffset: 1, ownership: "owned" }, + { label: "group-readable", mode: 0o640, ownership: "owned" }, + { label: "oversized", padding: 9000, ownership: "adopted" }, + ] as const; + for (const entry of cases) { + const spec = specFor({ root: await scratch(), port: freePort() }); + const portOffset = "portOffset" in entry ? entry.portOffset : 0; + const claim = + entry.ownership === "owned" + ? { + version: 1, + helper: spec.name, + port: spec.port + portOffset, + ownership: "owned", + phase: "ready", + pid: bystander.pid, + birth: identity?.birth ?? "", + holders: ["branch-a"], + } + : { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "adopted", + holders: ["branch-a"], + }; + const { path, text } = await writeClaimFile(spec, claim, { + mode: "mode" in entry ? entry.mode : undefined, + padding: "padding" in entry ? entry.padding : undefined, + }); + const seen = recorder(); + expect({ + label: entry.label, + result: await ensureSharedHostHelper({ + spec, + holder: "branch-b", + observe: seen.observe, + }), + }).toMatchObject({ + label: entry.label, + result: { state: "failed", reason: "claim-invalid" }, + }); + expect(seen.events).not.toContain("spawned"); + expect(seen.events).not.toContain("claim-reused"); + if (portOffset === 0) { + expect( + await releaseSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "failed", reason: "claim-invalid" }); + } + expect(await readFile(path, "utf8")).toBe(text); + } + await Bun.sleep(200); + expect(bystander.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a helper that exits during startup reports child-exited and leaves no claim", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), mode: "exit" }); + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }); + expect(result).toMatchObject({ state: "failed", reason: "child-exited" }); + expect(seen.events).toContain("gate-opened"); + expect(seen.events).toContain("stopped-after-failure"); + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); + +test( + "cancellation after the claim and before the gate never runs the helper", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const controller = new AbortController(); + let gated: { pid: number; birth: string } | null = null; + const seen = recorder(); + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + signal: controller.signal, + observe: async (event) => { + seen.observe(event); + if (event === "claimed") { + gated = await trackClaimedHelper(spec); + controller.abort(); + } + }, + }); + expect(result).toMatchObject({ state: "failed", reason: "cancelled" }); + expect(seen.events).not.toContain("gate-opened"); + expect(await waitGone(gated ?? { pid: 0, birth: "" })).toBe(true); + expect(await readClaim(spec)).toBeNull(); + expect( + await fetch(`http://127.0.0.1:${spec.port}/`).catch(() => null) + ).toBeNull(); + }, + TIMEOUT +); + +test( + "a helper that ignores SIGTERM is stopped by SIGKILL after the grace period", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort(), mode: "stubborn" }); + expect( + await ensureSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "ready", ownership: "owned" }); + const owned = await trackClaimedHelper(spec); + expect( + await releaseSharedHostHelper({ spec, holder: "branch-a", graceMs: 200 }) + ).toEqual({ state: "stopped" }); + expect(await isAlive(owned)).toBe(false); + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); + +test( + "an owned helper runs in its own session with only the caller's environment", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const saved = process.env.HACK_SHARED_HOST_RUNNER_ONLY; + process.env.HACK_SHARED_HOST_RUNNER_ONLY = "runner-value"; + try { + expect( + await ensureSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "ready", ownership: "owned" }); + } finally { + restoreEnv("HACK_SHARED_HOST_RUNNER_ONLY", saved); + } + const owned = await trackClaimedHelper(spec); + const keys = (await ( + await fetch(`http://127.0.0.1:${spec.port}/__fixture/env`) + ).json()) as string[]; + expect(keys).toContain("HOME"); + expect(keys).not.toContain("HACK_SHARED_HOST_RUNNER_ONLY"); + const helperGroup = await processGroupOf(owned.pid); + expect(helperGroup).toBe(owned.pid); + expect(helperGroup).not.toBe(await processGroupOf(process.pid)); + }, + TIMEOUT +); + +test( + "cancellation interrupts the readiness backoff sleep", + async () => { + const root = await scratch(); + const spec = specFor({ + root, + port: freePort(), + mode: "placeholder", + deadlineMs: 20_000, + backoff: { initialMs: 5000, maxMs: 5000 }, + }); + const controller = new AbortController(); + let abortedAt = 0; + let owned: { pid: number; birth: string } | null = null; + const result = await ensureSharedHostHelper({ + spec, + holder: "branch-a", + signal: controller.signal, + observe: async (event) => { + if (event === "claimed") { + owned = await trackClaimedHelper(spec); + } + if (event === "probe-retry") { + setTimeout(() => { + abortedAt = performance.now(); + controller.abort(); + }, 50); + } + }, + }); + expect(result).toMatchObject({ state: "failed", reason: "cancelled" }); + expect(abortedAt).toBeGreaterThan(0); + expect(performance.now() - abortedAt).toBeLessThan(2500); + expect(await isAlive(owned ?? { pid: 0, birth: "" })).toBe(false); + }, + TIMEOUT +); + +test( + "release waits for the helper lock and fails closed when it is held past the timeout", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + expect( + await ensureSharedHostHelper({ spec, holder: "branch-a" }) + ).toMatchObject({ state: "ready", ownership: "owned" }); + const owned = await trackClaimedHelper(spec); + const claimFile = join(spec.stateDir, `${spec.name}.claim.json`); + const before = await readFile(claimFile, "utf8"); + const blocked = await withProjectsRegistryLock({ + lockPath: join(spec.stateDir, `${spec.name}.lock`), + label: "test holder", + run: () => + releaseSharedHostHelper({ spec, holder: "branch-a", timeoutMs: 300 }), + }); + expect(blocked).toMatchObject({ + state: "failed", + reason: "lock-unavailable", + }); + expect(await isAlive(owned)).toBe(true); + expect(await readFile(claimFile, "utf8")).toBe(before); + expect(await releaseSharedHostHelper({ spec, holder: "branch-a" })).toEqual( + { state: "stopped" } + ); + }, + TIMEOUT +); + +test( + "an identity answer larger than 4 KiB is not identity: the listener is refused and preserved", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const oversized = await startForeign({ + mode: "oversized", + port: spec.port, + home: root, + }); + const seen = recorder(); + expect( + await ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: seen.observe, + }) + ).toMatchObject({ state: "failed", reason: "foreign-listener" }); + expect(seen.events).not.toContain("adopted"); + expect(await readClaim(spec)).toBeNull(); + await Bun.sleep(200); + expect(oversized.exitCode).toBeNull(); + }, + TIMEOUT +); + +test( + "a stop that cannot be confirmed keeps the claim so the process stays recorded", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + // The starter stays alive and is then frozen, so the helper it spawned + // cannot be reaped: after SIGKILL it remains a zombie in the census. + const starter = startStarter({ spec, holder: "branch-a", linger: true }); + expect(await starterResult(starter)).toMatchObject({ + state: "ready", + ownership: "owned", + }); + const owned = await trackClaimedHelper(spec); + starter.proc.kill("SIGSTOP"); + try { + expect( + await releaseSharedHostHelper({ + spec, + holder: "branch-a", + graceMs: 100, + }) + ).toEqual({ state: "stop-unconfirmed" }); + expect(await readClaim(spec)).toMatchObject({ + ownership: "owned", + pid: owned.pid, + birth: owned.birth, + holders: [], + }); + } finally { + starter.proc.kill("SIGCONT"); + } + }, + TIMEOUT +); + +test( + "an observer error is rethrown after the waiting gate is closed and its claim removed", + async () => { + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const children = async () => + (await readProcessIdentities()) + .filter((row) => row.parentPid === process.pid) + .map((row) => ({ pid: row.pid, birth: row.birth })); + const before = new Set((await children()).map((row) => row.pid)); + let gates: { pid: number; birth: string }[] = []; + const failure = new Error("observer failed on purpose"); + await expect( + ensureSharedHostHelper({ + spec, + holder: "branch-a", + observe: async (event) => { + if (event === "spawned") { + // Each census sees its own short-lived `ps` child; the gate is + // the new child present in two consecutive censuses. + const first = await children(); + const second = await children(); + gates = first.filter( + (row) => + !before.has(row.pid) && + second.some( + (entry) => entry.pid === row.pid && entry.birth === row.birth + ) + ); + claimedHelpers.push(...gates); + throw failure; + } + }, + }) + ).rejects.toBe(failure); + expect(gates.length).toBeGreaterThan(0); + for (const gate of gates) { + expect(await waitGone(gate)).toBe(true); + } + expect(await readClaim(spec)).toBeNull(); + }, + TIMEOUT +); From d056fc85e491e28b055b88ef050299e3f2771433 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:40:35 -0400 Subject: [PATCH 13/21] fix(lifecycle): confirm helper port absence with a bind probe on every platform The hosted Linux job showed that a refused loopback fetch does not carry the macOS ECONNREFUSED code there, so every free port was misread as a foreign listener. A failed connection now never proves absence by its error code: the loopback bind probe decides (occupied is foreign-listener, free spawns). Add a listener that drops every connection to the foreign-listener fixture. Refs: HACK-1142 Co-Authored-By: Claude Opus 5.5 --- docs/reference/shared-host-lifecycle.md | 7 +++++-- src/lib/shared-host-lifecycle.ts | 14 +++++++------- tests/fixtures/shared-host-helper.ts | 16 ++++++++++++++++ tests/shared-host-lifecycle.test.ts | 3 ++- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index 601bf98c4..bd55812aa 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -100,7 +100,10 @@ replaced atomically, and parsed strictly (exact keys, version 1): Either way the listener keeps running and no child is spawned; - a probe whose timeout was shortened by the deadline and then timed out proves nothing about the listener: `deadline`; - - connection refused: confirm with a loopback bind probe, then spawn. + - connection failed (refused, reset or unreachable): confirm with a loopback + bind probe. An occupied port is `foreign-listener`; a free port is spawned. + The refused-connection error code differs by platform, so the error itself + never proves absence. 5. Spawn through an exec gate in a new session: `/bin/sh -c 'IFS= read -r hack_gate || exit 0; [ "$hack_gate" = open ] || exit 0; exec "$@" undefined, 60_000); break; diff --git a/tests/shared-host-lifecycle.test.ts b/tests/shared-host-lifecycle.test.ts index 87c44a907..960850cd8 100644 --- a/tests/shared-host-lifecycle.test.ts +++ b/tests/shared-host-lifecycle.test.ts @@ -196,7 +196,7 @@ function lineReader(stream: ReadableStream) { } async function startForeign(opts: { - readonly mode: "identity" | "tcp" | "placeholder" | "oversized"; + readonly mode: "identity" | "tcp" | "reset" | "placeholder" | "oversized"; readonly port: number; readonly helper?: string; readonly upstream?: string; @@ -450,6 +450,7 @@ test( async () => { for (const fixture of [ { mode: "tcp" as const }, + { mode: "reset" as const }, { mode: "identity" as const, helper: "unrelated-dev-server" }, ]) { const root = await scratch(); From d5d96ed2a7ad3951dc90107e1617f667e734e411 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:43:17 -0400 Subject: [PATCH 14/21] fix(runtime): test cache quarantine orchestration and add explicit resolution Route admit/inspect/quarantine through a Port trait so a synthetic provider runs the shipped orchestration and guest scripts against a fixture volume. Add resolve-cache-quarantine, which settles a pending record as quarantined or abandoned without moving bytes, so a restart or a persistently refusing move no longer blocks a volume. Require the same findmnt mount ID before mv so it cannot copy-then-unlink, report retained quarantine bytes on every result, bound records per volume, and add an ignored native bad-cache fixture. Document the commands in the runtime README and the storage-inventory undercount. Refs: HACK-1198 Co-Authored-By: Claude Opus 5.5 --- docs/guides/native-candidate.md | 62 ++- packages/runtime-core/README.md | 31 ++ packages/runtime-core/src/graph_cli.rs | 13 +- packages/runtime-core/src/main.rs | 1 + .../src/provider/graph/cache_quarantine.rs | 408 +++++++++++---- .../graph/cache_quarantine/orchestration.rs | 481 ++++++++++++++++++ .../provider/graph/cache_quarantine/tests.rs | 149 +++++- .../provider/graph/foreground/native_test.rs | 1 + .../native_test/cache_quarantine.rs | 293 +++++++++++ .../runtime-core/src/provider/graph/mod.rs | 4 +- .../src/provider/graph/volume_subpaths.rs | 17 +- .../tests/cache_quarantine_cli_contract.rs | 21 +- 12 files changed, 1348 insertions(+), 133 deletions(-) create mode 100644 packages/runtime-core/src/provider/graph/cache_quarantine/orchestration.rs create mode 100644 packages/runtime-core/src/provider/graph/foreground/native_test/cache_quarantine.rs diff --git a/docs/guides/native-candidate.md b/docs/guides/native-candidate.md index 9c5c76a9d..eb15b2556 100644 --- a/docs/guides/native-candidate.md +++ b/docs/guides/native-candidate.md @@ -1559,8 +1559,8 @@ target directory identity and the measured bytes. Review it, then quarantine: --expect-selection --json ``` -Both commands hold the provider mutation lease. They refuse when any of these is -true: +These commands, and `resolve-cache-quarantine` below, hold the provider mutation +lease. They refuse when any of these is true: - the graph is active, its journal is incomplete, or interrupted cleanup is pending - the path is not exactly a declared entry, or is absolute, traversing or empty @@ -1573,24 +1573,58 @@ true: The guest renames the directory under a lock on the volume root into `/var/lib/docker/volumes//hack-quarantine/`. That directory is beside -the volume's `_data` root on the same filesystem, so the move never copies, and no -container can mount it. The result reports `quarantined_bytes` and +the volume's `_data` root, and no container can mount it. Before the rename the +guest checks that the source and the quarantine directory are on the same mount +(the same `findmnt` mount ID, not only the same device number). A rename within +one mount cannot fail with a cross-device error, so `mv` never falls back to +copying and then deleting the source. The result reports `quarantined_bytes` and `reclaimed_bytes: 0`. The bytes remain for diagnosis until the owning volume itself is explicitly removed (for example `graph cleanup --remove-data`). Restart, restore and ordinary cleanup never collect them. Then use a normal retaining restart. The framework regenerates its cache. +Quarantined bytes are outside the volume's `_data` directory. Docker's disk-usage +report, and therefore `graph storage-inventory`, does not count them, so the inventory +undercounts a volume that holds a quarantine. Every `inspect-cache-quarantine` +result, including `target_absent`, reports `retained_quarantine_records` and +`retained_quarantine_bytes`: the recorded bytes still held in the current +incarnation of that volume. + A private record under the candidate home's `run/cache-quarantine/` is written -before the guest effect and completed after the guest syncs the move. If the -command is interrupted, repeat it with the same selection: `inspect-cache-quarantine` -reports `resume_required` with the recorded digest. Resumption accepts the recorded -directory either still at its source or already at its destination with the same -inode. It refuses a regenerated directory, a different destination or both -present, and never moves new bytes. A torn record write refuses automatic replay. -Resume before restarting the graph. A restart changes the receipt, so the pending -record can no longer be resumed and further quarantines of that volume are -refused. The bytes stay where the interruption left them. -Records are retained (at most 256) and are not collected automatically. +before the guest effect and marked `quarantined` after the guest syncs the move. +If the command is interrupted, `inspect-cache-quarantine` reports the recorded +digest with one of two phases: + +- `resume_required`: the receipt and volume are unchanged. Repeat + `quarantine-cache` with the same selection. Resumption accepts the recorded + directory either still at its source or already at its destination with the + same inode. It refuses a regenerated directory, a different destination or + both present, and never moves new bytes. +- `resolve_required`: resumption is no longer possible, for example because a + restart changed the receipt, or because the move keeps refusing. Settle the + record without moving anything: + + ```sh + ./hack-native --candidate-root /absolute/private/candidate-home graph resolve-cache-quarantine \ + --run-id --volume www_next --path dev/cache/turbopack \ + --expect-selection --json + ``` + + Under the same lease, consumer and ownership checks, the guest reads the + recorded destination under the volume root lock. The record becomes + `quarantined` when the recorded directory (same inode) is there, and + `abandoned` when the destination is absent, which means nothing was moved. Any + other layout, such as a symbolic-link quarantine directory or a different + directory at the destination, refuses. After resolution, inspect again for a + fresh selection. + +You can also resolve a record that could still be resumed. While a record is +pending, other quarantines of that volume are refused. A torn record write +refuses automatic replay. Records are retained as diagnosis evidence and are not +collected automatically. Each volume admits at most 16 records, and the candidate +home holds at most 1024. Volume names are specific to a run, so a new run starts +with a fresh per-volume budget. When the global limit is reached, every new +quarantine is refused; this candidate has no command that collects records. This operation does not classify the cause of the corruption. One checksum error does not show an interrupted write or a storage integrity fault. Container health diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index eb308245d..fae108dc1 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -1464,6 +1464,37 @@ and the acquired startup lease recheck the selected owner; changed ownership refuses, and a reserve-qualified request cannot enter VM create or boot. Stopped, missing or unproved capacity retains fresh-allocation requirements or refuses. +### Disposable cache quarantine + +A volume can declare regenerable directories with `x-hack-disposable-cache` (at +most eight distinct, non-nested, volume-relative paths). The receipt records them +at admission. For a stopped retained graph, `graph inspect-cache-quarantine +--run-id RUN --volume VOLUME --path PATH --json` measures one declared directory +read-only and returns a `selection_sha256`. That digest binds the receipt bytes, +volume incarnation, root and target directory identities, and measured bytes. +`graph quarantine-cache ... --expect-selection SHA --json` renames exactly that +directory into `/var/lib/docker/volumes//hack-quarantine/`, under the +cleanup-only lease and a lock on the volume root. It requires owned volume labels, +no container consumers (stopped containers count), and source and destination on +the same mount ID, so `mv` cannot fall back to copy-then-unlink. Nothing is +deleted. Results report `reclaimed_bytes: 0`, and the bytes stay until the volume +itself is removed. Docker disk usage, and therefore `graph storage-inventory`, +does not count them. Inspect reports the recorded `retained_quarantine_bytes` +instead. + +A private record in `run/cache-quarantine/` is written before the guest effect +(`pending`) and settled after the synchronized rename (`quarantined`). When the +receipt and volume are unchanged, repeating `quarantine-cache` resumes it. +Otherwise inspect reports `resolve_required`, and `graph resolve-cache-quarantine +... --expect-selection SHA --json` settles the record without moving bytes. The +record becomes `quarantined` when the recorded inode is at the destination, and +`abandoned` when the destination is absent. Records are retained, with at most 16 +per run-scoped volume and 1024 in total. Unit tests drive this inspect, quarantine, +resume and resolve flow through a synthetic provider that runs the production +guest scripts on a fixture volume. The ignored native fixture +`synthetic_bad_cache_is_quarantined_and_a_retaining_restore_keeps_application_data` +covers the real engine, guest and restore path. + ### Native persistent-data identity codec `provider::graph::native::persistent_data` defines one closed, private-candidate diff --git a/packages/runtime-core/src/graph_cli.rs b/packages/runtime-core/src/graph_cli.rs index 5f8b8bf85..277c38e32 100644 --- a/packages/runtime-core/src/graph_cli.rs +++ b/packages/runtime-core/src/graph_cli.rs @@ -354,7 +354,7 @@ pub fn command(candidate: &Candidate, args: &[&str]) -> Result Result (run, volume, path, expected), _ => return Err(invalid()), }; - return graph::quarantine_cache( - candidate, - graph::CacheQuarantineTarget { run, volume, path }, - expected, - ); + let target = graph::CacheQuarantineTarget { run, volume, path }; + return if *action == "quarantine-cache" { + graph::quarantine_cache(candidate, target, expected) + } else { + graph::resolve_cache_quarantine(candidate, target, expected) + }; } if *action == "inspect-source-device-rebind" { let run = match *args { diff --git a/packages/runtime-core/src/main.rs b/packages/runtime-core/src/main.rs index 6e1d9a41c..f9d42fffa 100644 --- a/packages/runtime-core/src/main.rs +++ b/packages/runtime-core/src/main.rs @@ -42,6 +42,7 @@ Usage: hack-local graph inspect-source-device-rebind --run-id <32-hex> [--json] hack-local graph inspect-cache-quarantine --run-id <32-hex> --volume --path [--json] hack-local graph quarantine-cache --run-id <32-hex> --volume --path --expect-selection <64-hex> [--json] + hack-local graph resolve-cache-quarantine --run-id <32-hex> --volume --path --expect-selection <64-hex> [--json] hack-local graph recover-source-device-rebind --run-id <32-hex> --expect-selection <64-hex> --accept-legacy-device-rebind [--json] hack-local graph logs --run-id <32-hex> --service [--tail <1..1000>] [--json] hack-local graph exec --run-id <32-hex> --service [--workdir /path] [--timeout-seconds <1..120>] [--json] -- [args...] diff --git a/packages/runtime-core/src/provider/graph/cache_quarantine.rs b/packages/runtime-core/src/provider/graph/cache_quarantine.rs index c8dc69b54..6b78f45ea 100644 --- a/packages/runtime-core/src/provider/graph/cache_quarantine.rs +++ b/packages/runtime-core/src/provider/graph/cache_quarantine.rs @@ -6,24 +6,30 @@ //! selection digest (receipt bytes, volume incarnation, directory identities and //! measured bytes), the provider mutation lease, owned volume labels, and the //! absence of every container consumer. Nothing is deleted. The directory is -//! renamed on the same filesystem into a Hack-owned `hack-quarantine` sibling of -//! the volume's `_data` root, which no container mounts. Quarantined bytes stay +//! renamed within one mount into a Hack-owned `hack-quarantine` sibling of the +//! volume's `_data` root, which no container mounts. Quarantined bytes stay //! until the owning volume itself is explicitly removed; ordinary cleanup, //! restart and restore never collect them, so reclaimed bytes are always zero. //! //! Recovery: a durable host record is committed before the guest rename and is -//! marked complete only after the guest reports a synchronized move. A pending -//! record admits only its identical selection. The guest accepts the target at -//! its original path or already at its destination with the recorded inode, and -//! refuses every other layout. A torn record write refuses automatic replay. +//! marked `quarantined` only after the guest reports a synchronized move. A +//! pending record resumes only for its identical selection (same receipt bytes +//! and volume incarnation). Any pending record, including one whose receipt has +//! since changed through a restart, can be resolved explicitly: the guest reads +//! the recorded destination under the volume root lock and the record becomes +//! `quarantined` (the recorded inode is there) or `abandoned` (nothing was +//! moved). Resolution never moves bytes. A torn record write refuses replay. use super::*; use sha2::{Digest, Sha256}; use std::os::unix::fs::DirBuilderExt; use std::path::Path; const RECORD_LIMIT: u64 = 16 * 1024; -/// Records are retained as diagnosis evidence; there is no automatic collection. -const MAX_RECORDS: usize = 256; +/// Records are retained as diagnosis evidence; there is no automatic +/// collection. Volume names are run-scoped, so the per-volume budget renews +/// with each run while the global ceiling bounds the private directory scan. +const MAX_RECORDS: usize = 1024; +const MAX_VOLUME_RECORDS: usize = 16; const QUARANTINE_DIRECTORY: &str = "hack-quarantine"; fn refused() -> CandidateError { @@ -75,13 +81,26 @@ impl Selection { } } +/// Durable state of one selected operation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum Outcome { + /// Intent is durable; the guest rename may or may not have happened. + Pending, + /// The recorded directory is retained at its quarantine destination. + Quarantined, + /// Explicit resolution found no recorded directory at the destination; + /// nothing was moved by this operation. + Abandoned, +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct Record { version: u32, selection_sha256: String, selection: Selection, - completed: bool, + outcome: Outcome, } impl Record { fn valid(&self) -> bool { @@ -93,6 +112,9 @@ impl Record { .sha256() .is_ok_and(|value| value == self.selection_sha256) } + fn id(&self) -> &str { + Selection::quarantine_id(&self.selection_sha256) + } } /// Pure admission over the loaded receipt. Only an admitted ordinary volume of a @@ -189,7 +211,7 @@ impl Records { } let record: Record = state::read_bounded(&entry.path(), RECORD_LIMIT).map_err(|_| refused())?; - if !record.valid() || Selection::quarantine_id(&record.selection_sha256) != id { + if !record.valid() || record.id() != id { return Err(refused()); } records.push(record); @@ -198,18 +220,55 @@ impl Records { } Ok(Self { directory, records }) } - /// At most one incomplete operation per volume; it must be resumed first. + /// At most one unresolved operation per volume; it must be resumed or + /// resolved first. fn pending(&self, volume_name: &str) -> Result, CandidateError> { let mut pending = self .records .iter() - .filter(|r| !r.completed && r.selection.volume_name == volume_name); + .filter(|r| r.outcome == Outcome::Pending && r.selection.volume_name == volume_name); let first = pending.next(); if pending.next().is_some() { return Err(refused()); } Ok(first) } + /// A new selection never reuses a pending or quarantined record and stays + /// within the per-volume and global budgets. An abandoned identical + /// selection moved nothing, so selecting it again replaces that record. + fn admits(&self, selection: &Selection, sha: &str) -> bool { + let id = Selection::quarantine_id(sha); + let existing = self.records.iter().find(|r| r.id() == id); + if existing.is_some_and(|r| r.outcome != Outcome::Abandoned) { + return false; + } + let added = usize::from(existing.is_none()); + self.records.len() + added <= MAX_RECORDS + && self + .records + .iter() + .filter(|r| r.selection.volume_name == selection.volume_name) + .count() + + added + <= MAX_VOLUME_RECORDS + } + /// Recorded bytes still retained in this volume incarnation, counting + /// `replacement` in place of any stored record with the same id. + fn retained(&self, admitted: &Admitted, replacement: Option<&Record>) -> (u64, u64) { + self.records + .iter() + .filter(|r| replacement.is_none_or(|n| n.id() != r.id())) + .chain(replacement) + .filter(|r| { + r.outcome == Outcome::Quarantined + && r.selection.volume_name == admitted.resource.name + && r.selection.volume_created == admitted.created + && r.selection.root_identity == admitted.root_identity + }) + .fold((0, 0), |(count, bytes), r| { + (count + 1, bytes.saturating_add(r.selection.target_bytes)) + }) + } fn write(&self, record: &Record) -> Result<(), CandidateError> { if !record.valid() { return Err(refused()); @@ -232,8 +291,11 @@ impl Records { Ok(_) => {} } state::check_private_directory(&self.directory).map_err(|_| refused())?; - let id = Selection::quarantine_id(&record.selection_sha256); - state::write(&self.directory.join(format!("{id}.json")), record).map_err(|_| refused()) + state::write( + &self.directory.join(format!("{}.json", record.id())), + record, + ) + .map_err(|_| refused()) } } @@ -249,27 +311,61 @@ fn journal_incomplete(root: &Path, receipt: &Receipt) -> Result Result<(Receipt, PathBuf), CandidateError>; + fn inspect( + &self, + receipt: &Receipt, + resource: &Resource, + ) -> Result, CandidateError>; + fn containers(&self) -> Result; + fn execute(&self, script: &str, arguments: &[&str]) -> Result; +} +impl Port for Engine<'_> { + fn load(&self, candidate: &Candidate, run: &str) -> Result<(Receipt, PathBuf), CandidateError> { + load(candidate, self, run) + } + fn inspect( + &self, + receipt: &Receipt, + resource: &Resource, + ) -> Result, CandidateError> { + // Owner/graph/plan labels are verified here; a foreign volume that + // reuses the recorded name fails with its own ownership diagnostic. + inspect_resource(self, receipt, resource) + } + fn containers(&self) -> Result { + self.request(Method::GET, "/v1.53/containers/json?all=true", None) + } + fn execute(&self, script: &str, arguments: &[&str]) -> Result { + self.guest().execute(script, arguments, None) + } +} + /// Evidence observed under the provider lease for one admitted target. struct Admitted { receipt: Receipt, resource: Resource, inspected: Value, mountpoint: String, + quarantine: String, created: String, root_identity: String, receipt_sha256: String, } fn admit( candidate: &Candidate, - engine: &Engine<'_>, + port: &impl Port, target: &CacheQuarantineTarget<'_>, ) -> Result { - let (receipt, root) = load(candidate, engine, target.run)?; + let (receipt, root) = port.load(candidate, target.run)?; let incomplete = journal_incomplete(&root, &receipt)?; let resource = authorize(&receipt, incomplete, target)?.clone(); - // Owner/graph/plan labels are verified here; a foreign volume that reuses - // the recorded name fails with its own ownership diagnostic. - let inspected = inspect_resource(engine, &receipt, &resource)?.ok_or_else(refused)?; + let inspected = port.inspect(&receipt, &resource)?.ok_or_else(refused)?; let mountpoint = format!("/var/lib/docker/volumes/{}/_data", resource.name); let created = inspected["CreatedAt"] .as_str() @@ -283,17 +379,23 @@ fn admit( return Err(refused()); } // Stopped containers count: any consumer could start and observe the move. - volume_subpaths::no_consumers( - &engine.request(Method::GET, "/v1.53/containers/json?all=true", None)?, - &resource.name, + volume_subpaths::no_consumers(&port.containers()?, &resource.name)?; + let root_identity = volume_subpaths::directory_identity_with( + |script, arguments| port.execute(script, arguments), + &resource, + &inspected, )?; - let root_identity = volume_subpaths::directory_identity(engine, &resource, &inspected)?; let receipt_sha256 = digest(&serde_json::to_vec_pretty(&receipt).map_err(|_| refused())?); + let quarantine = format!( + "/var/lib/docker/volumes/{}/{QUARANTINE_DIRECTORY}", + resource.name + ); Ok(Admitted { receipt, resource, inspected, mountpoint, + quarantine, created, root_identity, receipt_sha256, @@ -302,13 +404,12 @@ fn admit( impl Admitted { fn observe( &self, - engine: &Engine<'_>, + port: &impl Port, target: &CacheQuarantineTarget<'_>, ) -> Result, CandidateError> { - let output = engine.guest().execute( + let output = port.execute( OBSERVE, &[&self.mountpoint, &self.root_identity, target.path], - None, )?; let Some((target_identity, target_bytes, allocated_bytes, entries)) = parse_observation(&output)? @@ -346,146 +447,237 @@ impl Admitted { && selection.path == target.path && same_device(&selection.target_identity, &self.root_identity) } + /// Volume identity must be unchanged across every guest effect. + fn recheck(&self, port: &impl Port) -> Result<(), CandidateError> { + if port.inspect(&self.receipt, &self.resource)?.as_ref() != Some(&self.inspected) { + return Err(refused()); + } + Ok(()) + } } -fn report(selection: &Selection, sha: &str, phase: &str, mutation: bool) -> Value { +fn report(record: &Record, phase: &str, retained: (u64, u64)) -> Value { + let selection = &record.selection; json!({ "run":selection.run, "volume":selection.volume, "path":selection.path, "phase":phase, - "selection_sha256":sha, + "selection_sha256":record.selection_sha256, "receipt_sha256":selection.receipt_sha256, - "quarantine":Selection::quarantine_id(sha), + "quarantine":record.id(), "target_bytes":selection.target_bytes, "allocated_bytes":selection.allocated_bytes, "entries":selection.entries, - "quarantined_bytes":if mutation {selection.target_bytes} else {0}, + "quarantined_bytes":if record.outcome == Outcome::Quarantined {selection.target_bytes} else {0}, + "retained_quarantine_records":retained.0, + "retained_quarantine_bytes":retained.1, "reclaimed_bytes":0, "data_retained":true, "quarantine_retained_until":"explicit_volume_removal", - "mutation":mutation + "mutation":false }) } -/// Read-only selection for `quarantine_cache`. It measures the exact admitted -/// target under the provider lease and returns the digest to review; a pending -/// interrupted operation for the volume is reported for resumption instead. -pub fn inspect_cache_quarantine( +fn inspect_with( candidate: &Candidate, - target: CacheQuarantineTarget<'_>, + port: &impl Port, + target: &CacheQuarantineTarget<'_>, ) -> Result { - let engine = Engine::connect_cleanup(candidate)?; - let admitted = admit(candidate, &engine, &target)?; + let admitted = admit(candidate, port, target)?; let records = Records::load(candidate)?; + let retained = records.retained(&admitted, None); if let Some(record) = records.pending(&admitted.resource.name)? { - if !admitted.resumes(&record.selection, &target) { - return Err(refused()); - } - return Ok(report( - &record.selection, - &record.selection_sha256, - "resume_required", - false, - )); + // The operator sees the recorded digest; resumption needs the exact + // receipt, while resolution is always available and never moves bytes. + let phase = if admitted.resumes(&record.selection, target) { + "resume_required" + } else { + "resolve_required" + }; + return Ok(report(record, phase, retained)); } - let Some(selection) = admitted.observe(&engine, &target)? else { - return Ok( - json!({"run":target.run,"volume":target.volume,"path":target.path,"phase":"target_absent","reclaimed_bytes":0,"mutation":false}), - ); + let Some(selection) = admitted.observe(port, target)? else { + return Ok(json!({ + "run":target.run,"volume":target.volume,"path":target.path, + "phase":"target_absent", + "retained_quarantine_records":retained.0, + "retained_quarantine_bytes":retained.1, + "reclaimed_bytes":0,"mutation":false + })); + }; + let record = Record { + version: 1, + selection_sha256: selection.sha256()?, + selection, + outcome: Outcome::Pending, }; - let sha = selection.sha256()?; - Ok(report(&selection, &sha, "selectable", false)) + Ok(report(&record, "selectable", retained)) } -/// Move one admitted disposable cache directory into the volume's owned -/// quarantine. Requires the exact digest from `inspect_cache_quarantine`; -/// repeating it after interruption resumes the same recorded operation. -pub fn quarantine_cache( +fn quarantine_with( candidate: &Candidate, - target: CacheQuarantineTarget<'_>, + port: &impl Port, + target: &CacheQuarantineTarget<'_>, expected_selection: &str, ) -> Result { if !hex(expected_selection, 64) { return Err(refused()); } - let engine = Engine::connect_cleanup(candidate)?; - let admitted = admit(candidate, &engine, &target)?; + let admitted = admit(candidate, port, target)?; let records = Records::load(candidate)?; let (mut record, resumed) = match records.pending(&admitted.resource.name)? { Some(record) => { if record.selection_sha256 != expected_selection - || !admitted.resumes(&record.selection, &target) + || !admitted.resumes(&record.selection, target) { return Err(refused()); } (record.clone(), true) } None => { - let selection = admitted.observe(&engine, &target)?.ok_or_else(refused)?; + let selection = admitted.observe(port, target)?.ok_or_else(refused)?; let sha = selection.sha256()?; - // An identical completed selection can never be replayed. - if sha != expected_selection - || records.records.len() >= MAX_RECORDS - || records - .records - .iter() - .any(|r| Selection::quarantine_id(&r.selection_sha256) == &sha[..32]) - { + // An identical retained selection can never be replayed. + if sha != expected_selection || !records.admits(&selection, &sha) { return Err(refused()); } let record = Record { version: 1, selection_sha256: sha, selection, - completed: false, + outcome: Outcome::Pending, }; // Commit point 1: intent is durable before any guest effect. records.write(&record)?; (record, false) } }; - let id = Selection::quarantine_id(&record.selection_sha256).to_owned(); - let quarantine = format!( - "/var/lib/docker/volumes/{}/{QUARANTINE_DIRECTORY}", - admitted.resource.name - ); - let output = engine.guest().execute( + let output = port.execute( MOVE, &[ &admitted.mountpoint, - &quarantine, - &id, + &admitted.quarantine, + record.id(), target.path, &admitted.root_identity, &record.selection.target_identity, ], - None, )?; let moved = match output.as_str() { "cache-quarantine-v1 moved\n" => true, "cache-quarantine-v1 resumed\n" => false, _ => return Err(refused()), }; - if inspect_resource(&engine, &admitted.receipt, &admitted.resource)?.as_ref() - != Some(&admitted.inspected) - { - return Err(refused()); - } + admitted.recheck(port)?; // Commit point 2: completion follows the guest's synchronized rename. - record.completed = true; + record.outcome = Outcome::Quarantined; records.write(&record)?; let mut value = report( - &record.selection, - &record.selection_sha256, + &record, "quarantined", - true, + records.retained(&admitted, Some(&record)), ); + value["mutation"] = json!(true); value["resumed"] = json!(resumed); value["moved_now"] = json!(moved); Ok(value) } +fn resolve_with( + candidate: &Candidate, + port: &impl Port, + target: &CacheQuarantineTarget<'_>, + expected_selection: &str, +) -> Result { + if !hex(expected_selection, 64) { + return Err(refused()); + } + let admitted = admit(candidate, port, target)?; + let records = Records::load(candidate)?; + let mut record = records + .pending(&admitted.resource.name)? + .filter(|r| r.selection_sha256 == expected_selection && r.selection.path == target.path) + .ok_or_else(refused)? + .clone(); + // Read-only under the volume root lock. The current root identity is + // checked; a replaced volume cannot hold the recorded destination inode. + let output = port.execute( + LOCATE, + &[ + &admitted.mountpoint, + &admitted.quarantine, + record.id(), + &admitted.root_identity, + &record.selection.target_identity, + ], + )?; + record.outcome = match output.as_str() { + "cache-quarantine-v1 present\n" => Outcome::Quarantined, + "cache-quarantine-v1 absent\n" => Outcome::Abandoned, + _ => return Err(refused()), + }; + admitted.recheck(port)?; + records.write(&record)?; + let phase = match record.outcome { + Outcome::Quarantined => "quarantined", + _ => "abandoned", + }; + let mut value = report(&record, phase, records.retained(&admitted, Some(&record))); + value["resolved"] = json!(true); + Ok(value) +} + +/// Read-only selection for `quarantine_cache`. It measures the exact admitted +/// target under the provider lease and returns the digest to review. A pending +/// operation for the volume is reported as `resume_required` or +/// `resolve_required` instead. Every result reports the recorded bytes still +/// retained in this volume's quarantine. +pub fn inspect_cache_quarantine( + candidate: &Candidate, + target: CacheQuarantineTarget<'_>, +) -> Result { + inspect_with(candidate, &Engine::connect_cleanup(candidate)?, &target) +} + +/// Move one admitted disposable cache directory into the volume's owned +/// quarantine. Requires the exact digest from `inspect_cache_quarantine`; +/// repeating it after interruption resumes the same recorded operation. +pub fn quarantine_cache( + candidate: &Candidate, + target: CacheQuarantineTarget<'_>, + expected_selection: &str, +) -> Result { + if !hex(expected_selection, 64) { + return Err(refused()); + } + quarantine_with( + candidate, + &Engine::connect_cleanup(candidate)?, + &target, + expected_selection, + ) +} + +/// Settle the volume's pending record without moving bytes: `quarantined` when +/// the recorded directory is at its destination, otherwise `abandoned`. Use it +/// when resumption is refused, for example after a restart changed the receipt. +pub fn resolve_cache_quarantine( + candidate: &Candidate, + target: CacheQuarantineTarget<'_>, + expected_selection: &str, +) -> Result { + if !hex(expected_selection, 64) { + return Err(refused()); + } + resolve_with( + candidate, + &Engine::connect_cleanup(candidate)?, + &target, + expected_selection, + ) +} + // Read-only. Rejects symlinked or non-directory components and reports the // target's identity plus apparent file bytes, allocated bytes and entry count // on the volume filesystem. Symlinks inside the target are counted, not followed. @@ -512,10 +704,11 @@ test "$(stat -c %d:%i "$root")" = "$root_identity" printf 'present %s %s %s\n' "$identity" "$bytes" "$allocated" "#; -// The only mutation: one same-filesystem rename(2) under the volume root lock. -// `mv` cannot fall back to copying because both ends are checked to share the -// root's device. Accepts the exact target at its source or already at its -// destination; any other layout refuses without effects. +// The only mutation: one rename(2) under the volume root lock. Source and +// quarantine must resolve to the same mount (findmnt mount ID), so rename +// cannot fail with EXDEV and `mv` never reaches its copy-then-unlink fallback. +// Accepts the exact target at its source or already at its destination; any +// other layout refuses without effects. const MOVE: &str = r#"set -efu root=$1; quarantine=$2; id=$3; relative=$4; root_identity=$5; target_identity=$6 case "$id" in *[!0-9a-f]*|'') exit 64;; esac @@ -550,6 +743,9 @@ if test -e "$path" || test -L "$path"; then test "$(stat -c %d:%i "$path")" = "$target_identity" test ! -e "$destination" test ! -L "$destination" + mount=$(findmnt -n -o ID -T "$path") + test -n "$mount" + test "$(findmnt -n -o ID -T "$quarantine")" = "$mount" mv "$path" "$destination" state=moved else @@ -563,6 +759,32 @@ sync -f "$root" printf 'cache-quarantine-v1 %s\n' "$state" "#; +// Read-only resolution of a pending record. Under the volume root lock (which +// MOVE holds across its rename and sync), the recorded destination either +// holds the recorded directory or is absent. Anything else refuses. +const LOCATE: &str = r#"set -efu +root=$1; quarantine=$2; id=$3; root_identity=$4; target_identity=$5 +case "$id" in *[!0-9a-f]*|'') exit 64;; esac +test "${#id}" -eq 32 +test "$quarantine" = "${root%/*}/hack-quarantine" +parent=$root +while test "$parent" != /; do test ! -L "$parent"; test -d "$parent"; parent=${parent%/*}; test -n "$parent" || parent=/; done +test "$(stat -c %d:%i "$root")" = "$root_identity" +exec 9<"$root" +flock -w 5 9 +test "$(stat -Lc %d:%i /proc/self/fd/9)" = "$root_identity" +destination=$quarantine/$id +if test ! -e "$quarantine" && test ! -L "$quarantine"; then printf 'cache-quarantine-v1 absent\n'; exit 0; fi +test ! -L "$quarantine" +test -d "$quarantine" +test "$(stat -c %d "$quarantine")" = "${root_identity%%:*}" +if test ! -e "$destination" && test ! -L "$destination"; then printf 'cache-quarantine-v1 absent\n'; exit 0; fi +test ! -L "$destination" +test -d "$destination" +test "$(stat -c %d:%i "$destination")" = "$target_identity" +printf 'cache-quarantine-v1 present\n' +"#; + #[cfg(test)] #[path = "cache_quarantine/tests.rs"] mod tests; diff --git a/packages/runtime-core/src/provider/graph/cache_quarantine/orchestration.rs b/packages/runtime-core/src/provider/graph/cache_quarantine/orchestration.rs new file mode 100644 index 000000000..8112dddf8 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/cache_quarantine/orchestration.rs @@ -0,0 +1,481 @@ +//! The inspect -> quarantine -> resume/resolve orchestration, exercised as +//! shipped against a synthetic provider. Engine answers are held in memory; +//! the production guest scripts run on a fixture volume. +use super::*; +use std::cell::{Cell, RefCell}; + +/// Where a guest effect is interrupted relative to the MOVE script. +#[derive(Clone, Copy, PartialEq)] +enum Fault { + None, + /// The host loses the guest before the script runs. + Before, + /// The script completes, but its reply is lost. + After, +} + +/// A real graph receipt under the candidate state root, engine answers held in +/// memory, and guest scripts run against a fixture whose +/// `/var/lib/docker/volumes` prefix maps into the fixture directory. +struct Provider { + fixture: crate::provider::graph::tests::Fixture, + candidate: Candidate, + volumes: PathBuf, + name: String, + inspected: RefCell, + /// Volume inspection that replaces the current one after the named guest call. + after: RefCell>, + containers: RefCell, + observe_reply: RefCell>, + fault: Cell, + calls: RefCell>, +} +impl Provider { + fn new() -> Self { + let fixture = crate::provider::graph::tests::Fixture::new(); + let candidate = Candidate::discover(&fixture.0).unwrap(); + let name = format!("hkg-{RUN}-volume-0"); + let volumes = fixture.0.join("volumes"); + let root = volumes.join(&name).join("_data"); + fs::create_dir_all(root.join(PATH).join("sst")).unwrap(); + fs::write(root.join(PATH).join("sst/00001.sst"), vec![7; 1000]).unwrap(); + fs::write(root.join(PATH).join("CURRENT"), b"corrupt-checksum").unwrap(); + fs::write(root.join("dev/cache/marker"), b"persistent").unwrap(); + fs::write(root.join("BUILD_ID"), b"keep").unwrap(); + let redis = volumes.join(format!("hkg-{RUN}-volume-1/_data")); + fs::create_dir_all(&redis).unwrap(); + fs::write(redis.join("dump.rdb"), b"redis").unwrap(); + let provider = Self { + candidate, + inspected: RefCell::new(json!({ + "Name":name,"Driver":"local","CreatedAt":"2026-10-19T00:00:00Z", + "Mountpoint":format!("/var/lib/docker/volumes/{name}/_data") + })), + name, + volumes, + fixture, + after: RefCell::new(None), + containers: RefCell::new(json!([ + {"State":"running","Mounts":[{"Type":"volume","Name":"unrelated"}]} + ])), + observe_reply: RefCell::new(None), + fault: Cell::new(Fault::None), + calls: RefCell::new(Vec::new()), + }; + provider.receipt("stopped-data-retained", "created"); + provider + } + /// Write the graph receipt. `redis` varies the receipt bytes the way a + /// retaining restart rewrites them, without changing the cache volume. + fn receipt(&self, phase: &str, redis: &str) { + let root = directory(&self.candidate, RUN).unwrap(); + state::private_directory(&root).unwrap(); + let volume = |key: &str, index: u8, phase: &str, caches: &[&str]| { + json!({"kind":"volume","key":key,"name":format!("hkg-{RUN}-volume-{index}"), + "id":null,"image":null,"phase":phase,"disposable_caches":caches}) + }; + state::write( + &root.join("state.json"), + &json!({ + "version":1,"run":RUN,"owner":"owner","namespace":"a".repeat(64), + "plan_id":"b".repeat(64),"phase":phase,"readiness":{}, + "resources":{ + "volume:www_next":volume("www_next", 0, "created", &[PATH, IMAGES]), + "volume:redis":volume("redis", 1, redis, &[]) + } + }), + ) + .unwrap(); + } + fn root(&self) -> PathBuf { + self.volumes.join(&self.name).join("_data") + } + fn destination(&self, sha: &str) -> PathBuf { + self.volumes + .join(&self.name) + .join("hack-quarantine") + .join(&sha[..32]) + } + fn inspect_path(&self, path: &str) -> Result { + inspect_with(&self.candidate, self, &target("www_next", path)) + } + fn inspect(&self) -> Result { + self.inspect_path(PATH) + } + fn quarantine(&self, sha: &str) -> Result { + quarantine_with(&self.candidate, self, &target("www_next", PATH), sha) + } + fn resolve_path(&self, path: &str, sha: &str) -> Result { + resolve_with(&self.candidate, self, &target("www_next", path), sha) + } + fn resolve(&self, sha: &str) -> Result { + self.resolve_path(PATH, sha) + } + fn records(&self) -> Vec { + Records::load(&self.candidate).unwrap().records + } + fn guest_calls(&self) -> Vec<&'static str> { + self.calls + .borrow() + .iter() + .copied() + .filter(|c| ["observe", "move", "locate"].contains(c)) + .collect() + } + fn untouched_application_data(&self) { + assert_eq!( + fs::read(self.root().join("dev/cache/marker")).unwrap(), + b"persistent" + ); + assert_eq!(fs::read(self.root().join("BUILD_ID")).unwrap(), b"keep"); + assert_eq!( + fs::read( + self.volumes + .join(format!("hkg-{RUN}-volume-1/_data/dump.rdb")) + ) + .unwrap(), + b"redis" + ); + } +} +const IMAGES: &str = "dev/cache/images"; + +impl Port for Provider { + fn load(&self, candidate: &Candidate, run: &str) -> Result<(Receipt, PathBuf), CandidateError> { + self.calls.borrow_mut().push("load"); + load_at(directory(candidate, run)?, run, "owner") + } + fn inspect(&self, _: &Receipt, resource: &Resource) -> Result, CandidateError> { + self.calls.borrow_mut().push("inspect"); + assert_eq!(resource.name, self.name); + let value = self.inspected.borrow().clone(); + Ok((!value.is_null()).then_some(value)) + } + fn containers(&self) -> Result { + self.calls.borrow_mut().push("containers"); + Ok(self.containers.borrow().clone()) + } + fn execute(&self, program: &str, arguments: &[&str]) -> Result { + let lost = || CandidateError::new("guest_lost", "Synthetic guest reply was lost."); + let call = match program { + OBSERVE => "observe", + MOVE => "move", + LOCATE => "locate", + _ => "identity", + }; + self.calls.borrow_mut().push(call); + if call == "observe" + && let Some(reply) = self.observe_reply.borrow().clone() + { + return Ok(reply); + } + if call == "move" && self.fault.get() == Fault::Before { + return Err(lost()); + } + let prefix = format!("{}/", self.volumes.display()); + let mapped: Vec = arguments + .iter() + .map(|a| a.replacen("/var/lib/docker/volumes/", &prefix, 1)) + .collect(); + let mapped: Vec<&str> = mapped.iter().map(String::as_str).collect(); + let output = script(&self.fixture.0, program, &mapped); + if !output.status.success() { + return Err(CandidateError::new( + "guest_script", + "Synthetic guest script refused.", + )); + } + let changed = self.after.borrow_mut().take_if(|(after, _)| *after == call); + if let Some((_, changed)) = changed { + *self.inspected.borrow_mut() = changed; + } + if call == "move" && self.fault.get() == Fault::After { + return Err(lost()); + } + Ok(String::from_utf8(output.stdout).unwrap()) + } +} +fn selection_sha(value: &Value) -> String { + value["selection_sha256"].as_str().unwrap().to_owned() +} + +#[test] +fn inspect_then_quarantine_retains_exact_bytes_and_a_regenerated_cache_is_reselected() { + let provider = Provider::new(); + let inspected = provider.inspect().unwrap(); + assert_eq!(inspected["phase"], "selectable"); + assert_eq!(inspected["mutation"], false); + assert_eq!(inspected["target_bytes"], 1016); + assert_eq!(inspected["entries"], 4); + assert_eq!(inspected["quarantined_bytes"], 0); + assert_eq!(inspected["retained_quarantine_bytes"], 0); + assert_eq!(provider.guest_calls(), ["observe"]); + assert!( + !provider + .candidate + .state_root + .join("run/cache-quarantine") + .exists() + ); + let sha = selection_sha(&inspected); + let identity_before = identity(&provider.root().join(PATH)); + + // A digest that was not reviewed is refused before any guest effect. + assert!(provider.quarantine(&"0".repeat(64)).is_err()); + assert!(!provider.guest_calls().contains(&"move")); + assert!(provider.records().is_empty()); + + let done = provider.quarantine(&sha).unwrap(); + assert_eq!(done["phase"], "quarantined"); + assert_eq!(done["selection_sha256"], sha); + assert_eq!(done["mutation"], true); + assert_eq!(done["moved_now"], true); + assert_eq!(done["resumed"], false); + assert_eq!(done["quarantined_bytes"], 1016); + assert_eq!(done["retained_quarantine_records"], 1); + assert_eq!(done["retained_quarantine_bytes"], 1016); + assert_eq!(done["reclaimed_bytes"], 0); + assert_eq!(done["data_retained"], true); + assert!(!provider.root().join(PATH).exists()); + let destination = provider.destination(&sha); + assert_eq!(identity(&destination), identity_before); + assert_eq!( + fs::read(destination.join("CURRENT")).unwrap(), + b"corrupt-checksum" + ); + provider.untouched_application_data(); + let records = provider.records(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].outcome, Outcome::Quarantined); + + // A retaining restart regenerates the cache: it is a new selection, and + // the earlier quarantine stays counted for this volume incarnation. + fs::create_dir(provider.root().join(PATH)).unwrap(); + fs::write(provider.root().join(PATH).join("fresh"), b"ok").unwrap(); + let again = provider.inspect().unwrap(); + assert_eq!(again["phase"], "selectable"); + assert_ne!(selection_sha(&again), sha); + assert_eq!(again["retained_quarantine_records"], 1); + assert_eq!(again["retained_quarantine_bytes"], 1016); + // The old digest no longer describes the source and moves nothing. + assert!(provider.quarantine(&sha).is_err()); + assert_eq!( + fs::read(provider.root().join(PATH).join("fresh")).unwrap(), + b"ok" + ); + + // An identical, already retained selection can never be replayed. + fs::remove_dir_all(provider.root().join(PATH)).unwrap(); + fs::rename(&destination, provider.root().join(PATH)).unwrap(); + assert_eq!(selection_sha(&provider.inspect().unwrap()), sha); + assert!(provider.quarantine(&sha).is_err()); + assert_eq!(identity(&provider.root().join(PATH)), identity_before); + assert!(!destination.exists()); +} + +#[test] +fn consumers_journal_and_volume_identity_refuse_before_any_guest_observation() { + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + // Any container, stopped or running, that mounts the volume. + for state in ["exited", "running"] { + *provider.containers.borrow_mut() = json!([ + {"State":state,"Mounts":[{"Type":"volume","Name":provider.name}]} + ]); + provider.calls.borrow_mut().clear(); + assert!(provider.inspect().is_err()); + assert!(provider.quarantine(&sha).is_err()); + assert!(provider.resolve(&sha).is_err()); + assert!(provider.guest_calls().is_empty(), "{state}"); + } + *provider.containers.borrow_mut() = json!([]); + + // An incomplete graph journal. + let journal = directory(&provider.candidate, RUN) + .unwrap() + .join("state.pending"); + fs::write(&journal, b"").unwrap(); + provider.calls.borrow_mut().clear(); + assert!(provider.inspect().is_err()); + assert!(provider.quarantine(&sha).is_err()); + assert_eq!(*provider.calls.borrow(), ["load", "load"]); + fs::remove_file(&journal).unwrap(); + + // An active graph. + provider.receipt("ready-observed", "created"); + assert!(provider.inspect().is_err()); + provider.receipt("stopped-data-retained", "created"); + + // A missing or differently identified volume. + let good = provider.inspected.borrow().clone(); + for (field, value) in [ + ("Driver", json!("other")), + ("Name", json!("hkg-foreign")), + ( + "Mountpoint", + json!("/var/lib/docker/volumes/hkg-foreign/_data"), + ), + ("CreatedAt", json!("")), + ] { + let mut changed = good.clone(); + changed[field] = value; + *provider.inspected.borrow_mut() = changed; + provider.calls.borrow_mut().clear(); + assert!(provider.inspect().is_err(), "{field}"); + assert!(provider.quarantine(&sha).is_err(), "{field}"); + assert!(provider.guest_calls().is_empty(), "{field}"); + } + *provider.inspected.borrow_mut() = Value::Null; + assert!(provider.inspect().is_err()); + *provider.inspected.borrow_mut() = good; + + // A target reported on another device than the volume root. + let root_device: u64 = identity(&provider.root()) + .split_once(':') + .unwrap() + .0 + .parse() + .unwrap(); + *provider.observe_reply.borrow_mut() = + Some(format!("present {}:5 1016 8192 4\n", root_device + 1)); + assert!(provider.inspect().is_err()); + assert!(provider.quarantine(&sha).is_err()); + *provider.observe_reply.borrow_mut() = None; + + assert!(provider.records().is_empty()); + assert!(provider.root().join(PATH).join("CURRENT").exists()); + provider.untouched_application_data(); + // The unchanged reviewed selection is still admitted afterwards. + assert_eq!(provider.quarantine(&sha).unwrap()["phase"], "quarantined"); +} + +#[test] +fn interrupted_moves_resume_only_the_recorded_selection() { + // Lost before the guest rename: resumption moves the recorded source. + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + provider.fault.set(Fault::Before); + assert!(provider.quarantine(&sha).is_err()); + assert!(provider.root().join(PATH).exists()); + assert_eq!(provider.records()[0].outcome, Outcome::Pending); + let pending = provider.inspect().unwrap(); + assert_eq!(pending["phase"], "resume_required"); + assert_eq!(selection_sha(&pending), sha); + provider.fault.set(Fault::None); + assert!(provider.quarantine(&"0".repeat(64)).is_err()); + let resumed = provider.quarantine(&sha).unwrap(); + assert_eq!(resumed["resumed"], true); + assert_eq!(resumed["moved_now"], true); + assert_eq!(provider.records()[0].outcome, Outcome::Quarantined); + + // Lost after the guest rename: resumption adopts the recorded inode. + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + let original = identity(&provider.root().join(PATH)); + provider.fault.set(Fault::After); + assert!(provider.quarantine(&sha).is_err()); + assert!(!provider.root().join(PATH).exists()); + assert_eq!(provider.inspect().unwrap()["phase"], "resume_required"); + provider.fault.set(Fault::None); + let resumed = provider.quarantine(&sha).unwrap(); + assert_eq!(resumed["resumed"], true); + assert_eq!(resumed["moved_now"], false); + assert_eq!(identity(&provider.destination(&sha)), original); + provider.untouched_application_data(); +} + +#[test] +fn a_restart_before_resumption_requires_explicit_resolution_without_moving_bytes() { + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + provider.fault.set(Fault::Before); + assert!(provider.quarantine(&sha).is_err()); + provider.fault.set(Fault::None); + // A retaining restart rewrote the receipt before the operator resumed. + provider.receipt("stopped-data-retained", "absent"); + let pending = provider.inspect().unwrap(); + assert_eq!(pending["phase"], "resolve_required"); + assert_eq!(selection_sha(&pending), sha); + // Resumption needs the reviewed receipt; nothing is moved. + assert!(provider.quarantine(&sha).is_err()); + assert!(provider.root().join(PATH).join("CURRENT").exists()); + // Another declared path of the same volume waits for this resolution. + fs::create_dir(provider.root().join(IMAGES)).unwrap(); + fs::write(provider.root().join(IMAGES).join("a"), b"i").unwrap(); + let other = provider.inspect_path(IMAGES).unwrap(); + assert_eq!(other["phase"], "resolve_required"); + assert_eq!(other["path"], PATH); + // Resolution needs the recorded digest and path. + assert!(provider.resolve(&"0".repeat(64)).is_err()); + assert!(provider.resolve_path(IMAGES, &sha).is_err()); + provider.calls.borrow_mut().clear(); + let resolved = provider.resolve(&sha).unwrap(); + assert_eq!(resolved["phase"], "abandoned"); + assert_eq!(resolved["resolved"], true); + assert_eq!(resolved["mutation"], false); + assert_eq!(resolved["quarantined_bytes"], 0); + assert_eq!(provider.guest_calls(), ["locate"]); + assert!(provider.root().join(PATH).join("CURRENT").exists()); + assert_eq!(provider.records()[0].outcome, Outcome::Abandoned); + // Nothing is pending; a fresh review of the current receipt proceeds. + assert!(provider.resolve(&sha).is_err()); + let fresh = provider.inspect().unwrap(); + assert_eq!(fresh["phase"], "selectable"); + assert_ne!(selection_sha(&fresh), sha); + let done = provider.quarantine(&selection_sha(&fresh)).unwrap(); + assert_eq!(done["phase"], "quarantined"); + assert_eq!(done["retained_quarantine_records"], 1); + assert_eq!( + fs::read(provider.root().join(IMAGES).join("a")).unwrap(), + b"i" + ); + provider.untouched_application_data(); +} + +#[test] +fn resolution_records_a_rename_that_completed_before_the_restart() { + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + let original = identity(&provider.root().join(PATH)); + provider.fault.set(Fault::After); + assert!(provider.quarantine(&sha).is_err()); + provider.fault.set(Fault::None); + provider.receipt("stopped-data-retained", "absent"); + // The restarted application regenerated its cache at the source. + fs::create_dir(provider.root().join(PATH)).unwrap(); + fs::write(provider.root().join(PATH).join("fresh"), b"ok").unwrap(); + assert_eq!(provider.inspect().unwrap()["phase"], "resolve_required"); + let resolved = provider.resolve(&sha).unwrap(); + assert_eq!(resolved["phase"], "quarantined"); + assert_eq!(resolved["quarantined_bytes"], 1016); + assert_eq!(resolved["retained_quarantine_bytes"], 1016); + assert_eq!(identity(&provider.destination(&sha)), original); + assert_eq!( + fs::read(provider.root().join(PATH).join("fresh")).unwrap(), + b"ok" + ); + assert_eq!(provider.records()[0].outcome, Outcome::Quarantined); + // A settled record cannot be resolved again. + assert!(provider.resolve(&sha).is_err()); +} + +#[test] +fn a_volume_changed_during_a_guest_effect_is_never_settled() { + let provider = Provider::new(); + let sha = selection_sha(&provider.inspect().unwrap()); + let original = provider.inspected.borrow().clone(); + let mut replaced = original.clone(); + replaced["CreatedAt"] = json!("2026-10-19T00:00:01Z"); + *provider.after.borrow_mut() = Some(("move", replaced.clone())); + assert!(provider.quarantine(&sha).is_err()); + assert_eq!(provider.records()[0].outcome, Outcome::Pending); + // The changed incarnation cannot resume the reviewed selection. + assert_eq!(provider.inspect().unwrap()["phase"], "resolve_required"); + assert!(provider.quarantine(&sha).is_err()); + assert_eq!(provider.records()[0].outcome, Outcome::Pending); + // Resolution observed under one incarnation is not recorded under another. + *provider.inspected.borrow_mut() = original; + *provider.after.borrow_mut() = Some(("locate", replaced)); + assert!(provider.resolve(&sha).is_err()); + assert_eq!(provider.records()[0].outcome, Outcome::Pending); +} diff --git a/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs b/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs index 9d1f60ad2..a78ba83fc 100644 --- a/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs +++ b/packages/runtime-core/src/provider/graph/cache_quarantine/tests.rs @@ -171,13 +171,13 @@ fn selection(target_identity: &str) -> Selection { entries: 3, } } -fn record(target_identity: &str, completed: bool) -> Record { +fn record(target_identity: &str, outcome: Outcome) -> Record { let selection = selection(target_identity); Record { version: 1, selection_sha256: selection.sha256().unwrap(), selection, - completed, + outcome, } } @@ -188,7 +188,7 @@ fn durable_records_are_private_exact_bounded_and_refuse_torn_or_forged_state() { state::private_directory(&candidate.state_root.join("run")).unwrap(); let records = Records::load(&candidate).unwrap(); assert!(records.pending("hkg-www").unwrap().is_none()); - let pending = record("7:200", false); + let pending = record("7:200", Outcome::Pending); records.write(&pending).unwrap(); let loaded = Records::load(&candidate).unwrap(); assert_eq!( @@ -216,7 +216,7 @@ fn durable_records_are_private_exact_bounded_and_refuse_torn_or_forged_state() { assert!(Records::load(&candidate).is_err()); fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); // Two unresolved operations for one volume are ambiguous. - let second = record("7:300", false); + let second = record("7:300", Outcome::Pending); records.write(&second).unwrap(); assert!( Records::load(&candidate) @@ -224,33 +224,58 @@ fn durable_records_are_private_exact_bounded_and_refuse_torn_or_forged_state() { .pending("hkg-www") .is_err() ); - // Completion is retained as diagnosis evidence, not collected. + // Settled outcomes are retained as diagnosis evidence, not collected. records .write(&Record { - completed: true, + outcome: Outcome::Quarantined, ..second.clone() }) .unwrap(); records .write(&Record { - completed: true, + outcome: Outcome::Abandoned, ..pending.clone() }) .unwrap(); let done = Records::load(&candidate).unwrap(); assert!(done.pending("hkg-www").unwrap().is_none()); assert_eq!(done.records.len(), 2); + // A quarantined selection never re-admits; an abandoned one moved nothing. + assert!(!done.admits(&second.selection, &second.selection_sha256)); + assert!(done.admits(&pending.selection, &pending.selection_sha256)); let mut invalid = pending.clone(); invalid.selection.target_bytes += 1; assert!(records.write(&invalid).is_err()); + // The per-volume budget refuses new selections for that volume only. + for inode in 0..MAX_VOLUME_RECORDS - 2 { + records + .write(&record( + &format!("7:{}", 1000 + inode), + Outcome::Quarantined, + )) + .unwrap(); + } + let full = Records::load(&candidate).unwrap(); + let next = record("7:999", Outcome::Pending); + assert!(!full.admits(&next.selection, &next.selection_sha256)); + let mut other = next.selection.clone(); + other.volume_name = "hkg-other".into(); + assert!(full.admits(&other, &other.sha256().unwrap())); } /// Run a production guest script on this host. Linux runs it unchanged; macOS /// receives PATH shims translating GNU `stat -c`, util-linux `flock`, and /// `sync -f` to the same syscalls (lstat, fstat of fd 9, flock, fsync). fn script(fixture: &Path, program: &str, args: &[&str]) -> Output { + script_with(fixture, program, args, false) +} +/// `split_mount` places a `findmnt` shim first in PATH that reports a distinct +/// mount for the quarantine root, as a separate bind mount of the same +/// filesystem would; equal device numbers alone cannot exclude that layout. +fn script_with(fixture: &Path, program: &str, args: &[&str], split_mount: bool) -> Output { let mut command = Command::new("sh"); command.args(["-c", program, "sh"]).args(args); + let mut path = std::env::var("PATH").unwrap(); if cfg!(target_os = "macos") { let bin = fixture.join("shim"); if !bin.exists() { @@ -292,19 +317,42 @@ while True: r#"#!/bin/sh test "$1" = -f || exit 70 exec python3 -c 'import os,sys; fd=os.open(sys.argv[1],os.O_RDONLY); os.fsync(fd); os.close(fd)' "$2" +"#, + ), + ( + // The mount containing the path, as statfs(2) reports it. + "findmnt", + r#"#!/bin/sh +test "$1 $2 $3 $4" = "-n -o ID -T" || exit 70 +exec python3 -c 'import os,sys; print(os.statvfs(sys.argv[1]).f_fsid)' "$5" "#, ), ] { - let path = bin.join(name); - fs::write(&path, body).unwrap(); - fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).unwrap(); + let shim = bin.join(name); + fs::write(&shim, body).unwrap(); + fs::set_permissions(&shim, fs::Permissions::from_mode(0o755)).unwrap(); } } - command.env( - "PATH", - format!("{}:{}", bin.display(), std::env::var("PATH").unwrap()), - ); + path = format!("{}:{path}", bin.display()); } + if split_mount { + let bin = fixture.join("split-mount"); + if !bin.exists() { + fs::create_dir(&bin).unwrap(); + let shim = bin.join("findmnt"); + fs::write( + &shim, + r#"#!/bin/sh +test "$1 $2 $3 $4" = "-n -o ID -T" || exit 70 +case "$5" in */hack-quarantine) echo 2;; *) echo 1;; esac +"#, + ) + .unwrap(); + fs::set_permissions(&shim, fs::Permissions::from_mode(0o755)).unwrap(); + } + path = format!("{}:{path}", bin.display()); + } + command.env("PATH", path); command.output().unwrap() } fn identity(path: &Path) -> String { @@ -581,3 +629,76 @@ fn foreign_quarantine_root_lock_and_identity_refuse_without_effects() { ); volume.untouched_application_data(); } + +#[test] +fn move_refuses_a_quarantine_on_another_mount_before_any_rename() { + // Equal st_dev does not exclude two bind mounts of one filesystem, where + // rename(2) fails with EXDEV and `mv` would copy then unlink the source. + let volume = Volume::new(); + let original = identity(&volume.target); + let output = script_with( + &volume.fixture.0, + MOVE, + &[ + volume.root.to_str().unwrap(), + volume.quarantine.to_str().unwrap(), + ID, + PATH, + &identity(&volume.root), + &original, + ], + true, + ); + assert!(!output.status.success()); + assert_eq!(identity(&volume.target), original); + assert!(!volume.quarantine.join(ID).exists()); + volume.untouched_application_data(); +} + +#[test] +fn locate_reports_only_the_recorded_destination_and_never_moves() { + let volume = Volume::new(); + let original = identity(&volume.target); + let locate = |root_identity: &str, target_identity: &str| { + script( + &volume.fixture.0, + LOCATE, + &[ + volume.root.to_str().unwrap(), + volume.quarantine.to_str().unwrap(), + ID, + root_identity, + target_identity, + ], + ) + }; + let root = identity(&volume.root); + // No quarantine root, then an empty one: the rename never happened. + assert_eq!( + stdout(locate(&root, &original)), + "cache-quarantine-v1 absent\n" + ); + fs::create_dir(&volume.quarantine).unwrap(); + fs::set_permissions(&volume.quarantine, fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!( + stdout(locate(&root, &original)), + "cache-quarantine-v1 absent\n" + ); + assert_eq!(identity(&volume.target), original); + fs::rename(&volume.target, volume.quarantine.join(ID)).unwrap(); + assert_eq!( + stdout(locate(&root, &original)), + "cache-quarantine-v1 present\n" + ); + // A different directory at the destination, or a replaced root, refuses. + assert!(!locate(&root, "1:1").status.success()); + assert!(!locate("0:0", &original).status.success()); + let moved = volume.fixture.0.join("moved-quarantine"); + fs::rename(&volume.quarantine, &moved).unwrap(); + std::os::unix::fs::symlink(&moved, &volume.quarantine).unwrap(); + assert!(!locate(&root, &original).status.success()); + volume.untouched_application_data(); +} + +#[path = "orchestration.rs"] +mod orchestration; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test.rs b/packages/runtime-core/src/provider/graph/foreground/native_test.rs index 9eabfe23c..fdd3db761 100644 --- a/packages/runtime-core/src/provider/graph/foreground/native_test.rs +++ b/packages/runtime-core/src/provider/graph/foreground/native_test.rs @@ -548,6 +548,7 @@ mod dependency_slots; mod startup_cancellation; mod absent_publication_recovery; +mod cache_quarantine; mod host_pin_recovery; #[cfg(feature = "native-http-probe")] mod retired_rebind_history; diff --git a/packages/runtime-core/src/provider/graph/foreground/native_test/cache_quarantine.rs b/packages/runtime-core/src/provider/graph/foreground/native_test/cache_quarantine.rs new file mode 100644 index 000000000..d380b5d14 --- /dev/null +++ b/packages/runtime-core/src/provider/graph/foreground/native_test/cache_quarantine.rs @@ -0,0 +1,293 @@ +//! Owned synthetic bad-cache fixture. A corrupt marker inside a declared +//! disposable cache directory fails a synthetic route check. After a retaining +//! stop, explicit quarantine and a retaining restore, the check passes while a +//! separate persistent marker and every other byte of the cache volume survive. +//! This qualifies the path through `Engine::connect_cleanup`, guest execution, +//! Docker volume inspection and the container list; it does not classify the +//! cause of a real framework cache corruption. +use super::absent_publication_recovery::{ready, refused_cli}; +use super::dependency_rebind::{checked_cli, exec, snapshot}; +use super::*; +use base64::Engine as _; + +const CACHE: &str = "dev/cache/turbopack"; +/// The synthetic route succeeds only without the corrupt cache marker, and +/// prints a sibling build marker from the same volume. +const ROUTE: &str = "test ! -e /cache/dev/cache/turbopack/CURRENT && cat /cache/BUILD_ID"; + +fn shell( + binary: &Path, + candidate: &Candidate, + run: &str, + script: &str, + deadline: Instant, +) -> (u64, Vec) { + let mut process = Process::start( + binary, + candidate, + &[ + "graph", + "exec", + "--run-id", + run, + "--service", + "web", + "--timeout-seconds", + "12", + "--json", + "--", + "/bin/sh", + "-ec", + script, + ], + None, + ); + let status = process.wait(deadline); + let result: Value = serde_json::from_slice(&process.out).expect("admitted exec result"); + let code = result["exit_code"].as_u64().expect("application exit code"); + assert_eq!(status.code().map(i64::from), Some(code as i64)); + assert_eq!(result["truncated"], false); + let stdout = base64::engine::general_purpose::STANDARD + .decode(result["stdout_base64"].as_str().unwrap()) + .unwrap(); + (code, stdout) +} + +fn target_args<'a>(action: &'a str, run: &'a str, volume: &'a str, path: &'a str) -> Vec<&'a str> { + vec![ + "graph", action, "--run-id", run, "--volume", volume, "--path", path, + ] +} + +#[test] +#[ignore = "Owned development VM, matching all-feature CLI, pinned static image and external 300s watchdog required"] +fn synthetic_bad_cache_is_quarantined_and_a_retaining_restore_keeps_application_data() { + let deadline = Instant::now() + Duration::from_secs(270); + let candidate = + Candidate::discover(Path::new(&std::env::var("HACK_LOCAL_TEST_ROOT").unwrap())).unwrap(); + let binary = PathBuf::from(std::env::var("HACK_LOCAL_TEST_BINARY").unwrap()); + let image = std::env::var("HACK_LOCAL_TEST_IMAGE").unwrap(); + let fixture = graph::tests::Fixture::new(); + state::write( + &fixture.0.join("compose.yaml"), + &json!({ + "services":{"web":{"image":image,"read_only":true,"network_mode":"none", + "init":true,"user":"0:0","entrypoint":["/bin/sleep","300"],"command":[], + "volumes":["data:/data","cache:/cache"], + "healthcheck":{"test":["CMD","/bin/hack-graph-startup-app","complete"], + "interval":"200ms","timeout":"2s","retries":10,"start_period":"500ms"}}}, + "volumes":{"data":{},"cache":{"x-hack-disposable-cache":[CACHE]}} + }), + ) + .unwrap(); + let review = project::plan( + &candidate, + project::PlanOptions { + branch: None, + project: &fixture.0, + compose_file: Path::new("compose.yaml"), + profiles: &[], + }, + ) + .unwrap(); + let dependencies_path = fixture.0.join("dependencies.json"); + state::write( + &dependencies_path, + &json!({"version":1,"plan":review.plan_id, + "artifact":"/tmp/unused-control-only-artifact","artifact_sha256":"a".repeat(64), + "dependencies":[]}), + ) + .unwrap(); + let dependency = checked_cli( + "dependency-plan", + &binary, + &candidate, + &[ + "graph", + "dependency-plan", + "--dependencies", + dependencies_path.to_str().unwrap(), + "--json", + ], + deadline, + ); + let run = graph::probes::token().unwrap(); + let normalized = fixture.0.join("compose.yaml"); + let start = |generation: Option<&str>| { + let mut args = vec![ + "graph", + if generation.is_some() { + "serve-restore" + } else { + "serve" + }, + "--project", + fixture.0.to_str().unwrap(), + "--file", + "compose.yaml", + "--expect-plan", + &review.plan_id, + "--run-id", + &run, + "--ready", + "web=healthy", + "--timeout-seconds", + "90", + "--dependencies", + dependencies_path.to_str().unwrap(), + "--expect-dependencies", + dependency["dependency_plan_id"].as_str().unwrap(), + "--normalized-file", + normalized.to_str().unwrap(), + "--expect-original", + &review.plan.compose_sha256, + "--expect-namespace", + &review.plan.namespace, + "--json", + ]; + if let Some(generation) = generation { + args.extend(["--expect-generation", generation]); + } + Process::start(&binary, &candidate, &args, None) + }; + let mut owner = start(None); + let mut cleanup = Cleanup { + binary: &binary, + candidate: &candidate, + run: &run, + done: false, + }; + ready(&mut owner, &run, deadline); + assert_eq!( + exec(&binary, &candidate, &run, "web", "write-data", deadline)["exit_code"], + 0 + ); + let (code, _) = shell( + &binary, + &candidate, + &run, + "mkdir -p /cache/dev/cache/turbopack/sst \ + && printf corrupt-checksum > /cache/dev/cache/turbopack/CURRENT \ + && printf 1234567 > /cache/dev/cache/turbopack/sst/00001.sst \ + && printf keep > /cache/BUILD_ID", + deadline, + ); + assert_eq!(code, 0); + // Container health is green while the synthetic route fails. + assert_ne!(shell(&binary, &candidate, &run, ROUTE, deadline).0, 0); + let ready_receipt = snapshot(&candidate, &run, deadline).receipt; + assert_eq!( + ready_receipt.resources["volume:cache"].disposable_caches, + [CACHE] + ); + let cache_volume = ready_receipt.resources["volume:cache"].name.clone(); + let data_volume = ready_receipt.resources["volume:data"].name.clone(); + let mut inspect = target_args("inspect-cache-quarantine", &run, "cache", CACHE); + inspect.push("--json"); + + let stopped = checked_cli( + "retaining-cleanup", + &binary, + &candidate, + &["graph", "cleanup", "--run-id", &run, "--json"], + deadline, + ); + assert_eq!(stopped["phase"], "stopped-data-retained"); + assert!(owner.wait(deadline).success()); + // Undeclared paths and application-data volumes are never selectable. + for (volume, path) in [ + ("cache", "dev/cache"), + ("cache", "BUILD_ID"), + ("data", CACHE), + ] { + let mut args = target_args("inspect-cache-quarantine", &run, volume, path); + args.push("--json"); + refused_cli( + &binary, + &candidate, + &args, + "graph_cache_quarantine", + deadline, + ); + } + let selection = checked_cli("inspect-cache", &binary, &candidate, &inspect, deadline); + assert_eq!(selection["phase"], "selectable"); + assert_eq!(selection["target_bytes"], 16 + 7); + assert_eq!(selection["entries"], 4); + assert_eq!(selection["mutation"], false); + let sha = selection["selection_sha256"].as_str().unwrap(); + let unreviewed = "0".repeat(64); + let mut wrong = target_args("quarantine-cache", &run, "cache", CACHE); + wrong.extend(["--expect-selection", &unreviewed, "--json"]); + refused_cli( + &binary, + &candidate, + &wrong, + "graph_cache_quarantine", + deadline, + ); + let mut quarantine = target_args("quarantine-cache", &run, "cache", CACHE); + quarantine.extend(["--expect-selection", sha, "--json"]); + let done = checked_cli( + "quarantine-cache", + &binary, + &candidate, + &quarantine, + deadline, + ); + assert_eq!(done["phase"], "quarantined"); + assert_eq!(done["quarantined_bytes"], 16 + 7); + assert_eq!(done["reclaimed_bytes"], 0); + assert_eq!(done["moved_now"], true); + // The reviewed selection cannot be replayed. + refused_cli( + &binary, + &candidate, + &quarantine, + "graph_cache_quarantine", + deadline, + ); + let absent = checked_cli("inspect-absent", &binary, &candidate, &inspect, deadline); + assert_eq!(absent["phase"], "target_absent"); + assert_eq!(absent["retained_quarantine_bytes"], 16 + 7); + + let next = checked_cli( + "restore-selection", + &binary, + &candidate, + &["graph", "restore-selection", "--run-id", &run, "--json"], + deadline, + ); + owner = start(Some(next["generation"].as_str().unwrap())); + ready(&mut owner, &run, deadline); + // The route recovers; the sibling build marker and data volume survive. + assert_eq!( + shell(&binary, &candidate, &run, ROUTE, deadline), + (0, b"keep".to_vec()) + ); + assert_eq!( + exec(&binary, &candidate, &run, "web", "read-data", deadline)["exit_code"], + 0 + ); + let restored = snapshot(&candidate, &run, deadline).receipt; + assert_eq!(restored.resources["volume:cache"].name, cache_volume); + assert_eq!(restored.resources["volume:data"].name, data_volume); + + let removed = checked_cli( + "remove-selected-data", + &binary, + &candidate, + &[ + "graph", + "cleanup", + "--run-id", + &run, + "--remove-data", + "--json", + ], + deadline, + ); + assert_eq!(removed["phase"], "removed"); + assert!(owner.wait(deadline).success()); + cleanup.done = true; +} diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index 65497ffa9..995548dac 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -53,7 +53,9 @@ pub use source_device_rebind::{ }; mod cache_quarantine; mod initializer_cache; -pub use cache_quarantine::{CacheQuarantineTarget, inspect_cache_quarantine, quarantine_cache}; +pub use cache_quarantine::{ + CacheQuarantineTarget, inspect_cache_quarantine, quarantine_cache, resolve_cache_quarantine, +}; mod volume_subpaths; pub use dependency_hosts::dependency_address; #[cfg(target_os = "macos")] diff --git a/packages/runtime-core/src/provider/graph/volume_subpaths.rs b/packages/runtime-core/src/provider/graph/volume_subpaths.rs index 08a8e360c..987610212 100644 --- a/packages/runtime-core/src/provider/graph/volume_subpaths.rs +++ b/packages/runtime-core/src/provider/graph/volume_subpaths.rs @@ -311,14 +311,25 @@ pub(super) fn directory_identity( engine: &Engine<'_>, resource: &Resource, inspected: &Value, +) -> Result { + directory_identity_with( + |script, arguments| engine.guest().execute(script, arguments, None), + resource, + inspected, + ) +} +/// The same check through an explicit guest executor, so a caller that +/// abstracts its provider effects runs the identical script and validation. +pub(super) fn directory_identity_with( + execute: impl FnOnce(&str, &[&str]) -> Result, + resource: &Resource, + inspected: &Value, ) -> Result { let expected = format!("/var/lib/docker/volumes/{}/_data", resource.name); if inspected["Mountpoint"].as_str() != Some(expected.as_str()) { return Err(refused()); } - let identity = engine - .guest() - .execute(IDENTITY, &[&expected, "existing"], None)?; + let identity = execute(IDENTITY, &[&expected, "existing"])?; let identity = identity.trim_end_matches('\n').to_owned(); if !valid_identity(&identity) { return Err(refused()); diff --git a/packages/runtime-core/tests/cache_quarantine_cli_contract.rs b/packages/runtime-core/tests/cache_quarantine_cli_contract.rs index 5ede4e0c3..d9fb5a677 100644 --- a/packages/runtime-core/tests/cache_quarantine_cli_contract.rs +++ b/packages/runtime-core/tests/cache_quarantine_cli_contract.rs @@ -78,6 +78,15 @@ fn cache_quarantine_requires_one_exact_target_and_reviewed_selection() { inspect.truncate(8); inspect.push("--expect-selection".into()); cases.push(inspect); + // Resolution takes the same exact shape and never a mutation flag. + let mut resolve = full.clone(); + resolve[1] = "resolve-cache-quarantine".into(); + let mut unreviewed = resolve.clone(); + unreviewed.drain(8..10); + cases.push(unreviewed); + let mut destructive = resolve.clone(); + destructive.insert(10, "--remove-data".into()); + cases.push(destructive); for arguments in cases { let (status, failure) = run(&home, &arguments); assert_eq!(status, Some(2), "{arguments:?}"); @@ -85,13 +94,21 @@ fn cache_quarantine_requires_one_exact_target_and_reviewed_selection() { assert_eq!(fs::read_dir(&home).unwrap().count(), 0); } // A malformed selection digest refuses before the provider is contacted. - for selection in [ + for (action, selection) in [ "B".repeat(64), "b".repeat(63), "b".repeat(65), String::new(), - ] { + ] + .into_iter() + .flat_map(|s| { + [ + ("quarantine-cache", s.clone()), + ("resolve-cache-quarantine", s), + ] + }) { let mut arguments = full.clone(); + arguments[1] = action.into(); arguments[9] = selection; let (status, failure) = run(&home, &arguments); assert_eq!(status, Some(2), "{arguments:?}"); From 536b348a34758560be97bea55a101c0e96146475 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:49:38 -0400 Subject: [PATCH 15/21] test(native): answer image admission in the branch startup tripwire Plan-time image admission now runs `runtime admit-image` before lifecycle hooks. The branch startup fixture's fake runtime treated every unknown call as a runtime effect and failed it, so admission refused before hooks ran. Answer admission like the runtime does, without touching the native-ran tripwire, so the test still proves hooks run before runtime up. Co-Authored-By: Claude Opus 5.5 --- tests/native-project-branch-cli.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/native-project-branch-cli.test.ts b/tests/native-project-branch-cli.test.ts index 3e696aa62..ba1a80217 100644 --- a/tests/native-project-branch-cli.test.ts +++ b/tests/native-project-branch-cli.test.ts @@ -59,6 +59,12 @@ if [ "$4" = "check-project-share" ]; then touch "$HOME/preflight-ran" ${approveSource ? `printf '%s\\n' '{"source_admitted":true,"pool_initialized":false}'\n exit 0` : "exit 71"} fi +# Image admission is a pure runtime check that runs before hooks; it is not a +# runtime effect, so it answers without touching the native-ran tripwire. +if [ "$4" = "admit-image" ]; then + printf '{"reference":"%s","kind":"docker_hub"}\n' "$6" + exit 0 +fi touch "$HOME/native-ran" exit 71 `, From 11436cd7777a54c987faac2b3468b112cd775ce0 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:53:00 -0400 Subject: [PATCH 16/21] fix(setup): rename setup migrate rollback flag to --rollback-manifest argv is parsed once against the union of every command's options keyed by long name. Declaring `setup migrate --rollback ` as a value option retyped the existing boolean `config adopt --rollback`, so `hack config adopt --rollback --json` consumed `--json` as its value and failed with a usage error. That broke every legacy-compose adoption rollback in the docker E2E shards. Rename the migration flag to `--rollback-manifest ` and add a command-tree guard that each long option keeps one value type, plus a parse regression for `config adopt --rollback --json`. Co-Authored-By: Claude Opus 5.5 --- docs/cli.md | 2 +- docs/integrations.md | 4 +- docs/reference/cli.md | 2 +- src/commands/setup-migrate.ts | 20 ++++----- tests/cli-command.test.ts | 69 +++++++++++++++++++++++++++++ tests/setup-migrate-command.test.ts | 11 +++-- 6 files changed, 90 insertions(+), 18 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 48571f110..15b14b465 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -146,7 +146,7 @@ Hack-owned integrations; add `--install-missing` to create missing ones (see repair; restart only if the client cannot reload changed hooks or skills. To move from standalone skill and rule files to the `hack` plugin, preview with `hack setup migrate`, then run `hack setup migrate --apply` after the plugin loads; apply writes a -backup manifest that `hack setup migrate --rollback ` restores (see +backup manifest that `hack setup migrate --rollback-manifest ` restores (see [migrate standalone files](integrations.md#migrate-standalone-files-to-the-plugin)). Ordinary commands, `hack update`, and `hack doctor --fix` never render, repair, remove, or otherwise mutate integration files. CLI upgrades do not trigger integration updates. diff --git a/docs/integrations.md b/docs/integrations.md index 6afd07c6a..e05ecac61 100644 --- a/docs/integrations.md +++ b/docs/integrations.md @@ -160,7 +160,7 @@ with their own `hack setup` command if you no longer want them. hack setup migrate # preview this project; changes nothing hack setup migrate --global --codex # preview user-scope Codex files only hack setup migrate --apply # back up and remove the previewed files -hack setup migrate --rollback ~/.hack/agent-migrations/.json +hack setup migrate --rollback-manifest ~/.hack/agent-migrations/.json ``` The preview (the default) is read-only. For each file it reports one action: @@ -194,7 +194,7 @@ is empty afterwards. A file edited in between is reported as `changed`, left in place, and makes the command exit 1. With nothing Hack-owned to remove, apply writes no manifest. -`--rollback ` restores every recorded file whose path is now empty, with +`--rollback-manifest ` restores every recorded file whose path is now empty, with its original bytes and mode. It never overwrites an existing file: identical bytes are reported as `already-present`, and different content as `conflict` (exit 1), left unchanged. Rollback can be repeated, and it also recovers an interrupted diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 5abc04f81..f27f6501d 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -1970,7 +1970,7 @@ hack setup migrate [options] | `--cursor` | Limit to Cursor files (default: all clients) | | `--skip-plugin-check ` | Treat the plugin as verified for these comma-separated clients (claude, codex, cursor, or all) after checking it loads yourself | | `--apply` | Remove the previewed Hack-owned standalone files after backing them up (default: read-only preview) | -| `--rollback ` | Restore the exact files recorded in a migration backup manifest | +| `--rollback-manifest ` | Restore the exact files recorded in a migration backup manifest | | `--json` | Output JSON (machine-readable) | | `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | | `--help, -h` | Show help | diff --git a/src/commands/setup-migrate.ts b/src/commands/setup-migrate.ts index 383ed8eb7..8a9c29724 100644 --- a/src/commands/setup-migrate.ts +++ b/src/commands/setup-migrate.ts @@ -29,7 +29,7 @@ import { logger } from "../ui/logger.ts"; /** * `hack setup migrate`: preview (default, read-only), `--apply`, or - * `--rollback ` for moving from standalone Hack skill and rule + * `--rollback-manifest ` for moving from standalone Hack skill and rule * files to the skills-only `hack` plugin. See `plugin-migration.ts` for the * ownership, verification and backup contract. */ @@ -42,10 +42,10 @@ const optApply = defineOption({ "Remove the previewed Hack-owned standalone files after backing them up (default: read-only preview)", } as const); -const optRollback = defineOption({ - name: "rollback", +const optRollbackManifest = defineOption({ + name: "rollbackManifest", type: "string", - long: "--rollback", + long: "--rollback-manifest", valueHint: "", description: "Restore the exact files recorded in a migration backup manifest", @@ -104,7 +104,7 @@ const setupMigrateOptions = [ optMigrateCursor, optSkipPluginCheck, optApply, - optRollback, + optRollbackManifest, optJson, ] as const; @@ -191,10 +191,10 @@ export async function handleSetupMigrate({ readonly args: SetupMigrateArgs; }): Promise { const options = args.options; - const rollback = options.rollback?.trim(); - if (options.rollback !== undefined) { + const rollback = options.rollbackManifest?.trim(); + if (options.rollbackManifest !== undefined) { if (!rollback) { - throw new CliUsageError("--rollback needs a manifest path."); + throw new CliUsageError("--rollback-manifest needs a manifest path."); } if ( options.apply || @@ -207,7 +207,7 @@ export async function handleSetupMigrate({ options.path !== undefined ) { throw new CliUsageError( - "--rollback takes only the manifest path (and --json); the manifest records the scopes and files." + "--rollback-manifest takes only the manifest path (and --json); the manifest records the scopes and files." ); } const result = await rollbackPluginMigration({ @@ -472,7 +472,7 @@ async function reportApply(opts: { }); logger.info({ message: `Backup manifest: ${result.manifestPath}` }); logger.info({ - message: `Undo with: hack setup migrate --rollback ${result.manifestPath}`, + message: `Undo with: hack setup migrate --rollback-manifest ${result.manifestPath}`, }); return exitCode; } diff --git a/tests/cli-command.test.ts b/tests/cli-command.test.ts index e63a10ae4..d3eb7c4c4 100644 --- a/tests/cli-command.test.ts +++ b/tests/cli-command.test.ts @@ -1,6 +1,10 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { + type AnyCommandSpec, + builtinOptions, collectAllowedOptionNames, + type OptionSpec, + parseCliArgv, parseOptionsForCommand, parsePositionalsForCommand, resolveCommand, @@ -86,6 +90,71 @@ test("resolveCommand exposes host exec path", () => { expect(resolved.remainingPositionals).toEqual(["bun", "test"]); }); +/** + * argv is parsed once against the union of every command's options, keyed by + * long name, so one long name declared as a boolean in one command and as a + * value in another changes how every command parses it. A value-typed + * `setup migrate --rollback` once made `config adopt --rollback --json` consume + * `--json` as its value and fail with a usage error. + */ +test("each long option keeps one value type across the command tree", () => { + const declarations = new Map>(); + const record = (opts: { + readonly owner: string; + readonly options: readonly OptionSpec[]; + }) => { + for (const option of opts.options) { + const kind = option.type === "boolean" ? "boolean" : "value"; + const owners = declarations.get(option.long) ?? new Set(); + owners.add(`${kind} (${opts.owner})`); + declarations.set(option.long, owners); + } + }; + record({ owner: "builtin", options: builtinOptions() }); + record({ owner: "global", options: CLI_SPEC.globalOptions }); + const walk = ( + commands: readonly AnyCommandSpec[], + parent: readonly string[] + ) => { + for (const command of commands) { + const path = [...parent, command.name]; + record({ owner: path.join(" "), options: command.options }); + walk(command.subcommands, path); + } + }; + walk(CLI_SPEC.commands, []); + + const conflicts = [...declarations] + .filter( + ([, owners]) => + new Set([...owners].map((owner) => owner.split(" ")[0])).size > 1 + ) + .map(([long, owners]) => `${long}: ${[...owners].sort().join(", ")}`); + expect(conflicts).toEqual([]); +}); + +test("boolean config adopt --rollback does not consume the next flag", () => { + const adopt = parseCliArgv(CLI_SPEC, [ + "config", + "adopt", + "--rollback", + "--json", + ]); + expect(adopt.values.rollback).toBe(true); + expect(adopt.values.json).toBe(true); + expect(adopt.positionals).toEqual(["config", "adopt"]); + + const migrate = parseCliArgv(CLI_SPEC, [ + "setup", + "migrate", + "--rollback-manifest", + "manifest.json", + "--json", + ]); + expect(migrate.values["rollback-manifest"]).toBe("manifest.json"); + expect(migrate.values.json).toBe(true); +}); + test("dispatch rejects retired GitHub PR automation flags as unknown", async () => { const result = await runCliWithCapturedOutput([ "dispatch", diff --git a/tests/setup-migrate-command.test.ts b/tests/setup-migrate-command.test.ts index 0de7dfc58..500b8bd03 100644 --- a/tests/setup-migrate-command.test.ts +++ b/tests/setup-migrate-command.test.ts @@ -152,7 +152,7 @@ test( await expect(lstat(owned)).rejects.toThrow(); expect(await readFile(rule, "utf8")).toContain("team addition"); - const rollback = await runCli(["--rollback", manifestPath]); + const rollback = await runCli(["--rollback-manifest", manifestPath]); expect(rollback.exitCode).toBe(0); expect(rollback.stdout).toContain("1 restored"); expect(await snapshotRepo()).toEqual(before); @@ -163,7 +163,10 @@ test( test( "a nonexistent manifest is refused without output of file content", async () => { - const result = await runCli(["--rollback", join(root, "missing.json")]); + const result = await runCli([ + "--rollback-manifest", + join(root, "missing.json"), + ]); expect(result.exitCode).toBe(1); expect(`${result.stdout}${result.stderr}`).toContain( "nothing was restored" @@ -176,10 +179,10 @@ test( "rollback rejects scope, client and apply flags", async () => { for (const extra of [["--apply"], ["--global"], ["--codex"]]) { - const result = await runCli(["--rollback", "m.json", ...extra]); + const result = await runCli(["--rollback-manifest", "m.json", ...extra]); expect(result.exitCode).not.toBe(0); expect(`${result.stdout}${result.stderr}`).toContain( - "--rollback takes only the manifest path" + "--rollback-manifest takes only the manifest path" ); } }, From 7b171775cd3ecd422914060aa6b25c11c50d1cfb Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 12:59:51 -0400 Subject: [PATCH 17/21] fix(runtime): give detached Compose image build/pull its own budget Detached startup ran one `docker compose up -d` under the 90 s HACK_COMPOSE_STARTUP_TIMEOUT_MS budget, so a first-time image build or large pull was killed before any container started. Split detached startup into an image phase (`up --no-start`, HACK_COMPOSE_IMAGE_TIMEOUT_MS, default 30 min) and a start phase (`up -d --no-build`, the existing startup budget). Each phase keeps its own owned process-group deadline, and E_STARTUP_TIMEOUT now names the phase, its budget and the variable that raises it. Refs HACK-1274 Co-Authored-By: Claude Opus 5.5 --- docs/cli.md | 51 ++++-- src/backends/runtime-backend.ts | 65 +++++-- src/commands/project.ts | 150 ++++++++++------ src/lib/compose-startup-budget.ts | 151 +++++++++++++++-- src/lib/dependency-cache-bootstrap.ts | 6 +- tests/compose-startup-budget.test.ts | 32 ++++ tests/compose-startup-phases.test.ts | 226 +++++++++++++++++++++++++ tests/project-up-startup-state.test.ts | 99 +++++++++++ tests/runtime-backend.test.ts | 46 +++-- 9 files changed, 716 insertions(+), 110 deletions(-) create mode 100644 tests/compose-startup-phases.test.ts diff --git a/docs/cli.md b/docs/cli.md index 562f75097..c47813776 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -209,27 +209,45 @@ Hack scans `.npmrc`, `.yarnrc.yml`, and `bunfig.toml` for `${VAR}` credential re with `E_ENV_KEY_MISSING` when the selected Hack overlay cannot supply them. Values are never printed. -Detached Compose startup has a 90-second budget by default. For a slow first -install or container launch, set the host CLI environment explicitly: +Detached Compose startup runs in two phases, each with its own budget: + +| Phase | Compose invocation | Default | Variable | Range (ms) | +| --- | --- | --- | --- | --- | +| Image build/pull and container creation | `up --no-start` | 30 minutes | `HACK_COMPOSE_IMAGE_TIMEOUT_MS` | 1,000–14,400,000 | +| Start and dependency wait | `up -d --no-build` | 90 seconds | `HACK_COMPOSE_STARTUP_TIMEOUT_MS` | 1,000–3,600,000 | + +A first-time build or a large image pull therefore does not consume the start +budget. Raise the budget for the phase that was hit, on the host CLI invocation +rather than in a container's environment: ```sh +# Slow first build or pull of a large image +HACK_COMPOSE_IMAGE_TIMEOUT_MS=3600000 hack up --detach +# Slow container start or a dependency that is slow to become healthy HACK_COMPOSE_STARTUP_TIMEOUT_MS=300000 hack up --detach HACK_COMPOSE_STARTUP_TIMEOUT_MS=300000 hack restart ``` -The value must be an integer from 1,000 to 3,600,000 milliseconds; detached startup -and restart reject invalid values before project preparation or teardown. Full and service-scoped -startup/restart use the same setting. The startup phase notice and -`E_STARTUP_TIMEOUT` message report the effective budget. It covers each detached -Compose invocation, including image/container launch and dependency waiting, not -lifecycle hooks or an end-to-end application readiness deadline. A restart repair -attempt has its own invocation budget and is reported separately. Foreground -`up` stays unbounded; `down` retains its separate 90-second limit. - -On timeout, Hack terminates the owned Compose process group. Containers may still -be running: inspect `hack ps` and `hack logs` before retrying. Raising the budget -does not fix a failed installer or prove readiness. Set this on the host invocation, -not merely in a container's environment. +Detached startup and restart reject invalid values before project preparation or +teardown. Full and service-scoped startup/restart use the same settings. The +startup notice reports both effective budgets. On expiry, `E_STARTUP_TIMEOUT` +names the phase that was hit, its budget, and the variable that raises it; with +`--json`, `error.detail` carries `composePhase` (`images` or `start`), +`budgetMs`, and `budgetEnv`. The start budget covers container start and Compose +`depends_on` waiting, not lifecycle hooks or an end-to-end application readiness +deadline. A restart repair attempt has its own budgets and is reported +separately. Foreground `up` stays a single unbounded Compose invocation; `down` +retains its separate 90-second limit. + +On timeout, Hack terminates the owned Compose process group of that phase. After an +image-phase timeout no container was started by that attempt, but containers may +have been created or recreated. After a start-phase timeout containers may still be +running. Inspect `hack ps` and `hack logs` before retrying. Raising a budget does not +fix a failed installer or prove readiness. + +Services with `pull_policy: always` still pull again in the start phase because +Compose applies that policy on every `up`; such pulls count against the start +budget. To share dependency data only across worktrees with the same lockfile/runtime fingerprint, label the bootstrap service with a logical top-level Compose volume: @@ -292,7 +310,8 @@ be mistaken for a ready one. No project lifecycle hook or unrelated dependency i A failed installer leaves existing consumer containers untouched; JSON lifecycle commands return `E_DEPENDENCY_BOOTSTRAP_FAILED`. Each initializer in detached scoped `up` and targeted `restart` uses the configured `HACK_COMPOSE_STARTUP_TIMEOUT_MS` budget and reports `E_STARTUP_TIMEOUT` on expiry. -The subsequent Compose launch has its own equal budget. Consumer `run` and foreground +That budget also covers any installer image pull or build, because `run` acquires +the image itself. The subsequent Compose launch has its own phased budgets. Consumer `run` and foreground scoped `up` retain a separate ten-minute initializer deadline. ## Branch instances and linked worktrees diff --git a/src/backends/runtime-backend.ts b/src/backends/runtime-backend.ts index b08a9248a..76c047ec6 100644 --- a/src/backends/runtime-backend.ts +++ b/src/backends/runtime-backend.ts @@ -1,5 +1,7 @@ import { + type ComposeStartupPhase, DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS, + resolveComposeImageTimeoutMs, resolveComposeStartupTimeoutMs, } from "../lib/compose-startup-budget.ts"; import { withDependencyStartupProgress } from "../lib/dependency-startup-progress.ts"; @@ -39,7 +41,15 @@ export interface RuntimeUpOptions extends RuntimeBaseOptions { readonly services: readonly string[]; }; readonly detach: boolean; + /** Detached `start` phase budget (container start and dependency wait). */ readonly startupTimeoutMs?: number; + /** Detached `images` phase budget (image build/pull and container creation). */ + readonly imageTimeoutMs?: number; + /** + * Called before each detached phase begins, so a caller can attribute a + * non-zero exit (including the 124 deadline) to the phase that produced it. + */ + readonly onStartupPhase?: (phase: ComposeStartupPhase) => void; readonly services?: readonly string[]; readonly noDeps?: boolean; readonly forceRecreate?: boolean; @@ -85,26 +95,52 @@ function buildComposeArgs(opts: RuntimeBaseOptions): string[] { export const composeRuntimeBackend: RuntimeBackend = { name: "compose", async up(opts) { - const cmd = [ - ...buildComposeArgs(opts), - "up", - ...(opts.detach ? ["-d"] : []), - ...(opts.noDeps ? ["--no-deps"] : []), - ...(opts.forceRecreate ? ["--force-recreate"] : []), - ...(opts.services ?? []), - ]; if (opts.detach) { + const budgetEnv = { ...process.env, ...opts.env }; const startupTimeoutMs = resolveComposeStartupTimeoutMs({ startupTimeoutMs: opts.startupTimeoutMs, - env: { ...process.env, ...opts.env }, + env: budgetEnv, }); - const launch = () => + const imageTimeoutMs = resolveComposeImageTimeoutMs({ + imageTimeoutMs: opts.imageTimeoutMs, + env: budgetEnv, + }); + const scope = [ + ...(opts.noDeps ? ["--no-deps"] : []), + ...(opts.services ?? []), + ]; + const runPhase = (cmd: readonly string[], timeoutMs: number) => run(cmd, { cwd: opts.cwd, env: opts.env, stdout: opts.routeStdoutToStderr ? "stderr" : "inherit", - timeoutMs: startupTimeoutMs, + timeoutMs, }); + // Image build/pull can take far longer than starting containers, so it + // runs (with container creation/recreation) under its own budget. The + // start invocation then only starts the prepared containers and waits on + // Compose dependency conditions; --no-build keeps it from rebuilding. + const launch = async () => { + opts.onStartupPhase?.("images"); + const prepared = await runPhase( + [ + ...buildComposeArgs(opts), + "up", + "--no-start", + ...(opts.forceRecreate ? ["--force-recreate"] : []), + ...scope, + ], + imageTimeoutMs + ); + if (prepared !== 0) { + return prepared; + } + opts.onStartupPhase?.("start"); + return await runPhase( + [...buildComposeArgs(opts), "up", "-d", "--no-build", ...scope], + startupTimeoutMs + ); + }; const progress = opts.dependencyProgress; if (!progress) { return await launch(); @@ -129,6 +165,13 @@ export const composeRuntimeBackend: RuntimeBackend = { }); } + const cmd = [ + ...buildComposeArgs(opts), + "up", + ...(opts.noDeps ? ["--no-deps"] : []), + ...(opts.forceRecreate ? ["--force-recreate"] : []), + ...(opts.services ?? []), + ]; const env = mergeSpawnEnv(opts.env); const proc = Bun.spawn(cmd, { cwd: opts.cwd, diff --git a/src/commands/project.ts b/src/commands/project.ts index 073ce8ee2..96b1756b4 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -154,7 +154,14 @@ import { import { withComposeServiceEnvFingerprints } from "../lib/compose-env-fingerprint.ts"; import { renderManagedComposeEnvOverride } from "../lib/compose-managed-env.ts"; import { + type ComposeStartupBudgets, + type ComposeStartupFailure, + type ComposeStartupPhase, + composeStartupFailure, + DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS, DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS, + describeComposeStartupBudgets, + resolveComposeImageTimeoutMs, resolveComposeStartupTimeoutMs, } from "../lib/compose-startup-budget.ts"; import { @@ -912,36 +919,59 @@ function startupSnapshotIsIncomplete(opts: { } function reportComposeStartupBudget(opts: { - readonly startupTimeoutMs: number; + readonly budgets: ComposeStartupBudgets; readonly detach: boolean; readonly repair?: boolean; }): void { if (opts.detach) { logger.step({ - message: `Compose ${opts.repair ? "repair" : "startup"}: launch and dependency wait (budget ${opts.startupTimeoutMs} ms)`, + message: `Compose ${opts.repair ? "repair" : "startup"}: ${describeComposeStartupBudgets(opts.budgets)}`, }); } } -function composeStartupFailure( - code: number, - action: string, - startupTimeoutMs: number -): { - readonly code: "E_COMPOSE_FAILED" | "E_STARTUP_TIMEOUT"; - readonly message: string; -} { - return code === 124 +/** Resolve both detached Compose budgets before any project side effect. */ +function resolveDetachedComposeBudgets(detach: boolean): ComposeStartupBudgets { + return detach ? { - code: "E_STARTUP_TIMEOUT", - message: `${action} exceeded its detached Compose startup budget of ${startupTimeoutMs} ms (HACK_COMPOSE_STARTUP_TIMEOUT_MS). The Compose process group was terminated; inspect hack ps and hack logs before retrying. Containers may still be running. This limit covers Compose launch and dependency waiting, not lifecycle hooks or application readiness.`, + imageTimeoutMs: resolveComposeImageTimeoutMs(), + startupTimeoutMs: resolveComposeStartupTimeoutMs(), } : { - code: "E_COMPOSE_FAILED", - message: `${action} failed (exit ${code})`, + imageTimeoutMs: DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS, + startupTimeoutMs: DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS, }; } +/** + * Record the last detached Compose phase the backend entered, so a failure is + * attributed to it. A backend that never reports a phase is treated as `start`. + */ +function trackComposeStartupPhase(): { + readonly onStartupPhase: (phase: ComposeStartupPhase) => void; + readonly phase: () => ComposeStartupPhase; +} { + let current: ComposeStartupPhase = "start"; + return { + onStartupPhase: (phase) => { + current = phase; + }, + phase: () => current, + }; +} + +function composeStartupFailureDetail(opts: { + readonly failure: ComposeStartupFailure; + readonly exitCode: number; +}): Record { + return { + exitCode: opts.exitCode, + composePhase: opts.failure.phase, + budgetMs: opts.failure.budgetMs, + budgetEnv: opts.failure.budgetEnv, + }; +} + /** * Emit the `--json` envelope for a lifecycle command and map it to the * standard 0/1 exit semantics. @@ -5244,9 +5274,8 @@ async function runUpCommand({ readonly startedAtMs?: number; }): Promise { const detach = args.options.detach || json; - const startupTimeoutMs = detach - ? resolveComposeStartupTimeoutMs() - : DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS; + const budgets = resolveDetachedComposeBudgets(detach); + const { startupTimeoutMs } = budgets; let project: Awaited>; try { project = await resolveProjectForArgs({ @@ -5497,7 +5526,7 @@ async function runUpCommand({ ...(runtimeMetadataOverride ? [runtimeMetadataOverride] : []), ...envOverrides.composeFiles, ]; - reportComposeStartupBudget({ startupTimeoutMs, detach }); + reportComposeStartupBudget({ budgets, detach }); const bootstrapCode = await bootstrapDependencyCaches({ timeoutMs: detach ? startupTimeoutMs : undefined, services: cacheBootstrapServices, @@ -5521,6 +5550,7 @@ async function runUpCommand({ logger.error({ message: failure.message }); return bootstrapCode; } + const upPhase = trackComposeStartupPhase(); const upCode = await composeRuntimeBackend.up({ dependencyProgress: resolveDependencyCacheProgress({ cache: dependencyCache, @@ -5528,6 +5558,8 @@ async function runUpCommand({ baseProject: baseProjectName, }), startupTimeoutMs, + imageTimeoutMs: budgets.imageTimeoutMs, + onStartupPhase: upPhase.onStartupPhase, composeFiles: composeFilesWithEnv, composeProject: composeProjectName, profiles, @@ -5544,28 +5576,23 @@ async function runUpCommand({ projectDir: project.projectDir, composeProject: lifecycleComposeProject, }); + const failure = composeStartupFailure({ + code: upCode, + action: "docker compose up", + phase: upPhase.phase(), + budgets, + }); if (json) { - const failure = composeStartupFailure( - upCode, - "docker compose up", - startupTimeoutMs - ); return emitLifecycleResult({ result: errorResult({ code: failure.code, message: failure.message, - detail: { exitCode: upCode }, + detail: composeStartupFailureDetail({ failure, exitCode: upCode }), }), exitCode: upCode, }); } - logger.error({ - message: composeStartupFailure( - upCode, - "docker compose up", - startupTimeoutMs - ).message, - }); + logger.error({ message: failure.message }); return upCode; } @@ -6250,7 +6277,7 @@ async function runRestartDownPhase(opts: { } async function runRestartUpPhase(opts: { - readonly startupTimeoutMs: number; + readonly budgets: ComposeStartupBudgets; readonly project: Awaited>; readonly cfg: Awaited>; readonly baseProjectName: string; @@ -6265,6 +6292,7 @@ async function runRestartUpPhase(opts: { }): Promise<{ readonly code: number; readonly phase: "compose" | "lifecycle"; + readonly composePhase?: ComposeStartupPhase; }> { await maybeSyncOauthAliasesInCompose({ project: opts.project }); @@ -6380,16 +6408,19 @@ async function runRestartUpPhase(opts: { ...envOverrides.composeFiles, ]; reportComposeStartupBudget({ - startupTimeoutMs: opts.startupTimeoutMs, + budgets: opts.budgets, detach: opts.detach === true, }); + const upPhase = trackComposeStartupPhase(); const upCode = await composeRuntimeBackend.up({ dependencyProgress: resolveDependencyCacheProgress({ cache: dependencyCache, project: opts.composeProjectName, baseProject: opts.baseProjectName, }), - startupTimeoutMs: opts.startupTimeoutMs, + startupTimeoutMs: opts.budgets.startupTimeoutMs, + imageTimeoutMs: opts.budgets.imageTimeoutMs, + onStartupPhase: upPhase.onStartupPhase, composeFiles: composeFilesWithEnv, composeProject: opts.composeProjectName, profiles: opts.profiles, @@ -6401,7 +6432,7 @@ async function runRestartUpPhase(opts: { }); if (upCode !== 0) { reportComposeStartupBudget({ - startupTimeoutMs: opts.startupTimeoutMs, + budgets: opts.budgets, detach: true, repair: true, }); @@ -6411,7 +6442,8 @@ async function runRestartUpPhase(opts: { project: opts.composeProjectName, baseProject: opts.baseProjectName, }), - startupTimeoutMs: opts.startupTimeoutMs, + startupTimeoutMs: opts.budgets.startupTimeoutMs, + imageTimeoutMs: opts.budgets.imageTimeoutMs, composeFiles: composeFilesWithEnv, composeProject: opts.composeProjectName, profiles: opts.profiles, @@ -6437,7 +6469,7 @@ async function runRestartUpPhase(opts: { composeProject: opts.lifecycleComposeProject, }); } - return { code: upCode, phase: "compose" }; + return { code: upCode, phase: "compose", composePhase: upPhase.phase() }; } const upAfter = await runLifecycleCommands({ @@ -6490,7 +6522,7 @@ type TargetedServiceRestartResult = * individual Compose service. */ async function runTargetedServiceRestart(opts: { - readonly startupTimeoutMs: number; + readonly budgets: ComposeStartupBudgets; readonly project: Awaited>; readonly cfg: Awaited>; readonly baseProjectName: string; @@ -6572,7 +6604,7 @@ async function runTargetedServiceRestart(opts: { ...envOverrides.composeFiles, ]; const bootstrapCode = await bootstrapDependencyCaches({ - timeoutMs: opts.startupTimeoutMs, + timeoutMs: opts.budgets.startupTimeoutMs, services: cacheBootstrapServices, composeFiles: composeFilesWithEnv, composeProject: opts.composeProjectName, @@ -6583,7 +6615,7 @@ async function runTargetedServiceRestart(opts: { if (bootstrapCode !== 0) { const failure = dependencyCacheBootstrapFailure({ code: bootstrapCode, - timeoutMs: opts.startupTimeoutMs, + timeoutMs: opts.budgets.startupTimeoutMs, }); return { ok: false, @@ -6595,16 +6627,19 @@ async function runTargetedServiceRestart(opts: { }; } reportComposeStartupBudget({ - startupTimeoutMs: opts.startupTimeoutMs, + budgets: opts.budgets, detach: true, }); + const upPhase = trackComposeStartupPhase(); const code = await composeRuntimeBackend.up({ dependencyProgress: resolveDependencyCacheProgress({ cache: dependencyCache, project: opts.composeProjectName, baseProject: opts.baseProjectName, }), - startupTimeoutMs: opts.startupTimeoutMs, + startupTimeoutMs: opts.budgets.startupTimeoutMs, + imageTimeoutMs: opts.budgets.imageTimeoutMs, + onStartupPhase: upPhase.onStartupPhase, composeFiles: composeFilesWithEnv, composeProject: opts.composeProjectName, profiles: opts.profiles, @@ -6630,11 +6665,12 @@ async function runTargetedServiceRestart(opts: { outcome: code === 0 ? "applied" : "unknown", }); if (code !== 0) { - const failure = composeStartupFailure( + const failure = composeStartupFailure({ code, - "Targeted service restart", - opts.startupTimeoutMs - ); + action: "Targeted service restart", + phase: upPhase.phase(), + budgets: opts.budgets, + }); return { ok: false, code, @@ -6800,7 +6836,7 @@ async function runRestartCommand({ readonly json: boolean; readonly startedAtMs?: number; }): Promise { - const startupTimeoutMs = resolveComposeStartupTimeoutMs(); + const budgets = resolveDetachedComposeBudgets(true); const project = await resolveProjectForArgs({ ctx, pathOpt: args.options.path, @@ -6860,7 +6896,7 @@ async function runRestartCommand({ }); if (requestedServices.length > 0) { const result = await runTargetedServiceRestart({ - startupTimeoutMs, + budgets, project, cfg, baseProjectName, @@ -6960,7 +6996,7 @@ async function runRestartCommand({ }); const upOutcome = await runRestartUpPhase({ - startupTimeoutMs, + budgets, project, cfg, baseProjectName, @@ -6978,7 +7014,12 @@ async function runRestartCommand({ if (upCode !== 0) { const failure = upOutcome.phase === "compose" - ? composeStartupFailure(upCode, "Restart up phase", startupTimeoutMs) + ? composeStartupFailure({ + code: upCode, + action: "Restart up phase", + phase: upOutcome.composePhase ?? "start", + budgets, + }) : { code: "E_LIFECYCLE_FAILED" as const, message: `Restart lifecycle hook failed (exit ${upCode})`, @@ -6991,7 +7032,12 @@ async function runRestartCommand({ result: errorResult({ code: failure.code, message: failure.message, - detail: { exitCode: upCode, phase: "up" }, + detail: { + ...("phase" in failure + ? composeStartupFailureDetail({ failure, exitCode: upCode }) + : { exitCode: upCode }), + phase: "up", + }, }), exitCode: upCode, }); diff --git a/src/lib/compose-startup-budget.ts b/src/lib/compose-startup-budget.ts index 93d635b72..a811968ef 100644 --- a/src/lib/compose-startup-budget.ts +++ b/src/lib/compose-startup-budget.ts @@ -1,40 +1,159 @@ import { CliUsageError } from "../cli/command.ts"; +/** Default budget for starting containers and waiting on Compose dependencies. */ export const DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS = 90_000; -const MIN_STARTUP_TIMEOUT_MS = 1000; +/** Default budget for image build/pull and container creation before start. */ +export const DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS = 1_800_000; +export const COMPOSE_STARTUP_TIMEOUT_ENV = "HACK_COMPOSE_STARTUP_TIMEOUT_MS"; +export const COMPOSE_IMAGE_TIMEOUT_ENV = "HACK_COMPOSE_IMAGE_TIMEOUT_MS"; +const MIN_TIMEOUT_MS = 1000; const MAX_STARTUP_TIMEOUT_MS = 3_600_000; +const MAX_IMAGE_TIMEOUT_MS = 14_400_000; const INTEGER_TEXT = /^[0-9]+$/; -/** Total detached Compose invocation budget, not a readiness or per-phase guarantee. */ +/** + * Detached Compose startup runs as two bounded invocations, each in its own + * owned process group that is terminated (exit 124) when its budget expires: + * - `images`: `up --no-start` builds or pulls missing images and creates or + * recreates containers without starting them. Bounded by + * {@link resolveComposeImageTimeoutMs}. + * - `start`: `up -d --no-build` starts containers and waits for Compose + * `depends_on` conditions. Bounded by {@link resolveComposeStartupTimeoutMs}. + * Image acquisition therefore never consumes the start/dependency-wait budget, + * while a genuine hang in either phase is still terminated. + */ +export type ComposeStartupPhase = "images" | "start"; + +/** + * Budget for the `start` phase of detached Compose startup: container start + * and dependency waiting. It excludes image build/pull and is not an + * application readiness guarantee. + */ export function resolveComposeStartupTimeoutMs( opts: { readonly startupTimeoutMs?: number; readonly env?: Readonly>; } = {} ): number { - const configured = (opts.env ?? process.env).HACK_COMPOSE_STARTUP_TIMEOUT_MS; - let timeout = opts.startupTimeoutMs; + return resolveBudget({ + explicit: opts.startupTimeoutMs, + configured: (opts.env ?? process.env)[COMPOSE_STARTUP_TIMEOUT_ENV], + fallback: DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS, + max: MAX_STARTUP_TIMEOUT_MS, + invalid: () => + new CliUsageError( + `Compose startup timeout must be an integer from ${MIN_TIMEOUT_MS} to ${MAX_STARTUP_TIMEOUT_MS} milliseconds (${COMPOSE_STARTUP_TIMEOUT_ENV})` + ), + }); +} + +/** Budget for the `images` phase: image build/pull and container creation. */ +export function resolveComposeImageTimeoutMs( + opts: { + readonly imageTimeoutMs?: number; + readonly env?: Readonly>; + } = {} +): number { + return resolveBudget({ + explicit: opts.imageTimeoutMs, + configured: (opts.env ?? process.env)[COMPOSE_IMAGE_TIMEOUT_ENV], + fallback: DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS, + max: MAX_IMAGE_TIMEOUT_MS, + invalid: () => + new CliUsageError( + `Compose image build/pull timeout must be an integer from ${MIN_TIMEOUT_MS} to ${MAX_IMAGE_TIMEOUT_MS} milliseconds (${COMPOSE_IMAGE_TIMEOUT_ENV})` + ), + }); +} + +function resolveBudget(opts: { + readonly explicit: number | undefined; + readonly configured: string | undefined; + readonly fallback: number; + readonly max: number; + readonly invalid: () => CliUsageError; +}): number { + let timeout = opts.explicit; if (timeout === undefined) { - if (configured === undefined) { - return DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS; + if (opts.configured === undefined) { + return opts.fallback; } - if (!INTEGER_TEXT.test(configured)) { - throw invalidBudget(); + if (!INTEGER_TEXT.test(opts.configured)) { + throw opts.invalid(); } - timeout = Number(configured); + timeout = Number(opts.configured); } if ( !Number.isSafeInteger(timeout) || - timeout < MIN_STARTUP_TIMEOUT_MS || - timeout > MAX_STARTUP_TIMEOUT_MS + timeout < MIN_TIMEOUT_MS || + timeout > opts.max ) { - throw invalidBudget(); + throw opts.invalid(); } return timeout; } -function invalidBudget(): CliUsageError { - return new CliUsageError( - "Compose startup timeout must be an integer from 1000 to 3600000 milliseconds (HACK_COMPOSE_STARTUP_TIMEOUT_MS)" - ); +export interface ComposeStartupBudgets { + readonly startupTimeoutMs: number; + readonly imageTimeoutMs: number; +} + +const PHASE_LABEL: Readonly> = { + images: "image build/pull and container creation", + start: "start and dependency wait", +}; + +/** One-line summary of both detached phases and their effective budgets. */ +export function describeComposeStartupBudgets( + budgets: ComposeStartupBudgets +): string { + return `${PHASE_LABEL.images} (budget ${budgets.imageTimeoutMs} ms, ${COMPOSE_IMAGE_TIMEOUT_ENV}), then ${PHASE_LABEL.start} (budget ${budgets.startupTimeoutMs} ms, ${COMPOSE_STARTUP_TIMEOUT_ENV})`; +} + +export interface ComposeStartupFailure { + readonly code: "E_COMPOSE_FAILED" | "E_STARTUP_TIMEOUT"; + readonly message: string; + readonly phase: ComposeStartupPhase; + readonly budgetMs: number; + readonly budgetEnv: string; +} + +/** + * Map a detached Compose exit code to the CLI failure contract. Exit 124 is + * the owned process-group deadline; its message names the phase that hit it, + * the effective budget, and the host variable that raises it. + */ +export function composeStartupFailure(opts: { + readonly code: number; + readonly action: string; + readonly phase: ComposeStartupPhase; + readonly budgets: ComposeStartupBudgets; +}): ComposeStartupFailure { + const images = opts.phase === "images"; + const budgetMs = images + ? opts.budgets.imageTimeoutMs + : opts.budgets.startupTimeoutMs; + const budgetEnv = images + ? COMPOSE_IMAGE_TIMEOUT_ENV + : COMPOSE_STARTUP_TIMEOUT_ENV; + const phaseLabel = PHASE_LABEL[opts.phase]; + if (opts.code !== 124) { + return { + code: "E_COMPOSE_FAILED", + message: `${opts.action} failed (exit ${opts.code}) during Compose ${phaseLabel}`, + phase: opts.phase, + budgetMs, + budgetEnv, + }; + } + const message = images + ? `${opts.action} exceeded its Compose image build/pull budget of ${budgetMs} ms (phase: ${phaseLabel}). Raise it with ${budgetEnv}= on the host invocation (max ${MAX_IMAGE_TIMEOUT_MS}); it is separate from ${COMPOSE_STARTUP_TIMEOUT_ENV}. The Compose process group was terminated before this attempt started containers; containers may have been created or recreated but not started. Inspect hack ps before retrying.` + : `${opts.action} exceeded its detached Compose startup budget of ${budgetMs} ms (phase: ${phaseLabel}). Raise it with ${budgetEnv}= on the host invocation (max ${MAX_STARTUP_TIMEOUT_MS}). The Compose process group was terminated; inspect hack ps and hack logs before retrying. Containers may still be running. Image build/pull is not counted against this budget, and it does not cover lifecycle hooks or application readiness.`; + return { + code: "E_STARTUP_TIMEOUT", + message, + phase: opts.phase, + budgetMs, + budgetEnv, + }; } diff --git a/src/lib/dependency-cache-bootstrap.ts b/src/lib/dependency-cache-bootstrap.ts index a5b735469..403f1e8fa 100644 --- a/src/lib/dependency-cache-bootstrap.ts +++ b/src/lib/dependency-cache-bootstrap.ts @@ -2,6 +2,7 @@ import { composeRuntimeBackend, type RuntimeBaseOptions, } from "../backends/runtime-backend.ts"; +import { COMPOSE_STARTUP_TIMEOUT_ENV } from "./compose-startup-budget.ts"; const BOOTSTRAP_TIMEOUT_MS = 600_000; @@ -44,7 +45,10 @@ export function dependencyCacheBootstrapFailure(opts: { return opts.code === 124 ? { code: "E_STARTUP_TIMEOUT", - message: `Dependency cache initialization exceeded its startup budget of ${opts.timeoutMs ?? BOOTSTRAP_TIMEOUT_MS} ms; consumers were not changed`, + message: + opts.timeoutMs === undefined + ? `Dependency cache initialization exceeded its startup budget of ${BOOTSTRAP_TIMEOUT_MS} ms (phase: dependency cache initialization; fixed for foreground and consumer runs); consumers were not changed` + : `Dependency cache initialization exceeded its startup budget of ${opts.timeoutMs} ms (phase: dependency cache initialization, including any installer image pull; raise ${COMPOSE_STARTUP_TIMEOUT_ENV}= on the host invocation); consumers were not changed`, } : { code: "E_DEPENDENCY_BOOTSTRAP_FAILED", diff --git a/tests/compose-startup-budget.test.ts b/tests/compose-startup-budget.test.ts index 4b5635e4a..20964cb71 100644 --- a/tests/compose-startup-budget.test.ts +++ b/tests/compose-startup-budget.test.ts @@ -1,6 +1,8 @@ import { expect, test } from "bun:test"; import { + DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS, DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS, + resolveComposeImageTimeoutMs, resolveComposeStartupTimeoutMs, } from "../src/lib/compose-startup-budget.ts"; @@ -58,3 +60,33 @@ test("startup budget rejects malformed or out-of-bounds settings without echoing ).toThrow(); } }); + +test("image build/pull budget is separate, defaults to 30 minutes and allows up to 4 hours", () => { + expect(resolveComposeImageTimeoutMs({ env: {} })).toBe( + DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS + ); + expect(DEFAULT_COMPOSE_IMAGE_TIMEOUT_MS).toBe(1_800_000); + // The startup budget never sizes the image phase, and vice versa. + expect( + resolveComposeImageTimeoutMs({ + env: { HACK_COMPOSE_STARTUP_TIMEOUT_MS: "1000" }, + }) + ).toBe(1_800_000); + expect( + resolveComposeStartupTimeoutMs({ + env: { HACK_COMPOSE_IMAGE_TIMEOUT_MS: "14400000" }, + }) + ).toBe(DEFAULT_COMPOSE_STARTUP_TIMEOUT_MS); + expect( + resolveComposeImageTimeoutMs({ + env: { HACK_COMPOSE_IMAGE_TIMEOUT_MS: "14400000" }, + }) + ).toBe(14_400_000); + for (const value of ["999", "14400001", "1e6", "private-input"]) { + expect(() => + resolveComposeImageTimeoutMs({ + env: { HACK_COMPOSE_IMAGE_TIMEOUT_MS: value }, + }) + ).toThrow("HACK_COMPOSE_IMAGE_TIMEOUT_MS"); + } +}); diff --git a/tests/compose-startup-phases.test.ts b/tests/compose-startup-phases.test.ts new file mode 100644 index 000000000..9c595da05 --- /dev/null +++ b/tests/compose-startup-phases.test.ts @@ -0,0 +1,226 @@ +import { afterEach, expect, test } from "bun:test"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { composeRuntimeBackend } from "../src/backends/runtime-backend.ts"; +import { + type ComposeStartupPhase, + composeStartupFailure, +} from "../src/lib/compose-startup-budget.ts"; + +/** + * Stub `docker` that models Compose image handling without a Docker engine: + * any `up` without `--no-build` "builds" missing images (sleeping + * STUB_BUILD_SECONDS, then marking them present); `up --no-build` refuses when + * images are missing; `up -d` then "starts" containers, and with + * STUB_START_HANG=1 waits forever on a grandchild like a dependency that never + * becomes healthy. It records each argv and the hanging grandchild's pid. + */ +const STUB_DOCKER = `#!/bin/sh +[ "$1" = warmup ] && exit 0 +dir="$(dirname "$0")" +echo "$*" >> "$dir/calls.log" +no_build=0; no_start=0; detach=0 +for arg in "$@"; do + case "$arg" in + --no-build) no_build=1 ;; + --no-start) no_start=1 ;; + -d) detach=1 ;; + esac +done +if [ ! -f "$dir/images-ready" ]; then + if [ "$no_build" = 1 ]; then + echo "image missing" >&2 + exit 17 + fi + sleep "$STUB_BUILD_SECONDS" + if [ "$STUB_BUILD_HANG" = 1 ]; then + sleep 30 & + echo $! > "$dir/hang.pid" + wait + fi + : > "$dir/images-ready" +fi +if [ "$no_start" = 1 ] || [ "$detach" = 0 ]; then + exit 0 +fi +if [ "$STUB_START_HANG" = 1 ]; then + sleep 30 & + echo $! > "$dir/hang.pid" + wait +fi +exit 0 +`; + +const dirs: string[] = []; + +afterEach(async () => { + for (const dir of dirs.splice(0)) { + await rm(dir, { recursive: true, force: true }); + } +}); + +async function stubCompose(opts: { + readonly buildSeconds: number; + readonly buildHang?: boolean; + readonly startHang?: boolean; +}) { + const dir = await mkdtemp(join(tmpdir(), "hack-compose-phases-")); + dirs.push(dir); + await writeFile(join(dir, "docker"), STUB_DOCKER); + await chmod(join(dir, "docker"), 0o700); + // Execute the new script once outside any deadline: the first exec of a + // freshly written executable can be slow (for example, macOS policy scans) + // and must not be charged to the 1 s budgets under test. + expect(Bun.spawnSync([join(dir, "docker"), "warmup"]).exitCode).toBe(0); + return { + dir, + env: { + // The stub shadows any engine client; no socket or HOME is reachable. + PATH: `${dir}:/usr/bin:/bin`, + HOME: dir, + DOCKER_HOST: `unix://${join(dir, "no-engine.sock")}`, + STUB_BUILD_SECONDS: String(opts.buildSeconds), + STUB_BUILD_HANG: opts.buildHang ? "1" : "0", + STUB_START_HANG: opts.startHang ? "1" : "0", + }, + }; +} + +async function detachedUp(opts: { + readonly dir: string; + readonly env: Record; + readonly startupTimeoutMs: number; + readonly imageTimeoutMs: number; +}) { + const phases: ComposeStartupPhase[] = []; + const started = Date.now(); + const code = await composeRuntimeBackend.up({ + composeFiles: ["compose.yml"], + composeProject: "phases", + cwd: opts.dir, + env: opts.env, + detach: true, + startupTimeoutMs: opts.startupTimeoutMs, + imageTimeoutMs: opts.imageTimeoutMs, + routeStdoutToStderr: true, + onStartupPhase: (phase) => { + phases.push(phase); + }, + }); + const calls = (await readFile(join(opts.dir, "calls.log"), "utf8")) + .trim() + .split("\n"); + return { code, phases, calls, elapsedMs: Date.now() - started }; +} + +function processExists(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function expectProcessGone(pid: number): Promise { + const deadline = Date.now() + 4000; + while (processExists(pid) && Date.now() < deadline) { + await Bun.sleep(50); + } + expect(processExists(pid)).toBe(false); +} + +test("a first-time build longer than the startup budget still starts", async () => { + // Build takes 1.6 s; the start/dependency-wait budget is only 1 s. + const stub = await stubCompose({ buildSeconds: 1.6 }); + const result = await detachedUp({ + ...stub, + startupTimeoutMs: 1000, + imageTimeoutMs: 10_000, + }); + + expect(result.code).toBe(0); + expect(result.phases).toEqual(["images", "start"]); + expect(result.calls).toEqual([ + "compose -p phases -f compose.yml up --no-start", + "compose -p phases -f compose.yml up -d --no-build", + ]); +}, 15_000); + +test("a genuine readiness hang still times out in the named start phase", async () => { + const stub = await stubCompose({ buildSeconds: 0, startHang: true }); + const result = await detachedUp({ + ...stub, + startupTimeoutMs: 1000, + imageTimeoutMs: 10_000, + }); + + expect(result.code).toBe(124); + expect(result.phases).toEqual(["images", "start"]); + // Bounded by the 1 s start budget, not the 10 s image budget. + expect(result.elapsedMs).toBeLessThan(5000); + // The owned process group, including the hanging grandchild, is terminated. + const hangPid = Number(await readFile(join(stub.dir, "hang.pid"), "utf8")); + await expectProcessGone(hangPid); + + const failure = composeStartupFailure({ + code: result.code, + action: "docker compose up", + phase: result.phases.at(-1) ?? "start", + budgets: { startupTimeoutMs: 1000, imageTimeoutMs: 10_000 }, + }); + expect(failure.code).toBe("E_STARTUP_TIMEOUT"); + expect(failure.message).toContain( + "startup budget of 1000 ms (phase: start and dependency wait)" + ); + expect(failure.message).toContain( + "Raise it with HACK_COMPOSE_STARTUP_TIMEOUT_MS=" + ); +}, 15_000); + +test("a hung image build times out in the named image phase without starting", async () => { + const stub = await stubCompose({ buildSeconds: 0, buildHang: true }); + const result = await detachedUp({ + ...stub, + startupTimeoutMs: 60_000, + imageTimeoutMs: 1000, + }); + + expect(result.code).toBe(124); + expect(result.phases).toEqual(["images"]); + expect(result.calls).toEqual([ + "compose -p phases -f compose.yml up --no-start", + ]); + expect(result.elapsedMs).toBeLessThan(5000); + const hangPid = Number(await readFile(join(stub.dir, "hang.pid"), "utf8")); + await expectProcessGone(hangPid); + + const failure = composeStartupFailure({ + code: result.code, + action: "docker compose up", + phase: result.phases.at(-1) ?? "start", + budgets: { startupTimeoutMs: 60_000, imageTimeoutMs: 1000 }, + }); + expect(failure.code).toBe("E_STARTUP_TIMEOUT"); + expect(failure.message).toContain( + "image build/pull budget of 1000 ms (phase: image build/pull and container creation)" + ); + expect(failure.message).toContain( + "Raise it with HACK_COMPOSE_IMAGE_TIMEOUT_MS=" + ); +}, 15_000); + +test("non-timeout failures keep E_COMPOSE_FAILED and name the phase", () => { + const failure = composeStartupFailure({ + code: 17, + action: "docker compose up", + phase: "images", + budgets: { startupTimeoutMs: 90_000, imageTimeoutMs: 1_800_000 }, + }); + expect(failure).toMatchObject({ + code: "E_COMPOSE_FAILED", + message: + "docker compose up failed (exit 17) during Compose image build/pull and container creation", + }); +}); diff --git a/tests/project-up-startup-state.test.ts b/tests/project-up-startup-state.test.ts index f2524eafb..c7148f5fe 100644 --- a/tests/project-up-startup-state.test.ts +++ b/tests/project-up-startup-state.test.ts @@ -33,6 +33,9 @@ const psEnvs: Array> | undefined> = []; const upDetachSelections: Array = []; const upStartupTimeouts: Array = []; let upExitCode = 0; +/** Detached phases the mocked backend reports before returning upExitCode. */ +let upReportedPhases: readonly ("images" | "start")[] = []; +const upImageTimeouts: Array = []; let downCalls = 0; const upServiceSelections: Array = []; const tempDirs = new Set(); @@ -89,7 +92,11 @@ const runtimeBackendMock = await registerScopedModuleMock({ upEnvs.push(opts.env); upServiceSelections.push(opts.services); upStartupTimeouts.push(opts.startupTimeoutMs); + upImageTimeouts.push(opts.imageTimeoutMs); upDetachSelections.push(opts.detach); + for (const phase of upReportedPhases) { + opts.onStartupPhase?.(phase); + } return upExitCode; }, down: async () => { @@ -156,8 +163,11 @@ beforeAll(() => { loggerMock.activate(); }); +const originalImageTimeout = process.env.HACK_COMPOSE_IMAGE_TIMEOUT_MS; + beforeEach(() => { Reflect.deleteProperty(process.env, "HACK_COMPOSE_STARTUP_TIMEOUT_MS"); + Reflect.deleteProperty(process.env, "HACK_COMPOSE_IMAGE_TIMEOUT_MS"); }); afterEach(async () => { @@ -171,6 +181,9 @@ afterEach(async () => { upStartupTimeouts.length = 0; upDetachSelections.length = 0; upExitCode = 0; + upReportedPhases = []; + upImageTimeouts.length = 0; + restoreEnv("HACK_COMPOSE_IMAGE_TIMEOUT_MS", originalImageTimeout); downCalls = 0; if (originalStartupTimeout === undefined) { Reflect.deleteProperty(process.env, "HACK_COMPOSE_STARTUP_TIMEOUT_MS"); @@ -664,6 +677,92 @@ for (const action of ["up", "restart", "scoped restart"] as const) { }); } +for (const action of ["up", "restart", "scoped restart"] as const) { + const run = (projectRoot: string) => + action === "up" + ? runDetachedUp({ projectRoot }) + : runRestart({ + projectRoot, + ...(action === "scoped restart" ? { services: ["api"] } : {}), + }); + + test(`${action} forwards a separate image build/pull budget to Compose`, async () => { + process.env.HACK_COMPOSE_IMAGE_TIMEOUT_MS = "2400000"; + const projectRoot = await createProject(); + psRows.push( + JSON.stringify({ Service: "api", State: "running", ExitCode: 0 }) + ); + + expect(await run(projectRoot)).toBe(0); + expect(upImageTimeouts).toEqual([2_400_000]); + expect(upStartupTimeouts).toEqual([90_000]); + }); + + test(`${action} names the image phase and its variable when image build/pull times out`, async () => { + process.env.HACK_COMPOSE_IMAGE_TIMEOUT_MS = "2400000"; + upReportedPhases = ["images"]; + upExitCode = 124; + const projectRoot = await createProject(); + + expect(await run(projectRoot)).toBe(124); + const errors = errorMessages.join("\n"); + expect(errors).toContain( + "image build/pull budget of 2400000 ms (phase: image build/pull and container creation)" + ); + expect(errors).toContain("HACK_COMPOSE_IMAGE_TIMEOUT_MS="); + }); + + test(`${action} names the start phase when dependency wait times out after a build`, async () => { + upReportedPhases = ["images", "start"]; + upExitCode = 124; + const projectRoot = await createProject(); + + expect(await run(projectRoot)).toBe(124); + const errors = errorMessages.join("\n"); + expect(errors).toContain( + "startup budget of 90000 ms (phase: start and dependency wait)" + ); + expect(errors).toContain("HACK_COMPOSE_STARTUP_TIMEOUT_MS="); + expect(errors).not.toContain("HACK_COMPOSE_IMAGE_TIMEOUT_MS="); + }); + + test(`${action} rejects an invalid image budget before hooks or backend effects`, async () => { + process.env.HACK_COMPOSE_IMAGE_TIMEOUT_MS = "soon"; + const projectRoot = await createProject({ + lifecycleCommand: "touch image-budget-hook-ran", + }); + + await expect(run(projectRoot)).rejects.toThrow( + "HACK_COMPOSE_IMAGE_TIMEOUT_MS" + ); + expect( + await Bun.file(resolve(projectRoot, "image-budget-hook-ran")).exists() + ).toBe(false); + expect(upComposeContents).toEqual([]); + expect(downCalls).toBe(0); + }); +} + +test("up JSON reports the timed-out Compose phase and its budget", async () => { + upReportedPhases = ["images"]; + upExitCode = 124; + const projectRoot = await createProject(); + const result = await runJsonUp({ projectRoot }); + expect(result.exitCode).toBe(124); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: false, + error: { + code: "E_STARTUP_TIMEOUT", + detail: { + exitCode: 124, + composePhase: "images", + budgetMs: 1_800_000, + budgetEnv: "HACK_COMPOSE_IMAGE_TIMEOUT_MS", + }, + }, + }); +}); + test("foreground up ignores an invalid detached startup budget", async () => { process.env.HACK_COMPOSE_STARTUP_TIMEOUT_MS = "invalid"; const projectRoot = await createProject(); diff --git a/tests/runtime-backend.test.ts b/tests/runtime-backend.test.ts index 7370336cc..5106dc713 100644 --- a/tests/runtime-backend.test.ts +++ b/tests/runtime-backend.test.ts @@ -94,7 +94,7 @@ test("composeRuntimeBackend.up builds compose args with profiles and detach", as cwd: "/tmp", }); - expect(runCalls[0]).toEqual([ + const base = [ "docker", "compose", "-p", @@ -105,8 +105,11 @@ test("composeRuntimeBackend.up builds compose args with profiles and detach", as "b.yml", "--profile", "ops", - "up", - "-d", + ]; + // Image build/pull and container creation, then start and dependency wait. + expect(runCalls).toEqual([ + [...base, "up", "--no-start"], + [...base, "up", "-d", "--no-build"], ]); }); @@ -123,19 +126,26 @@ test("composeRuntimeBackend.up targets services without dependencies", async () cwd: "/tmp", }); - expect(runCalls[0]).toEqual([ + const base = [ "docker", "compose", "-p", "myproj", "-f", "docker-compose.yml", - "up", - "-d", - "--no-deps", - "--force-recreate", - "chat-q", - "sim-runner", + ]; + // Recreation happens once, in the image phase; the start phase only starts. + expect(runCalls).toEqual([ + [ + ...base, + "up", + "--no-start", + "--force-recreate", + "--no-deps", + "chat-q", + "sim-runner", + ], + [...base, "up", "-d", "--no-build", "--no-deps", "chat-q", "sim-runner"], ]); }); @@ -404,11 +414,19 @@ test("detached startup uses selected budget and down keeps its fixed budget", as env: { HACK_COMPOSE_STARTUP_TIMEOUT_MS: "600000" }, }; await backend.up({ ...base, detach: true }); - expect(runOpts[0]?.timeoutMs).toBe(600_000); - await backend.up({ ...base, detach: true, startupTimeoutMs: 120_000 }); - expect(runOpts[1]?.timeoutMs).toBe(120_000); + // The image phase keeps its own default; only the start phase uses this. + expect(runOpts.map((opts) => opts.timeoutMs)).toEqual([1_800_000, 600_000]); + await backend.up({ + ...base, + detach: true, + startupTimeoutMs: 120_000, + imageTimeoutMs: 2_400_000, + }); + expect(runOpts.slice(2).map((opts) => opts.timeoutMs)).toEqual([ + 2_400_000, 120_000, + ]); await backend.down(base); - expect(runOpts[2]?.timeoutMs).toBe(90_000); + expect(runOpts[4]?.timeoutMs).toBe(90_000); }); test("invalid startup budget refuses before invoking Compose", async () => { From f8cbd52c51ab9f99e2482a3bf7a6baf904ce83e7 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:23:36 -0400 Subject: [PATCH 18/21] test: make process-timing tests tolerate host load Replace wall-clock guesses in three tests that failed intermittently on loaded hosts (HACK-1280): - The hanging-probe test now proves the kill only after the fake Docker publishes its pid through an atomic marker handshake. A probe timeout that fires before the handshake is inconclusive and escalates through named rungs (500 ms, 2 s, 8 s); every rung still requires the timeout refusal, and the reap check polls to a named deadline instead of a fixed 50 ms sleep. - The native compose ownership file sets a named 60 s default test budget: its scenarios launch many cold fake-Docker processes, and an expired test kept writing into the next test's shared root. - The initial forwarding trace test uses named bounds for each forwarded command (30 s) and for the test (150 s) instead of 3 s and 5 s. No product timeout or assertion changes. Refs HACK-1280 Co-Authored-By: Claude Opus 5.5 --- tests/native-compose-ownership.test.ts | 97 ++++++-- ...ative-process-policy-initial-trace.test.ts | 227 ++++++++++-------- 2 files changed, 201 insertions(+), 123 deletions(-) diff --git a/tests/native-compose-ownership.test.ts b/tests/native-compose-ownership.test.ts index 3e278349a..207d778f9 100644 --- a/tests/native-compose-ownership.test.ts +++ b/tests/native-compose-ownership.test.ts @@ -1,4 +1,11 @@ -import { afterEach, beforeEach, expect, spyOn, test } from "bun:test"; +import { + afterEach, + beforeEach, + expect, + setDefaultTimeout, + spyOn, + test, +} from "bun:test"; import { chmod, mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -26,6 +33,15 @@ const VOLUME = "hack-21-hack-fixture-instance-4-data"; const CREATED = "2026-10-08T00:00:00.123456789Z"; const REBORN = "2026-10-08T00:00:01.123456789Z"; const CANARY = "synthetic-private-env-image-canary"; +/** + * Every fake Docker query here is a cold Bun process, and one scenario can launch + * dozens. Bun's 5 s default then budgets host process start-up rather than the + * product, and an expired test keeps running against the next test's shared + * root. The probe keeps its own default (or test-chosen) timeout in every query. + */ +const FAKE_DOCKER_TEST_TIMEOUT_MS = 60_000; +setDefaultTimeout(FAKE_DOCKER_TEST_TIMEOUT_MS); + const options: NativeComposeOwnershipOptions = { composeProject: PROJECT, runtimeIdentity: PROJECT, @@ -50,7 +66,7 @@ beforeEach(async () => { await Bun.write( join(root, "docker"), `#!${process.execPath} -import {appendFileSync, existsSync, readFileSync, writeFileSync} from "node:fs"; +import {appendFileSync, existsSync, readFileSync, renameSync, writeFileSync} from "node:fs"; const root = ${JSON.stringify(root)}; const args = process.argv.slice(2); appendFileSync(root + "/commands", JSON.stringify(args) + "\\n"); @@ -59,10 +75,12 @@ const [kind, action] = args; if (!["container","volume","network"].includes(kind) || !["ls","inspect"].includes(action) || !args.includes("--format")) { writeFileSync(root + "/mutated", "unexpected engine mutation"); process.exit(99); } -if (fixture.mode === "hang") { writeFileSync(root + "/started", String(process.pid)); await Bun.sleep(60_000); } +// Handshake markers appear atomically: a reader never observes a partial pid. +const signal = (name, pid) => { writeFileSync(root + "/" + name + ".tmp", String(pid)); renameSync(root + "/" + name + ".tmp", root + "/" + name); }; +if (fixture.mode === "hang") { signal("started", process.pid); await Bun.sleep(60_000); } if (fixture.mode === "descendant") { const child = Bun.spawn([process.execPath, "-e", 'await Bun.sleep(60_000)'], {stdout: "inherit", stderr: "inherit"}); - writeFileSync(root + "/descendant", String(child.pid)); process.exit(0); + signal("descendant", child.pid); process.exit(0); } if (fixture.mode === "fail") { console.error(${JSON.stringify(CANARY)}); process.exit(23); } if (fixture.mode === "stderr-overflow") { await Bun.write(Bun.stderr, "x".repeat(17 * 1024)); process.exit(0); } @@ -1057,21 +1075,66 @@ test("inspect argv batching does not cap the number of owned resources", async ( ).toBeLessThanOrEqual(16 * 1024); } }); +/** + * Probe timeouts tried in order for one hanging fake Docker. The kill proof needs + * the fake Docker to publish its pid before the probe fires; on a loaded host a + * cold Bun start can exceed the fast rung. An attempt without that handshake is + * inconclusive (its process died before it could be observed) and escalates; it + * never counts as a pass. Every attempt still has to refuse with the timeout code. + * Both modes at every rung plus reaping stay within FAKE_DOCKER_TEST_TIMEOUT_MS. + */ +const HANG_PROBE_TIMEOUTS_MS = [500, 2000, 8000] as const; +/** Bound for the kernel to reap a SIGKILLed orphan after the probe refuses. */ +const HANG_REAP_DEADLINE_MS = 10_000; + +async function hangHandshakePid(marker: string): Promise { + if (!(await Bun.file(marker).exists())) { + return; + } + const pid = Number(await readFile(marker, "utf8")); + if (!(Number.isSafeInteger(pid) && pid > 1)) { + throw new Error(`Malformed ${marker} handshake`); + } + return pid; +} + +async function exitedWithin(pid: number, deadlineMs: number): Promise { + const deadline = Date.now() + deadlineMs; + while (true) { + try { + process.kill(pid, 0); + } catch { + return true; + } + if (Date.now() >= deadline) { + return false; + } + await Bun.sleep(10); + } +} + test("a real hanging process times out and its inherited-pipe descendant is killed", async () => { for (const mode of ["hang", "descendant"]) { - await prepare({ mode }); - await expectRefusal( - { ...options, timeoutMs: 500 }, - "E_NATIVE_COMPOSE_PROBE_TIMEOUT" - ); - const pid = Number( - await readFile( - join(root, mode === "hang" ? "started" : "descendant"), - "utf8" - ) - ); - await Bun.sleep(50); - expect(() => process.kill(pid, 0)).toThrow(); + const marker = join(root, mode === "hang" ? "started" : "descendant"); + let pid: number | undefined; + for (const timeoutMs of HANG_PROBE_TIMEOUTS_MS) { + await rm(marker, { force: true }); + await prepare({ mode }); + await expectRefusal( + { ...options, timeoutMs }, + "E_NATIVE_COMPOSE_PROBE_TIMEOUT" + ); + pid = await hangHandshakePid(marker); + if (pid !== undefined) { + break; + } + } + if (pid === undefined) { + throw new Error( + `Fake Docker never published its ${mode} pid before the longest probe timeout` + ); + } + expect(await exitedWithin(pid, HANG_REAP_DEADLINE_MS)).toBe(true); } }); test("abort before spawn and during a real read-only query refuses and reaps the child", async () => { diff --git a/tests/native-process-policy-initial-trace.test.ts b/tests/native-process-policy-initial-trace.test.ts index c9690bd5d..46685832f 100644 --- a/tests/native-process-policy-initial-trace.test.ts +++ b/tests/native-process-policy-initial-trace.test.ts @@ -105,121 +105,136 @@ test("initial trace accepts only exact ownership read protocols", () => { } }); -test("initial forwarding preserves bytes/exits but records no unknown output, args or stderr", async () => { - const root = await mkdtemp(join(tmpdir(), "process-initial-forwarding-")); - const previous = process.env.PATH; - const binary = join(root, "docker"); - const traceRoot = join(root, "trace"); - try { - await Bun.write( - binary, - `#!${process.execPath} --no-env-file +/** + * Each forwarded command cold-starts two Bun processes (the trace forwarder and + * the fake Docker). These bounds only stop a wedged child from hanging the run; + * exits, bytes and trace order are still asserted exactly. + */ +const FORWARDED_DOCKER_EXEC_TIMEOUT_MS = 30_000; +/** Four forwarded commands at their exec bound, plus trace reads and rewrites. */ +const INITIAL_FORWARDING_TEST_TIMEOUT_MS = 150_000; + +test( + "initial forwarding preserves bytes/exits but records no unknown output, args or stderr", + async () => { + const root = await mkdtemp(join(tmpdir(), "process-initial-forwarding-")); + const previous = process.env.PATH; + const binary = join(root, "docker"); + const traceRoot = join(root, "trace"); + try { + await Bun.write( + binary, + `#!${process.execPath} --no-env-file const args=process.argv.slice(2); await Bun.sleep(20); if(args[0]==="compose"){await Bun.write(Bun.stdout,"original compose bytes\\n");await Bun.write(Bun.stderr,${JSON.stringify(CANARY)});process.exit(17);} if(args[1]==="ls"){await Bun.write(Bun.stdout,"original list bytes\\n");process.exit(0);} await Bun.write(Bun.stdout,${JSON.stringify(CANARY)});await Bun.write(Bun.stderr,${JSON.stringify(CANARY)});process.exit(23); ` - ); - await chmod(binary, 0o700); - process.env.PATH = root; - const prepared = await prepareProcessPolicyInitialTrace({ - directory: traceRoot, - }); - const run = async (args: readonly string[]) => - await exec([join(traceRoot, "docker"), ...args], { - env: { PATH: prepared.path }, - stdin: "ignore", - timeoutMs: 3000, + ); + await chmod(binary, 0o700); + process.env.PATH = root; + const prepared = await prepareProcessPolicyInitialTrace({ + directory: traceRoot, + }); + const run = async (args: readonly string[]) => + await exec([join(traceRoot, "docker"), ...args], { + env: { PATH: prepared.path }, + stdin: "ignore", + timeoutMs: FORWARDED_DOCKER_EXEC_TIMEOUT_MS, + }); + expect(await run(list("container"))).toEqual({ + exitCode: 0, + stdout: "original list bytes\n", + stderr: "", }); - expect(await run(list("container"))).toEqual({ - exitCode: 0, - stdout: "original list bytes\n", - stderr: "", - }); - expect( - await run(["compose", "-p", project, "-f", CANARY, "up", "-d"]) - ).toEqual({ - exitCode: 17, - stdout: "original compose bytes\n", - stderr: CANARY, - }); - expect( - await run(["inspect", id, "--format", "{{json .Config.Env}}"]) - ).toEqual({ exitCode: 23, stdout: CANARY, stderr: CANARY }); - expect(await run(list("network"))).toEqual({ - exitCode: 0, - stdout: "original list bytes\n", - stderr: "", - }); - const trace = await readProcessPolicyInitialTrace(prepared.directory); - expect(trace.queries.map((row) => row.args)).toEqual([ - list("container"), - list("network"), - ]); - expect((trace.queries[0]?.startOrder ?? -1) < trace.upStartOrder).toBe( - true - ); - expect((trace.queries[1]?.startOrder ?? -1) > trace.upReapOrder).toBe(true); - for (const name of await readdir(prepared.directory)) { expect( - await Bun.file(join(prepared.directory, name)).text() - ).not.toContain(CANARY); - } - const journal = join(prepared.directory, "events.jsonl"); - const originalJournal = await Bun.file(journal).text(); - const second = trace.queries[1]; - const first = trace.queries[0]; - if (!(first && second)) { - throw new Error("Missing synthetic query pair"); - } - // Complete but overlapping original reads cannot be grouped into a faithful - // single-owner replay, even if every capsule and reply is still present. - const effect = originalJournal - .trimEnd() - .split("\n") - .filter( - (line) => !(line.includes(first.token) || line.includes(second.token)) + await run(["compose", "-p", project, "-f", CANARY, "up", "-d"]) + ).toEqual({ + exitCode: 17, + stdout: "original compose bytes\n", + stderr: CANARY, + }); + expect( + await run(["inspect", id, "--format", "{{json .Config.Env}}"]) + ).toEqual({ exitCode: 23, stdout: CANARY, stderr: CANARY }); + expect(await run(list("network"))).toEqual({ + exitCode: 0, + stdout: "original list bytes\n", + stderr: "", + }); + const trace = await readProcessPolicyInitialTrace(prepared.directory); + expect(trace.queries.map((row) => row.args)).toEqual([ + list("container"), + list("network"), + ]); + expect((trace.queries[0]?.startOrder ?? -1) < trace.upStartOrder).toBe( + true ); - const overlapping = [ - { token: first.token, event: "start" }, - { token: second.token, event: "start" }, - { token: first.token, event: "reap" }, - { token: second.token, event: "reap" }, - ].map((event) => JSON.stringify(event)); - await Bun.write(journal, `${[...overlapping, ...effect].join("\n")}\n`); - await expect( - readProcessPolicyInitialTrace(prepared.directory) - ).rejects.toThrow("trace unavailable"); - await Bun.write(journal, originalJournal.slice(0, -1)); - await expect( - readProcessPolicyInitialTrace(prepared.directory) - ).rejects.toThrow("trace unavailable"); - await Bun.write(journal, originalJournal); - expect( - (await readProcessPolicyInitialTrace(prepared.directory)).queries.map( - (row) => row.args - ) - ).toEqual([list("container"), list("network")]); - const row = trace.queries[0]; - if (!row) { - throw new Error("Missing synthetic trace"); + expect((trace.queries[1]?.startOrder ?? -1) > trace.upReapOrder).toBe( + true + ); + for (const name of await readdir(prepared.directory)) { + expect( + await Bun.file(join(prepared.directory, name)).text() + ).not.toContain(CANARY); + } + const journal = join(prepared.directory, "events.jsonl"); + const originalJournal = await Bun.file(journal).text(); + const second = trace.queries[1]; + const first = trace.queries[0]; + if (!(first && second)) { + throw new Error("Missing synthetic query pair"); + } + // Complete but overlapping original reads cannot be grouped into a faithful + // single-owner replay, even if every capsule and reply is still present. + const effect = originalJournal + .trimEnd() + .split("\n") + .filter( + (line) => !(line.includes(first.token) || line.includes(second.token)) + ); + const overlapping = [ + { token: first.token, event: "start" }, + { token: second.token, event: "start" }, + { token: first.token, event: "reap" }, + { token: second.token, event: "reap" }, + ].map((event) => JSON.stringify(event)); + await Bun.write(journal, `${[...overlapping, ...effect].join("\n")}\n`); + await expect( + readProcessPolicyInitialTrace(prepared.directory) + ).rejects.toThrow("trace unavailable"); + await Bun.write(journal, originalJournal.slice(0, -1)); + await expect( + readProcessPolicyInitialTrace(prepared.directory) + ).rejects.toThrow("trace unavailable"); + await Bun.write(journal, originalJournal); + expect( + (await readProcessPolicyInitialTrace(prepared.directory)).queries.map( + (row) => row.args + ) + ).toEqual([list("container"), list("network")]); + const row = trace.queries[0]; + if (!row) { + throw new Error("Missing synthetic trace"); + } + const reap = join(prepared.directory, `${row.token}.json.reap`); + const originalReap = await Bun.file(reap).text(); + await unlink(reap); + await expect( + readProcessPolicyInitialTrace(prepared.directory) + ).rejects.toThrow(); + await Bun.write(reap, originalReap); + await unlink(join(prepared.directory, `${row.token}.json.start`)); + await expect( + readProcessPolicyInitialTrace(prepared.directory) + ).rejects.toThrow(); + } finally { + restoreEnv("PATH", previous); + await rm(root, { recursive: true, force: true }); } - const reap = join(prepared.directory, `${row.token}.json.reap`); - const originalReap = await Bun.file(reap).text(); - await unlink(reap); - await expect( - readProcessPolicyInitialTrace(prepared.directory) - ).rejects.toThrow(); - await Bun.write(reap, originalReap); - await unlink(join(prepared.directory, `${row.token}.json.start`)); - await expect( - readProcessPolicyInitialTrace(prepared.directory) - ).rejects.toThrow(); - } finally { - restoreEnv("PATH", previous); - await rm(root, { recursive: true, force: true }); - } -}); + }, + INITIAL_FORWARDING_TEST_TIMEOUT_MS +); test("initial forwarding preserves a multicall Docker entry and refuses its replacement", async () => { const root = await mkdtemp(join(tmpdir(), "process-initial-entry-")); From 907726ee4ce6c21b4e8ef386497b0b9eaecbb8f2 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:32:02 -0400 Subject: [PATCH 19/21] fix(runtime-core): compact relay-owned graph lease history at restore Every environment-bound graph the CLI creates is relay-owned, so ordinary graph cleanup never ran compaction for it. Foreground redelivery and fresh-owner restore now compact the run's retired history under the engine lease. Compaction runs after every prior slot is re-retired and after the acknowledged stopped receipt is superseded on disk, before the new generation allocates. Enrolled relay cleanup still does not compact, because the acknowledged-publisher check re-verifies its inventory. A compaction refusal after a committed cleanup is now Settled::Retained instead of an error. The records stay active and the next cleanup, restore or archive retries. graph::cleanup no longer reports a cleanup that succeeded on disk as failed. New tests cover the retire guard (live_intent), archived-copy refusal, residue with a pending record, the 16384 retired-slot cap, and 260 generations of a 14-service graph across 9 boots. A source-order wiring guard covers the trigger points. The README now describes the trigger points, refusal semantics, current-boot growth and tombstone retention. Refs: HACK-1193 Co-Authored-By: Claude Opus 5.5 --- packages/runtime-core/README.md | 38 +++- .../src/provider/environment_recovery.rs | 21 +- .../environment_recovery/compaction.rs | 29 +++ .../environment_recovery/compaction/tests.rs | 215 +++++++++++++++++- .../src/provider/graph/environment.rs | 40 ++-- .../src/provider/graph/environment/tests.rs | 95 ++++++++ .../runtime-core/src/provider/graph/mod.rs | 10 +- .../src/provider/graph/restore.rs | 18 ++ 8 files changed, 429 insertions(+), 37 deletions(-) diff --git a/packages/runtime-core/README.md b/packages/runtime-core/README.md index 3342e1d79..33772d04b 100644 --- a/packages/runtime-core/README.md +++ b/packages/runtime-core/README.md @@ -916,17 +916,35 @@ Graph cleanup uses that same history capacity, counting active and archived records together, so repeated restores cannot exceed a smaller cleanup-only ceiling after allocation succeeds. Matching ownership, container bindings and unambiguous records are still required. -After ordinary `graph cleanup` commits, the same engine lease compacts that run's -records: each must match a receipt container that is absent and a guest slot that is -absent, or nothing is written. Their exact intent digests move into one value-free -per-run tombstone in `run/environment-lease-tombstones`, bound to the VM incarnation, -and only then are the active records unlinked. Prior boots fold into one sealed -summary each; the current boot keeps per-slot reservations (bounded at 16384). A -retired slot or any slot of a sealed boot is never recorded, delivered, retired as -live or counted again, so history no longer consumes the 4096 active-record budget. +Retired graph history is compacted under the engine mutation lease where no retained +effect identity still binds the run's environment inventory: right after an ordinary +(non-relay) `graph cleanup` commits, and in every graph restore (foreground +`owner-restore` redelivery and fresh-owner `serve-restore`) after all prior slots are +re-retired and the acknowledged stopped receipt is superseded on disk, before the new +generation allocates. Enrolled relay cleanup (`graph cleanup` of a relay-owned graph, +or owner shutdown) does not compact, because its acknowledged effect is re-verified +against the inventory; its last generation stays active until the next restore or +archive. Each compacted record must match a receipt container that is absent and a +guest slot that is absent, or nothing is written. Exact intent digests move into one +value-free per-run tombstone in `run/environment-lease-tombstones`, bound to the VM +incarnation, and only then are the active records unlinked. Prior boots fold into one +sealed summary each; the current boot keeps per-slot reservations (at most 16384, about +1170 generations of a 14-service graph in one boot). A retired slot or any slot of a +sealed boot is never recorded, delivered, retired as live or counted again. Restore and +cleanup therefore scan only the active generation, and repeated restores no longer +consume the 4096 active-record budget; tombstone reads still grow with the current +boot's per-slot reservations until the next boot seals them. Pending, live, changed, misbound or foreign-incarnation records refuse compaction, -and other runs' records are untouched. An interruption is resumed by the next -cleanup or archive; exact residue reads as retired, changed residue refuses. +and other runs' records are untouched. A refusal never fails or reverts the cleanup +or restore that triggered it: the records stay active as exact evidence and the next +cleanup, restore or archive retries; a run that keeps refusing eventually meets the +active-record budget's `environment_recovery` admission refusal. An interruption is +resumed by the next compaction or by archive; exact residue reads as retired, changed +residue refuses. Tombstones are retained permanently as slot-ID reservations, like +standalone retention exports: archive and prune do not remove them, so the directory +holds one file per environment-attached run: about 160 bytes per current-boot +retired slot plus one summary per sealed boot, capped at 4 MiB and refused, never +truncated, beyond that. No retained evidence is automatically deleted to admit new work. This environment budget does not increase the runtime's separate container, CPU or memory limits and does not establish 32-branch qualification. diff --git a/packages/runtime-core/src/provider/environment_recovery.rs b/packages/runtime-core/src/provider/environment_recovery.rs index a1e4cb831..9d6d82295 100644 --- a/packages/runtime-core/src/provider/environment_recovery.rs +++ b/packages/runtime-core/src/provider/environment_recovery.rs @@ -3,7 +3,7 @@ mod compaction; mod retention; use super::{environment::EnvironmentLease, lifecycle::OwnedGuest, state}; use crate::{Candidate, CandidateError}; -pub(super) use compaction::{compact_graph, resume_graph}; +pub(super) use compaction::{Settled, compact_graph, resume_graph}; pub use retention::{RetiredExport, export_retired}; use serde::{Deserialize, Serialize}; use std::{fs, path::PathBuf}; @@ -594,14 +594,23 @@ pub(super) fn retire( if retention::reserved(candidate, slot)? { return retention::verify_retired(candidate, guest, slot, lease); } - // A compacted graph slot is retained history; never treat it as a live allocation. - if let Some(binding) = read_mode(candidate, slot, guest.incarnation(), lease, false)?.graph { - if compaction::reserved(candidate, &binding.run, guest.incarnation(), slot)? { + let intent = live_intent(candidate, guest.incarnation(), slot, lease)?; + retire_intent(guest, &intent) +} +/// The recorded intent a retirement may act on. A slot reserved by its graph's compaction +/// tombstone is retained history (exact residue or a replay), never a live allocation. +fn live_intent( + candidate: &Candidate, + incarnation: &str, + slot: &str, + lease: Option<&EnvironmentLease>, +) -> Result { + if let Some(binding) = read_mode(candidate, slot, incarnation, lease, false)?.graph { + if compaction::reserved(candidate, &binding.run, incarnation, slot)? { return Err(error()); } } - let intent = read(candidate, slot, guest.incarnation(), lease)?; - retire_intent(guest, &intent) + read(candidate, slot, incarnation, lease) } fn retire_intent(guest: &OwnedGuest<'_>, intent: &Intent) -> Result<(), CandidateError> { let slot = &intent.slot; diff --git a/packages/runtime-core/src/provider/environment_recovery/compaction.rs b/packages/runtime-core/src/provider/environment_recovery/compaction.rs index d3ac05792..ba7310599 100644 --- a/packages/runtime-core/src/provider/environment_recovery/compaction.rs +++ b/packages/runtime-core/src/provider/environment_recovery/compaction.rs @@ -16,6 +16,17 @@ //! 3. Retired slots of prior guest boots are folded into one sealed summary per boot. A prior //! boot cannot allocate again, so the whole boot stays reserved without per-slot entries. //! +//! Trigger points. Compaction rewrites the run's environment inventory, so it runs only where no +//! retained effect identity still binds that inventory: +//! - ordinary `graph cleanup` (no relay enrollment), right after its cleaned receipt commits; +//! - graph restore (foreground redelivery and fresh-owner serve-restore), after every prior slot +//! is re-retired and the acknowledged stopped receipt is superseded on disk, before the new +//! generation allocates. Enrolled relay cleanup itself never compacts: its acknowledged +//! effect, which includes the inventory, is re-verified until a restore supersedes it; +//! - archive, which only resumes an interrupted compaction. +//! +//! A refusal after a committed cleanup is [`Settled::Retained`], never a failure of that cleanup. +//! //! Reservation: a retired slot, or any slot of a sealed boot, can never be recorded again, //! selected for delivery, retired as live, or counted as a live allocation. Records are never //! deleted to make room: an uncertain (pending), live, changed, misbound or foreign-owned record @@ -70,6 +81,24 @@ pub(crate) struct Compaction { pub sealed_boots: usize, } +/// Outcome of compaction attempted after the caller's cleanup already committed. A refusal +/// (uncertain or changed record, retired-slot cap, sealed-boot cap, malformed tombstone, I/O) +/// reverts nothing and is not reported as a failure of that cleanup: every record it did not +/// durably retire stays active as exact evidence, and the next confirmed cleanup, restore or +/// archive retries. A run that keeps refusing eventually meets the active-record budget, whose +/// admission refusal is the closed `environment_recovery` error. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Settled { + Compacted(Compaction), + Retained, +} + +impl From> for Settled { + fn from(result: Result) -> Self { + result.map_or(Self::Retained, Self::Compacted) + } +} + /// `Resume` only completes an interrupted compaction: it selects no new record and seals nothing, /// so it is safe where live allocations of the run may remain. #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs b/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs index f4a9de77a..fdc039aa1 100644 --- a/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs +++ b/packages/runtime-core/src/provider/environment_recovery/compaction/tests.rs @@ -1,6 +1,8 @@ //! Filesystem-only tests. Guest absence is injected through `confirm`; no VM, engine, //! real HOME or process interruption is involved. Faults stop the state machine in-process. -use super::super::{active_exec_slot_for, graph_slots_for, preflight_records, record_intent}; +use super::super::{ + active_exec_slot_for, graph_slots_for, live_intent, preflight_records, record_intent, +}; use super::*; use std::{ cell::Cell, @@ -666,3 +668,214 @@ fn record_changed_after_tombstone_commit_is_retained_not_unlinked() { assert!(fixture.compact_now(RUN).is_err()); assert_eq!(fs::read(&target).unwrap(), replacement); } + +#[test] +fn compacted_residue_and_replay_are_never_retired_as_live() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + let slot = intents[0].slot.clone(); + // Before compaction the record is a live allocation that retirement may act on. + assert!( + live_intent(&fixture.candidate, INCARNATION, &slot, None).unwrap() == intents[0], + "live record is retirable" + ); + // Commit point 1 only: every record is exact residue of a durable tombstone. + assert!( + fixture + .compact(Mode::Compact, INCARNATION, CURRENT, RUN, stop_at(0)) + .is_err() + ); + assert!(fixture.active(&slot).exists()); + let before = fixture.state(); + assert!(live_intent(&fixture.candidate, INCARNATION, &slot, None).is_err()); + assert_eq!(fixture.state(), before, "refusal is read-only"); + // After completion, an exact replay of the retired record is refused the same way. + fixture.compact_now(RUN).unwrap(); + write_raw( + &fixture.active(&slot), + &serde_json::to_vec_pretty(&intents[0]).unwrap(), + ); + assert!(live_intent(&fixture.candidate, INCARNATION, &slot, None).is_err()); +} + +#[cfg(target_os = "macos")] +#[test] +fn archived_copy_of_a_compacted_record_is_refused_by_inventory() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + fixture.compact_now(RUN).unwrap(); + let archive = fixture + .candidate + .state_root + .join("graph-environment-archive"); + let inventory = || { + graph_inventory( + &fixture.candidate, + INCARNATION, + RUN, + &containers(RUN), + &archive, + ) + }; + state::private_directory(&archive).unwrap(); + assert!(inventory().unwrap().is_empty()); + // The archived directory has no residue rule: a compacted identity there is a copy. + state::write( + &archive.join(format!("{}.json", intents[0].slot)), + &intents[0], + ) + .unwrap(); + assert!(inventory().is_err()); + fs::remove_file(archive.join(format!("{}.json", intents[0].slot))).unwrap(); + // A fresh, unreserved archived record of the run is still ordinary inventory. + let fresh = fixture.intent(Some(RUN), CURRENT, 0); + state::write(&archive.join(format!("{}.json", fresh.slot)), &fresh).unwrap(); + assert_eq!(inventory().unwrap().intents.len(), 1); +} + +#[test] +fn residue_with_an_uncertain_pending_record_refuses() { + let fixture = Fixture::new(); + let intents = fixture.generation(RUN, CURRENT); + assert!( + fixture + .compact(Mode::Compact, INCARNATION, CURRENT, RUN, stop_at(0)) + .is_err() + ); + let tombstone = load(&fixture.candidate, RUN, INCARNATION).unwrap(); + let slot = &intents[0].slot; + assert!(residue(&fixture.candidate, tombstone.as_ref(), slot).unwrap()); + let pending = fixture.active(slot).with_extension("pending"); + write_raw(&pending, &serde_json::to_vec(&intents[0]).unwrap()); + assert!(residue(&fixture.candidate, tombstone.as_ref(), slot).is_err()); + fs::remove_file(&pending).unwrap(); + assert!(residue(&fixture.candidate, tombstone.as_ref(), slot).unwrap()); +} + +#[test] +fn retired_slot_cap_refuses_rather_than_discards() { + let filled = |entries: usize| { + let fixture = Fixture::new(); + let mut tombstone = Tombstone::new(RUN, INCARNATION); + for index in 0..entries { + tombstone.retired.insert( + format!( + "hack-env-lease-{CURRENT}-{:032x}", + u32::MAX as usize - index + ), + "e".repeat(64), + ); + } + commit( + &directory(&fixture.candidate), + &fixture.tombstone_path(RUN), + &tombstone, + ) + .unwrap(); + fixture.generation(RUN, CURRENT); + fixture + }; + // Exactly at the cap the compaction is admitted. + let fixture = filled(MAX_RETIRED_SLOTS - SERVICES); + assert_eq!(fixture.compact_now(RUN).unwrap().compacted, SERVICES); + let tombstone = load(&fixture.candidate, RUN, INCARNATION).unwrap().unwrap(); + assert_eq!(tombstone.retired.len(), MAX_RETIRED_SLOTS); + // One past the cap refuses before any write: nothing is discarded or unlinked. + let fixture = filled(MAX_RETIRED_SLOTS - SERVICES + 1); + let before = fixture.state(); + let refused = fixture.compact_now(RUN); + assert!(refused.is_err()); + assert_eq!(fixture.state(), before); + assert_eq!( + fixture.slots(RUN).unwrap().len(), + SERVICES, + "still live inventory" + ); + // After a committed cleanup that refusal is retained evidence, not a cleanup failure. + assert_eq!(Settled::from(refused), Settled::Retained); +} + +/// HACK-1193 state-machine proof: more than 256 start/cleanup/restore generations of a +/// 14-service graph across several guest boots. Every generation is admitted through the +/// ordinary record path; active records and per-generation scans stay at one generation; +/// retained bytes stay bounded. Guest and container absence is injected; this does not +/// measure native cleanup time or VM disk allocation. +#[test] +fn repeated_generations_beyond_256_stay_bounded_by_active_allocations() { + const GENERATIONS: usize = 260; + const PER_BOOT: usize = 32; + let fixture = Fixture::new(); + let foreign = fixture.foreign(); + let untouched = fixture.snapshot(&root(&fixture.candidate)); + let boot = + |generation: usize| format!("{:08x}-2222-4222-8222-222222222222", generation / PER_BOOT); + let mut peak_tombstone = 0; + for generation in 0..GENERATIONS { + let boot = boot(generation); + let intents: Vec = (0..SERVICES) + .map(|service| fixture.intent(Some(RUN), &boot, service)) + .collect(); + // The whole batch is admitted; one record goes through the full durable record path + // (budget plus standalone and tombstone reservations). The others are written without + // per-file fsync only to keep this long test fast; their bytes are identical. + preflight_records(&fixture.candidate, SERVICES).unwrap(); + for (index, intent) in intents.iter().enumerate() { + if index == 0 { + record_intent(&fixture.candidate, intent).unwrap(); + } else { + assert!(!reserved(&fixture.candidate, RUN, INCARNATION, &intent.slot).unwrap()); + write_raw( + &fixture.active(&intent.slot), + &serde_json::to_vec_pretty(intent).unwrap(), + ); + } + } + // Cleanup and the next restore see only the active generation, never history. + assert_eq!( + fixture.slots(RUN).unwrap().len(), + SERVICES, + "generation {generation}" + ); + let outcome = fixture + .compact(Mode::Compact, INCARNATION, &boot, RUN, |_| Ok(())) + .unwrap(); + assert_eq!(outcome.compacted, SERVICES); + assert_eq!( + outcome.sealed_boots, + usize::from(generation % PER_BOOT == 0 && generation > 0) + ); + assert!(fixture.slots(RUN).unwrap().is_empty()); + assert_eq!( + fs::read_dir(root(&fixture.candidate)).unwrap().count(), + foreign.len(), + "generation {generation}" + ); + let metadata = fs::metadata(fixture.tombstone_path(RUN)).unwrap(); + peak_tombstone = peak_tombstone.max(metadata.blocks() * 512); + } + let tombstone = load(&fixture.candidate, RUN, INCARNATION).unwrap().unwrap(); + let boots = GENERATIONS.div_ceil(PER_BOOT); + assert_eq!(tombstone.sealed.len(), boots - 1); + assert!( + tombstone + .sealed + .values() + .all(|sealed| sealed.slots as usize == PER_BOOT * SERVICES) + ); + assert_eq!( + tombstone.retired.len(), + (GENERATIONS - (boots - 1) * PER_BOOT) * SERVICES + ); + // Retained bytes follow the current boot's generations, not total history: at most one + // boot of per-slot entries plus one summary per sealed boot. + assert!( + peak_tombstone <= 256 * 1024, + "allocated {peak_tombstone} bytes" + ); + assert!(preflight_records(&fixture.candidate, MAX_INTENT_ENTRIES - foreign.len()).is_ok()); + let now = fixture.snapshot(&root(&fixture.candidate)); + assert_eq!(now, untouched, "foreign records unchanged"); + eprintln!( + "generations={GENERATIONS} services={SERVICES} boots={boots} peak_tombstone_allocated_bytes={peak_tombstone}" + ); +} diff --git a/packages/runtime-core/src/provider/graph/environment.rs b/packages/runtime-core/src/provider/graph/environment.rs index bd470fa8f..e8c52d904 100644 --- a/packages/runtime-core/src/provider/graph/environment.rs +++ b/packages/runtime-core/src/provider/graph/environment.rs @@ -134,25 +134,31 @@ pub(super) fn verify_cleanup( environment_recovery::verify_graph_retired(engine.guest(), expected) } -/// Compact this graph's confirmed-retired allocation records after ordinary cleanup. The caller -/// holds the engine's mutation lease and has committed the cleaned receipt; every record must -/// match a receipt container that is absent, and its guest slot must be absent. -pub(super) fn compact_retired( +/// Service-to-container bindings an environment-attached receipt's records must match; `None` +/// when the graph never recorded an environment allocation. +pub(super) fn bound_containers(receipt: &Receipt) -> Option> { + receipt.environment_attached.then(|| { + receipt + .resources + .values() + .filter(|r| r.kind == Kind::Container) + .map(|r| (r.key.clone(), r.name.clone())) + .collect() + }) +} + +/// Compact `run`'s confirmed-retired allocation records. The caller holds the engine's mutation +/// lease, has already committed a cleanup or retirement, and no retained effect identity still +/// binds this run's environment inventory (see `environment_recovery::compaction`). Every +/// record must match `containers` with that container and its guest slot absent. A refusal is +/// retained evidence, never a failure of the caller's committed cleanup. +pub(super) fn compact_history( candidate: &Candidate, engine: &Engine<'_>, - receipt: &Receipt, -) -> Result<(), CandidateError> { - if !receipt.environment_attached { - return Ok(()); - } - let containers = receipt - .resources - .values() - .filter(|r| r.kind == Kind::Container) - .map(|r| (r.key.clone(), r.name.clone())) - .collect(); - environment_recovery::compact_graph(candidate, engine.guest(), &receipt.run, &containers)?; - Ok(()) + run: &str, + containers: &BTreeMap, +) -> environment_recovery::Settled { + environment_recovery::compact_graph(candidate, engine.guest(), run, containers).into() } pub(super) fn require_replay_supported(receipt: &Receipt) -> Result<(), CandidateError> { diff --git a/packages/runtime-core/src/provider/graph/environment/tests.rs b/packages/runtime-core/src/provider/graph/environment/tests.rs index 616afbbcb..19ce3086b 100644 --- a/packages/runtime-core/src/provider/graph/environment/tests.rs +++ b/packages/runtime-core/src/provider/graph/environment/tests.rs @@ -27,6 +27,101 @@ fn bound_receipt_never_silently_loses_its_redelivery_gate() { ); } #[test] +fn history_bindings_cover_only_attached_graph_containers() { + let mut receipt: Receipt = serde_json::from_value(json!({"version":1,"run":"a".repeat(32),"owner":"b".repeat(32),"namespace":"c".repeat(64),"plan_id":"d".repeat(64),"phase":"stopped-data-retained","readiness":{},"resources":{ + "container:app":{"key":"app","kind":"container","name":"hkg-app","phase":"absent"}, + "container:db":{"key":"db","kind":"container","name":"hkg-db","phase":"absent"}, + "volume:data":{"key":"data","kind":"volume","name":"hkg-data","phase":"created"} + }})).unwrap(); + assert_eq!( + bound_containers(&receipt), + None, + "no environment, no history" + ); + receipt.environment_attached = true; + assert_eq!( + bound_containers(&receipt).unwrap(), + BTreeMap::from([ + ("app".to_owned(), "hkg-app".to_owned()), + ("db".to_owned(), "hkg-db".to_owned()), + ]) + ); +} + +/// Wiring guard. The trigger points need an owned VM and engine, so their effects are proven +/// natively, not here; this only fails if a trigger is removed or moved past the point where +/// the new generation allocates. The state machine itself is covered by compaction's tests. +#[test] +fn compaction_triggers_stay_wired_at_their_lease_points() { + fn body<'a>(source: &'a str, start: &str, end: &str) -> &'a str { + let from = source.find(start).expect("function start"); + let to = from + source[from..].find(end).expect("function end"); + &source[from..to] + } + let cleanup = body( + include_str!("../mod.rs"), + "pub fn cleanup(", + "fn cleanup_owned(", + ); + let committed = cleanup.find("cleanup_owned(").expect("cleanup effect"); + let compacted = cleanup + .find("environment::compact_history(") + .expect("ordinary cleanup compacts"); + assert!( + committed < compacted, + "compaction follows the committed cleanup" + ); + + let restore = body( + include_str!("../restore.rs"), + "fn restore_inputs(", + "fn retain_restore_failure(", + ); + let retired = restore + .find("environment_recovery::retire(") + .expect("prior slots re-retired"); + let captured = restore + .find("environment::bound_containers(&receipt)") + .expect("history bindings captured"); + let replaced = restore + .find("*resource = prepared.resources[key].clone()") + .expect("fresh resources"); + let prepared = restore + .find("driver.prepare(") + .expect("fresh owner prepare"); + let compacted = restore + .find("environment::compact_history(") + .expect("restore compacts"); + let allocated = restore + .find("session.create_resources(true)") + .expect("new generation allocates"); + assert!( + retired < captured && captured < replaced, + "bindings of the retired generation" + ); + assert!( + prepared < compacted && compacted < allocated, + "superseded, then allocate" + ); + + let relay = body( + include_str!("../host_relay.rs"), + "pub(super) fn cleanup_with_relay_expected(", + "pub fn resume_relay_cleanup(", + ); + assert!( + !relay.contains("compact_history("), + "acknowledged relay cleanup re-verifies its inventory; it must not compact" + ); + + let archive = body( + include_str!("../environment.rs"), + "pub(super) fn archive_slots(", + "environment_recovery::archive_graph(", + ); + assert!(archive.contains("environment_recovery::resume_graph(")); +} +#[test] #[ignore = "Manual owned development VM, pinned Bun image, explicit candidate and external watchdog"] fn container_attachment_records_ownership_before_create_and_omits_values_from_metadata() { let candidate = candidate(); diff --git a/packages/runtime-core/src/provider/graph/mod.rs b/packages/runtime-core/src/provider/graph/mod.rs index f94177fab..0ea71c9ce 100644 --- a/packages/runtime-core/src/provider/graph/mod.rs +++ b/packages/runtime-core/src/provider/graph/mod.rs @@ -1625,9 +1625,13 @@ pub fn cleanup( let (receipt, root) = load(candidate, &engine, run)?; cleanup_enrollment::ordinary_mutation(&root, &receipt)?; let receipt = cleanup_owned(candidate, &engine, receipt, &root, remove_data)?; - // Only ordinary cleanup compacts. Recovery flows retain an exact environment inventory - // across their confirmation and use the fenced path without this effect. - environment::compact_retired(candidate, &engine, &receipt)?; + // The cleaned receipt is committed and ordinary cleanup binds no relay effect identity to + // its environment inventory, so history is compacted under the same lease. A refusal keeps + // the records for the next cleanup, restore or archive; it never reports this cleanup failed. + // Relay-enrolled graphs compact in restore instead (see `environment_recovery::compaction`). + if let Some(containers) = environment::bound_containers(&receipt) { + environment::compact_history(candidate, &engine, &receipt.run, &containers); + } Ok(receipt) } diff --git a/packages/runtime-core/src/provider/graph/restore.rs b/packages/runtime-core/src/provider/graph/restore.rs index f60b9dfab..fcf4a5dad 100644 --- a/packages/runtime-core/src/provider/graph/restore.rs +++ b/packages/runtime-core/src/provider/graph/restore.rs @@ -478,6 +478,8 @@ fn restore_inputs( for slot in environment::cleanup_slots(candidate, &engine, &receipt)? { super::super::environment_recovery::retire(candidate, engine.guest(), &slot, None)?; } + // Bindings of the retired generations, captured before fresh resources replace them. + let retired_history = environment::bound_containers(&receipt); let launcher = if environments.is_empty() { None } else { @@ -528,6 +530,10 @@ fn restore_inputs( if let Some(selected) = &source_rebind { selected.reverify(&session.engine)?; } + // A redelivery restore committed its restoring receipt (relay marker dormant) above; a + // fresh owner's prepare commits its own enrollment. Only then is the acknowledged + // stopped receipt, whose cleanup effect binds the environment inventory, superseded. + let mut superseded = fresh.is_none(); if fresh.is_some() && let Some(driver) = session.startup.as_mut() { @@ -537,6 +543,18 @@ fn restore_inputs( &session.root, &mut session.configs, )?; + superseded = true; + } + // Every prior slot was re-retired under this lease and each prior container is absent: + // compact that history before the new generation allocates, so restore and the next + // cleanup scan only active allocations. A refusal retains the records. + if superseded && let Some(containers) = &retired_history { + environment::compact_history( + candidate, + &session.engine, + &session.receipt.run, + containers, + ); } session.create_resources(true)?; execution::run(&prepared.graph, &mut session, options.timeout) From 73b09904651d4358812f7c3f7b89fce36f8602b3 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 13:44:16 -0400 Subject: [PATCH 20/21] fix(lifecycle): count an unreaped zombie helper as stopped The "stop that cannot be confirmed" test froze its starter with Subprocess.kill("SIGSTOP"). On macOS, the pinned Bun 1.4.2 delivers SIGCONT for that call, so the starter kept running, reaped the helper after TERM, and release reported `stopped`. CI uses the pinned Bun, so the test failed there; local runs passed only on an unpinned Bun 1.4.3. The scenario also showed a product defect. The census counted a zombie (an exited, unreaped process) as live. A stop that left the helper a zombie reported `stop-unconfirmed`. A `ready` claim whose helper crashed under a suspended starter looked live, so other clients could not restart the helper until that starter resumed. A zombie runs no code and holds no listener, and its PID cannot be reused until it is reaped. The census now reads `ps` state and treats zombies as exited. The unconfirmed-stop test now injects the census and signal boundary through a narrow `processes` seam on release, because no real process can portably survive SIGKILL. A separate real-process test freezes the starter with process.kill and asserts that the helper is an unreaped zombie and that release reports `stopped`. Refs HACK-1142 Co-Authored-By: Claude Opus 5.5 --- docs/reference/shared-host-lifecycle.md | 9 ++- src/lib/process-tree.ts | 16 ++++- src/lib/shared-host-lifecycle.ts | 64 +++++++++++++++--- tests/shared-host-lifecycle.test.ts | 89 ++++++++++++++++++++++--- 4 files changed, 152 insertions(+), 26 deletions(-) diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index bd55812aa..3652190b6 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -63,6 +63,12 @@ replaced atomically, and parsed strictly (exact keys, version 1): - The claim is live only when the census shows that exact pair. A missing PID, or the same PID with a different birth time (PID reuse), makes the claim stale. A stale claim grants nothing. Hack discards it and never signals that PID. +- A zombie (a process that has exited but that its parent has not reaped yet, + `ps` state `Z`) counts as exited. It runs no code and holds no listener, and its + PID cannot be reused until it is reaped. So a stop that leaves the helper a zombie + is `stopped`, and a claim whose helper is a zombie is stale. Otherwise a helper + that crashed under a suspended starter would block every restart until that + starter resumed. - One exception: if a stale owned claim's PID still answers the identity endpoint as the expected helper, ownership is uncertain. Hack keeps the claim and fails with `claim-invalid`. It neither adopts the helper (which would drop the owned @@ -205,7 +211,8 @@ Linux pass is a receipt only once that hosted job has run it. | lost parent after ready | The helper stays claimed; another client reuses it; the release by both holders stops it. | | owned descendants | Release stops the helper and its grandchild; a bystander survives. | | SIGTERM ignored | Release escalates to SIGKILL after the grace period: `stopped`. | -| stop unconfirmed | A helper that stays in the census after KILL (a zombie under a frozen parent) gives `stop-unconfirmed` and keeps the claim. | +| stop unconfirmed | With a simulated census and signal boundary (no real process can portably survive KILL), a tree that stays in the census gets TERM then KILL, gives `stop-unconfirmed` and keeps the claim with no holders. | +| unreaped zombie | A helper whose starter is frozen with `SIGSTOP` exits on TERM and stays an unreaped zombie; release reports `stopped` and removes the claim. | | session and environment | The helper leads its own process group and does not see a runner-only variable. | | impostor identity | A live `ready` claim whose port is answered by another PID fails `foreign-listener`; neither process is signalled. | | invalid claim / no census | Malformed, other-port, group-readable and oversized claims, and an unavailable census, fail closed; nothing is spawned or signalled; the claim is kept. | diff --git a/src/lib/process-tree.ts b/src/lib/process-tree.ts index a6bb346b7..d51ba0fb7 100644 --- a/src/lib/process-tree.ts +++ b/src/lib/process-tree.ts @@ -2,8 +2,14 @@ export type ProcessIdentityRow = { readonly pid: number; readonly parentPid: number; readonly birth: string; + /** + * The process has exited but its parent has not reaped it yet (`ps` state + * `Z`). A zombie runs no code and holds no listener; its PID cannot be + * reused until it is reaped. + */ + readonly zombie: boolean; }; -const IDENTITY_ROW = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/; +const IDENTITY_ROW = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/; /** * Capture only process identity and lineage; never command arguments or environment. @@ -13,10 +19,13 @@ const IDENTITY_ROW = /^\s*(\d+)\s+(\d+)\s+(.+?)\s*$/; * different `TZ` values, or one host whose zone changes, would see different * births for the same live process. Births may be persisted and compared across * processes, so the zone is pinned here (POSIX `UTC0` needs no zoneinfo). + * + * Zombie rows are kept and flagged, not dropped: each caller decides whether + * an exited, unreaped process still counts for its purpose. */ export async function readProcessIdentities(): Promise { try { - const proc = Bun.spawn(["ps", "-A", "-o", "pid=,ppid=,lstart="], { + const proc = Bun.spawn(["ps", "-A", "-o", "pid=,ppid=,stat=,lstart="], { env: { ...process.env, LC_ALL: "C", TZ: "UTC0" }, stdin: "ignore", stdout: "pipe", @@ -34,7 +43,8 @@ export async function readProcessIdentities(): Promise { { pid: Number(match[1]), parentPid: Number(match[2]), - birth: (match[3] ?? "").replace(/\s+/g, " "), + birth: (match[4] ?? "").replace(/\s+/g, " "), + zombie: (match[3] ?? "").startsWith("Z"), }, ] : []; diff --git a/src/lib/shared-host-lifecycle.ts b/src/lib/shared-host-lifecycle.ts index c28eb8236..8671e2c15 100644 --- a/src/lib/shared-host-lifecycle.ts +++ b/src/lib/shared-host-lifecycle.ts @@ -31,9 +31,10 @@ import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; * left running: this module never signals a process it cannot prove it owns. * - Ownership is a persisted PID plus process birth time (`ps lstart`, read in * UTC so clients with different time zones agree). A claim whose PID is - * gone or now belongs to a different process (PID reuse) is stale and is - * discarded without signalling anything. A stale claim whose PID still - * answers as the helper is uncertain: it is kept, never adopted. + * gone, an unreaped zombie, or now belongs to a different process (PID + * reuse) is stale and is discarded without signalling anything. A stale + * claim whose PID still answers as the helper is uncertain: it is kept, + * never adopted. * - A child is spawned behind an exec gate: it waits on stdin until its claim * is durable. If the starter dies first, the gate reads EOF and exits, so a * spawned helper is never left running without a claim. @@ -454,13 +455,47 @@ async function removeClaim( } } +/** + * The process census and signal boundary. Production callers omit it and get + * {@link systemProcesses}; tests replace it to simulate states real processes + * cannot produce portably, such as a process that stays in the census after + * SIGKILL. + */ +export type SharedHostProcesses = { + /** Every process row, including zombies; `[]` when `ps` failed. */ + readonly census: () => Promise; + /** Signal the members of a captured tree that still carry their birth. */ + readonly signalTree: ( + tree: readonly ProcessIdentityRow[], + signal: NodeJS.Signals + ) => Promise; +}; + +const systemProcesses: SharedHostProcesses = { + census: readProcessIdentities, + signalTree: signalVerifiedProcessTree, +}; + /** * A census is usable only when it observed this process; an empty or failed * `ps` must never be read as "every claimed process is gone". + * + * Zombies are left out: an exited process that its parent has not reaped yet + * runs no code, holds no listener or open file, and cannot be stopped any + * further. Its PID cannot be reused until it is reaped, so treating it as + * exited can never confuse it with another process. Counting it as live would + * report a finished stop as unconfirmed, and would make a `ready` claim whose + * helper crashed under a suspended (for example Ctrl-Z) starter look live, so + * no other client could restart that helper until the starter resumed. */ -async function readCensus(): Promise { - const rows = await readProcessIdentities(); - return rows.some((row) => row.pid === process.pid) ? rows : null; +async function readCensus( + processes: SharedHostProcesses = systemProcesses +): Promise { + const rows = await processes.census(); + if (!rows.some((row) => row.pid === process.pid)) { + return null; + } + return rows.filter((row) => !row.zombie); } function isLive( @@ -581,14 +616,17 @@ function classifyIdentity( /** * Stop a birth-verified process tree: TERM, bounded grace, then KILL for the * members that still carry their captured birth time. Nothing is signalled - * when the root no longer matches its claim. + * when the root no longer matches its claim. A member that has exited counts + * as stopped even before its parent reaps it (see {@link readCensus}). */ export async function stopVerifiedSharedHostProcess(opts: { readonly pid: number; readonly birth: string; readonly graceMs: number; + readonly processes?: SharedHostProcesses; }): Promise<"stopped" | "absent" | "unconfirmed" | "census-unavailable"> { - const census = await readCensus(); + const processes = opts.processes ?? systemProcesses; + const census = await readCensus(processes); if (census === null) { return "census-unavailable"; } @@ -597,13 +635,13 @@ export async function stopVerifiedSharedHostProcess(opts: { } const tree = selectProcessTree(census, opts.pid); const remaining = async (): Promise => { - const live = await readCensus(); + const live = await readCensus(processes); if (live === null) { return null; } return tree.filter((row) => isLive(live, row)); }; - await signalVerifiedProcessTree(tree, "SIGTERM"); + await processes.signalTree(tree, "SIGTERM"); const termDeadline = performance.now() + opts.graceMs; let left = await remaining(); while (left !== null && left.length > 0 && performance.now() < termDeadline) { @@ -613,7 +651,7 @@ export async function stopVerifiedSharedHostProcess(opts: { if (left !== null && left.length === 0) { return "stopped"; } - await signalVerifiedProcessTree(left ?? tree, "SIGKILL"); + await processes.signalTree(left ?? tree, "SIGKILL"); const killDeadline = performance.now() + 1000; left = await remaining(); while (left !== null && left.length > 0 && performance.now() < killDeadline) { @@ -1122,6 +1160,8 @@ export async function releaseSharedHostHelper(opts: { readonly graceMs?: number; readonly timeoutMs?: number; readonly signal?: AbortSignal; + /** Test seam for the census and signals; production callers omit it. */ + readonly processes?: SharedHostProcesses; }): Promise { if (!NAME.test(opts.spec.name)) { throw new Error("Invalid shared host helper specification"); @@ -1149,6 +1189,7 @@ async function releaseLocked(opts: { readonly spec: Pick; readonly holder: string; readonly graceMs?: number; + readonly processes?: SharedHostProcesses; }): Promise { const read = await readClaim(opts.spec); if (read.kind === "absent") { @@ -1171,6 +1212,7 @@ async function releaseLocked(opts: { pid: claim.pid, birth: claim.birth, graceMs: opts.graceMs ?? 2000, + processes: opts.processes, }); switch (stopped) { case "census-unavailable": diff --git a/tests/shared-host-lifecycle.test.ts b/tests/shared-host-lifecycle.test.ts index 960850cd8..f6b684783 100644 --- a/tests/shared-host-lifecycle.test.ts +++ b/tests/shared-host-lifecycle.test.ts @@ -22,6 +22,7 @@ import { releaseSharedHostHelper, type SharedHostEvent, type SharedHostHelperSpec, + type SharedHostProcesses, } from "../src/lib/shared-host-lifecycle.ts"; import { restoreEnv } from "./helpers/env.ts"; @@ -1431,35 +1432,101 @@ test( test( "a stop that cannot be confirmed keeps the claim so the process stays recorded", + async () => { + // No real process can portably survive SIGKILL, so the census and signal + // boundary is simulated: the claimed helper and its child stay in every + // census whatever is sent to them. + const root = await scratch(); + const spec = specFor({ root, port: freePort() }); + const birth = "Thu Jan 1 00:00:00 1970"; + const helper = { pid: 4242, parentPid: 1, birth, zombie: false }; + const child = { pid: 4243, parentPid: 4242, birth, zombie: false }; + const self = { + pid: process.pid, + parentPid: process.ppid, + birth, + zombie: false, + }; + await writeClaimFile(spec, { + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: helper.pid, + birth, + holders: ["branch-a"], + }); + const signals: [NodeJS.Signals, number[]][] = []; + const processes: SharedHostProcesses = { + census: async () => [self, helper, child], + signalTree: async (tree, signal) => { + signals.push([signal, tree.map((row) => row.pid)]); + }, + }; + expect( + await releaseSharedHostHelper({ + spec, + holder: "branch-a", + graceMs: 100, + processes, + }) + ).toEqual({ state: "stop-unconfirmed" }); + // TERM, then KILL, each to the whole captured tree and nothing else. + expect(signals).toEqual([ + ["SIGTERM", [helper.pid, child.pid]], + ["SIGKILL", [helper.pid, child.pid]], + ]); + // The claim keeps the exact process recorded, now with no holders. + expect(await readClaim(spec)).toEqual({ + version: 1, + helper: spec.name, + port: spec.port, + ownership: "owned", + phase: "ready", + pid: helper.pid, + birth, + holders: [], + }); + }, + TIMEOUT +); + +test( + "an exited helper its parent has not reaped yet is stopped, not unconfirmed", async () => { const root = await scratch(); const spec = specFor({ root, port: freePort() }); // The starter stays alive and is then frozen, so the helper it spawned - // cannot be reaped: after SIGKILL it remains a zombie in the census. + // cannot be reaped: once it exits on SIGTERM it is a zombie in the census. const starter = startStarter({ spec, holder: "branch-a", linger: true }); expect(await starterResult(starter)).toMatchObject({ state: "ready", ownership: "owned", }); const owned = await trackClaimedHelper(spec); - starter.proc.kill("SIGSTOP"); + // process.kill, not Subprocess.kill: Bun 1.4.2 on macOS delivers SIGCONT + // for Subprocess.kill("SIGSTOP"), which would leave the starter running. + process.kill(starter.proc.pid, "SIGSTOP"); try { expect( await releaseSharedHostHelper({ spec, holder: "branch-a", - graceMs: 100, + graceMs: 1000, }) - ).toEqual({ state: "stop-unconfirmed" }); - expect(await readClaim(spec)).toMatchObject({ - ownership: "owned", - pid: owned.pid, - birth: owned.birth, - holders: [], - }); + ).toEqual({ state: "stopped" }); + expect(await readClaim(spec)).toBeNull(); + // The helper really is an unreaped zombie, so this run exercised the + // zombie path rather than an ordinary reaped exit. + const row = (await readProcessIdentities()).find( + (entry) => entry.pid === owned.pid && entry.birth === owned.birth + ); + expect(row).toMatchObject({ parentPid: starter.proc.pid, zombie: true }); } finally { - starter.proc.kill("SIGCONT"); + process.kill(starter.proc.pid, "SIGCONT"); } + expect(await waitGone(owned)).toBe(true); }, TIMEOUT ); From e2dab3a82e9f02831fc1fb00fe0dc66faa00faf1 Mon Sep 17 00:00:00 2001 From: Dimitri Kennedy Date: Mon, 19 Oct 2026 14:20:04 -0400 Subject: [PATCH 21/21] docs(lifecycle): require one shared stateDir and name per helper port Clients of a helper port coordinate only through the claim and lock under `/`. The previous JSDoc called stateDir a "caller-owned private directory", which suggested one directory per caller. A client with its own directory finds no claim. It adopts the running owned helper, and the owner's last release then stops a helper that client still holds. Its starts are also not serialized with the owner's. The spec JSDoc, the module contract and the reference doc now state that every client of one port must pass the same stateDir and name, derived from the helper and port rather than the branch. Refs HACK-1142 Co-Authored-By: Claude Opus 5.5 --- docs/reference/shared-host-lifecycle.md | 11 +++++++++++ src/lib/shared-host-lifecycle.ts | 24 ++++++++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/reference/shared-host-lifecycle.md b/docs/reference/shared-host-lifecycle.md index 3652190b6..e07b83b03 100644 --- a/docs/reference/shared-host-lifecycle.md +++ b/docs/reference/shared-host-lifecycle.md @@ -56,6 +56,17 @@ replaced atomically, and parsed strictly (exact keys, version 1): {"version":1,"helper":"issuer-bridge","port":4100,"ownership":"adopted","holders":["branch-a"]} ``` +The lock is `/.lock`. Every branch or client that uses one helper +port must pass the same `stateDir` and `name`: the state directory belongs to the +helper, not to each caller. The claim and lock are the only coordination between +clients, and Hack cannot see a claim in another directory. A client with a +different `stateDir` or `name` takes a different lock and finds no claim. If the +running owned helper answers its identity check, that client adopts it as if +someone else had started it. The owner's last release then stops the helper while +the other client still holds it, and the two clients' starts are not serialized. +Derive both values from the helper and its port, never from the branch or +checkout. + - Ownership is the pair `pid` + `birth`. `birth` is the process start time from `ps lstart`, always read with `TZ=UTC0` and `LC_ALL=C`. `ps` prints `lstart` in local time, so without a fixed zone two clients with different `TZ` values, or a diff --git a/src/lib/shared-host-lifecycle.ts b/src/lib/shared-host-lifecycle.ts index 8671e2c15..20a47d7e6 100644 --- a/src/lib/shared-host-lifecycle.ts +++ b/src/lib/shared-host-lifecycle.ts @@ -24,8 +24,10 @@ import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; * - Ownership is `owned` (Hack started it and holds an exact process claim), * `adopted` (an identity-verified helper someone else started) or `disabled` * (no lock, no claim, no child). Phases are `starting`, `ready` and `failed`. - * - Starters for one helper are serialized by an exclusive lock in the - * caller's private state directory; only one of them may spawn. + * - Starters for one helper are serialized by an exclusive lock next to its + * claim; only one of them may spawn. Every client of one helper port must + * therefore use the same `stateDir` and `name` (see + * {@link SharedHostHelperSpec.stateDir}). * - A listener is only ever adopted after its identity endpoint names the * expected helper and upstream. Anything else on the port is refused and * left running: this module never signals a process it cannot prove it owns. @@ -102,11 +104,25 @@ export type SharedHostIdentity = { }; export type SharedHostHelperSpec = { - /** Claim and lock name; `[A-Za-z0-9][A-Za-z0-9_.-]{0,127}`. */ + /** + * Claim and lock name; `[A-Za-z0-9][A-Za-z0-9_.-]{0,127}`. Every client of + * one helper port must use the same name (see {@link stateDir}). + */ readonly name: string; /** Disabled helpers take no lock, write no claim and start no child. */ readonly enabled: boolean; - /** Caller-owned private directory for the claim and lock. */ + /** + * Private (owner-only) directory that holds this helper's claim and lock. + * + * It is shared, not per caller: every branch or client that uses the helper + * on `port` must pass the same `stateDir` and `name`. The claim and lock are + * the only coordination between clients, and nothing here can see a claim in + * another directory. A client with a different `stateDir` or `name` takes a + * different lock and finds no claim. If the running owned helper answers its + * identity check, that client adopts it, so the owner's last release stops a + * helper this client still holds, and their starts are not serialized. + * Derive both values from the helper and its port, never from the branch. + */ readonly stateDir: string; /** Loopback port the helper serves. */ readonly port: number;