diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4e2de881..94bfc84e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,6 +74,18 @@ env: flake.lock tools/toolchain/microvm-vmm-env.nix .github/workflows/release.yml + # The guest artifact's own closure set, SEPARATE from the two above: this + # publishes the three guest assets, so an agent- or runner-only change must + # not republish it. Derived from guest-image/moon.yml's build.inputs plus the + # publish lane itself; the agent pin travels inside guest-image/. + GUEST_IMAGE_CLOSURE_PATHS: | + guest-image/** + tools/guest-image/** + go/go.mod + go/go.sum + go/cmd/compass-guestd/** + go/internal/** + .github/workflows/release.yml jobs: release-pr: @@ -1278,3 +1290,175 @@ jobs: --repo ghcr.io/rigelbuild/compass-runner --sha "$sha12")" echo "published $ref" echo "runner image: \`$ref\`" >> "$GITHUB_STEP_SUMMARY" + + publish-guest-image: + name: publish-guest-image + runs-on: ubuntu-latest + # Least privilege: read the tree, write the GHCR package, nothing else. + permissions: + contents: read + packages: write + # Its own group: a different package from the agent and runner images, so + # serializing against those would only add latency. Within this package, + # publishes must serialize — two runs pushing one tag race over which bytes + # it names. + concurrency: + group: publish-guest-image + cancel-in-progress: false + queue: max + # A dispatch from a feature branch must never publish assets for unmerged + # code. Main pushes satisfy this trivially. + if: github.ref == 'refs/heads/main' + # The rootfs is a ~2 GiB erofs built from source; that nix build sizes this. + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Decide whether this push touches the guest-image closure + id: gate + # The sibling publish jobs' push-event tree diff, over + # GUEST_IMAGE_CLOSURE_PATHS. Every fallback errs toward publishing, so + # a missing diff base never silently drops a closure change. + env: + EVENT_NAME: ${{ github.event_name }} + BEFORE_SHA: ${{ github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + echo "should_publish=true" >> "$GITHUB_OUTPUT" + echo "workflow_dispatch: force-publish (no push range to diff)" + exit 0 + fi + + if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then + echo "should_publish=true" >> "$GITHUB_OUTPUT" + echo "no diff base (first push to branch): force-publish" + exit 0 + fi + + git fetch --no-tags --depth=1 origin "$BEFORE_SHA" >/dev/null 2>&1 || true + + if ! changed="$(git diff --name-only "$BEFORE_SHA" "$HEAD_SHA" 2>/dev/null)"; then + echo "should_publish=true" >> "$GITHUB_OUTPUT" + echo "before-sha unreachable: force-publish (no-drop errs toward publishing)" + exit 0 + fi + + should_publish=false + while IFS= read -r pattern; do + [ -n "$pattern" ] || continue + case "$pattern" in + *'/**') + prefix="${pattern%'/**'}/" + while IFS= read -r f; do + [ -n "$f" ] || continue + case "$f" in + "$prefix"*) should_publish=true ;; + esac + done <<< "$changed" + ;; + *) + while IFS= read -r f; do + [ "$f" = "$pattern" ] && should_publish=true + done <<< "$changed" + ;; + esac + [ "$should_publish" = true ] && break + done <<< "$GUEST_IMAGE_CLOSURE_PATHS" + + echo "should_publish=$should_publish" >> "$GITHUB_OUTPUT" + echo "changed-path gate over the guest-image closure set: should_publish=$should_publish" + + - uses: cachix/install-nix-action@630ae543ea3a38a9a4166f03376c02c50f408342 # v31 + if: steps.gate.outputs.should_publish == 'true' + with: + # Declared here rather than via `accept-flake-config`, which would + # make nix trust the nixConfig of any flake it evaluates. + extra_nix_config: | + experimental-features = nix-command flakes + extra-substituters = https://devenv.cachix.org https://cachix.cachix.org + extra-trusted-public-keys = devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw= cachix.cachix.org-1:eWNHQldwUO7G2VkjpnjDbWwy4KQ/HNxht7H4SSoMckM= + + - name: Put the pinned bun and moon on PATH + if: steps.gate.outputs.should_publish == 'true' + # Both come from the gate-tools pin, so the lane runs byte-identical + # binaries here and on a dev box. moon is required, not incidental: the + # lane realises its assets through the gate's own build task. + run: | + set -euo pipefail + for lang in bun moon; do + # `jq -r` renders a missing `.store` as the string `null`, a value + # rather than an absence, so test for it explicitly. + store=$(nix eval --json -f tools/toolchain/gate-tools.nix "langs.$lang" | jq -r '.store') + if [ -z "$store" ] || [ "$store" = null ]; then + echo "::error::gate-tools.nix langs.$lang produced no store path" >&2 + exit 1 + fi + nix build --no-link "$store" + echo "$store/bin" >>"$GITHUB_PATH" + done + + - name: Put the fork's patched skopeo on PATH + if: steps.gate.outputs.should_publish == 'true' + # The publish lane probes and copies with a plain `skopeo` (the + # RigelBuild/nix2container fork's patched build). Resolve it from the + # shared pinned helper so this privileged job cannot run a mutable + # upstream build, the same pattern the sibling publish jobs use. + run: | + set -euo pipefail + # `--print-out-paths` prints every output (skopeo ships a `-man` output + # too); take the one carrying bin/skopeo, not a fixed line. + skopeo_bin="" + for store in $(nix build --no-link --print-out-paths \ + -f tools/toolchain/skopeo-nix2container-env.nix skopeo); do + if [ -x "$store/bin/skopeo" ]; then + skopeo_bin="$store/bin" + break + fi + done + if [ -z "$skopeo_bin" ]; then + echo "::error::skopeo-nix2container-env.nix produced no output carrying bin/skopeo" >&2 + exit 1 + fi + echo "$skopeo_bin" >> "$GITHUB_PATH" + + - name: Pin the registry auth file + if: steps.gate.outputs.should_publish == 'true' + # LOAD-BEARING. The login and the lane's own skopeo calls are SEPARATE + # processes and must resolve the SAME creds file; the default location + # is environment-dependent on hosted runners, and a mismatch greens the + # login then 401s the push. + run: echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/ghcr-auth.json" >> "$GITHUB_ENV" + + - name: Log in to GHCR + if: steps.gate.outputs.should_publish == 'true' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Pass the actor through env rather than interpolating ${{ }} into the + # shell, keeping context values off the run: command line. + ACTOR: ${{ github.actor }} + # The token is passed via env and `--password-stdin` only — never on a + # command line or in a log. + run: | + skopeo \ + login ghcr.io -u "$ACTOR" --password-stdin \ + --authfile "$REGISTRY_AUTH_FILE" <<< "$GITHUB_TOKEN" + + - name: Publish the guest artifact by digest + if: steps.gate.outputs.should_publish == 'true' + # The lane realises the three assets, assembles the layout, scans its + # annotations before any push, and asserts the registry resolved the + # manifest it built. The deployable `repo@sha256:…` lands in the summary: + # GHCR has no server-side tag immutability, so nothing resolves by tag. + run: | + set -euo pipefail + # `--short=12` returns the shortest UNIQUE length >= 12, so a collision + # would yield 13+ chars and fail the lane's strict 12-hex check. A + # deterministic truncation keeps the contract exact. + sha12="$(git rev-parse HEAD | cut -c1-12)" + ref="$(bun tools/guest-image/publish.ts \ + --repo ghcr.io/rigelbuild/compass-guest-image --sha "$sha12")" + echo "published $ref" + echo "guest image: \`$ref\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 982c5a8e..4a488c81 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,7 @@ result-* # entrypoint symlink) and the OCI layout, realised by tools/runner-image/build.ts. /runner-image/store/ /runner-image/out/ + +# guest-image publish output: the OCI layout a local dry run writes, multi-GiB +# because it holds a copy of the rootfs blob. +/guest-image/oci-layout/ diff --git a/tools/guest-image/moon.yml b/tools/guest-image/moon.yml index 609d0775..7e1d5422 100644 --- a/tools/guest-image/moon.yml +++ b/tools/guest-image/moon.yml @@ -1,13 +1,13 @@ # yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json # -# guest-image: the agent-image pin the microVM guest rootfs derives from. +# guest-image: the agent-image pin the microVM guest rootfs derives from, and +# the lane that publishes the guest triple as an OCI artifact. # TypeScript rather than a shell script (the no-bash-gate CI task). # -# The pin CLI itself talks to a registry, so it is not a gate task. The -# typecheck/test tasks here ARE ordinary bun gates: they cover the pure core — -# the provenance rejections and the layer-descriptor extraction whose drift -# would silently make the rootfs unreproducible or stack a different -# filesystem. +# Both CLIs talk to a registry, so neither is a gate task. The typecheck/test +# tasks here ARE ordinary bun gates: they cover the pure cores — the provenance +# rejections, the layer-descriptor extraction whose drift would silently make +# the rootfs unreproducible, and the artifact mapping a registry would reject. # # A bun/TypeScript CLI, hoisted root-workspace member (`bun` tag): install is # inherited via .moon/tasks/tag-bun.yml and lint/format are whole-repo root @@ -22,9 +22,10 @@ tasks: deps: ['install'] inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] test: - # The pure core (pin-core.ts): every provenance rejection (foreign repo, - # moving tag, malformed digest), the manifest-shape refusals, and the - # layer order that determines the stacked filesystem. + # The pure cores: every provenance rejection (foreign repo, moving tag, + # malformed digest), the manifest-shape refusals, the layer order that + # determines the stacked filesystem, and the artifact descriptors plus + # fail-closed tag disposition the publish lane maps. inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] # The CI aggregate. Without it ci-matrix contributes NO target for this # project (it emits `:ci` only for a project defining a `ci` task), so @@ -34,3 +35,10 @@ tasks: deps: ['typecheck', 'test'] options: cache: false + # Publishing needs network and GHCR credentials, so it runs in the release + # workflow, never the gate — the gate's cost stays the guest-image build. + publish: + command: 'bun publish.ts' + options: + runInCI: false + cache: false diff --git a/tools/guest-image/publish-core.test.ts b/tools/guest-image/publish-core.test.ts new file mode 100644 index 00000000..8570c242 --- /dev/null +++ b/tools/guest-image/publish-core.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, test } from "bun:test"; +import { + AGENT_DIGEST_ANNOTATION, + ARTIFACT_TYPE, + annotationViolations, + buildTag, + digestRef, + EMPTY_CONFIG_MEDIA_TYPE, + EXIT, + layoutPlan, + manifestDigest, + tagDisposition, +} from "./publish-core.ts"; + +const hex = (c: string) => c.repeat(64); +const assets = { + kernel: { digest: `sha256:${hex("1")}`, size: 11_534_336 }, + rootfs: { digest: `sha256:${hex("2")}`, size: 2_264_924_160 }, + initrd: { digest: `sha256:${hex("3")}`, size: 9_437_184 }, +} as const; +const provenance = { + revision: "d3adb33fd3ad", + agentImageDigest: `sha256:${hex("a")}`, +}; +const plan = () => layoutPlan(assets, provenance); + +describe("layoutPlan", () => { + test("marks the artifact non-runnable: artifactType plus the empty config", () => { + const manifest = JSON.parse(plan().manifest); + expect(manifest.artifactType).toBe(ARTIFACT_TYPE); + expect(manifest.config.mediaType).toBe(EMPTY_CONFIG_MEDIA_TYPE); + // The two-byte `{}` blob, so nothing can resolve a runnable config. + expect(manifest.config.size).toBe(2); + expect(manifest.config.digest).toBe(manifestDigest("{}")); + }); + + test("carries schemaVersion 2, which a conformant registry requires", () => { + expect(JSON.parse(plan().manifest).schemaVersion).toBe(2); + expect(JSON.parse(plan().index).schemaVersion).toBe(2); + }); + + test("every layer descriptor declares the asset's real byte length", () => { + expect( + JSON.parse(plan().manifest).layers.map((l: { size: number }) => l.size), + ).toEqual([11_534_336, 2_264_924_160, 9_437_184]); + }); + + test("layers are kernel, rootfs, initrd in that order with their own media types", () => { + expect( + JSON.parse(plan().manifest).layers.map( + (l: { mediaType: string }) => l.mediaType, + ), + ).toEqual([ + "application/vnd.compass.guest-kernel.v1", + "application/vnd.compass.guest-rootfs.v1+erofs", + "application/vnd.compass.guest-initrd.v1+cpio.zst", + ]); + }); + + test("annotations key on the materialised FILENAME, never a hash-prefixed store basename", () => { + const { annotations } = plan(); + expect( + annotations["org.compass.guest.layer.compass-guest-rootfs.erofs"], + ).toBe(hex("2")); + expect(annotations["org.compass.guest.layer.bzImage"]).toBe(hex("1")); + expect(annotations["org.compass.guest.layer.compass-guest-initrd"]).toBe( + hex("3"), + ); + // A store basename would carry a content hash and never match + // verifyImages' filepath.Base lookup. + expect( + Object.keys(annotations).some((k) => /layer\.[a-z0-9]{32}-/.test(k)), + ).toBe(false); + expect(annotations[AGENT_DIGEST_ANNOTATION]).toBe(`sha256:${hex("a")}`); + expect(annotations["org.opencontainers.image.revision"]).toBe( + "d3adb33fd3ad", + ); + }); + + test("the index descriptor's size and digest match the manifest bytes", () => { + const built = plan(); + const entry = JSON.parse(built.index).manifests[0]; + expect(entry.digest).toBe(manifestDigest(built.manifest)); + expect(entry.size).toBe( + new TextEncoder().encode(built.manifest).byteLength, + ); + }); + + test.each([ + [ + "a tag instead of a digest", + { ...assets, rootfs: { ...assets.rootfs, digest: "latest" } }, + ], + [ + "a truncated digest", + { ...assets, rootfs: { ...assets.rootfs, digest: "sha256:abc" } }, + ], + ["a zero size", { ...assets, kernel: { ...assets.kernel, size: 0 } }], + ["a negative size", { ...assets, kernel: { ...assets.kernel, size: -1 } }], + [ + "a fractional size", + { ...assets, initrd: { ...assets.initrd, size: 1.5 } }, + ], + ])( + "rejects %s rather than publishing a manifest a registry will refuse", + (_label, broken) => { + expect(() => layoutPlan(broken, provenance)).toThrow(); + }, + ); + + test("rejects provenance that would publish an unusable pin", () => { + expect(() => + layoutPlan(assets, { ...provenance, agentImageDigest: "git-abc" }), + ).toThrow(); + expect(() => layoutPlan(assets, { ...provenance, revision: "" })).toThrow(); + }); +}); + +describe("manifestDigest", () => { + test("hashes the raw bytes, matching the registry's manifest identity", () => { + expect(manifestDigest("{}")).toBe( + "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + ); + }); + + test("a one-byte manifest change moves the digest", () => { + expect(manifestDigest('{"a":1}')).not.toBe(manifestDigest('{"a":2}')); + }); +}); + +describe("annotationViolations", () => { + test("blocks secret-shaped names and values", () => { + expect( + annotationViolations({ GITHUB_TOKEN: "x", note: "DEPLOY_KEY" }), + ).toEqual(["GITHUB_TOKEN", "note"]); + }); + + test.each([ + "org.compass.guest.registry-token", + "org.compass.guest.password", + "org.compass.guest.api-key", + "org.compass.guest.auth", + "org.compass.guest.session", + ])("catches %s, the dot-and-dash shape this lane emits", (name) => { + // An underscore-only boundary is inert against every real key here, so + // the scan would report clean while leaking. + expect(annotationViolations({ [name]: "value" })).toEqual([name]); + }); + + test("the real provenance annotation set is clean", () => { + expect(annotationViolations(plan().annotations)).toEqual([]); + }); + + test("word boundaries keep PATH and KEYBOARD from firing", () => { + expect(annotationViolations({ PATH: "/usr/bin", KEYBOARD: "us" })).toEqual( + [], + ); + }); +}); + +describe("tagDisposition", () => { + const local = `sha256:${hex("b")}`; + + test.each(["manifest unknown", "name unknown", "manifest not found"])( + "%s is a definitive absence, so the tag is free", + (stderr) => { + expect( + tagDisposition({ exitCode: 1, stdout: "", stderr }, local).action, + ).toBe("publish"); + }, + ); + + test.each([ + [ + "a local credentials fault", + "creds was not found in the docker config file", + ], + ["an auth rejection", "unauthorized: authentication required"], + ["a transport fault", "i/o timeout"], + ["an unrelated missing file", "error: file was not found"], + ])("%s aborts rather than reading as absence", (_label, stderr) => { + expect( + tagDisposition({ exitCode: 1, stdout: "", stderr }, local).action, + ).toBe("abort"); + }); + + test("a tag already holding this exact manifest is an idempotent skip", () => { + const body = '{"schemaVersion":2}'; + expect( + tagDisposition( + { exitCode: 0, stdout: body, stderr: "" }, + manifestDigest(body), + ).action, + ).toBe("skip"); + }); + + test("a tag holding different content aborts rather than overwriting", () => { + expect( + tagDisposition( + { exitCode: 0, stdout: '{"other":true}', stderr: "" }, + local, + ).action, + ).toBe("abort"); + }); + + test("an unreadable probe failure aborts, so a transient fault cannot overwrite a published artifact", () => { + const probe = { + exitCode: 1, + stdout: "", + stderr: "i/o timeout talking to registry", + }; + expect(tagDisposition(probe, local).action).toBe("abort"); + }); +}); + +describe("references", () => { + test("digestRef pins by digest and buildTag only addresses the build", () => { + expect(digestRef("ghcr.io/x/y", `sha256:${hex("c")}`)).toBe( + `ghcr.io/x/y@sha256:${hex("c")}`, + ); + expect(buildTag("ghcr.io/x/y", "0123456789ab")).toBe( + "ghcr.io/x/y:git-0123456789ab", + ); + }); + + test("rejects a tag-shaped digest and a non-12-hex sha", () => { + expect(() => digestRef("ghcr.io/x/y", "latest")).toThrow(); + expect(() => buildTag("ghcr.io/x/y", "not-hex")).toThrow(); + }); +}); + +test("exit codes distinguish usage, secret, transport, mismatch, and layout faults", () => { + expect(EXIT).toEqual({ + usage: 2, + secretFound: 3, + pushFailed: 4, + digestMismatch: 5, + badLayout: 6, + }); +}); diff --git a/tools/guest-image/publish-core.ts b/tools/guest-image/publish-core.ts new file mode 100644 index 00000000..470164cf --- /dev/null +++ b/tools/guest-image/publish-core.ts @@ -0,0 +1,257 @@ +// The pure core of the guest-image publish lane: no I/O, so every mapping and +// fail-closed edge is unit-testable without a registry. GHCR has no +// server-side tag immutability, so a tag only addresses a build and the +// deployed contract is `repo@sha256:…`. + +/** Exit codes, numbered so a failed run names its own fault in CI. The + * recoveries differ: 3 is a real secret to rotate, 4 is a registry/transport + * fault, 5 means the registry resolved something other than what was built, 6 + * means the local layout is malformed (the artifact is wrong, not the argv). */ +export const EXIT = { + usage: 2, + secretFound: 3, + pushFailed: 4, + digestMismatch: 5, + badLayout: 6, +} as const; + +/** The empty config every non-runnable artifact points at: the literal two + * bytes `{}`, per the OCI 1.1 guidance for a manifest with no runnable config. */ +export const EMPTY_CONFIG_BYTES = new TextEncoder().encode("{}"); + +export const ARTIFACT_TYPE = "application/vnd.compass.guest-image.v1"; +export const EMPTY_CONFIG_MEDIA_TYPE = "application/vnd.oci.empty.v1+json"; +const MANIFEST_MEDIA_TYPE = "application/vnd.oci.image.manifest.v1+json"; +const INDEX_MEDIA_TYPE = "application/vnd.oci.image.index.v1+json"; + +const LAYER_MEDIA_TYPES = { + kernel: "application/vnd.compass.guest-kernel.v1", + rootfs: "application/vnd.compass.guest-rootfs.v1+erofs", + initrd: "application/vnd.compass.guest-initrd.v1+cpio.zst", +} as const; + +export type AssetName = keyof typeof LAYER_MEDIA_TYPES; + +/** Layer order is contract: a materialiser reads the three positionally. */ +export const ASSET_ORDER: readonly AssetName[] = ["kernel", "rootfs", "initrd"]; + +/** One realised asset. */ +export type Asset = { + readonly digest: string; + readonly size: number; +}; + +export type Descriptor = { + readonly mediaType: string; + readonly digest: string; + readonly size: number; +}; + +export type LayoutPlan = { + readonly config: Descriptor; + readonly configBytes: Uint8Array; + readonly layers: readonly Descriptor[]; + readonly annotations: Readonly>; + readonly manifest: string; + readonly manifestDescriptor: Descriptor; + readonly index: string; +}; + +/** Credential-shaped NAME segments whose presence must block the push. The + * boundary spans `_`, `.` and `-`, because OCI annotation keys are + * dot-and-dash separated (`org.compass.guest.registry-token`) — an + * underscore-only boundary, the shape this pattern has in env-var land, is + * inert against every key this lane actually emits. Matching on the name, not + * the value: a value-shaped heuristic both misses an unusual token and fires + * on a harmless path. The boundary still keeps a keyword from matching + * mid-word, so `PAT` never fires on `PATH`. */ +const SECRET_NAME_PATTERN = + /(^|[_.-])(TOKEN|SECRET|PASSWORD|PASSWD|PASSPHRASE|APIKEY|API_KEY|KEY|CREDENTIAL|CREDENTIALS|PRIVATE_KEY|SESSION|AUTH|PAT|BEARER)([_.-]|$)/i; + +const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +/** The on-disk filenames a materialiser writes. `verifyImages` keys its + * sha256sum manifest on `filepath.Base` of each configured path, so these — + * not the nix store basenames, which carry a content hash — are what the + * annotations must name. */ +export const ASSET_FILENAMES = { + kernel: "bzImage", + rootfs: "compass-guest-rootfs.erofs", + initrd: "compass-guest-initrd", +} as const; + +/** Per-asset digest annotations carry the same filename→hex facts the Runner's + * `--microvm-image-manifest` consumes, so a materialiser can write that file + * straight from the manifest it pulled. Bare hex, not `sha256:…`, because that + * is the sha256sum format the Runner parses. */ +export const LAYER_ANNOTATION_PREFIX = "org.compass.guest.layer."; +export const AGENT_DIGEST_ANNOTATION = "org.compass.guest.agent-image-digest"; + +export function isDigest(value: string): boolean { + return DIGEST_PATTERN.test(value); +} + +/** + * Map the three realised assets into the OCI layout that publishes them. + * + * Every descriptor carries its real byte length: a registry validates the + * declared size against the blob it receives, so a placeholder size is + * rejected at push — the one place a local `skopeo copy` will not catch it, + * because the local path reads the blob and never compares. + */ +export function layoutPlan( + assets: Readonly>, + provenance: { revision: string; agentImageDigest: string }, +): LayoutPlan { + for (const name of ASSET_ORDER) { + const asset = assets[name]; + if (!isDigest(asset.digest)) { + throw new Error( + `${name}: digest must be sha256:<64 hex>, got ${asset.digest}`, + ); + } + if (!Number.isSafeInteger(asset.size) || asset.size <= 0) { + throw new Error( + `${name}: size must be a positive integer, got ${asset.size}`, + ); + } + } + if (!isDigest(provenance.agentImageDigest)) { + throw new Error( + `agent image digest must be sha256:<64 hex>, got ${provenance.agentImageDigest}`, + ); + } + if (provenance.revision === "") throw new Error("revision must not be empty"); + + const layers = ASSET_ORDER.map((name) => ({ + mediaType: LAYER_MEDIA_TYPES[name], + digest: assets[name].digest, + size: assets[name].size, + })); + const annotations: Record = { + "org.opencontainers.image.revision": provenance.revision, + [AGENT_DIGEST_ANNOTATION]: provenance.agentImageDigest, + }; + for (const name of ASSET_ORDER) { + annotations[`${LAYER_ANNOTATION_PREFIX}${ASSET_FILENAMES[name]}`] = assets[ + name + ].digest.slice("sha256:".length); + } + + const config: Descriptor = { + mediaType: EMPTY_CONFIG_MEDIA_TYPE, + digest: digestBytes(EMPTY_CONFIG_BYTES), + size: EMPTY_CONFIG_BYTES.byteLength, + }; + // schemaVersion is REQUIRED and must be 2; a manifest without it is + // rejected by a conformant registry even though a local copy accepts it. + const manifest = JSON.stringify({ + schemaVersion: 2, + mediaType: MANIFEST_MEDIA_TYPE, + artifactType: ARTIFACT_TYPE, + config, + layers, + annotations, + }); + const manifestBytes = new TextEncoder().encode(manifest); + const manifestDescriptor: Descriptor = { + mediaType: MANIFEST_MEDIA_TYPE, + digest: digestBytes(manifestBytes), + size: manifestBytes.byteLength, + }; + const index = JSON.stringify({ + schemaVersion: 2, + mediaType: INDEX_MEDIA_TYPE, + manifests: [{ ...manifestDescriptor, artifactType: ARTIFACT_TYPE }], + }); + return { + config, + configBytes: EMPTY_CONFIG_BYTES, + layers, + annotations, + manifest, + manifestDescriptor, + index, + }; +} + +/** The manifest's identity is the sha256 of its RAW bytes — the same identity + * the registry computes, so re-serialising before hashing would diverge. */ +export function manifestDigest(manifest: string): string { + return digestBytes(new TextEncoder().encode(manifest)); +} + +/** Annotation entries that must block the push. The config is empty and the + * layers are opaque binaries, so annotations are the only place this lane + * could leak a credential. */ +export function annotationViolations( + annotations: Readonly>, +): string[] { + const found: string[] = []; + for (const [name, value] of Object.entries(annotations)) { + if (SECRET_NAME_PATTERN.test(name) || SECRET_NAME_PATTERN.test(value)) + found.push(name); + } + return found; +} + +/** The build-addressability tag for a commit. Nothing deployed resolves + * through it. */ +export function buildTag(repo: string, sha12: string): string { + if (!/^[0-9a-f]{12}$/.test(sha12)) + throw new Error(`sha must be 12 lowercase hex characters, got ${sha12}`); + if (repo === "") throw new Error("repo must not be empty"); + return `${repo}:git-${sha12}`; +} + +/** The immutable reference a deployment pins. Never `repo:tag`. */ +export function digestRef(repo: string, digest: string): string { + if (!isDigest(digest)) + throw new Error(`digest must be sha256:<64 hex>, got ${digest}`); + if (repo === "") throw new Error("repo must not be empty"); + return `${repo}@${digest}`; +} + +/** + * What to do about an existing `:git-` tag, given what the registry + * returned for it. + * + * `skip` is the idempotent re-run of the same commit. `abort` covers both a tag + * holding different content and any inspect failure we cannot read as a plain + * absence — an ambiguous registry answer must never be treated as "free to + * push", or a transient fault would silently overwrite a published artifact. + */ +export function tagDisposition( + probe: { exitCode: number; stdout: string; stderr: string }, + localManifestDigest: string, +): { action: "publish" | "skip" | "abort"; reason?: string } { + if (probe.exitCode === 0) { + const remote = manifestDigest(probe.stdout); + if (remote === localManifestDigest) return { action: "skip" }; + return { + action: "abort", + reason: `tag already holds ${remote}, refusing to overwrite`, + }; + } + // Only a registry's own "this name/manifest does not exist" answer means the + // tag is free. Anything else — auth, transport, a local creds problem — is + // ambiguous, and treating it as absence would overwrite a published + // artifact, so it aborts. + if ( + /(^|\W)(manifest unknown|name unknown|manifest not found)(\W|$)/i.test( + probe.stderr, + ) + ) { + return { action: "publish" }; + } + return { + action: "abort", + reason: `registry probe failed ambiguously: ${probe.stderr.trim()}`, + }; +} + +function digestBytes(bytes: Uint8Array): string { + const hasher = new Bun.CryptoHasher("sha256"); + hasher.update(bytes); + return `sha256:${hasher.digest("hex")}`; +} diff --git a/tools/guest-image/publish.test.ts b/tools/guest-image/publish.test.ts new file mode 100644 index 00000000..e22e54a9 --- /dev/null +++ b/tools/guest-image/publish.test.ts @@ -0,0 +1,270 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { EXIT } from "./publish-core.ts"; + +// The shell's observable contract: stdout carries the digest reference and +// nothing else, the layout holds one blob per declared digest, and a registry +// that resolves something else fails closed. The pure-core suite cannot cover +// any of it, so a regression here would otherwise ship green. + +const cli = join(import.meta.dir, "publish.ts"); +const REPO = "ghcr.io/rigelbuild/compass-guest-image"; +const SHA = "0123456789ab"; + +let sandbox = ""; +let binDir = ""; +let storeDir = ""; + +/** A stub on PATH ahead of the real tool, so no test touches nix or a registry. */ +function stub(name: string, body: string): void { + const path = join(binDir, name); + writeFileSync(path, `#!/usr/bin/env bash\n${body}\n`); + chmodSync(path, 0o755); +} + +function runCli( + args: readonly string[], + env: Readonly> = {}, +) { + const result = spawnSync("bun", [cli, ...args], { + encoding: "utf8", + env: { ...process.env, PATH: `${binDir}:${process.env.PATH}`, ...env }, + }); + return { + code: result.status ?? -1, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + +beforeEach(() => { + sandbox = mkdtempSync(join(tmpdir(), "guest-publish-")); + binDir = join(sandbox, "bin"); + storeDir = join(sandbox, "store"); + mkdirSync(binDir, { recursive: true }); + // The kernel attr yields a DIRECTORY whose bzImage is the blob; the other + // two attrs are the blobs themselves. + mkdirSync(join(storeDir, "linux-0.0.0"), { recursive: true }); + writeFileSync(join(storeDir, "linux-0.0.0", "bzImage"), "KERNEL-BYTES"); + writeFileSync(join(storeDir, "rootfs.erofs"), "ROOTFS-BYTES"); + writeFileSync(join(storeDir, "initrd"), "INITRD-BYTES"); + stub( + "moon", + `echo " banner line moon prints" +echo "${join(storeDir, "linux-0.0.0")}" +echo "${join(storeDir, "rootfs.erofs")}" +echo "${join(storeDir, "initrd")}" +echo "Tasks: 1 completed"`, + ); + stub("git", 'echo "d3adb33fd3adb33fd3adb33fd3adb33fd3adb33f"'); +}); + +afterEach(() => { + rmSync(sandbox, { recursive: true, force: true }); +}); + +function layoutPath(): string { + return join(sandbox, "layout"); +} + +function dryRun(env: Readonly> = {}) { + return runCli( + ["--repo", REPO, "--sha", SHA, "--layout", layoutPath(), "--dry-run"], + env, + ); +} + +describe("the dry-run contract", () => { + test("stdout is exactly one digest reference and nothing else", () => { + const { code, stdout } = dryRun(); + expect(code).toBe(0); + expect(stdout.trimEnd().split("\n")).toHaveLength(1); + expect(stdout.trim()).toMatch(new RegExp(`^${REPO}@sha256:[0-9a-f]{64}$`)); + }); + + test("writes one blob per declared digest, so the layout a registry reads is complete", () => { + dryRun(); + const index = JSON.parse( + readFileSync(join(layoutPath(), "index.json"), "utf8"), + ); + const manifestDigest = index.manifests[0].digest.slice("sha256:".length); + const manifest = JSON.parse( + readFileSync( + join(layoutPath(), "blobs", "sha256", manifestDigest), + "utf8", + ), + ); + for (const descriptor of [...manifest.layers, manifest.config]) { + const blob = join( + layoutPath(), + "blobs", + "sha256", + descriptor.digest.slice("sha256:".length), + ); + expect(existsSync(blob)).toBe(true); + expect(readFileSync(blob).byteLength).toBe(descriptor.size); + } + }); + + test("selects store paths from moon's framed output rather than line positions", () => { + dryRun(); + const index = JSON.parse( + readFileSync(join(layoutPath(), "index.json"), "utf8"), + ); + const manifestDigest = index.manifests[0].digest.slice("sha256:".length); + const manifest = JSON.parse( + readFileSync( + join(layoutPath(), "blobs", "sha256", manifestDigest), + "utf8", + ), + ); + // The kernel layer must be the bzImage inside the directory attr. + expect(manifest.layers[0].size).toBe("KERNEL-BYTES".length); + expect(manifest.layers[1].size).toBe("ROOTFS-BYTES".length); + }); + + test("clears a stale layout, so the digest assertion cannot read a previous run's blobs", () => { + mkdirSync(join(layoutPath(), "blobs", "sha256"), { recursive: true }); + // The marker is what identifies this as a layout the lane may replace. + writeFileSync( + join(layoutPath(), "oci-layout"), + '{"imageLayoutVersion":"1.0.0"}', + ); + const stale = join(layoutPath(), "blobs", "sha256", "stale"); + writeFileSync(stale, "LEFTOVER"); + dryRun(); + expect(existsSync(stale)).toBe(false); + }); + + test("refuses to recursively delete a --layout path that is not a layout", () => { + mkdirSync(layoutPath(), { recursive: true }); + const bystander = join(layoutPath(), "important.txt"); + writeFileSync(bystander, "DO NOT DELETE"); + const { code } = dryRun(); + expect(code).toBe(EXIT.usage); + expect(readFileSync(bystander, "utf8")).toBe("DO NOT DELETE"); + }); + test("a build failure is a layout fault, not a push attempt", () => { + stub("moon", 'echo "boom" >&2; exit 1'); + const { code, stdout } = dryRun(); + expect(code).toBe(EXIT.badLayout); + expect(stdout.trim()).toBe(""); + }); + + test("an empty --repo is a usage error before any realisation work", () => { + const { code } = runCli([ + "--repo", + "", + "--sha", + SHA, + "--layout", + layoutPath(), + "--dry-run", + ]); + expect(code).toBe(EXIT.usage); + // The multi-GiB realisation must not have run to reject an empty flag. + expect(existsSync(layoutPath())).toBe(false); + }); + + test("a malformed agent-oci.lock is a layout fault, not a stack trace", () => { + const lock = join( + import.meta.dir, + "..", + "..", + "guest-image", + "agent-oci.lock", + ); + const saved = readFileSync(lock, "utf8"); + writeFileSync(lock, "{ not json"); + try { + expect(dryRun().code).toBe(EXIT.badLayout); + } finally { + writeFileSync(lock, saved); + } + }); + + test("an asset path that stats but is not a regular file fails closed", () => { + // A directory where a blob belongs: stat succeeds, reading it does not. + stub( + "moon", + `echo "${join(storeDir, "linux-0.0.0")}" +echo "${storeDir}" +echo "${join(storeDir, "initrd")}"`, + ); + expect(dryRun().code).toBe(EXIT.badLayout); + }); + + test("a build emitting the wrong number of store paths fails closed", () => { + stub("moon", `echo "${join(storeDir, "rootfs.erofs")}"`); + expect(dryRun().code).toBe(EXIT.badLayout); + }); +}); + +describe("the push contract", () => { + function publish(env: Readonly> = {}) { + return runCli( + ["--repo", REPO, "--sha", SHA, "--layout", layoutPath()], + env, + ); + } + + /** Records argv so a test can assert what skopeo was actually asked to do. */ + function stubSkopeo(script: string): string { + const log = join(sandbox, "skopeo.log"); + stub("skopeo", `echo "$@" >> "${log}"\n${script}`); + return log; + } + + test("publishes when the tag is absent, then prints the registry-resolved digest", () => { + stubSkopeo(`if [ "$1" = inspect ]; then + if [ -f "${join(sandbox, "pushed")}" ]; then cat "${join(sandbox, "manifest")}"; exit 0; fi + echo "manifest unknown" >&2; exit 1 +fi +if [ "$1" = copy ]; then touch "${join(sandbox, "pushed")}"; exit 0; fi`); + // The registry echoes back the manifest this run built. + dryRun(); + const index = JSON.parse( + readFileSync(join(layoutPath(), "index.json"), "utf8"), + ); + const digest = index.manifests[0].digest.slice("sha256:".length); + writeFileSync( + join(sandbox, "manifest"), + readFileSync(join(layoutPath(), "blobs", "sha256", digest)), + ); + const { code, stdout } = publish(); + expect(code).toBe(0); + expect(stdout.trim()).toBe(`${REPO}@${index.manifests[0].digest}`); + }); + + test("a registry resolving a different manifest exits digestMismatch", () => { + stubSkopeo(`if [ "$1" = inspect ]; then + if [ -f "${join(sandbox, "pushed")}" ]; then echo '{"schemaVersion":2,"impostor":true}'; exit 0; fi + echo "manifest unknown" >&2; exit 1 +fi +if [ "$1" = copy ]; then touch "${join(sandbox, "pushed")}"; exit 0; fi`); + expect(publish().code).toBe(EXIT.digestMismatch); + }); + + test("an ambiguous probe failure refuses to push at all", () => { + const log = stubSkopeo('echo "i/o timeout" >&2; exit 1'); + expect(publish().code).toBe(EXIT.pushFailed); + expect(readFileSync(log, "utf8")).not.toContain("copy"); + }); + + test("threads --authfile to every registry call when the login pinned one", () => { + const log = stubSkopeo('echo "manifest unknown" >&2; exit 1'); + publish({ REGISTRY_AUTH_FILE: "/tmp/creds.json" }); + expect(readFileSync(log, "utf8")).toContain("--authfile /tmp/creds.json"); + }); +}); diff --git a/tools/guest-image/publish.ts b/tools/guest-image/publish.ts new file mode 100755 index 00000000..323a1811 --- /dev/null +++ b/tools/guest-image/publish.ts @@ -0,0 +1,286 @@ +#!/usr/bin/env bun +// The I/O shell of the guest-image publish lane: realise the three assets, +// assemble the OCI layout, guard the build tag, push, and assert the registry +// resolved what was built. Every mapping and fail-closed decision lives in +// publish-core.ts so it is unit-testable without a registry. + +import { spawnSync } from "node:child_process"; +import { + closeSync, + copyFileSync, + existsSync, + mkdirSync, + openSync, + readFileSync, + readSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + ASSET_ORDER, + type Asset, + type AssetName, + annotationViolations, + buildTag, + digestRef, + EXIT, + type LayoutPlan, + layoutPlan, + manifestDigest, + tagDisposition, +} from "./publish-core.ts"; + +const USAGE = + "usage: publish.ts --repo --sha [--layout ] [--dry-run]"; + +function fail(code: number, message: string): never { + console.error(`::error::guest-image publish: ${message}`); + process.exit(code); +} + +function flag(name: string): string | undefined { + const at = process.argv.indexOf(`--${name}`); + if (at === -1) return undefined; + const value = process.argv[at + 1]; + // A following flag means this one's value is missing, not that it is `--sha`. + return value === undefined || value.startsWith("--") ? undefined : value; +} + +function run( + command: string, + args: readonly string[], + cwd: string, +): { ok: boolean; stdout: string; stderr: string } { + const result = spawnSync(command, [...args], { + cwd, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.error) + return { + ok: false, + stdout: "", + stderr: `${command}: ${result.error.message}`, + }; + return { + ok: result.status === 0, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + }; +} + +/** Hash in chunks: the rootfs is multi-GiB, so reading it whole to hash it + * would put all of it in this process's heap. */ +function sha256OfFile(path: string): string { + const hasher = new Bun.CryptoHasher("sha256"); + const fd = openSync(path, "r"); + try { + const buffer = Buffer.allocUnsafe(4 * 1024 * 1024); + let read = readSync(fd, buffer, 0, buffer.byteLength, null); + while (read > 0) { + hasher.update(buffer.subarray(0, read)); + read = readSync(fd, buffer, 0, buffer.byteLength, null); + } + } finally { + closeSync(fd); + } + return `sha256:${hasher.digest("hex")}`; +} + +const repo = flag("repo"); +const sha = flag("sha"); +const dryRun = process.argv.includes("--dry-run"); +// An empty value is a usage error, not a late throw: without this the lane +// does the whole multi-GiB realisation before digestRef rejects it. +if (repo === undefined || sha === undefined || repo === "") + fail(EXIT.usage, USAGE); +if (!/^[0-9a-f]{12}$/.test(sha)) + fail(EXIT.usage, `--sha must be 12 lowercase hex characters, got ${sha}`); + +const here = dirname(fileURLToPath(import.meta.url)); +const workspaceRoot = join(here, "..", ".."); +const guestDir = join(workspaceRoot, "guest-image"); +const layoutDir = flag("layout") ?? join(guestDir, "oci-layout"); + +// Realise through the gate's own build task, so one command definition governs +// both the pre-merge build and what gets published; a second copy of the nix +// invocation here could drift from the gate without either side noticing. +const built = run("moon", ["run", "compass-guest-image:build"], workspaceRoot); +if (!built.ok) + fail( + EXIT.badLayout, + `realising the guest assets failed: ${built.stderr.trim()}`, + ); +// moon frames task output with banner lines carrying its block glyphs and a +// trailing summary, so drop those rather than trusting line positions. +const outPaths = built.stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.startsWith("/")); +const [kernelDir, rootfsPath, initrdPath] = outPaths; +if ( + kernelDir === undefined || + rootfsPath === undefined || + initrdPath === undefined || + outPaths.length !== 3 +) { + fail( + EXIT.badLayout, + `expected three store paths from the build, got ${outPaths.length}`, + ); +} + +// The kernel attr yields a directory; its bzImage is the blob. +const assetPaths: Readonly> = { + kernel: join(kernelDir, "bzImage"), + rootfs: rootfsPath, + initrd: initrdPath, +}; + +const assets: Record = {} as Record; +for (const name of ASSET_ORDER) { + const path = assetPaths[name]; + // One try covers stat AND the hashing read: a path that stats but is not a + // readable regular file (a directory, most obviously) must still be a + // layout fault with its own exit code, not an uncaught throw. + try { + const stats = statSync(path); + if (!stats.isFile()) throw new Error("not a regular file"); + assets[name] = { digest: sha256OfFile(path), size: stats.size }; + } catch (error) { + fail( + EXIT.badLayout, + `${name}: ${path} is not a readable file (${String(error)})`, + ); + } +} + +const revision = run("git", ["rev-parse", "HEAD"], workspaceRoot); +if (!revision.ok) + fail( + EXIT.badLayout, + `reading the source revision failed: ${revision.stderr.trim()}`, + ); + +let lock: unknown; +try { + lock = JSON.parse(readFileSync(join(guestDir, "agent-oci.lock"), "utf8")); +} catch (error) { + fail(EXIT.badLayout, `reading agent-oci.lock failed: ${String(error)}`); +} +if ( + typeof lock !== "object" || + lock === null || + !("digest" in lock) || + typeof lock.digest !== "string" +) { + fail( + EXIT.badLayout, + "agent-oci.lock has no string digest; the artifact's agent provenance would be unset", + ); +} +const agentImageDigest = lock.digest; + +let plan: LayoutPlan; +try { + plan = layoutPlan(assets, { + revision: revision.stdout.trim(), + agentImageDigest, + }); +} catch (error) { + fail(EXIT.badLayout, error instanceof Error ? error.message : String(error)); +} + +const violations = annotationViolations(plan.annotations); +if (violations.length > 0) + fail(EXIT.secretFound, `secret-shaped annotations: ${violations.join(", ")}`); + +// Refuse to clear anything but a layout this lane made: `--layout` is +// caller-supplied and this is a recursive delete. +if (existsSync(layoutDir) && !existsSync(join(layoutDir, "oci-layout"))) { + fail( + EXIT.usage, + `${layoutDir} exists and is not an OCI layout; refusing to delete it`, + ); +} +// A stale layout would let the digest assertion below compare against blobs +// this run never wrote. +rmSync(layoutDir, { recursive: true, force: true }); +const blobs = join(layoutDir, "blobs", "sha256"); +mkdirSync(blobs, { recursive: true }); +writeFileSync( + join(layoutDir, "oci-layout"), + JSON.stringify({ imageLayoutVersion: "1.0.0" }), +); +writeFileSync( + join(blobs, plan.config.digest.slice("sha256:".length)), + plan.configBytes, +); +for (const name of ASSET_ORDER) { + // copyFile keeps the kernel/rootfs/initrd out of this process's heap; the + // rootfs alone is ~2.3 GiB, and it was already read once to hash it. + copyFileSync( + assetPaths[name], + join(blobs, assets[name].digest.slice("sha256:".length)), + ); +} +writeFileSync( + join(blobs, plan.manifestDescriptor.digest.slice("sha256:".length)), + plan.manifest, +); +writeFileSync(join(layoutDir, "index.json"), plan.index); + +const local = plan.manifestDescriptor.digest; +if (dryRun) { + console.error(`guest-image publish: dry run wrote ${layoutDir}`); + console.log(digestRef(repo, local)); + process.exit(0); +} + +const tag = buildTag(repo, sha); +// The login runs in a separate process, so every call must name the same creds +// file; skopeo's default location is environment-dependent on hosted runners. +const authFile = process.env.REGISTRY_AUTH_FILE; +const auth = authFile === undefined ? [] : ["--authfile", authFile]; +const probe = run( + "skopeo", + ["inspect", ...auth, "--raw", `docker://${tag}`], + workspaceRoot, +); +const disposition = tagDisposition( + { exitCode: probe.ok ? 0 : 1, stdout: probe.stdout, stderr: probe.stderr }, + local, +); +if (disposition.action === "abort") + fail(EXIT.pushFailed, disposition.reason ?? "refusing to publish"); +if (disposition.action === "skip") { + console.error(`guest-image publish: ${tag} already holds this artifact`); + console.log(digestRef(repo, local)); + process.exit(0); +} + +const pushed = run( + "skopeo", + ["copy", ...auth, `oci:${layoutDir}`, `docker://${tag}`], + workspaceRoot, +); +if (!pushed.ok) + fail(EXIT.pushFailed, `pushing ${tag} failed: ${pushed.stderr.trim()}`); + +// The registry is authoritative: a transport that rewrote the manifest must +// fail closed rather than have its digest published as ours. +const resolved = run( + "skopeo", + ["inspect", ...auth, "--raw", `docker://${tag}`], + workspaceRoot, +); +if (!resolved.ok) + fail(EXIT.pushFailed, `re-reading ${tag} failed: ${resolved.stderr.trim()}`); +const remote = manifestDigest(resolved.stdout); +if (remote !== local) + fail(EXIT.digestMismatch, `registry resolved ${remote}, built ${local}`); + +console.log(digestRef(repo, remote));