Fix six pivot-path blockers + add real-kernel integration test

.github/workflows/ci.yml

@@ -24,6 +24,32 @@ jobs:
       - name: Compile binary via nix
         run: nix build
 
+  integration:
+    name: pivot integration (NixOS VM)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: DeterminateSystems/nix-installer-action@main
+
+      - uses: cachix/cachix-action@v15
+        with:
+          name: ananthb
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      # NixOS VM tests boot a guest under QEMU/KVM; expose /dev/kvm to the job.
+      - name: Enable KVM
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
+            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+
+      # Real-kernel coverage of the pivot path: pivot_root + mount ordering
+      # and the tar-extraction containment tests, run as root inside the VM.
+      - name: pivot_root + extraction VM test
+        run: nix build -L .#checks.x86_64-linux.nixos-pivot
+
   goreleaser-check:
     name: goreleaser check + snapshot smoke test
     runs-on: ubuntu-latest

README.md

@@ -14,10 +14,10 @@ replacement that stays reachable over Tailscale.
 # Nix
 nix run github:ananthb/xmorph -- --help
 
-# Static binary (each release ships SHA256SUMS)
-curl -LO https://github.com/ananthb/xmorph/releases/latest/download/xmorph-x86_64-linux
-chmod +x xmorph-x86_64-linux
-sudo mv xmorph-x86_64-linux /usr/local/bin/xmorph
+# Static binary (each release ships a SHA256SUMS file alongside the archives)
+curl -LO https://github.com/ananthb/xmorph/releases/latest/download/xmorph-x86_64-linux.tar.gz
+tar xzf xmorph-x86_64-linux.tar.gz
+sudo mv xmorph/xmorph /usr/local/bin/xmorph
 
 # From source (Go 1.26+)
 CGO_ENABLED=0 go build -o xmorph ./cmd/xmorph
@@ -30,5 +30,6 @@ reprovisioning recipes at **[ananthb.github.io/xmorph](https://ananthb.github.io
 
 - [Rescue an unreachable host](docs/rescue.md)
 - [Reprovision to Flatcar / FCOS / Alpine / NixOS / Ubuntu](docs/reprovision/)
+- [Testing (unit, NixOS VM, real-kernel dev loop)](docs/testing.md)
 
 Licensed under the [AGPL-3.0](LICENSE).

docs/testing.md

@@ -0,0 +1,65 @@
+# Testing
+
+xmorph has three tiers of tests, matching how much of a real Linux kernel
+each one needs.
+
+## 1. Unit tests + lint (any OS)
+
+```sh
+nix run .#build      # gofmt + go vet (GOOS=linux) + go test
+```
+
+On Linux this runs `go test ./...`; on macOS it runs the pure-Go tests
+natively and cross-compiles the rest. This is what the `build` CI job runs.
+
+Tests that need root and mount namespaces (`internal/pivot`, and the
+as-root tar-extraction cases in `internal/oci`) are guarded with
+`if os.Geteuid() != 0 { t.Skip(...) }`, so they skip here rather than fail.
+
+## 2. Integration tests on a real kernel (CI)
+
+The pivot path — `pivot_root(2)`, mount ordering, tar extraction running as
+root — can only be exercised against a real kernel with `CAP_SYS_ADMIN`.
+The nix build sandbox can't do mounts, so the integration test binaries are
+compiled by the `xmorphIntegrationTests` derivation and run inside a NixOS
+VM test:
+
+```sh
+nix build -L .#checks.x86_64-linux.nixos-pivot   # pivot_root + extraction
+nix flake check                                  # all checks + every VM test
+```
+
+The `integration` CI job runs `nixos-pivot` on every push and PR (GitHub's
+Linux runners expose `/dev/kvm`). This is the project's real-kernel
+regression coverage — notably it guards the mount-ordering invariant that
+`/proc`, `/sys`, and `/dev` stay visible after the pivot.
+
+## 3. Fast real-kernel iteration on macOS (developer option)
+
+NixOS VM tests are the source of truth, but they're slow to iterate on. On
+Apple Silicon you can run the same Linux tests directly against a real
+kernel using [Apple `container`](https://github.com/apple/container), which
+puts each container in its own lightweight VM.
+
+One-time setup:
+
+```sh
+container system start
+container system kernel set --recommended
+```
+
+Then run any package's tests as root with mount capabilities:
+
+```sh
+container run --rm --cap-add ALL \
+  -v "$PWD:/src" \
+  -v "$HOME/go/pkg/mod:/go/pkg/mod:ro" \
+  -w /src \
+  docker.io/library/golang:1.26 \
+  sh -c 'GOFLAGS=-mod=mod GOPROXY=off go test -race ./...'
+```
+
+`--cap-add ALL` grants the `CAP_SYS_ADMIN` that `unshare`/`pivot_root`
+require; mounting `$HOME/go/pkg/mod` read-only reuses your module cache so
+the run is offline and fast. This is a convenience for local development —
+CI relies on the NixOS VM test in tier 2, not on `container`.

flake.nix

@@ -58,6 +58,32 @@
           };
         };
 
+        # Integration test binaries that need a real kernel + root to run
+        # (mount namespaces, pivot_root). They are compiled here but executed
+        # inside the NixOS VM test below — the nix build sandbox can't do
+        # mounts, and GitHub CI has no Apple `container`. Developers on macOS
+        # can run the same binaries under `container`; see docs/testing.md.
+        xmorphIntegrationTests = buildGoModule {
+          pname = "xmorph-integration-tests";
+          inherit version vendorHash;
+          src = ./.;
+          env.CGO_ENABLED = "0";
+          doCheck = false;
+          buildPhase = ''
+            runHook preBuild
+            for pkg in pivot oci; do
+              go test -c -o "xmorph-$pkg.test" "./internal/$pkg"
+            done
+            runHook postBuild
+          '';
+          installPhase = ''
+            runHook preInstall
+            mkdir -p "$out/bin"
+            install -m555 xmorph-*.test "$out/bin/"
+            runHook postInstall
+          '';
+        };
+
         # xmorphLint: gofmt + go vet gate. Exposed as `apps.lint` so the
         # pre-commit hook and CI both go through `nix run .#lint`.
         xmorphLint = pkgs.writeShellApplication {
@@ -201,6 +227,29 @@
             '';
           };
 
+          # NixOS VM test: real pivot_root + mount ordering on a live kernel.
+          # Runs the pivot/oci integration test binaries as root inside the VM
+          # — this is the project's real-kernel coverage of the pivot path
+          # (the nix sandbox can't do mount namespaces). Guards the
+          # mount-ordering fix: essentials must stay visible after the pivot.
+          nixos-pivot = pkgs.testers.nixosTest {
+            name = "xmorph-pivot-integration";
+
+            nodes.machine = { ... }: {
+              environment.systemPackages = [ xmorphIntegrationTests ];
+              virtualisation.memorySize = 2048;
+            };
+
+            testScript = ''
+              machine.wait_for_unit("multi-user.target")
+              # pivot_root + mount-ordering (needs root + CAP_SYS_ADMIN).
+              machine.succeed("xmorph-pivot.test -test.v")
+              # tar-extraction containment + setuid, exercised as root — the
+              # privilege level at which a symlink escape would actually bite.
+              machine.succeed("xmorph-oci.test -test.v")
+            '';
+          };
+
           # NixOS VM test: headscale integration (offline, ~2-3 min)
           nixos-headscale = import ./nix/tests/headscale.nix {
             inherit pkgs;

internal/cli/pivot.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"os/signal"
 	"path/filepath"
 	"runtime"
+	"syscall"
 	"time"
 
 	"github.com/ananthb/xmorph/internal/config"
@@ -135,6 +137,18 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface {
 		}
 	}
 
+	// From here on a dropped controlling session must not take xmorph with
+	// it. The transient-scope relocation above only fixes cgroup reaping; it
+	// does NOT detach the controlling TTY, so when the pivot severs the SSH
+	// connection (or init coordination stops sshd) the kernel delivers SIGHUP
+	// to this process group, whose default disposition is fatal. SIGPIPE is
+	// the same story for a slog write to a stderr fd whose reader just died.
+	// Ignore both now — we have already committed to proceeding (the SSH
+	// grace window above was the last chance to abort). SIGINT/SIGTERM are
+	// left alone until the destructive steps below so local Ctrl-C still
+	// aborts a long rootfs build.
+	signal.Ignore(syscall.SIGHUP, syscall.SIGPIPE)
+
 	if err := os.MkdirAll(cfg.WorkDir, 0o755); err != nil {
 		return fmt.Errorf("create work dir: %w", err)
 	}
@@ -207,6 +221,12 @@ func runPivot(ctx context.Context, cfg *config.Config, stdout interface {
 	// surfaced via the slog output — for now we just verify the file
 	// is readable.)
 
+	// Point of no safe abort: everything below stops services, terminates
+	// userspace, and pivots. A stray Ctrl-C or a SIGTERM from a supervising
+	// unit here would strand the machine with services down and no new root,
+	// so stop honoring them now (SIGHUP/SIGPIPE were already ignored above).
+	signal.Ignore(syscall.SIGINT, syscall.SIGTERM)
+
 	if !cfg.SystemdMode {
 		if !cfg.NoInitCoord && !initsys.ShouldSkipCoordination() {
 			coord := initsys.NewCoordinator(time.Duration(cfg.Timeout) * time.Second)