diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e370e9a..de579b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,15 @@ jobs: steps: - uses: actions/checkout@v7 - uses: DeterminateSystems/nix-installer-action@v22 + with: + determinate: false + diagnostic-endpoint: '' + - name: Use systemstart Cachix cache + uses: cachix/cachix-action@v17 + with: + name: systemstart + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + signingKey: ${{ secrets.CACHIX_SIGNING_KEY }} - run: nix flake check --impure - run: nix build --impure .#claude - run: nix build --impure .#gemini @@ -25,3 +34,4 @@ jobs: uses: ./.github/workflows/release.yml permissions: contents: write + diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9374043..7f3a970 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,15 @@ jobs: with: fetch-depth: 0 - uses: DeterminateSystems/nix-installer-action@v22 + with: + determinate: false + diagnostic-endpoint: '' + - name: Use systemstart Cachix cache + uses: cachix/cachix-action@v17 + with: + name: systemstart + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + signingKey: ${{ secrets.CACHIX_SIGNING_KEY }} - run: nix build --impure .#claude - run: nix build --impure .#gemini - run: nix build --impure .#codex diff --git a/.github/workflows/update-flake.yml b/.github/workflows/update-flake.yml index 25824fa..3901184 100644 --- a/.github/workflows/update-flake.yml +++ b/.github/workflows/update-flake.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v7 - uses: DeterminateSystems/nix-installer-action@v22 - - uses: DeterminateSystems/update-flake-lock@main + - uses: DeterminateSystems/update-flake-lock@v28 with: pr-title: "chore: nix flake update" pr-body: | diff --git a/README.md b/README.md index b26071e..bf7bf22 100644 --- a/README.md +++ b/README.md @@ -235,7 +235,7 @@ ENABLE_CRI=crun make claude.run ENABLE_CRI=podman make claude.run ``` -Container images and layers are stored on the host at `$AGENT_HOME/cri-storage/` via a dedicated virtiofs share at `/var/lib/containers`, so they persist across VM restarts and don't consume the VM's RAM-backed root filesystem. +Container images and layers are stored on a dedicated ext4 disk image at `$AGENT_HOME-cri/cri-storage.img` (sparse, up to 30 GiB), mounted at `/var/lib/containers`, so they persist across VM restarts and don't consume the VM's RAM-backed root filesystem. The image is kept in a host-only sibling directory next to agent home rather than inside it, so it is never exported through the agent-home virtiofs share and the guest cannot read or tamper with its own raw storage backing file. A real block-backed filesystem is required here rather than a virtiofs share: image unpack must `lchown` extracted layers to UID 0, which the host's rootless virtiofsd cannot do (it has a single-ID uid map). On a share, `docker info` reports `Backing Filesystem: fuse` and overlay2/KinD layer extraction fails with `lchown … operation not permitted`; on the ext4 volume it reports `extfs` and works. #### Available runtimes diff --git a/flake.lock b/flake.lock index ba27aa9..c9bb9d2 100644 --- a/flake.lock +++ b/flake.lock @@ -8,11 +8,11 @@ "spectrum": "spectrum" }, "locked": { - "lastModified": 1781389237, - "narHash": "sha256-Ne1/E5XNUq0gleaQz0vW5R4xf/0h/uEZ+bOW1aNjeQk=", + "lastModified": 1782764610, + "narHash": "sha256-CzCF8RD9a9kwRrpeQ+z67sqqADUG0tf+4+YBSUG30y0=", "owner": "microvm-nix", "repo": "microvm.nix", - "rev": "6ad601df0a07d9855c5e8f9b81135ecaf7c287eb", + "rev": "3904edd282a7166cdad6c597964718e6d018baa6", "type": "github" }, "original": { @@ -23,11 +23,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1781074563, - "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=", + "lastModified": 1783224372, + "narHash": "sha256-8i/87eeoqiGE4yOTjwSA3Eh/ziJRQEmd/unYU+K27sk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca", + "rev": "d407951447dcd00442e97087bf374aad70c04cea", "type": "github" }, "original": { @@ -46,11 +46,11 @@ "spectrum": { "flake": false, "locked": { - "lastModified": 1778940603, - "narHash": "sha256-voSM8dZNlaOWN3kbYFky+FNY6fFQOEw0xF+ZMpZKkCQ=", + "lastModified": 1782480951, + "narHash": "sha256-jrfyAgv5XfYDXJJ28OzANQOQIlsfNaI1kM9Mt13n+5k=", "ref": "refs/heads/main", - "rev": "367dd227f539267eae2b62770b4c17b88ac8c1f1", - "revCount": 1265, + "rev": "e4562d4c7646fea3cbb7306ff9b7ff41154c75c3", + "revCount": 1400, "type": "git", "url": "https://spectrum-os.org/git/spectrum" }, diff --git a/flake.nix b/flake.nix index 9315926..ab31404 100644 --- a/flake.nix +++ b/flake.nix @@ -101,18 +101,25 @@ fi AGENT_TEMP="" - # --- CRI storage share (virtiofsd) --- - CRI_SOCK="$RUNTIME/$VM_ID-virtiofs-$ID-cri-storage.sock" - CRI_UNIT="$VM_ID-virtiofsd-$ID-cri-storage" - CRI_STATE="$RUNTIME/$VM_ID-virtiofsd-$ID-cri-storage.dir" - CRI_DIR="$AGENT_DIR/cri-storage" - mkdir -p "$CRI_DIR" + # --- CRI storage volume (ext4 block image) --- + # Container runtimes need a real block-backed fs that permits lchown to + # arbitrary UIDs during image unpack — a virtiofs share cannot (rootless + # virtiofsd has a single-ID uid map). The image is a sparse ext4 disk + # created on first boot by microvm.nix's createVolumesScript. It must NOT + # live inside $AGENT_DIR: that directory is exported into the guest via + # the agent-home virtiofs share, so the guest could read or tamper with + # its own raw storage backing file. Keep it in a host-only sibling state + # dir that persists across runs but is never shared into the guest. The + # CRI module declares the image as "cri-storage.img"; rewrite it to this + # absolute path below. + CRI_STATE_DIR="$AGENT_DIR-cri" + mkdir -p "$CRI_STATE_DIR" + CRI_IMG="$CRI_STATE_DIR/cri-storage.img" cleanup() { ${pkgs.systemd}/bin/systemctl --user stop "$UNIT" 2>/dev/null || true ${pkgs.systemd}/bin/systemctl --user stop "$AGENT_UNIT" 2>/dev/null || true - ${pkgs.systemd}/bin/systemctl --user stop "$CRI_UNIT" 2>/dev/null || true - rm -f "$SOCK" "$AGENT_SOCK" "$STATE" "$AGENT_STATE" "$CRI_SOCK" "$CRI_STATE" + rm -f "$SOCK" "$AGENT_SOCK" "$STATE" "$AGENT_STATE" if [ -n "$AGENT_TEMP" ]; then rm -rf "$AGENT_TEMP" fi @@ -152,41 +159,6 @@ [ -S "$AGENT_SOCK" ] || { echo "error: agent-home virtiofsd socket did not appear"; exit 1; } fi - # --- CRI storage virtiofsd --- - CRI_NEED_START=1 - if ${pkgs.systemd}/bin/systemctl --user is-active "$CRI_UNIT" &>/dev/null; then - if [ -f "$CRI_STATE" ] && [ "$(cat "$CRI_STATE")" = "$CRI_DIR" ] && [ -S "$CRI_SOCK" ]; then - CRI_NEED_START=0 - else - ${pkgs.systemd}/bin/systemctl --user stop "$CRI_UNIT" 2>/dev/null || true - fi - fi - - if [ "$CRI_NEED_START" = "1" ]; then - rm -f "$CRI_SOCK" - - ${pkgs.systemd}/bin/systemd-run --user --unit="$CRI_UNIT" --collect \ - -- ${virtiofsd}/bin/virtiofsd \ - --socket-path="$CRI_SOCK" \ - --shared-dir="$CRI_DIR" \ - --sandbox=namespace \ - --uid-map ":0:$(id -u):1:" \ - --gid-map ":0:$(id -g):1:" \ - --translate-uid "map:1000:0:1" \ - --translate-gid "map:1000:0:1" \ - --socket-group="$(id -gn)" \ - --xattr \ - --xattrmap ":prefix:all:trusted.:user.virtiofs.::prefix:all:security.:user.virtiofs.::ok:all:::" - - echo "$CRI_DIR" > "$CRI_STATE" - - for i in $(seq 1 50); do - [ -S "$CRI_SOCK" ] && break - sleep 0.1 - done - [ -S "$CRI_SOCK" ] || { echo "error: cri-storage virtiofsd socket did not appear"; exit 1; } - fi - # Write host env vars for the VM echo "DIRENV_ALLOW=''${DIRENV_ALLOW:-0}" > "$AGENT_DIR/.microvm-env" echo "ENABLE_CRI=''${ENABLE_CRI:-}" >> "$AGENT_DIR/.microvm-env" @@ -249,6 +221,11 @@ || rm -f "$AGENT_DIR/.microvm-nix-db.sqlite" fi + # $CRI_IMG is used as sed replacement text below. Escape characters + # that are special on the replacement side (\, &) and the | delimiter + # so a path with such characters can't corrupt the generated command. + _CRI_IMG_ESC=$(printf '%s' "$CRI_IMG" | ${pkgs.gnused}/bin/sed -e 's/[\\&|]/\\&/g') + # Build sed arguments for QEMU runner _SED_ARGS=( # Process and QEMU name: inject project basename @@ -259,8 +236,16 @@ -e "s|${hostname}-virtiofs-work.sock|$SOCK|g" -e "s|/tmp/${hostname}-home|$AGENT_DIR|g" -e "s|${hostname}-virtiofs-agent-home.sock|$AGENT_SOCK|g" - -e "s|/tmp/${hostname}-cri-storage|$CRI_DIR|g" - -e "s|${hostname}-virtiofs-cri-storage.sock|$CRI_SOCK|g" + # CRI storage volume image: microvm.nix emits the relative path + # "cri-storage.img" in both the createVolumesScript and the QEMU + # -drive; point both at the persistent image in agent home. + -e "s|cri-storage.img|$_CRI_IMG_ESC|g" + # microvm.nix unconditionally sets cache=none (O_DIRECT) on volume + # drives, which fails when agent home is on a filesystem without + # O_DIRECT support (tmpfs, some network/virtiofs mounts). The CRI + # store is a rebuildable cache, so writeback (host page cache) is the + # portable choice and works on every backing filesystem. + -e "s|,cache=none|,cache=writeback|g" # Runtime mem/vcpu override (VM_MEM / VM_VCPU env vars) -e "s| -m ${toString defaultMem} | -m $VM_MEM |g" -e "s| -smp ${toString defaultVcpu} | -smp $VM_VCPU |g" diff --git a/modules/cri.nix b/modules/cri.nix index 83a6395..60de261 100644 --- a/modules/cri.nix +++ b/modules/cri.nix @@ -36,12 +36,21 @@ CRIEOF export CONTAINER_HOST=unix:///run/podman/podman.sock ''; - microvm.shares = [ + # Container storage must live on a real block-backed filesystem, NOT a + # virtiofs share. The host virtiofsd runs rootless with a single-ID uid/gid + # map, so it cannot lchown extracted layer files to UID 0 — overlay2/KinD + # image unpack fails with "lchown /bin: operation not permitted" and + # `docker info` reports `Backing Filesystem: fuse`. A dedicated ext4 volume + # gives the guest a genuine root-capable fs (Backing Filesystem: extfs). + # The sparse image is created on the host and persists in agent home; the + # runner script (flake.nix) rewrites the `cri-storage.img` path accordingly. + microvm.volumes = [ { - tag = "cri-storage"; - source = "/tmp/${config.networking.hostName}-cri-storage"; + image = "cri-storage.img"; + label = "cri-storage"; mountPoint = "/var/lib/containers"; - proto = "virtiofs"; + size = 30720; # MiB, sparse — max size, only consumes what is used + fsType = "ext4"; } ];