diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f55dfe2f9..794d5db8b 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -33,6 +33,7 @@ import ( "github.com/agent-substrate/substrate/internal/imagecache" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" + "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -47,11 +48,10 @@ import ( // - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot // file:// (config.json + state.json + sparse memory-ranges) // -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower + -// guest-tmpfs upper), so the writable upper lives in guest RAM and is captured by -// the memory snapshot — process memory and rootfs writes both persist across -// suspend/resume. The RO lower is reconstructed from the OCI image at restore, so -// nothing rootfs-related ships. Durable-dir volumes are host-backed rather than in -// guest RAM, so they ship alongside as a tar. +// disk-backed upper): the upper is host-backed like the durable-dir volumes and +// ships alongside as its own tar (see rootfsupper.go); process memory persists +// via the memory snapshot. The RO lower is reconstructed from the OCI image at +// restore, so it never ships. Durable-dir volumes ship alongside as a tar. // - DATA: the durable-dir volumes only, as that same tar. The guest is discarded, so // the actor cold-starts on restore with its volumes re-materialized. // @@ -120,27 +120,56 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec return nil, fmt.Errorf("while creating checkpoint dir %q: %w", checkpointDir, err) } - // Only a Full snapshot captures the guest. A Data snapshot deliberately - // captures no VM state — no memory image, and no base-id, since nothing will - // reattach to the frozen virtio-fs lower: at restore the actor cold-boots - // from the OCI image (or, under an OnGolden data resume policy, is combined - // with the golden snapshot's guest state) and gets its durable-dir volumes - // back from the tar below. - var dSnapshot time.Duration + // Capture the snapshot's pieces CONCURRENTLY: the CH snapshot, the + // durable-dir tar, and the rootfs upper tar read independent data from a + // quiesced guest and write distinct files into checkpointDir, so the paused + // window costs the slowest of them rather than their sum (the tars scale + // with the actor's data; suspend latency is the metric that matters). + // + // - CH snapshot (Full only): the guest memory + VM state. A Data snapshot + // deliberately captures no VM state — no memory image, and no base-id, + // since nothing will reattach to the frozen virtio-fs lower: at restore + // the actor cold-boots from the OCI image (or, under an OnGolden data + // resume policy, is combined with the golden snapshot's guest state). + // - Durable-dir tar (any scope, when declared): host-backed, so pausing + // the write-through share makes the tar coherent. + // - Rootfs upper tar (Full only): host-backed like the durable volumes — + // the memory snapshot does not carry rootfs writes. Under Data the + // workload cold-starts on restore, discarding rootfs state. Gated on + // the host dir a disk-upper boot creates (actorHasDiskUpper) so a + // legacy actor restored from a tmpfs-upper snapshot checkpoints + // correctly (its upper is inside the memory image). + var dSnapshot, dDurable, dUpper time.Duration + g, gctx := errgroup.WithContext(ctx) if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL { - var err error - if dSnapshot, err = s.snapshotVMState(ctx, client, ra, actorUID, checkpointDir); err != nil { - return nil, err - } + g.Go(func() error { + var err error + dSnapshot, err = s.snapshotVMState(gctx, client, ra, actorUID, checkpointDir) + return err + }) } - - var dDurable time.Duration if durable { - tDurable := time.Now() - if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil { - return nil, err - } - dDurable = time.Since(tDurable) + g.Go(func() error { + t := time.Now() + if err := tarDurableVolumes(gctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil { + return err + } + dDurable = time.Since(t) + return nil + }) + } + if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL && actorHasDiskUpper(actorUID) { + g.Go(func() error { + t := time.Now() + if err := tarRootfsUpper(gctx, ateompath.RootfsUpperDir(actorUID), checkpointDir); err != nil { + return err + } + dUpper = time.Since(t) + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err } // Report exactly the files we wrote so atelet ships precisely this snapshot: for @@ -180,9 +209,11 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles), slog.String("scope", scope.String()), slog.Duration("pause", dPause), slog.Duration("snapshot", dSnapshot), - // The durable-dir tar runs while the guest is paused, so its cost is part - // of the suspend latency and scales with the volume's contents. - slog.Duration("durable_dir", dDurable), slog.Duration("teardown", dTeardown)) + // The tars run while the guest is paused, CONCURRENTLY with the CH + // snapshot: the paused window costs max(snapshot, durable_dir, + // rootfs_upper), and the tar durations scale with the actor's data. + slog.Duration("durable_dir", dDurable), slog.Duration("rootfs_upper", dUpper), + slog.Duration("teardown", dTeardown)) return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil } @@ -232,9 +263,9 @@ func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, r slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge))) } - // Nothing rootfs-related ships: the overlay's writable upper is a guest tmpfs, so - // the actor's rootfs writes are already in the memory snapshot above, and the RO - // lower is reconstructed from the OCI image at restore (it never changes). + // The RO lower never ships (reconstructed from the OCI image at restore). + // The disk-backed upper ships as its own tar from CheckpointWorkload; a + // legacy tmpfs upper is already inside the memory snapshot above. return dSnapshot, nil } @@ -283,8 +314,8 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running _, _ = ra.chCmd.Process.Wait() } // Kill the virtiofsds (after CH, their only client): the overlay RO lower's - // and, when the actor has durable-dir volumes, the writable share's. - for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} { + // and, when present, the writable durable-dir and rootfs upper shares'. + for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd, ra.upperVfsdCmd} { if cmd != nil && cmd.Process != nil { _ = cmd.Process.Kill() _, _ = cmd.Process.Wait() @@ -292,6 +323,14 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running } } + // Remove the rootfs upper dir: ateom owns it — atelet's actor-dir reset + // doesn't know it — and its absence is what marks a worker as holding no + // disk-backed upper (actorHasDiskUpper). Runs after the checkpoint tar, + // which is already on disk. A no-op for legacy tmpfs-upper actors. + if err := os.RemoveAll(ateompath.RootfsUpperDir(id)); err != nil { + slog.WarnContext(ctx, "Failed to remove rootfs upper dir", slog.String("actorUID", id), slog.Any("err", err)) + } + // Sweep any leftover per-sandbox host-side state + orphaned per-sandbox // processes. This is ateom's own cleanup (process kill + unmount + rm). kata.CleanupSandboxState(ctx, id) diff --git a/cmd/ateom-microvm/internal/kata/agentclient.go b/cmd/ateom-microvm/internal/kata/agentclient.go index f7a242694..c8e80f015 100644 --- a/cmd/ateom-microvm/internal/kata/agentclient.go +++ b/cmd/ateom-microvm/internal/kata/agentclient.go @@ -137,7 +137,7 @@ func (a *AgentClient) Close() error { // CreateContainer asks the agent to create a container: mount its storages (in // order) and build the rootfs, then fork the parked init process. This is the // hook point — the agent mounts storages[] (here: a bind of the virtio-fs lower -// followed by the tmpfs-upper overlay) before init_rootfs consumes the rootfs. +// followed by the disk-backed-upper overlay) before init_rootfs consumes the rootfs. // Mirrors grpc.AgentService/CreateContainer (returns google.protobuf.Empty). func (a *AgentClient) CreateContainer(ctx context.Context, req *agentpb.CreateContainerRequest) error { if err := a.client.Call(ctx, "grpc.AgentService", "CreateContainer", req, &emptypb.Empty{}); err != nil { diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux.go b/cmd/ateom-microvm/internal/kata/overlay_linux.go index abf0995df..38e1b707c 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux.go @@ -17,9 +17,12 @@ package kata // Each container's rootfs is an overlay: its OCI image served read-only over virtio-fs -// (the lower) plus a guest tmpfs (the writable upper). The upper is in guest RAM, so -// rootfs writes ride along in the memory snapshot and persist across suspend/resume. -// This file holds the overlay-specific helpers. +// (the lower) plus a writable upper on the ateUpper virtio-fs share, backed by host +// disk (see cmd/ateom-microvm/rootfsupper.go). Rootfs writes therefore cost host disk, +// not guest RAM, and persist across suspend/resume via the snapshot's rootfs-upper +// tar. (Snapshots from the retired tmpfs-upper mode still restore: their upper rides +// inside the restored guest memory and needs nothing from this file.) This file holds +// the overlay-specific helpers. import ( "context" @@ -54,6 +57,17 @@ const ( // volume's contents live at / and are bind-mounted // from there into the containers that declare the volume. guestDurableDir = "/run/ateom-durable" + + // UpperFsTag is the virtio-fs tag for the actor's WRITABLE disk-backed + // rootfs upper share, served by a third virtiofsd from + // ateompath.RootfsUpperDir on the host. Every container's overlay upper + // lives on it, so rootfs writes cost host disk, not guest RAM. + UpperFsTag = "ateUpper" + // guestUpperDir is where the agent mounts UpperFsTag in the guest; each + // container's overlay upper/work then live under /. + // Deliberately distinct from the retired tmpfs upper's /run/ateom-upper + // prefix, which guests restored from old snapshots may still have mounted. + guestUpperDir = "/run/ateom-upper-disk" ) // GuestDurableVolumeDir is the in-guest path holding one durable volume's @@ -71,11 +85,22 @@ func SharedDir(id string) string { // VirtiofsdSocketPath is the vhost-user-fs socket CH connects to for the fs device. func VirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd.sock") } -// OverlayUpperBase is the in-guest mount point for one container's overlay upper/work. -// It lives under /run (tmpfs) so the upper's writes are in guest RAM and ride along in -// the memory-only snapshot (rootfs writes persist). Keyed on the container id, which is -// stable across the actor's restore lineage. -func OverlayUpperBase(containerID string) string { return "/run/ateom-upper/" + containerID } +// UpperBase is the in-guest mount point for one container's overlay upper/work: +// a subdirectory of the ateUpper virtio-fs share, so the upper's writes land on +// host disk (ateompath.RootfsUpperDir) instead of guest RAM — the memory +// snapshot stays lean and the upper ships as a tar instead (see +// cmd/ateom-microvm/rootfsupper.go). Keyed on the container id, which is stable +// across the actor's restore lineage. +func UpperBase(containerID string) string { return guestUpperDir + "/" + containerID } + +// upperWorkDirs returns the overlay upperdir and workdir for an upper base: +// SIBLING directories under the one base. Both properties are load-bearing — +// the kernel requires upperdir and workdir on the same filesystem, and rejects +// a workdir nested inside (or equal to) upperdir — so a layout change here +// breaks every overlay mount. Covered by a regression test. +func upperWorkDirs(upperBase string) (upper, work string) { + return upperBase + "/fs", upperBase + "/work" +} // GuestSharedRootfs is the in-guest path the kataShared mount exposes a container's // rootfs at. A carrier container with this as Root.Path makes the agent bind it to @@ -91,6 +116,13 @@ type VirtiofsdOptions struct { // Cache is virtiofsd's --cache mode. Empty defaults to "always", which is // only correct for a strictly read-only share (see virtiofsdArgs). Cache string + // Xattr enables xattr passthrough (--xattr). Required for a share hosting an + // overlayfs upper: overlay records whiteouts and opaque directories as + // user.overlay.* xattrs in the upper (userxattr mode), and without + // passthrough the guest's overlay mount cannot round-trip them to the host + // (deletes of lower files would fail or silently un-delete across + // suspend/resume). + Xattr bool Log io.Writer } @@ -106,7 +138,7 @@ func virtiofsdArgs(o VirtiofsdOptions) []string { // side changes underneath the guest (e.g. contents restored from a snapshot). cache = "always" } - return []string{ + args := []string{ "--socket-path=" + o.SocketPath, "--shared-dir=" + o.SharedDir, "--cache=" + cache, @@ -114,6 +146,10 @@ func virtiofsdArgs(o VirtiofsdOptions) []string { "--announce-submounts", "--migration-mode", "find-paths", } + if o.Xattr { + args = append(args, "--xattr") + } + return args } // StartVirtiofsd launches virtiofsd in find-paths migration mode serving o.SharedDir @@ -177,8 +213,8 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, for _, d := range []string{"proc", "sys", "dev"} { _ = os.MkdirAll(filepath.Join(dst, d), 0o755) } - // Remount read-only: the lower is immutable, so all writes go to the tmpfs upper and - // it stays byte-identical across reconstructions (required by find-paths migration). + // Remount read-only: the lower is immutable, so all writes go to the overlay upper + // and it stays byte-identical across reconstructions (required by find-paths migration). ro := exec.CommandContext(ctx, "mount", "-o", "remount,bind,ro", dst) var roErr strings.Builder ro.Stderr = &roErr @@ -189,17 +225,27 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, } // CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount -// (the RO base backing every container's rootfs). Mirrors kata startSandbox. +// (the RO base backing every container's rootfs) and the writable disk-backed rootfs +// upper share, under whose mount each container's overlay upper/work live (UpperBase). +// Mirrors kata startSandbox. // // withDurableShare additionally mounts the writable durable-dir share, whose // per-volume subdirectories the containers bind-mount at their declared paths. func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error { - storages := []*agentpb.Storage{{ - Driver: virtioFSDriver, - Source: FsTag, - Fstype: typeVirtioFS, - MountPoint: guestSharedDir, - }} + storages := []*agentpb.Storage{ + { + Driver: virtioFSDriver, + Source: FsTag, + Fstype: typeVirtioFS, + MountPoint: guestSharedDir, + }, + { + Driver: virtioFSDriver, + Source: UpperFsTag, + Fstype: typeVirtioFS, + MountPoint: guestUpperDir, + }, + } if withDurableShare { storages = append(storages, &agentpb.Storage{ Driver: virtioFSDriver, @@ -222,7 +268,7 @@ func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, host func (a *AgentClient) CreateCarrier(ctx context.Context, cid string, spec *specs.Spec) error { pbSpec := SpecToAgentPB(spec) // Readonly: the carrier only exists to materialize the base bind; its rootfs (the - // overlay lower) must stay immutable. Overlay writes go to the tmpfs upper. + // overlay lower) must stay immutable. Overlay writes go to the disk-backed upper. pbSpec.Root = &agentpb.Root{Path: GuestSharedRootfs(cid), Readonly: true} if pbSpec.Linux != nil { pbSpec.Linux.CgroupsPath = "/ateomchv/" + cid + "-carrier" @@ -239,17 +285,16 @@ func (a *AgentClient) CreateCarrier(ctx context.Context, cid string, spec *specs // StartOverlayWorkload creates + starts one container with an overlayfs rootfs: // lower = the carrier's resolved bind (/run/kata-containers//rootfs from the RO -// virtio-fs base), upper/work = /{fs,work} on a guest tmpfs so rootfs writes -// land in guest RAM (captured by the memory-only snapshot → persist). The agent creates -// the upper/work dirs (create_directory) before mounting the overlay. +// virtio-fs base), upper/work = /{fs,work} on the disk-backed ateUpper +// share (UpperBase: writes land on host disk, shipped as a tar at checkpoint). The +// agent creates the upper/work dirs (create_directory) before mounting the overlay. func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID, upperBase string, spec *specs.Spec) error { const createDir = "io.katacontainers.volume.overlayfs.create_directory" sharedBase := "/run/kata-containers/" + cid + "/rootfs" base := "/run/kata-containers/" + workloadID lower := base + "/lower" ovlRoot := base + "/rootfs" - upper := upperBase + "/fs" - work := upperBase + "/work" + upper, work := upperWorkDirs(upperBase) storages := []*agentpb.Storage{ { @@ -260,12 +305,18 @@ func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID, Options: []string{"bind"}, }, { - Driver: "overlayfs", - Source: "overlay", - Fstype: "overlay", - MountPoint: ovlRoot, + Driver: "overlayfs", + Source: "overlay", + Fstype: "overlay", + MountPoint: ovlRoot, DriverOptions: []string{createDir + "=" + upper, createDir + "=" + work}, - Options: []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work}, + // index=off,metacopy=off,userxattr: required for an upper on + // virtio-fs — the guest kernel rejects the mount (EINVAL) with + // file-handle indexing enabled, and whiteouts/opaque markers must + // use unprivileged user.overlay.* xattrs (which the snapshot tar + // round-trips as PAX records; see tarutil). + Options: []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work, + "index=off", "metacopy=off", "userxattr"}, }, } pbSpec := SpecToAgentPB(spec) diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux_test.go b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go index c6dfb56a8..2c47d2051 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux_test.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux_test.go @@ -17,15 +17,36 @@ package kata import ( + "path/filepath" "slices" + "strings" "testing" ) +// The kernel requires overlay upperdir and workdir on the same filesystem and +// rejects a workdir nested inside (or equal to) upperdir — so they must be +// SIBLINGS under the one upper base on the ateUpper share. A layout change +// here breaks every overlay mount. +func TestUpperWorkDirsAreSiblings(t *testing.T) { + base := UpperBase("app") + upper, work := upperWorkDirs(base) + if filepath.Dir(upper) != base || filepath.Dir(work) != base { + t.Errorf("upperWorkDirs(%q) = %q, %q; want both directly under the base", base, upper, work) + } + if upper == work { + t.Errorf("upperWorkDirs(%q): upper and work are the same directory %q", base, upper) + } + if strings.HasPrefix(work+"/", upper+"/") { + t.Errorf("upperWorkDirs(%q): work %q is nested inside upper %q", base, work, upper) + } +} + func TestVirtiofsdArgs(t *testing.T) { tests := []struct { name string opts VirtiofsdOptions wantCache string + wantXattr bool }{ { name: "RO lower defaults to cache=always", @@ -41,6 +62,17 @@ func TestVirtiofsdArgs(t *testing.T) { }, wantCache: "--cache=auto", }, + { + name: "rootfs upper share passes xattrs through", + opts: VirtiofsdOptions{ + SocketPath: "/run/vm/virtiofsd-upper.sock", + SharedDir: "/var/lib/ateom-gvisor/actors/uid/rootfs-upper", + Cache: "auto", + Xattr: true, + }, + wantCache: "--cache=auto", + wantXattr: true, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -48,6 +80,12 @@ func TestVirtiofsdArgs(t *testing.T) { if !slices.Contains(args, tc.wantCache) { t.Errorf("args %v do not contain %q", args, tc.wantCache) } + // Overlay whiteouts/opaque markers are user.overlay.* xattrs in the + // upper; a share hosting an upper must pass them through, and the + // others must not pay the passthrough cost. + if gotXattr := slices.Contains(args, "--xattr"); gotXattr != tc.wantXattr { + t.Errorf("args %v: --xattr present = %v, want %v", args, gotXattr, tc.wantXattr) + } for _, want := range []string{ "--socket-path=" + tc.opts.SocketPath, "--shared-dir=" + tc.opts.SharedDir, diff --git a/cmd/ateom-microvm/internal/kata/restore.go b/cmd/ateom-microvm/internal/kata/restore.go index 0b71bea9d..41a38f6be 100644 --- a/cmd/ateom-microvm/internal/kata/restore.go +++ b/cmd/ateom-microvm/internal/kata/restore.go @@ -35,3 +35,10 @@ func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.so func DurableVirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd-durable.sock") } + +// UpperVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable +// disk-backed rootfs upper share, served by a third virtiofsd alongside the RO +// lower's and the durable share's. +func UpperVirtiofsdSocketPath(id string) string { + return filepath.Join(VMDir(id), "virtiofsd-upper.sock") +} diff --git a/cmd/ateom-microvm/internal/tarutil/fifo_linux.go b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go index c7496fc9e..79b185681 100644 --- a/cmd/ateom-microvm/internal/tarutil/fifo_linux.go +++ b/cmd/ateom-microvm/internal/tarutil/fifo_linux.go @@ -15,6 +15,7 @@ package tarutil import ( + "archive/tar" "fmt" "os" "path/filepath" @@ -46,3 +47,31 @@ func createFifo(root *os.Root, name string, mode os.FileMode) error { } return nil } + +// createDevice creates a character or block device node at name, a path +// relative to root, using the same parent-directory containment as createFifo. +// Device nodes matter here because overlayfs records a deleted lower-layer +// file as a 0:0 character device ("whiteout") in the upper — dropping one at +// extraction would resurrect the deleted file on restore. mknod requires +// privilege; extraction runs as root in ateom, and the tests gate on it. +func createDevice(root *os.Root, name string, hdr *tar.Header, mode os.FileMode) error { + dir, base := filepath.Split(name) + if dir == "" { + dir = "." + } + parent, err := root.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("opening parent directory of %q: %w", name, err) + } + defer parent.Close() + + var typ uint32 = unix.S_IFCHR + if hdr.Typeflag == tar.TypeBlock { + typ = unix.S_IFBLK + } + dev := unix.Mkdev(uint32(hdr.Devmajor), uint32(hdr.Devminor)) + if err := unix.Mknodat(int(parent.Fd()), base, typ|uint32(mode.Perm()), int(dev)); err != nil { + return fmt.Errorf("creating device node %q: %w", name, err) + } + return nil +} diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil.go b/cmd/ateom-microvm/internal/tarutil/tarutil.go index 7b1a6d319..d7f921683 100644 --- a/cmd/ateom-microvm/internal/tarutil/tarutil.go +++ b/cmd/ateom-microvm/internal/tarutil/tarutil.go @@ -16,12 +16,17 @@ // Package tarutil archives and restores a directory tree as a tar file, // preserving the metadata a workload's data directory depends on: modes, -// ownership, modification times, symlinks, hardlinks, and FIFOs. +// ownership, modification times, symlinks, hardlinks, FIFOs, device nodes, +// and user.* extended attributes (as PAX SCHILY.xattr records). // -// It exists for snapshotting durable-dir volumes (see cmd/ateom-microvm): the -// contents are written by the sandboxed workload under arbitrary uids, shipped -// to object storage, and restored — possibly onto another node — where the -// workload must see them unchanged. +// It exists for snapshotting durable-dir volumes and rootfs overlay uppers +// (see cmd/ateom-microvm): the contents are written by the sandboxed workload +// under arbitrary uids, shipped to object storage, and restored — possibly +// onto another node — where the workload must see them unchanged. The upper +// is why device nodes and xattrs matter: overlayfs records a deleted file as +// a 0:0 character device (whiteout) and a replaced directory as a +// user.overlay.opaque xattr, and losing either silently resurrects deleted +// content after a resume. // // Extraction is confined to the destination with os.Root, so a crafted archive // cannot write outside it via "..", an absolute path, or a symlink. @@ -39,16 +44,18 @@ import ( "path/filepath" "sort" "strings" + + "golang.org/x/sys/unix" ) // Create writes a tar archive of srcDir's contents to tarPath. Entry names are // relative to srcDir, so extracting into another directory reproduces the tree. // srcDir itself is not an entry. // -// Regular files, directories, symlinks, and FIFOs are archived with their mode, -// ownership, and modification time. A file with multiple links inside srcDir is -// archived once and referenced as a hardlink thereafter. Sockets are skipped -// (see writeTree). A device node is an error rather than silent data loss. +// Regular files, directories, symlinks, FIFOs, and device nodes are archived +// with their mode, ownership, modification time, and user.* xattrs. A file +// with multiple links inside srcDir is archived once and referenced as a +// hardlink thereafter. Sockets are skipped (see writeTree). func Create(ctx context.Context, tarPath, srcDir string) error { f, err := os.Create(tarPath) if err != nil { @@ -127,6 +134,23 @@ func writeTree(ctx context.Context, tw *tar.Writer, srcDir string) error { } setOwner(hdr, info) + // user.* xattrs ride as PAX records — overlayfs stores its + // opaque-directory markers there (userxattr mode), and the round trip + // must preserve them or replaced directories un-replace on restore. + // Symlinks are exempt: Linux refuses user.* xattrs on them. + if info.Mode()&os.ModeSymlink == 0 { + xattrs, err := readUserXattrs(path) + if err != nil { + return fmt.Errorf("reading xattrs of %q: %w", path, err) + } + for attr, val := range xattrs { + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} + } + hdr.PAXRecords["SCHILY.xattr."+attr] = val + } + } + switch { case info.Mode().IsRegular(): // A second link to an already-archived inode: record the link and @@ -145,15 +169,15 @@ func writeTree(ctx context.Context, tw *tar.Writer, srcDir string) error { } return copyFileInto(tw, path) - case d.IsDir(), info.Mode()&os.ModeSymlink != 0, info.Mode()&os.ModeNamedPipe != 0: - // FileInfoHeader already gave a FIFO Typeflag TypeFifo and size 0. + case d.IsDir(), info.Mode()&os.ModeSymlink != 0, info.Mode()&os.ModeNamedPipe != 0, + info.Mode()&os.ModeDevice != 0: + // FileInfoHeader already populated the Typeflag (and, for devices, + // the major/minor) with size 0. Devices are archived rather than + // rejected because an overlay upper legitimately contains them: + // every deleted lower-layer file is a 0:0 char-device whiteout. return tw.WriteHeader(hdr) default: - // Device nodes reach here. They need privilege to create, so one in a - // workload's data directory means something unexpected happened — - // worth failing on rather than silently dropping or re-creating it - // under a later restore. return fmt.Errorf("unsupported file type %v at %q", info.Mode().Type(), path) } }) @@ -284,6 +308,15 @@ func extractEntry(root *os.Root, tr *tar.Reader, hdr *tar.Header, name string, d } return nil + case tar.TypeChar, tar.TypeBlock: + if err := replaceExisting(root, name); err != nil { + return err + } + if err := createDevice(root, name, hdr, mode); err != nil { + return err + } + return restoreMeta(root, name, hdr) + default: return fmt.Errorf("unsupported tar entry type %q at %q", string([]byte{hdr.Typeflag}), name) } @@ -352,9 +385,97 @@ func restoreMeta(root *os.Root, name string, hdr *tar.Header) error { return fmt.Errorf("restoring times on %q: %w", name, err) } } + return restoreUserXattrs(root, name, hdr) +} + +// restoreUserXattrs re-applies the user.* xattrs recorded in the entry's PAX +// records (writeTree's SCHILY.xattr.* — the overlayfs opaque-directory markers +// a userxattr-mode upper depends on). +// +// The target is addressed THROUGH its parent directory opened via root (the +// same containment pattern createFifo and createDevice use), never by joining +// a host path: a symlinked intermediate component in a crafted archive must +// not redirect the write outside the extraction dir. The final component is +// addressed path-wise under /proc/self/fd/ rather than by opening the +// entry itself — opening would block on a FIFO and fail on a whiteout device — +// and Lsetxattr does not follow a final-component symlink (nor do symlink +// entries take this path: extractEntry never calls restoreMeta for them, and +// Linux refuses user.* xattrs on symlinks regardless). +func restoreUserXattrs(root *os.Root, name string, hdr *tar.Header) error { + var attrs map[string]string + for k, v := range hdr.PAXRecords { + if strings.HasPrefix(k, "SCHILY.xattr.user.") { + if attrs == nil { + attrs = map[string]string{} + } + attrs[strings.TrimPrefix(k, "SCHILY.xattr.")] = v + } + } + if len(attrs) == 0 { + return nil + } + dir, base := filepath.Split(name) + if dir == "" { + dir = "." + } + parent, err := root.Open(filepath.Clean(dir)) + if err != nil { + return fmt.Errorf("opening parent directory of %q to restore xattrs: %w", name, err) + } + defer parent.Close() + for attr, val := range attrs { + target := fmt.Sprintf("/proc/self/fd/%d/%s", parent.Fd(), base) + if err := unix.Lsetxattr(target, attr, []byte(val), 0); err != nil { + return fmt.Errorf("restoring xattr %q on %q: %w", attr, name, err) + } + } return nil } +// readUserXattrs returns path's user.* extended attributes. Filesystems +// without xattr support report none rather than failing: tarutil archives +// arbitrary workload trees, and only the user.* namespace carries state the +// round trip must preserve. +func readUserXattrs(path string) (map[string]string, error) { + sz, err := unix.Llistxattr(path, nil) + if err != nil { + if errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EOPNOTSUPP) { + return nil, nil + } + return nil, err + } + if sz <= 0 { + return nil, nil + } + buf := make([]byte, sz) + if sz, err = unix.Llistxattr(path, buf); err != nil { + return nil, err + } + + var attrs map[string]string + for _, attr := range strings.Split(string(buf[:sz]), "\x00") { + if !strings.HasPrefix(attr, "user.") { + continue + } + vsz, err := unix.Lgetxattr(path, attr, nil) + if err != nil { + return nil, err + } + val := make([]byte, vsz) + if vsz > 0 { + if vsz, err = unix.Lgetxattr(path, attr, val); err != nil { + return nil, err + } + val = val[:vsz] + } + if attrs == nil { + attrs = map[string]string{} + } + attrs[attr] = string(val) + } + return attrs, nil +} + // cleanTarName validates an archive entry name and returns it relative and // slash-free of surprises. skip is true for entries that name nothing ("" or // "."), which some tar writers emit for the archive root. diff --git a/cmd/ateom-microvm/internal/tarutil/tarutil_test.go b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go index d8b72c4ae..d566d13f7 100644 --- a/cmd/ateom-microvm/internal/tarutil/tarutil_test.go +++ b/cmd/ateom-microvm/internal/tarutil/tarutil_test.go @@ -90,6 +90,12 @@ func TestRoundTrip(t *testing.T) { if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { t.Fatalf("mkdir empty: %v", err) } + // Chmod explicitly: the Mkdir mode is clipped by the process umask (0o750 + // under the 027 some workstations default to), which would skew the mode + // the round-trip below is expected to preserve. + if err := os.Chmod(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatalf("chmod empty: %v", err) + } if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { t.Fatalf("symlink: %v", err) } @@ -360,20 +366,83 @@ func TestRoundTripOwnership(t *testing.T) { } } -func TestCreateRejectsDeviceNode(t *testing.T) { +// Device nodes must round-trip: overlayfs records a deleted lower-layer file +// as a 0:0 character device (whiteout) in the rootfs upper, and dropping one +// at archive or extract time silently resurrects the deleted file on restore. +func TestRoundTripDeviceNode(t *testing.T) { roottest.Require(t, "creating a device node requires root") src := t.TempDir() - // /dev/null, the cheapest character device to plant. - if err := unix.Mknod(filepath.Join(src, "null"), unix.S_IFCHR|0o666, int(unix.Mkdev(1, 3))); err != nil { - t.Fatalf("creating device node: %v", err) + // An overlayfs whiteout: character device 0:0. + if err := unix.Mknod(filepath.Join(src, "deleted-file"), unix.S_IFCHR|0o600, int(unix.Mkdev(0, 0))); err != nil { + t.Fatalf("creating whiteout device node: %v", err) + } + tarPath := filepath.Join(t.TempDir(), "dev.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) } - err := Create(t.Context(), filepath.Join(t.TempDir(), "dev.tar"), src) - if err == nil { - t.Fatal("Create succeeded on a device node, want an error") + st, err := os.Lstat(filepath.Join(dst, "deleted-file")) + if err != nil { + t.Fatalf("stat extracted device node: %v", err) } - if !strings.Contains(err.Error(), "unsupported file type") { - t.Errorf("error = %v, want it to mention an unsupported file type", err) + if st.Mode()&os.ModeCharDevice == 0 { + t.Errorf("extracted node mode = %v, want a character device", st.Mode()) + } + stat, ok := st.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("no syscall.Stat_t for extracted device node") + } + if unix.Major(stat.Rdev) != 0 || unix.Minor(stat.Rdev) != 0 { + t.Errorf("extracted device = %d:%d, want the 0:0 whiteout", unix.Major(stat.Rdev), unix.Minor(stat.Rdev)) + } +} + +// user.* xattrs must round-trip: overlayfs records a replaced lower-layer +// directory as a user.overlay.opaque xattr on the upper directory (userxattr +// mode), and dropping it would merge the old lower contents back in after a +// restore. +func TestRoundTripUserXattrs(t *testing.T) { + src := t.TempDir() + if err := os.Mkdir(filepath.Join(src, "replaced-dir"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := unix.Lsetxattr(filepath.Join(src, "replaced-dir"), "user.overlay.opaque", []byte("y"), 0); err != nil { + t.Skipf("filesystem does not support user xattrs: %v", err) + } + if err := os.WriteFile(filepath.Join(src, "f.txt"), []byte("x"), 0o644); err != nil { + t.Fatalf("writing file: %v", err) + } + if err := unix.Lsetxattr(filepath.Join(src, "f.txt"), "user.custom", []byte("val"), 0); err != nil { + t.Fatalf("setting file xattr: %v", err) + } + + tarPath := filepath.Join(t.TempDir(), "xattr.tar") + if err := Create(t.Context(), tarPath, src); err != nil { + t.Fatalf("Create: %v", err) + } + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract: %v", err) + } + + for path, want := range map[string]struct{ attr, val string }{ + "replaced-dir": {"user.overlay.opaque", "y"}, + "f.txt": {"user.custom", "val"}, + } { + buf := make([]byte, 64) + n, err := unix.Lgetxattr(filepath.Join(dst, path), want.attr, buf) + if err != nil { + t.Errorf("reading %s on restored %q: %v", want.attr, path, err) + continue + } + if got := string(buf[:n]); got != want.val { + t.Errorf("restored %q %s = %q, want %q", path, want.attr, got, want.val) + } } } @@ -490,10 +559,20 @@ func TestExtractIntoPrecreatedVolumeDir(t *testing.T) { } } -func TestExtractRejectsUnsupportedType(t *testing.T) { +func TestExtractSupportsDeviceEntry(t *testing.T) { + roottest.Require(t, "creating a device node requires root") + tarPath := filepath.Join(t.TempDir(), "dev.tar") writeTar(t, tarPath, tar.Header{Name: "null", Typeflag: tar.TypeChar, Devmajor: 1, Devminor: 3}) - if err := Extract(tarPath, t.TempDir()); err == nil { - t.Fatal("Extract succeeded on a device entry, want an error") + dst := t.TempDir() + if err := Extract(tarPath, dst); err != nil { + t.Fatalf("Extract failed on a device entry: %v", err) + } + st, err := os.Lstat(filepath.Join(dst, "null")) + if err != nil { + t.Fatalf("stat extracted device node: %v", err) + } + if st.Mode()&os.ModeCharDevice == 0 { + t.Errorf("extracted node mode = %v, want a character device", st.Mode()) } } diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 93d6518a9..b0a0648db 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -130,15 +130,16 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // restoreFullScope restores a whole-guest snapshot: relaunch cloud-hypervisor // directly from it and resume. // -// Each container's rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper). Steps: +// Each container's rootfs is overlay(virtio-fs RO lower + disk-backed upper). Steps: // reconstruct each RO lower from the local OCI bundle (atelet re-unpacked the golden // image) at the frozen find-paths path and start the virtiofsd serving them; rewrite // the snapshot config's per-VMDir paths (vsock + serial + fs sockets) to this actor's; // rebuild the tap (the snapshot's virtio-net is fd-backed → fresh net_fds); relaunch -// CH with --restore (OnDemand), and resume. Guest RAM — incl. the actor's in-memory -// state, the tmpfs rootfs upper (so rootfs writes PERSIST), and the frozen network -// config — comes back from the memory snapshot. Durable-dir volumes are host-backed -// instead, and the caller has already restored them from the snapshot's tar. +// CH with --restore (OnDemand), and resume. Guest RAM — the actor's in-memory state +// and the frozen network config — comes back from the memory snapshot. The rootfs +// uppers and durable-dir volumes are host-backed: the durable volumes were restored +// by the caller from their tar, and the rootfs uppers are re-materialized from +// rootfs-upper.tar below (in the background, overlapped with the setup here). func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, restoreDir string, tStart time.Time) (retErr error) { actorUID := p.actorUID templateNS, templateName := p.templateNS, p.templateName @@ -165,14 +166,39 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, return fmt.Errorf("while creating VM dir: %w", err) } + // Disk-backed rootfs uppers: the snapshot says whether the guest expects the + // ateUpper share (its config.json references the fs device and the tar rides + // alongside). Start re-materializing the upper contents NOW, in the + // background: the untar scales with the actor's data and depends on nothing + // below, so it hides behind the lower staging, network, and tap setup and is + // joined right before the share's virtiofsd starts (the guest must never + // observe the directory mid-restore). Legacy tmpfs-upper snapshots have no + // tar and skip all of this — their upper rides inside the restored guest + // memory. + // + // An error return between here and the join MUST drain the goroutine (the + // deferred receive below): returning with the untar still writing would let + // a retried restore's own untar race it inside the same directory. + hasUpper := snapshotHasRootfsUpper(restoreDir) + untarDone := make(chan error, 1) + untarJoined := false + if hasUpper { + go func() { + untarDone <- untarRootfsUpper(ateompath.RootfsUpperDir(actorUID), restoreDir) + }() + defer func() { + if !untarJoined { + <-untarDone + } + }() + } + // Reconstruct each container's overlay RO lower from the LOCAL OCI bundle (atelet // re-unpacked the golden image; the lower is the immutable golden image) at the // frozen find-paths location SharedDir(id)//rootfs, and start the one virtiofsd - // serving them. The writable upper is a guest tmpfs restored from the memory - // snapshot (rootfs writes persist), so there is no disk to rebuild or repoint; the - // fs socket in the snapshot config is repointed to this VMDir by - // rewriteSnapshotSocketPaths above. cross-node consistency relies on a deterministic - // unpack of the same image at the same /rootfs path. + // serving them. The fs sockets in the snapshot config are repointed to this VMDir + // by rewriteSnapshotSocketPaths above. cross-node consistency relies on a + // deterministic unpack of the same image at the same /rootfs path. containers := p.containers if len(containers) == 0 { return status.Error(codes.InvalidArgument, "actor spec has no containers") @@ -263,6 +289,28 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, restoredNets = append(restoredNets, rn) } + // Join the background untar and serve the re-materialized uppers exactly + // like the durable share — the tar reproduces the paths find-paths re-opens. + // Staged as late as possible (CH first needs the socket at Restore below) so + // the untar hid behind all of the setup above. + var upperVfsdCmd *exec.Cmd + if hasUpper { + untarErr := <-untarDone + untarJoined = true + if untarErr != nil { + return untarErr + } + if upperVfsdCmd, err = s.stageRootfsUpperShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && upperVfsdCmd.Process != nil { + _ = upperVfsdCmd.Process.Kill() + _, _ = upperVfsdCmd.Process.Wait() + } + }() + } + // Relaunch CH and restore with the tap FDs attached (SCM_RIGHTS). CH reopens // /dev/vda (image) + each /dev/vd{b+i} (actor rootfs) from the snapshot config paths. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api-restore.sock") @@ -297,7 +345,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } ra := &runningActor{ - chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, + chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, upperVfsdCmd: upperVfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, } @@ -358,8 +406,8 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { // Each virtio-fs share is served by its own per-VMDir virtiofsd socket; the // snapshot recorded the golden actor's, so repoint them at this actor's VMDir. // Match on the device tag: the shares have separate sockets (the overlay RO - // lower's and, when the actor has durable-dir volumes, the writable share's), and - // crossing them would hand the guest the wrong filesystem. + // lower's and, when present, the writable durable-dir and rootfs upper shares'), + // and crossing them would hand the guest the wrong filesystem. if fss, ok := cfg["fs"].([]any); ok { for _, f := range fss { fm, ok := f.(map[string]any) @@ -371,6 +419,8 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { fm["socket"] = kata.VirtiofsdSocketPath(id) case kata.DurableFsTag: fm["socket"] = kata.DurableVirtiofsdSocketPath(id) + case kata.UpperFsTag: + fm["socket"] = kata.UpperVirtiofsdSocketPath(id) default: return fmt.Errorf("snapshot config %q has fs device with unknown tag %q", cfgPath, tag) } diff --git a/cmd/ateom-microvm/restore_test.go b/cmd/ateom-microvm/restore_test.go index 4ea92e102..c8643f557 100644 --- a/cmd/ateom-microvm/restore_test.go +++ b/cmd/ateom-microvm/restore_test.go @@ -86,26 +86,31 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { }) t.Run("each share keeps its own socket", func(t *testing.T) { - // Ordered durable-first to catch a rewrite that assumes the RO lower comes + // Ordered with the RO lower last to catch a rewrite that assumes it comes // first, which would hand the guest the wrong filesystem. dir := writeSnapshotConfig(t, []map[string]any{ {"tag": kata.DurableFsTag, "socket": "/run/vc/vm/golden/virtiofsd-durable.sock"}, + {"tag": kata.UpperFsTag, "socket": "/run/vc/vm/golden/virtiofsd-upper.sock"}, {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, }) if err := rewriteSnapshotSocketPaths(dir, id); err != nil { t.Fatalf("rewriteSnapshotSocketPaths: %v", err) } got := readFsSockets(t, dir) - for tag, want := range map[string]string{ + want := map[string]string{ kata.FsTag: kata.VirtiofsdSocketPath(id), kata.DurableFsTag: kata.DurableVirtiofsdSocketPath(id), - } { - if got[tag] != want { - t.Errorf("%s socket = %q, want %q", tag, got[tag], want) - } + kata.UpperFsTag: kata.UpperVirtiofsdSocketPath(id), } - if got[kata.FsTag] == got[kata.DurableFsTag] { - t.Error("both shares were pointed at the same socket") + seen := map[string]string{} + for tag, w := range want { + if got[tag] != w { + t.Errorf("%s socket = %q, want %q", tag, got[tag], w) + } + if prev, dup := seen[got[tag]]; dup { + t.Errorf("shares %s and %s were pointed at the same socket %q", prev, tag, got[tag]) + } + seen[got[tag]] = tag } }) diff --git a/cmd/ateom-microvm/rootfsupper.go b/cmd/ateom-microvm/rootfsupper.go new file mode 100644 index 000000000..49fe8e6a1 --- /dev/null +++ b/cmd/ateom-microvm/rootfsupper.go @@ -0,0 +1,156 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +// Disk-backed rootfs writes for the micro-VM runtime. +// +// Every container's overlay upper lives on a THIRD virtio-fs share +// (kata.UpperFsTag), served by its own virtiofsd from +// ateompath.RootfsUpperDir(actorUID) on the host: rootfs writes cost host disk, +// not guest RAM. (The retired alternative — a guest tmpfs upper — capped rootfs +// writes at the tmpfs size, a fifth of guest RAM, and pinned every written byte +// in memory; snapshots taken in that mode still restore, see below.) +// +// The share is served like the durable-dir one — write-through (no +// --writeback, so a paused guest's completed writes are already on the host), +// cache=auto (the host contents change underneath the guest on restore), and +// find-paths migration — plus --xattr, because overlayfs stores whiteouts and +// opaque-directory markers as user.overlay.* xattrs in the upper and the +// guest kernel must round-trip them through virtiofsd. Unlike the durable dirs +// (whose host side atelet owns), this directory is owned entirely by ateom: +// created fresh at cold boot, re-materialized from the snapshot at restore, +// and removed at teardown. +// +// Snapshots: the upper does not ride in guest memory, so a FULL snapshot +// ships it as a tar (rootfsUpperTarFile) exactly like the durable volumes, +// taken while the guest is paused. Restore is self-describing — the tar's +// presence in the snapshot is what says the guest expects the ateUpper share +// (the snapshot's config.json references its fs device) — which is also what +// keeps legacy tmpfs-upper snapshots restorable: no tar, no share, their upper +// rides inside the restored guest memory. A DATA snapshot deliberately +// excludes rootfs state: the workload cold-starts on restore. + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil" + "github.com/agent-substrate/substrate/internal/ateompath" +) + +// rootfsUpperTarFile is the snapshot file holding the tar of the actor's +// rootfs uppers. Its entries are /fs/... and /work/... +// relative to ateompath.RootfsUpperDir, so extraction restores the exact layout +// the guest's find-paths re-opens. +const rootfsUpperTarFile = "rootfs-upper.tar" + +// upperVirtiofsdLogPath is where the rootfs upper share's virtiofsd logs, +// beside the overlay lower's and the durable share's under the actor's VM dir. +func upperVirtiofsdLogPath(id string) string { + return filepath.Join(kata.VMDir(id), "virtiofsd-upper.log") +} + +// resetRootfsUpperDir gives a cold boot a pristine upper directory: a cold +// boot must start from the bare image, and atelet's actor-dir reset does not +// know about this directory, so ateom wipes any previous activation's contents +// itself. +func resetRootfsUpperDir(actorUID string) error { + dir := ateompath.RootfsUpperDir(actorUID) + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("while clearing rootfs upper dir %q: %w", dir, err) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating rootfs upper dir %q: %w", dir, err) + } + return nil +} + +// actorHasDiskUpper reports whether the running actor's rootfs uppers are +// disk-backed, by the host directory only a disk-upper boot/restore creates +// (and teardownActor removes). A LEGACY actor — restored from a snapshot taken +// by the retired tmpfs-upper implementation — has no directory: its upper +// lives inside guest memory, and its checkpoints must keep capturing it there. +func actorHasDiskUpper(actorUID string) bool { + _, err := os.Stat(ateompath.RootfsUpperDir(actorUID)) + return err == nil +} + +// snapshotHasRootfsUpper reports whether a snapshot carries disk-backed rootfs +// uppers — i.e. whether its guest expects the ateUpper share on restore. +func snapshotHasRootfsUpper(snapshotDir string) bool { + _, err := os.Stat(filepath.Join(snapshotDir, rootfsUpperTarFile)) + return err == nil +} + +// stageRootfsUpperShare starts the virtiofsd serving the actor's rootfs +// uppers. The caller has already created (cold boot) or re-materialized +// (restore) the host directory. +// +// The returned cmd outlives this call (CH talks to it for the VM's lifetime); +// the caller owns it (tracked on runningActor, killed in teardownActor). +func (s *AteomService) stageRootfsUpperShare(ctx context.Context, rr resolvedRuntime, actorUID string) (*exec.Cmd, error) { + shared := ateompath.RootfsUpperDir(actorUID) + if _, err := os.Stat(shared); err != nil { + return nil, fmt.Errorf("while checking rootfs upper dir %q: %w", shared, err) + } + log, _ := os.OpenFile(upperVirtiofsdLogPath(actorUID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + cmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ + Binary: rr.virtiofsd, + SocketPath: kata.UpperVirtiofsdSocketPath(actorUID), + SharedDir: shared, + Cache: "auto", + Xattr: true, + Log: log, + }) + if err != nil { + return nil, fmt.Errorf("while starting rootfs upper virtiofsd: %w", err) + } + return cmd, nil +} + +// tarRootfsUpper archives the actor's rootfs uppers (dir) into the checkpoint +// directory. The caller must have paused the guest first: virtiofsd is +// write-through, so a completed guest write is on the host by then, but a +// running guest could still add more after the walk. +func tarRootfsUpper(ctx context.Context, dir, checkpointDir string) error { + if err := tarutil.Create(ctx, filepath.Join(checkpointDir, rootfsUpperTarFile), dir); err != nil { + return fmt.Errorf("while archiving rootfs uppers from %q: %w", dir, err) + } + return nil +} + +// untarRootfsUpper restores the rootfs uppers from a snapshot into the actor's +// host directory. It must run before the upper share's virtiofsd starts, so +// the guest never observes the directory mid-restore. The directory is +// recreated from scratch: nothing else owns it, and stale contents from a +// previous activation would corrupt the overlay state find-paths re-opens. +func untarRootfsUpper(dir, snapshotDir string) error { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("while clearing rootfs upper dir %q: %w", dir, err) + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("while creating rootfs upper dir %q: %w", dir, err) + } + if err := tarutil.Extract(filepath.Join(snapshotDir, rootfsUpperTarFile), dir); err != nil { + return fmt.Errorf("while restoring rootfs uppers into %q: %w", dir, err) + } + return nil +} diff --git a/cmd/ateom-microvm/rootfsupper_test.go b/cmd/ateom-microvm/rootfsupper_test.go new file mode 100644 index 000000000..9477526f6 --- /dev/null +++ b/cmd/ateom-microvm/rootfsupper_test.go @@ -0,0 +1,87 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// upperDirWith returns a rootfs upper directory laid out the way the guest +// agent builds one: /{fs,work} per container, with the given +// files created under it (paths relative to the directory). +func upperDirWith(t *testing.T, files map[string]string) string { + t.Helper() + dir := t.TempDir() + for rel, content := range files { + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatalf("creating %q: %v", filepath.Dir(p), err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatalf("writing %q: %v", rel, err) + } + } + return dir +} + +func TestRootfsUpperRoundTrip(t *testing.T) { + // Checkpoint: every container's upper/work, archived while the guest is + // paused. The layout under the dir is exactly what find-paths re-opens. + files := map[string]string{ + "app_ovl/fs/home/agent/notes.txt": "rootfs write", + "app_ovl/work/index": "", + "sidecar_ovl/fs/var/log/s.log": "sidecar write", + } + src := upperDirWith(t, files) + checkpointDir := t.TempDir() + if err := tarRootfsUpper(t.Context(), src, checkpointDir); err != nil { + t.Fatalf("tarRootfsUpper: %v", err) + } + if !snapshotHasRootfsUpper(checkpointDir) { + t.Fatalf("snapshotHasRootfsUpper(%q) = false after tarRootfsUpper", checkpointDir) + } + + // Restore: onto a directory holding a stale previous activation's contents, + // which must not leak into the restored overlay state. + dst := upperDirWith(t, map[string]string{"app_ovl/fs/stale.txt": "stale"}) + if err := untarRootfsUpper(dst, checkpointDir); err != nil { + t.Fatalf("untarRootfsUpper: %v", err) + } + for rel, want := range files { + got, err := os.ReadFile(filepath.Join(dst, rel)) + if err != nil { + t.Errorf("reading restored %q: %v", rel, err) + continue + } + if string(got) != want { + t.Errorf("restored %q = %q, want %q", rel, got, want) + } + } + if _, err := os.Stat(filepath.Join(dst, "app_ovl/fs/stale.txt")); !os.IsNotExist(err) { + t.Errorf("stale pre-restore content survived untarRootfsUpper (stat err = %v), want it wiped", err) + } +} + +// A snapshot without the tar is a legacy tmpfs-upper snapshot: restore must +// not stage the ateUpper share (the guest has no fs device for it). +func TestSnapshotHasRootfsUpperAbsent(t *testing.T) { + if snapshotHasRootfsUpper(t.TempDir()) { + t.Error("snapshotHasRootfsUpper() = true for a snapshot without the tar") + } +} diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index eb4b34740..e5c920873 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -66,6 +66,10 @@ type runningActor struct { // durable-dir volumes. nil when the actor declares none. Owned and torn down // exactly like vfsdCmd. durableVfsdCmd *exec.Cmd + // upperVfsdCmd is the third virtiofsd, serving the actor's disk-backed + // rootfs uppers (see rootfsupper.go). nil only for a legacy actor restored + // from a tmpfs-upper snapshot. Owned and torn down exactly like vfsdCmd. + upperVfsdCmd *exec.Cmd // apiSocket is the CH api-socket for this ateom-owned VMM. apiSocket string @@ -125,7 +129,7 @@ func overlayWorkloadID(name string) string { return name + "_ovl" } // actorContainer is one of the actor's containers prepared for the shared micro-VM: // its name (also the kata containerID + the overlay lower's find-paths subdir), the // host OCI bundle rootfs that backs the RO lower, and its OCI spec. The writable -// overlay upper is a guest tmpfs (OverlayUpperBase(name)), so there is no host disk. +// overlay upper is a directory on the disk-backed ateUpper share (kata.UpperBase(name)). type actorContainer struct { name string bundleRootfs string @@ -188,8 +192,8 @@ func writeGuestResolvConf(rootfs string) error { // RunWorkload boots the actor as a cloud-hypervisor micro-VM and starts its containers. // // ateom boots cloud-hypervisor directly (no kata shim) and gives each container an -// overlay rootfs: its OCI image read-only over virtio-fs (the lower) plus a guest -// tmpfs (the writable upper). It drives the kata clh boot (vm.create kernel+image+fs, +// overlay rootfs: its OCI image read-only over virtio-fs (the lower) plus a writable +// upper on the disk-backed ateUpper share. It drives the kata clh boot (vm.create kernel+image+fs, // add-net, vm.boot) and the post-boot setup the shim would otherwise do (agent // CreateSandbox + guest network config) before having the kata-agent assemble and // start each container. @@ -354,7 +358,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() // Prepare each container's OCI spec + record its bundle rootfs (the overlay RO - // lower). No host disk — the rootfs is overlay(virtio-fs lower + guest-tmpfs upper). + // lower). The rootfs is overlay(virtio-fs RO lower + disk-backed ateUpper upper). ctrs, err := s.buildActorContainers(actorUID, containers) if err != nil { return err @@ -403,6 +407,23 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() } + // Disk-backed rootfs uppers: a third writable virtio-fs share, served from + // a per-actor host directory ateom prepares itself (pristine — a cold boot + // starts from the bare image). See rootfsupper.go. + if err := resetRootfsUpperDir(actorUID); err != nil { + return err + } + upperVfsdCmd, err := s.stageRootfsUpperShare(ctx, rr, actorUID) + if err != nil { + return err + } + defer func() { + if retErr != nil && upperVfsdCmd.Process != nil { + _ = upperVfsdCmd.Process.Kill() + _, _ = upperVfsdCmd.Process.Wait() + } + }() + // Launch a bare VMM (CH + api-socket); ateom owns this process for teardown. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api.sock") chCmd, client, err := ch.LaunchVMM(ctx, ch.LaunchVMMOptions{ @@ -422,8 +443,8 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() // Assemble the CH VmConfig (kata-compatible cmdline, RO kata image on /dev/vda + - // the virtio-fs device for the overlay RO lower; no actor virtio-blk disks — the - // writable upper is a guest tmpfs). serialLog is also read on a failed agent dial + // the virtio-fs devices; no actor virtio-blk disks — the writable upper is the + // disk-backed ateUpper share). serialLog is also read on a failed agent dial // below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable) @@ -489,7 +510,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} + ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, upperVfsdCmd: upperVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} if err := s.activateActorNetworking(p.actorRef.Atespace, p.actorRef.Name, egress); err != nil { return err } @@ -509,9 +530,9 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // buildActorContainers prepares each of the actor's containers for the shared // micro-VM: it loads the OCI spec from the per-container bundle, injects guest DNS, // and records the bundle rootfs that backs the overlay's RO lower. No host disk is -// built — the rootfs is overlay(virtio-fs RO lower + guest-tmpfs upper); the lowers -// are bound into virtiofsd's shared dir in stageOverlayLowers after the sandbox state -// is clean. Both RunWorkload and RestoreWorkload go through here. +// prepared here — the rootfs is overlay(virtio-fs RO lower + ateUpper upper); the +// lowers are bound into virtiofsd's shared dir in stageOverlayLowers after the sandbox +// state is clean. Both RunWorkload and RestoreWorkload go through here. func (s *AteomService) buildActorContainers(actorUID string, containers []*ateompb.Container) ([]actorContainer, error) { netnsPath := ateompath.AteomNetNSPath(s.podUID) ctrs := make([]actorContainer, len(containers)) @@ -527,7 +548,7 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom // overlay spec). Everything downstream — the resolv.conf write below, // the bind into virtiofsd's shared dir, the read-only remount — then // sees the composed tree, with host-side writes landing in the bundle's - // private upper. The guest still builds its own tmpfs upper on top. + // private upper. The guest still builds its own writable upper on top. if err := imagecache.SetupBundleRootfs(bundle); err != nil { return nil, fmt.Errorf("while composing rootfs for %q: %w", cn, err) } @@ -602,6 +623,7 @@ func (s *AteomService) guestConfig(rr resolvedRuntime) (memMiB, vcpus int, kpara // // withDurable adds a second virtio-fs device for the actor's writable durable-dir // volumes (see durable.go), served by its own virtiofsd on the same PCI segment. +// The disk-backed rootfs upper share (see rootfsupper.go) is always present. func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { @@ -629,13 +651,21 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i } // buildFsConfigs returns the VM's virtio-fs devices: the overlay RO lower's -// share, plus the writable durable-dir share when the actor has one. Both sit on -// PCI segment 1 (the segment buildVMConfig reserves for virtio-fs). +// share, the writable disk-backed rootfs upper share (always present — every +// container's overlay upper lives on it), plus the writable durable-dir share +// when the actor has one. All sit on PCI segment 1 (the segment buildVMConfig +// reserves for virtio-fs). func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { - fs := []ch.FsConfig{{ - Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), - NumQueues: 1, QueueSize: 1024, PciSegment: 1, - }} + fs := []ch.FsConfig{ + { + Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }, + { + Tag: kata.UpperFsTag, Socket: kata.UpperVirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }, + } if withDurable { fs = append(fs, ch.FsConfig{ Tag: kata.DurableFsTag, Socket: kata.DurableVirtiofsdSocketPath(id), @@ -654,8 +684,8 @@ func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { // the writable durable share, and each container binds the volumes it declared. func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durable bool) error { // Establish the agent sandbox + the kataShared virtio-fs mount (the RO base for - // every container's overlay lower). All containers share it, so use the first - // container's hostname. + // every container's overlay lower) + the writable rootfs upper share. All + // containers share them, so use the first container's hostname. sbCtx, sbCancel := context.WithTimeout(ctx, 20*time.Second) err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname, durable) sbCancel() @@ -683,9 +713,10 @@ func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentC } // startOverlayContainer brings up one container's rootfs as overlay(virtio-fs RO -// lower + guest-tmpfs upper): a carrier container (id == name) eager-binds the RO base +// lower + writable upper): a carrier container (id == name) eager-binds the RO base // to /run/kata-containers//rootfs, then the workload (id == _ovl) overlays -// it with a tmpfs upper. On failure it dumps the guest overlay state. +// it with a directory on the disk-backed ateUpper share. On failure it dumps the +// guest overlay state. // // With a durable-dir volume, the WORKLOAD also binds it at the container's declared // mount paths. The carrier deliberately does not: its rootfs is the read-only lower, @@ -700,7 +731,7 @@ func startOverlayContainer(ctx context.Context, ac *kata.AgentClient, vsockPath return fmt.Errorf("while creating carrier %q: %w", c.name, err) } - upperBase := kata.OverlayUpperBase(c.name) + upperBase := kata.UpperBase(c.name) wlCtx, wlCancel := context.WithTimeout(ctx, 30*time.Second) err = ac.StartOverlayWorkload(wlCtx, c.name, overlayWorkloadID(c.name), upperBase, workloadSpec(c)) wlCancel() @@ -784,6 +815,7 @@ func logGuestBootDiagnostics(ctx context.Context, actorUID, serialLog string) { {"serial", serialLog}, {"virtiofsd", virtiofsdLogPath(actorUID)}, {"virtiofsd-durable", durableVirtiofsdLogPath(actorUID)}, + {"virtiofsd-upper", upperVirtiofsdLogPath(actorUID)}, } { b, err := os.ReadFile(l.path) if err != nil || len(b) == 0 { diff --git a/internal/ateompath/ateompath.go b/internal/ateompath/ateompath.go index 680e329c1..1f1652c7a 100644 --- a/internal/ateompath/ateompath.go +++ b/internal/ateompath/ateompath.go @@ -172,6 +172,19 @@ func DurableDirVolumeMountPoint(actorUID, volumeName string) string { ) } +// RootfsUpperDir is the host directory backing the actor's rootfs overlay +// uppers in ateom-microvm: one subdirectory per container, served into the +// guest over virtio-fs so rootfs writes land on host disk instead of guest +// RAM. Unlike the durable-dir volumes it is owned entirely by ateom (created +// at cold boot, archived at checkpoint, removed at teardown); atelet never +// touches it. +func RootfsUpperDir(actorUID string) string { + return filepath.Join( + ActorPath(actorUID), + "rootfs-upper", + ) +} + // RestoreStateDir is the local directory to use to restore an actor from a // checkpoint downloaded from GCS. //