Skip to content

microvm: write container rootfs to host disk instead of guest memory #846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Lucky Abolorunke (Oneimu) wants to merge 1 commit into
base: main
Choose a base branch
from
Open

microvm: write container rootfs to host disk instead of guest memory #846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 70 additions & 31 deletions cmd/ateom-microvm/checkpoint.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/agent-substrate/substrate/internal/imagecache"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/resources"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
Expand All @@ -47,11 +48,10 @@ import (
// - FULL: the whole guest. ateom drives the CH REST api-socket: pause -> snapshot
// file://<CheckpointStateDir> (config.json + state.json + sparse memory-ranges)
// -> tear the VMM down. Each container's rootfs is overlay(virtio-fs RO lower +
// guest-tmpfs upper), so the writable upper lives in guest RAM and is captured by
// the memory snapshot — process memory and rootfs writes both persist across
// suspend/resume. The RO lower is reconstructed from the OCI image at restore, so
// nothing rootfs-related ships. Durable-dir volumes are host-backed rather than in
// guest RAM, so they ship alongside as a tar.
// disk-backed upper): the upper is host-backed like the durable-dir volumes and
// ships alongside as its own tar (see rootfsupper.go); process memory persists
// via the memory snapshot. The RO lower is reconstructed from the OCI image at
// restore, so it never ships. Durable-dir volumes ship alongside as a tar.
// - DATA: the durable-dir volumes only, as that same tar. The guest is discarded, so
// the actor cold-starts on restore with its volumes re-materialized.
//
Expand Down Expand Up @@ -120,27 +120,56 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
return nil, fmt.Errorf("while creating checkpoint dir %q: %w", checkpointDir, err)
}

// Only a Full snapshot captures the guest. A Data snapshot deliberately
// captures no VM state — no memory image, and no base-id, since nothing will
// reattach to the frozen virtio-fs lower: at restore the actor cold-boots
// from the OCI image (or, under an OnGolden data resume policy, is combined
// with the golden snapshot's guest state) and gets its durable-dir volumes
// back from the tar below.
var dSnapshot time.Duration
// Capture the snapshot's pieces CONCURRENTLY: the CH snapshot, the
// durable-dir tar, and the rootfs upper tar read independent data from a
// quiesced guest and write distinct files into checkpointDir, so the paused
// window costs the slowest of them rather than their sum (the tars scale
// with the actor's data; suspend latency is the metric that matters).
//
// - CH snapshot (Full only): the guest memory + VM state. A Data snapshot
// deliberately captures no VM state — no memory image, and no base-id,
// since nothing will reattach to the frozen virtio-fs lower: at restore
// the actor cold-boots from the OCI image (or, under an OnGolden data
// resume policy, is combined with the golden snapshot's guest state).
// - Durable-dir tar (any scope, when declared): host-backed, so pausing
// the write-through share makes the tar coherent.
// - Rootfs upper tar (Full only): host-backed like the durable volumes —
// the memory snapshot does not carry rootfs writes. Under Data the
// workload cold-starts on restore, discarding rootfs state. Gated on
// the host dir a disk-upper boot creates (actorHasDiskUpper) so a
// legacy actor restored from a tmpfs-upper snapshot checkpoints
// correctly (its upper is inside the memory image).
var dSnapshot, dDurable, dUpper time.Duration
g, gctx := errgroup.WithContext(ctx)
if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL {
var err error
if dSnapshot, err = s.snapshotVMState(ctx, client, ra, actorUID, checkpointDir); err != nil {
return nil, err
}
g.Go(func() error {
var err error
dSnapshot, err = s.snapshotVMState(gctx, client, ra, actorUID, checkpointDir)
return err
})
}

var dDurable time.Duration
if durable {
tDurable := time.Now()
if err := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil {
return nil, err
}
dDurable = time.Since(tDurable)
g.Go(func() error {
t := time.Now()
if err := tarDurableVolumes(gctx, ateompath.DurableDirVolumeMountsDir(actorUID), checkpointDir); err != nil {
return err
}
dDurable = time.Since(t)
return nil
})
}
if scope == ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL && actorHasDiskUpper(actorUID) {
g.Go(func() error {
t := time.Now()
if err := tarRootfsUpper(gctx, ateompath.RootfsUpperDir(actorUID), checkpointDir); err != nil {
return err
}
dUpper = time.Since(t)
return nil
})
}
if err := g.Wait(); err != nil {
return nil, err
}

// Report exactly the files we wrote so atelet ships precisely this snapshot: for
Expand Down Expand Up @@ -180,9 +209,11 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec
slog.InfoContext(ctx, "Actor checkpointed", slog.String("id", actorUID), slog.Any("snapshot_files", snapshotFiles),
slog.String("scope", scope.String()), slog.Duration("pause", dPause),
slog.Duration("snapshot", dSnapshot),
// The durable-dir tar runs while the guest is paused, so its cost is part
// of the suspend latency and scales with the volume's contents.
slog.Duration("durable_dir", dDurable), slog.Duration("teardown", dTeardown))
// The tars run while the guest is paused, CONCURRENTLY with the CH
// snapshot: the paused window costs max(snapshot, durable_dir,
// rootfs_upper), and the tar durations scale with the actor's data.
slog.Duration("durable_dir", dDurable), slog.Duration("rootfs_upper", dUpper),
slog.Duration("teardown", dTeardown))
return &ateompb.CheckpointWorkloadResponse{SnapshotFiles: snapshotFiles}, nil
}

Expand Down Expand Up @@ -232,9 +263,9 @@ func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, r
slog.String("id", actorUID), slog.Duration("merge", time.Since(tMerge)))
}

// Nothing rootfs-related ships: the overlay's writable upper is a guest tmpfs, so
// the actor's rootfs writes are already in the memory snapshot above, and the RO
// lower is reconstructed from the OCI image at restore (it never changes).
// The RO lower never ships (reconstructed from the OCI image at restore).
// The disk-backed upper ships as its own tar from CheckpointWorkload; a
// legacy tmpfs upper is already inside the memory snapshot above.
return dSnapshot, nil
}

Expand Down Expand Up @@ -283,15 +314,23 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running
_, _ = ra.chCmd.Process.Wait()
}
// Kill the virtiofsds (after CH, their only client): the overlay RO lower's
// and, when the actor has durable-dir volumes, the writable share's.
for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} {
// and, when present, the writable durable-dir and rootfs upper shares'.
for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd, ra.upperVfsdCmd} {
if cmd != nil && cmd.Process != nil {
_ = cmd.Process.Kill()
_, _ = cmd.Process.Wait()
}
}
}

// Remove the rootfs upper dir: ateom owns it — atelet's actor-dir reset
// doesn't know it — and its absence is what marks a worker as holding no
// disk-backed upper (actorHasDiskUpper). Runs after the checkpoint tar,
// which is already on disk. A no-op for legacy tmpfs-upper actors.
if err := os.RemoveAll(ateompath.RootfsUpperDir(id)); err != nil {
slog.WarnContext(ctx, "Failed to remove rootfs upper dir", slog.String("actorUID", id), slog.Any("err", err))
}

// Sweep any leftover per-sandbox host-side state + orphaned per-sandbox
// processes. This is ateom's own cleanup (process kill + unmount + rm).
kata.CleanupSandboxState(ctx, id)
Expand Down
2 changes: 1 addition & 1 deletion cmd/ateom-microvm/internal/kata/agentclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ func (a *AgentClient) Close() error {
// CreateContainer asks the agent to create a container: mount its storages (in
// order) and build the rootfs, then fork the parked init process. This is the
// hook point — the agent mounts storages[] (here: a bind of the virtio-fs lower
// followed by the tmpfs-upper overlay) before init_rootfs consumes the rootfs.
// followed by the disk-backed-upper overlay) before init_rootfs consumes the rootfs.
// Mirrors grpc.AgentService/CreateContainer (returns google.protobuf.Empty).
func (a *AgentClient) CreateContainer(ctx context.Context, req *agentpb.CreateContainerRequest) error {
if err := a.client.Call(ctx, "grpc.AgentService", "CreateContainer", req, &emptypb.Empty{}); err != nil {
Expand Down
109 changes: 80 additions & 29 deletions cmd/ateom-microvm/internal/kata/overlay_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@
package kata

// Each container's rootfs is an overlay: its OCI image served read-only over virtio-fs
// (the lower) plus a guest tmpfs (the writable upper). The upper is in guest RAM, so
// rootfs writes ride along in the memory snapshot and persist across suspend/resume.
// This file holds the overlay-specific helpers.
// (the lower) plus a writable upper on the ateUpper virtio-fs share, backed by host
// disk (see cmd/ateom-microvm/rootfsupper.go). Rootfs writes therefore cost host disk,
// not guest RAM, and persist across suspend/resume via the snapshot's rootfs-upper
// tar. (Snapshots from the retired tmpfs-upper mode still restore: their upper rides
// inside the restored guest memory and needs nothing from this file.) This file holds
// the overlay-specific helpers.

import (
"context"
Expand Down Expand Up @@ -54,6 +57,17 @@ const (
// volume's contents live at <guestDurableDir>/<volumeName> and are bind-mounted
// from there into the containers that declare the volume.
guestDurableDir = "/run/ateom-durable"

// UpperFsTag is the virtio-fs tag for the actor's WRITABLE disk-backed
// rootfs upper share, served by a third virtiofsd from
// ateompath.RootfsUpperDir on the host. Every container's overlay upper
// lives on it, so rootfs writes cost host disk, not guest RAM.
UpperFsTag = "ateUpper"
// guestUpperDir is where the agent mounts UpperFsTag in the guest; each
// container's overlay upper/work then live under <guestUpperDir>/<cid>.
// Deliberately distinct from the retired tmpfs upper's /run/ateom-upper
// prefix, which guests restored from old snapshots may still have mounted.
guestUpperDir = "/run/ateom-upper-disk"
)

// GuestDurableVolumeDir is the in-guest path holding one durable volume's
Expand All @@ -71,11 +85,22 @@ func SharedDir(id string) string {
// VirtiofsdSocketPath is the vhost-user-fs socket CH connects to for the fs device.
func VirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd.sock") }

// OverlayUpperBase is the in-guest mount point for one container's overlay upper/work.
// It lives under /run (tmpfs) so the upper's writes are in guest RAM and ride along in
// the memory-only snapshot (rootfs writes persist). Keyed on the container id, which is
// stable across the actor's restore lineage.
func OverlayUpperBase(containerID string) string { return "/run/ateom-upper/" + containerID }
// UpperBase is the in-guest mount point for one container's overlay upper/work:
// a subdirectory of the ateUpper virtio-fs share, so the upper's writes land on
// host disk (ateompath.RootfsUpperDir) instead of guest RAM — the memory
// snapshot stays lean and the upper ships as a tar instead (see
// cmd/ateom-microvm/rootfsupper.go). Keyed on the container id, which is stable
// across the actor's restore lineage.
func UpperBase(containerID string) string { return guestUpperDir + "/" + containerID }

// upperWorkDirs returns the overlay upperdir and workdir for an upper base:
// SIBLING directories under the one base. Both properties are load-bearing —
// the kernel requires upperdir and workdir on the same filesystem, and rejects
// a workdir nested inside (or equal to) upperdir — so a layout change here
// breaks every overlay mount. Covered by a regression test.
func upperWorkDirs(upperBase string) (upper, work string) {
return upperBase + "/fs", upperBase + "/work"
}

// GuestSharedRootfs is the in-guest path the kataShared mount exposes a container's
// rootfs at. A carrier container with this as Root.Path makes the agent bind it to
Expand All @@ -91,6 +116,13 @@ type VirtiofsdOptions struct {
// Cache is virtiofsd's --cache mode. Empty defaults to "always", which is
// only correct for a strictly read-only share (see virtiofsdArgs).
Cache string
// Xattr enables xattr passthrough (--xattr). Required for a share hosting an
// overlayfs upper: overlay records whiteouts and opaque directories as
// user.overlay.* xattrs in the upper (userxattr mode), and without
// passthrough the guest's overlay mount cannot round-trip them to the host
// (deletes of lower files would fail or silently un-delete across
// suspend/resume).
Xattr bool
Log io.Writer
}

Expand All @@ -106,14 +138,18 @@ func virtiofsdArgs(o VirtiofsdOptions) []string {
// side changes underneath the guest (e.g. contents restored from a snapshot).
cache = "always"
}
return []string{
args := []string{
"--socket-path=" + o.SocketPath,
"--shared-dir=" + o.SharedDir,
"--cache=" + cache,
"--thread-pool-size=1",
"--announce-submounts",
"--migration-mode", "find-paths",
}
if o.Xattr {
args = append(args, "--xattr")
}
return args
}

// StartVirtiofsd launches virtiofsd in find-paths migration mode serving o.SharedDir
Expand Down Expand Up @@ -177,8 +213,8 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID,
for _, d := range []string{"proc", "sys", "dev"} {
_ = os.MkdirAll(filepath.Join(dst, d), 0o755)
}
// Remount read-only: the lower is immutable, so all writes go to the tmpfs upper and
// it stays byte-identical across reconstructions (required by find-paths migration).
// Remount read-only: the lower is immutable, so all writes go to the overlay upper
// and it stays byte-identical across reconstructions (required by find-paths migration).
ro := exec.CommandContext(ctx, "mount", "-o", "remount,bind,ro", dst)
var roErr strings.Builder
ro.Stderr = &roErr
Expand All @@ -189,17 +225,27 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID,
}

// CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount
// (the RO base backing every container's rootfs). Mirrors kata startSandbox.
// (the RO base backing every container's rootfs) and the writable disk-backed rootfs
// upper share, under whose mount each container's overlay upper/work live (UpperBase).
// Mirrors kata startSandbox.
//
// withDurableShare additionally mounts the writable durable-dir share, whose
// per-volume subdirectories the containers bind-mount at their declared paths.
func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error {
storages := []*agentpb.Storage{{
Driver: virtioFSDriver,
Source: FsTag,
Fstype: typeVirtioFS,
MountPoint: guestSharedDir,
}}
storages := []*agentpb.Storage{
{
Driver: virtioFSDriver,
Source: FsTag,
Fstype: typeVirtioFS,
MountPoint: guestSharedDir,
},
{
Driver: virtioFSDriver,
Source: UpperFsTag,
Fstype: typeVirtioFS,
MountPoint: guestUpperDir,
},
}
if withDurableShare {
storages = append(storages, &agentpb.Storage{
Driver: virtioFSDriver,
Expand All @@ -222,7 +268,7 @@ func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, host
func (a *AgentClient) CreateCarrier(ctx context.Context, cid string, spec *specs.Spec) error {
pbSpec := SpecToAgentPB(spec)
// Readonly: the carrier only exists to materialize the base bind; its rootfs (the
// overlay lower) must stay immutable. Overlay writes go to the tmpfs upper.
// overlay lower) must stay immutable. Overlay writes go to the disk-backed upper.
pbSpec.Root = &agentpb.Root{Path: GuestSharedRootfs(cid), Readonly: true}
if pbSpec.Linux != nil {
pbSpec.Linux.CgroupsPath = "/ateomchv/" + cid + "-carrier"
Expand All @@ -239,17 +285,16 @@ func (a *AgentClient) CreateCarrier(ctx context.Context, cid string, spec *specs

// StartOverlayWorkload creates + starts one container with an overlayfs rootfs:
// lower = the carrier's resolved bind (/run/kata-containers/<cid>/rootfs from the RO
// virtio-fs base), upper/work = <upperBase>/{fs,work} on a guest tmpfs so rootfs writes
// land in guest RAM (captured by the memory-only snapshot → persist). The agent creates
// the upper/work dirs (create_directory) before mounting the overlay.
// virtio-fs base), upper/work = <upperBase>/{fs,work} on the disk-backed ateUpper
// share (UpperBase: writes land on host disk, shipped as a tar at checkpoint). The
// agent creates the upper/work dirs (create_directory) before mounting the overlay.
func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID, upperBase string, spec *specs.Spec) error {
const createDir = "io.katacontainers.volume.overlayfs.create_directory"
sharedBase := "/run/kata-containers/" + cid + "/rootfs"
base := "/run/kata-containers/" + workloadID
lower := base + "/lower"
ovlRoot := base + "/rootfs"
upper := upperBase + "/fs"
work := upperBase + "/work"
upper, work := upperWorkDirs(upperBase)

storages := []*agentpb.Storage{
{
Expand All @@ -260,12 +305,18 @@ func (a *AgentClient) StartOverlayWorkload(ctx context.Context, cid, workloadID,
Options: []string{"bind"},
},
{
Driver: "overlayfs",
Source: "overlay",
Fstype: "overlay",
MountPoint: ovlRoot,
Driver: "overlayfs",
Source: "overlay",
Fstype: "overlay",
MountPoint: ovlRoot,
DriverOptions: []string{createDir + "=" + upper, createDir + "=" + work},
Options: []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work},
// index=off,metacopy=off,userxattr: required for an upper on
// virtio-fs — the guest kernel rejects the mount (EINVAL) with
// file-handle indexing enabled, and whiteouts/opaque markers must
// use unprivileged user.overlay.* xattrs (which the snapshot tar
// round-trips as PAX records; see tarutil).
Options: []string{"lowerdir=" + lower, "upperdir=" + upper, "workdir=" + work,
"index=off", "metacopy=off", "userxattr"},
},
}
pbSpec := SpecToAgentPB(spec)
Expand Down
Loading
Loading