diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f55dfe2f9..fb5c56d03 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -101,7 +101,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec chSocket = ra.apiSocket } client := ch.NewClient(chSocket) - if err := client.WaitReady(ctx, 10*time.Second); err != nil { + if _, err := client.WaitReady(ctx, 10*time.Second); err != nil { return nil, fmt.Errorf("while waiting for CH api-socket: %w", err) } @@ -218,7 +218,13 @@ func (s *AteomService) snapshotVMState(ctx context.Context, client *ch.Client, r // source). Overlay it onto that source to rebuild a COMPLETE memory-ranges, so the // snapshot is self-contained and re-restorable. (A cold-run actor has no restore // source and its snapshot is already complete — no merge.) - if ra != nil && ra.restoreSourceDir != "" { + if ra != nil && ra.snapshotIsSelfContained { + // Eager restore already pulled every populated extent into guest memory, so + // what cloud-hypervisor just wrote is the whole guest, not a delta. Merging + // would copy the entire resident set onto the restore source for nothing. + slog.InfoContext(ctx, "Snapshot is self-contained (eager restore); skipping merge", + slog.String("id", actorUID)) + } else if ra != nil && ra.restoreSourceDir != "" { base := filepath.Join(ra.restoreSourceDir, "memory-ranges") delta := filepath.Join(checkpointDir, "memory-ranges") tMerge := time.Now() diff --git a/cmd/ateom-microvm/internal/ch/api.go b/cmd/ateom-microvm/internal/ch/api.go index 97bf6832a..1e2dd72c6 100644 --- a/cmd/ateom-microvm/internal/ch/api.go +++ b/cmd/ateom-microvm/internal/ch/api.go @@ -56,24 +56,6 @@ func newAPIClient(socketPath string) *apiClient { } } -// get issues a GET and checks for a 2xx status. -func (c *apiClient) get(ctx context.Context, path string) error { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiBase+path, nil) - if err != nil { - return err - } - resp, err := c.http.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - _, _ = io.Copy(io.Discard, resp.Body) - if resp.StatusCode >= 300 { - return fmt.Errorf("GET %s: status %d", path, resp.StatusCode) - } - return nil -} - // getJSON issues a GET and decodes the 2xx JSON response into out. func (c *apiClient) getJSON(ctx context.Context, path string, out any) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiBase+path, nil) diff --git a/cmd/ateom-microvm/internal/ch/ch.go b/cmd/ateom-microvm/internal/ch/ch.go index 4a011606a..0921635d7 100644 --- a/cmd/ateom-microvm/internal/ch/ch.go +++ b/cmd/ateom-microvm/internal/ch/ch.go @@ -35,6 +35,7 @@ import ( type Client struct { apiSocket string api *apiClient + info VMMInfo } // NewClient returns a Client bound to a cloud-hypervisor api-socket path. The @@ -43,24 +44,47 @@ func NewClient(apiSocket string) *Client { return &Client{apiSocket: apiSocket, api: newAPIClient(apiSocket)} } -// Ping returns nil if the VMM api-socket answers vmm.ping. -func (c *Client) Ping(ctx context.Context) error { - return c.api.get(ctx, "/api/v1/vmm.ping") +// Info returns what the VMM last reported about itself, zero until a successful +// Ping or WaitReady. A Client belongs to one actor's VMM and is used from that +// actor's goroutine, so this needs no synchronization. +func (c *Client) Info() VMMInfo { return c.info } + +// VMMInfo is what vmm.ping reports about the running VMM. Version is a semver +// ("53.0.0"); BuildVersion is the release tag it was built from ("v53.0"). +type VMMInfo struct { + Version string `json:"version"` + BuildVersion string `json:"build_version"` + Features []string `json:"features"` +} + +// Ping reports what the VMM says about itself, or an error if the api-socket does +// not answer vmm.ping. +func (c *Client) Ping(ctx context.Context) (VMMInfo, error) { + var info VMMInfo + if err := c.api.getJSON(ctx, "/api/v1/vmm.ping", &info); err != nil { + return VMMInfo{}, err + } + c.info = info + return info, nil } -// WaitReady blocks until the api-socket answers vmm.ping or the deadline passes. -func (c *Client) WaitReady(ctx context.Context, deadline time.Duration) error { +// WaitReady blocks until the api-socket answers vmm.ping or the deadline passes, +// returning what that answer said. Callers get the VMM's version for free this way: +// the handshake already happens before every boot and restore, so nothing has to +// run the binary again to ask. +func (c *Client) WaitReady(ctx context.Context, deadline time.Duration) (VMMInfo, error) { end := time.Now().Add(deadline) for { - if err := c.Ping(ctx); err == nil { - return nil + info, err := c.Ping(ctx) + if err == nil { + return info, nil } if !time.Now().Before(end) { - return fmt.Errorf("cloud-hypervisor api socket %q not ready after %s", c.apiSocket, deadline) + return VMMInfo{}, fmt.Errorf("cloud-hypervisor api socket %q not ready after %s", c.apiSocket, deadline) } select { case <-ctx.Done(): - return ctx.Err() + return VMMInfo{}, ctx.Err() case <-time.After(10 * time.Millisecond): } } diff --git a/cmd/ateom-microvm/internal/ch/ch_test.go b/cmd/ateom-microvm/internal/ch/ch_test.go index 4fbac88bd..6f4b1f596 100644 --- a/cmd/ateom-microvm/internal/ch/ch_test.go +++ b/cmd/ateom-microvm/internal/ch/ch_test.go @@ -61,6 +61,12 @@ func startFakeCH(t *testing.T) (*Client, *fakeCH) { f.mu.Lock() f.requests = append(f.requests, recordedReq{method: r.Method, path: r.URL.Path, body: string(body)}) f.mu.Unlock() + if r.URL.Path == "/api/v1/vmm.ping" { + // Mirror what a real VMM answers, which callers parse for its version. + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"build_version":"v52.0","version":"52.0.0","pid":1,"features":["kvm"]}`)) + return + } w.WriteHeader(http.StatusNoContent) }) f.srv = &http.Server{Handler: mux} @@ -82,7 +88,7 @@ func TestClientLifecycleCalls(t *testing.T) { client, fake := startFakeCH(t) ctx := context.Background() - if err := client.WaitReady(ctx, time.Second); err != nil { + if _, err := client.WaitReady(ctx, time.Second); err != nil { t.Fatalf("WaitReady: %v", err) } if err := client.Pause(ctx); err != nil { @@ -135,7 +141,7 @@ func TestClientLifecycleCalls(t *testing.T) { func TestWaitReadyTimesOut(t *testing.T) { // Socket that never exists -> WaitReady should time out, not hang. client := NewClient(filepath.Join(t.TempDir(), "nonexistent.sock")) - err := client.WaitReady(context.Background(), 50*time.Millisecond) + _, err := client.WaitReady(context.Background(), 50*time.Millisecond) if err == nil { t.Fatal("WaitReady returned nil for a dead socket, want timeout error") } diff --git a/cmd/ateom-microvm/internal/ch/prefault.go b/cmd/ateom-microvm/internal/ch/prefault.go new file mode 100644 index 000000000..c8387d74b --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/prefault.go @@ -0,0 +1,108 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +import ( + "strconv" + "strings" +) + +// Guest RAM restore modes accepted by vm.restore. +const ( + // MemRestoreOnDemand faults pages in as the guest touches them, so an idle + // restored guest holds its working set rather than its whole snapshot. + MemRestoreOnDemand = "OnDemand" + // MemRestoreEager reads the snapshot's populated extents up front. It registers + // no userfaultfd, so nothing prefaults and nothing gates a later snapshot. + MemRestoreEager = "Copy" +) + +// prefaultingSince is the first cloud-hypervisor release whose userfaultfd restore +// handler background-prefaults every registered page (PR #8150) and refuses +// vm.snapshot until that finishes (PR #8556, the fix for issue #8525). +var prefaultingSince = [3]int{53, 0, 0} + +// prefaultingUntil bounds the affected range, exclusive. It is deliberately open +// ({0,0,0} means "no known fix yet") rather than absent: OnDemand is the mode we +// actually want, so when a release stops prefaulting unconditionally — or lets a +// caller decline it — set this to that version and the smaller idle footprint comes +// back on its own, instead of every future release inheriting the workaround. +var prefaultingUntil = [3]int{0, 0, 0} + +// PrefaultsUnconditionally reports whether this VMM prefaults an OnDemand restore. +// +// It reports true when the version cannot be read or parsed. The two ways to be +// wrong are not equal: choosing OnDemand on an affected version leaves the guest +// unable to pass its readiness probe, because the prefault storm starves it, while +// choosing eager on an unaffected one merely costs memory. Callers should log when +// they fall back on an unknown version, since that cost is otherwise invisible. +func (i VMMInfo) PrefaultsUnconditionally() bool { + v, ok := i.semver() + if !ok { + return true + } + if compareVersions(v, prefaultingSince) < 0 { + return false + } + if prefaultingUntil != [3]int{0, 0, 0} && compareVersions(v, prefaultingUntil) >= 0 { + return false + } + return true +} + +// semver parses the reported version, preferring the semver field ("53.0.0") and +// falling back to the release tag ("v53.0"). +func (i VMMInfo) semver() ([3]int, bool) { + if v, ok := parseVersion(i.Version); ok { + return v, true + } + return parseVersion(i.BuildVersion) +} + +// parseVersion reads a dotted version, tolerating a leading "v", a missing patch +// component, and trailing build metadata ("53.0.0-dirty"). +func parseVersion(s string) ([3]int, bool) { + s = strings.TrimSpace(s) + s = strings.TrimPrefix(s, "v") + if s == "" { + return [3]int{}, false + } + // Drop any pre-release or build suffix. + if i := strings.IndexAny(s, "-+"); i >= 0 { + s = s[:i] + } + + var out [3]int + for idx, part := range strings.SplitN(s, ".", 3) { + n, err := strconv.Atoi(part) + if err != nil || n < 0 { + return [3]int{}, false + } + out[idx] = n + } + return out, true +} + +func compareVersions(a, b [3]int) int { + for i := range a { + switch { + case a[i] < b[i]: + return -1 + case a[i] > b[i]: + return 1 + } + } + return 0 +} diff --git a/cmd/ateom-microvm/internal/ch/prefault_test.go b/cmd/ateom-microvm/internal/ch/prefault_test.go new file mode 100644 index 000000000..432f75163 --- /dev/null +++ b/cmd/ateom-microvm/internal/ch/prefault_test.go @@ -0,0 +1,86 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ch + +import "testing" + +func TestPrefaultsUnconditionally(t *testing.T) { + for _, tc := range []struct { + name string + info VMMInfo + want bool + }{ + // Real payloads, as reported by the binaries we ship. + {"v52 unaffected", VMMInfo{Version: "52.0.0", BuildVersion: "v52.0"}, false}, + {"v53 affected", VMMInfo{Version: "53.0.0", BuildVersion: "v53.0"}, true}, + {"a later release stays affected until a fix is known", VMMInfo{Version: "60.1.2"}, true}, + {"much older", VMMInfo{Version: "41.0.0"}, false}, + + // The semver field is preferred, but the release tag is enough on its own. + {"tag only", VMMInfo{BuildVersion: "v52.0"}, false}, + {"tag only, affected", VMMInfo{BuildVersion: "v53.0"}, true}, + {"suffixed", VMMInfo{Version: "53.0.0-dirty"}, true}, + + // Unknown means eager: a wrong guess toward eager costs memory, a wrong guess + // toward OnDemand leaves the guest unable to pass its readiness probe. + {"empty", VMMInfo{}, true}, + {"garbage", VMMInfo{Version: "not-a-version"}, true}, + {"partial garbage", VMMInfo{Version: "53.x.0"}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := tc.info.PrefaultsUnconditionally(); got != tc.want { + t.Errorf("PrefaultsUnconditionally() = %v, want %v (info %+v)", got, tc.want, tc.info) + } + }) + } +} + +// TestPrefaultingUpperBound documents how to retire the workaround: setting the +// exclusive upper bound restores OnDemand on releases that carry the fix. +func TestPrefaultingUpperBound(t *testing.T) { + orig := prefaultingUntil + prefaultingUntil = [3]int{55, 0, 0} + t.Cleanup(func() { prefaultingUntil = orig }) + + if !(VMMInfo{Version: "54.0.0"}).PrefaultsUnconditionally() { + t.Error("54.0.0 is inside the affected range, want prefaulting") + } + if (VMMInfo{Version: "55.0.0"}).PrefaultsUnconditionally() { + t.Error("55.0.0 is at the fix, want OnDemand back") + } +} + +func TestParseVersion(t *testing.T) { + for _, tc := range []struct { + in string + want [3]int + ok bool + }{ + {"53.0.0", [3]int{53, 0, 0}, true}, + {"v53.0", [3]int{53, 0, 0}, true}, + {"53", [3]int{53, 0, 0}, true}, + {" 52.1.3 ", [3]int{52, 1, 3}, true}, + {"53.0.0+build7", [3]int{53, 0, 0}, true}, + {"", [3]int{}, false}, + {"v", [3]int{}, false}, + {"abc", [3]int{}, false}, + {"53.abc", [3]int{}, false}, + } { + got, ok := parseVersion(tc.in) + if ok != tc.ok || (ok && got != tc.want) { + t.Errorf("parseVersion(%q) = %v,%v want %v,%v", tc.in, got, ok, tc.want, tc.ok) + } + } +} diff --git a/cmd/ateom-microvm/internal/ch/restorefds.go b/cmd/ateom-microvm/internal/ch/restorefds.go index 88e2c96df..f023faf2b 100644 --- a/cmd/ateom-microvm/internal/ch/restorefds.go +++ b/cmd/ateom-microvm/internal/ch/restorefds.go @@ -72,7 +72,7 @@ func LaunchVMM(ctx context.Context, o LaunchVMMOptions) (*exec.Cmd, *Client, err return nil, nil, fmt.Errorf("while starting cloud-hypervisor: %w", err) } client := NewClient(o.APISocket) - if err := client.WaitReady(ctx, 15*time.Second); err != nil { + if _, err := client.WaitReady(ctx, 15*time.Second); err != nil { _ = cmd.Process.Kill() _, _ = cmd.Process.Wait() return nil, nil, fmt.Errorf("while waiting for VMM api-socket: %w", err) diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 93d6518a9..9443b17a5 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -40,6 +40,30 @@ import ( "google.golang.org/grpc/status" ) +// restoreMemMode picks how cloud-hypervisor should load guest RAM, from what the VMM +// just told us about itself over vmm.ping. +// +// OnDemand is what we want: it faults pages in as the guest touches them, so an idle +// restored actor holds its working set rather than its whole snapshot — on the counter +// demo, 16MiB against 158MiB. Eager gives that up, reading every populated extent up +// front. +// +// It is still the right choice on a VMM that prefaults, where OnDemand is not merely +// wasteful but unusable: the prefault storm starves the guest and its readiness probe +// never passes. +func restoreMemMode(ctx context.Context, info ch.VMMInfo) string { + if !info.PrefaultsUnconditionally() { + return ch.MemRestoreOnDemand + } + if info.Version == "" && info.BuildVersion == "" { + // Unknown version: eager works everywhere, so prefer a bigger idle footprint + // over an actor that cannot start. Say so, because that cost is invisible. + slog.WarnContext(ctx, "cloud-hypervisor did not report a version; restoring eagerly", + slog.String("mode", ch.MemRestoreEager)) + } + return ch.MemRestoreEager +} + // RestoreWorkload brings the actor back from a snapshot, on a possibly different // pod. What that means depends on the scope the snapshot was taken with: // @@ -277,14 +301,20 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, _ = chCmd.Process.Kill() } }() - // OnDemand (userfaultfd) memory restore: ~75ms vs ~1.8s eager, and it keeps the - // memfd SPARSE so the next suspend isn't the eager-copy-densified full-RAM scan. - // CH's OnDemand snapshot alone would be INCOMPLETE (it writes only faulted pages, - // dropping the un-faulted ones it demand-pages from this source) — so - // CheckpointWorkload overlays CH's delta onto this source (restoreSourceDir) to - // rebuild a complete snapshot. CH demand-pages from restoreDir for the VM's whole - // lifetime, so it must persist until teardown (atelet keeps it until reset). - if err := client.RestoreWithNetFDs(ctx, restoreDir, restoredNets, "OnDemand"); err != nil { + // How guest RAM comes back depends on the VMM (see restoreMemMode), and the rest + // of the actor's lifecycle follows from that choice: + // + // - OnDemand: cloud-hypervisor demand-pages from restoreDir for the VM's whole + // lifetime, so it must stay put, and the snapshot it writes later holds only + // the pages faulted in meanwhile — CheckpointWorkload overlays that delta onto + // this source to rebuild a complete one. + // - Eager: every populated extent is read here and now. Nothing pages from the + // source afterwards and nothing merges against it, so it is dropped below and + // the next snapshot stands on its own. + memMode := restoreMemMode(ctx, client.Info()) + slog.InfoContext(ctx, "restoring guest memory", + slog.String("mode", memMode), slog.String("vmm_version", client.Info().Version)) + if err := client.RestoreWithNetFDs(ctx, restoreDir, restoredNets, memMode); err != nil { return fmt.Errorf("while restoring VM with net FDs: %w", err) } if err := client.Resume(ctx); err != nil { @@ -296,9 +326,25 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, return fmt.Errorf("while waiting for container readyz: %w", err) } + // An eager restore has read the whole snapshot into guest memory, and nothing + // merges against it afterwards, so the staged copy is dead weight from here on — + // a second ~160MiB per running actor on top of the checkpoint it will write. + // Drop the memory image but keep the directory: atelet re-stages it wholesale + // before any later restore, and the small files beside it stay cheap to keep. + if memMode == ch.MemRestoreEager { + staged := filepath.Join(restoreDir, "memory-ranges") + if err := os.Remove(staged); err != nil && !os.IsNotExist(err) { + // Not fatal: it only costs disk until the actor is torn down. + slog.WarnContext(ctx, "could not drop the staged memory image", "error", err) + } else { + slog.InfoContext(ctx, "dropped the staged memory image (eager restore needs no merge base)") + } + } + ra := &runningActor{ chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, + snapshotIsSelfContained: memMode == ch.MemRestoreEager, } // Re-attach stdout/stderr forwarding for each container: the restored guest's diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index eb4b34740..c8f723d27 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -76,6 +76,12 @@ type runningActor struct { // un-faulted pages). Empty for cold-run actors (their snapshot is already complete). restoreSourceDir string + // snapshotIsSelfContained is set when this actor was restored eagerly, which + // reads every populated extent up front. Every page the snapshot had is then + // resident, so cloud-hypervisor's next snapshot already holds all of it and + // there is no delta to overlay onto restoreSourceDir. + snapshotIsSelfContained bool + // logAgent is the kata-agent ttrpc client kept open for the lifetime of the // stdout/stderr forwarding goroutines (they pump the container's output via // ReadStdout/ReadStderr on this connection). It is NOT closed when RunWorkload /