macOS guests 4/6: snapshots, schedules and stopped forks

cmd/vz-shim/main.go

@@ -141,6 +141,9 @@ func main() {
 			case newState := <-stateChanged:
 				slog.Info("VM state changed", "state", newState)
 				if newState == vz.VirtualMachineStateStopped || newState == vz.VirtualMachineStateError {
+					if newState == vz.VirtualMachineStateStopped && !server.forcedStop.Load() {
+						writeStopReceipt(config.StopReceiptPath)
+					}
 					slog.Info("VM stopped, shutting down shim")
 					cancel()
 					return
@@ -165,13 +168,25 @@ func main() {
 	if vm.State() == vz.VirtualMachineStateRunning {
 		slog.Info("stopping VM")
 		if vm.CanStop() {
+			server.forcedStop.Store(true)
 			vm.Stop()
 		}
 	}
 
 	slog.Info("vz-shim shutdown complete")
 }
 
+// writeStopReceipt records that the guest stopped the VM itself. The host counts a
+// macOS stop as graceful only with this receipt.
+func writeStopReceipt(path string) {
+	if path == "" {
+		return
+	}
+	if err := os.WriteFile(path, []byte(time.Now().UTC().Format(time.RFC3339Nano)+"\n"), 0600); err != nil {
+		slog.Error("failed to write stop receipt", "error", err, "path", path)
+	}
+}
+
 func setupLogging(logPath string) error {
 	if logPath == "" {
 		// Log to stderr if no path specified

cmd/vz-shim/server.go

@@ -13,6 +13,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 
 	"github.com/Code-Hex/vz/v3"
 	"github.com/kernel/hypeman/lib/hypervisor/vz/shimconfig"
@@ -24,6 +25,9 @@ type ShimServer struct {
 	vmConfig   *vz.VirtualMachineConfiguration
 	shimConfig shimconfig.ShimConfig
 	mu         sync.RWMutex
+	// forcedStop is set before the shim stops the VM itself, so that stop is not
+	// mistaken for the guest powering off.
+	forcedStop atomic.Bool
 }
 
 // NewShimServer creates a new shim server.
@@ -59,6 +63,7 @@ func (s *ShimServer) Handler() http.Handler {
 	mux.HandleFunc("PUT /api/v1/vm.resume", s.handleResume)
 	mux.HandleFunc("PUT /api/v1/vm.shutdown", s.handleShutdown)
 	mux.HandleFunc("PUT /api/v1/vm.snapshot", s.handleSnapshot)
+	mux.HandleFunc("GET /api/v1/vm.snapshot-support", s.handleSnapshotSupport)
 	mux.HandleFunc("GET /api/v1/vm.balloon", s.handleGetBalloon)
 	mux.HandleFunc("PUT /api/v1/vm.balloon", s.handleSetBalloon)
 	mux.HandleFunc("PUT /api/v1/vm.power-button", s.handlePowerButton)
@@ -138,6 +143,21 @@ func (s *ShimServer) handleShutdown(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
+func (s *ShimServer) handleSnapshotSupport(w http.ResponseWriter, r *http.Request) {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	response := struct {
+		Supported bool   `json:"supported"`
+		Reason    string `json:"reason,omitempty"`
+	}{Supported: true}
+	if err := validateSaveRestoreSupport(s.vmConfig); err != nil {
+		response.Supported = false
+		response.Reason = err.Error()
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(response)
+}
+
 func (s *ShimServer) handleSnapshot(w http.ResponseWriter, r *http.Request) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -289,6 +309,7 @@ func (s *ShimServer) handleVMMShutdown(w http.ResponseWriter, r *http.Request) {
 		defer s.mu.Unlock()
 
 		if s.vm.CanStop() {
+			s.forcedStop.Store(true)
 			s.vm.Stop()
 		}
 		// Process will exit when VM stops (monitored in main)

docs/macos-snapshots.md

@@ -0,0 +1,112 @@
+# macOS snapshots, schedules and restore (PR 4, draft)
+
+Experimental: stopped disk snapshots, same-instance memory checkpoints, snapshot
+schedules, and cold forks from gracefully stopped instances or snapshots.
+Independent warm memory forks are unsupported. Native validation of the memory
+snapshot and schedule paths is still required.
+
+## Stopped snapshots and cold forks
+
+- `kind: Stopped` requires a stopped source and a graceful shutdown receipt: after
+  the shutdown request, the vz-shim recorded that the guest powered the VM off
+  itself, and the owned VMM exited. VMM exit alone does not count, and a forced
+  stop is refused with 409. A shim started before this receipt existed never
+  writes one, so its guest needs a fresh boot and stop.
+- Restore targets are Stopped or Running. The source instance, hardware model,
+  machine identifier and MAC must match the snapshot.
+- A stopped-snapshot fork requires a graceful receipt and an enabled guest agent.
+  Disk and auxiliary storage are copied, the clone gets a fresh machine identifier
+  and MAC, and SSH host keys are rekeyed on first boot. The default target is
+  Stopped; Running cold-boots and completes rekeying before returning.
+- Forking a stopped instance follows the same identity and rekey contract. Hostnames,
+  account credentials and application identities are not rekeyed.
+- A failed rekey stops the clone and leaves rekey pending for a later retry.
+
+## Same-instance memory snapshots
+
+`POST /instances/{id}/snapshots` with `{"kind":"Standby"}`.
+
+- A Running source is paused, its VZ machine state saved, and the VMM shut down
+  with confirmed exit. The disk, auxiliary storage and machine state are copied
+  while the VMM is down, then the source resumes from its own checkpoint.
+- A Standby source is copied without being resumed. Stopped and other states are
+  rejected with 409.
+- Before pausing, the shim validates the exact VM configuration with Apple's
+  `validateSaveRestoreSupport`. Host and architecture capability checks alone do
+  not prove that the guest's devices support save and restore.
+- Pause waits for raw Paused state, not transitional Pausing, before save. Error-path
+  resume and source recovery use bounded cleanup contexts that survive cancellation
+  of the request. If recovery fails, the API reports it and the source may need
+  explicit recovery.
+- VZ machine-state files are uncompressed. Explicit enabled compression is rejected
+  before capture; Linux compression defaults do not apply.
+
+Restore with `POST /instances/{id}/snapshots/{snapshot_id}/restore`.
+
+- Same-instance only, not a clone. The source must be inactive (Stopped or Standby);
+  Running, Paused and other live states are refused.
+- The default target for a memory snapshot is Running. Explicit Standby installs the
+  checkpoint without resuming it. Stopped discards memory and creates no graceful
+  receipt, so a clean boot and stop are needed before a stopped snapshot or cold fork.
+- Preflight checks CPU/RAM, hardware model, machine identifier, NIC MAC, disk and
+  auxiliary paths, and the full memory/disk/auxiliary bundle before storage is
+  replaced. An invalid payload leaves current storage untouched.
+- Restore stages the payload and restored metadata (including the current name and
+  expiration) in `restore-staging/` inside the instance directory, then swaps the
+  instance's files in place. Deleting the instance always removes its staging. A
+  crash before the swap commits rolls back to the previous files; after it, server
+  start or the next restore finishes the install. Host phase accounting is preserved rather than replayed from the
+  checkpoint; a memory restore keeps its boot time. An explicit Stopped target
+  suppresses automatic restart.
+- A restore never mutates the persistent snapshot. If the framework restore fails
+  after installation, the restored checkpoint remains available in Standby.
+- Once the shim has loaded the VM, the checkpoint is retired before the guest
+  resumes, so a later crash cannot restore old memory onto a newer disk. If retiring
+  it fails, the VM is shut down and the checkpoint is kept. A resume that fails after
+  it was retired leaves the instance Stopped with no graceful receipt.
+- If the API dies during a restore, the shim it started keeps loading the VM. Until
+  it opens its socket the instance reads Initializing. Restore returns 409 while
+  that shim has not resumed the VM, and keeps the checkpoint. Once loaded it reads
+  Paused: `POST /restore` retires the checkpoint, resumes the VM and records the
+  shim, and DELETE kills it. As on Linux, stop does not accept Paused.
+- Saved state is not portable. Use the originating Mac and OS account, a compatible
+  framework version and the same VM configuration. API availability on macOS 14+
+  does not guarantee support for a particular guest configuration.
+
+## Schedules
+
+Schedules reuse the existing snapshot-schedule API, interval handling and retention.
+They require host save/restore support, and admission probes the exact configuration
+when a live VM is available.
+
+- A gracefully Stopped source produces a stopped snapshot. A Running source produces
+  a memory snapshot and is resumed afterwards. A Standby source produces a memory
+  snapshot and stays in Standby.
+- An ineligible state, a missing graceful receipt, or a configuration or capture
+  failure is recorded in `last_error`, and no snapshot ID is reported for that run.
+- Failed runs consume the interval, matching the existing scheduler. The next run
+  follows the configured cadence rather than retrying tightly.
+- Scheduling never stops a running guest to manufacture a graceful disk snapshot.
+
+## Warm memory forks are not supported
+
+`ForkSnapshot` rejects macOS memory snapshots with 501, and `ForkInstance` requires a
+gracefully stopped macOS source. VZ restores saved state only into an identical
+configuration: on Apple silicon, changing the MAC, machine identifier, NIC or
+auxiliary storage fails with "invalid argument", while relocating paths works.
+Apple documents that concurrent VMs sharing a machine identifier have undefined guest
+behavior, so copies of one saved state cannot run side by side. Repeated same-instance
+restore is the supported warm start.
+
+References:
+- [Apple: Mac platform configuration](https://developer.apple.com/documentation/virtualization/vzmacplatformconfiguration)
+- [Apple: restoring machine state](https://developer.apple.com/documentation/virtualization/vzvirtualmachine/restoremachinestatefrom(url:completionhandler:))
+
+## Validation
+
+Linux-runnable tests cover capture, restore, isolation, manifest checks, schedules,
+retention and crash-swap recovery. Mock HTTP fixtures cover VZ pause, cancellation
+and configuration probing. Native Apple-silicon runs have exercised Running capture
+and resume, same-instance rollback, repeated restore, schedules with retention,
+cancellation after pause, API crash during restore, and stopped clones with fresh
+identities.

docs/macos-support.md

@@ -15,14 +15,18 @@ owns the behavior.
 | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 |
 | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 |
 | Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 6 |
-| Stopped snapshots, same-instance restore | Partial | Graceful stop required. Running restore has an open readiness issue. | PR 4 |
-| Standby, memory snapshot, memory fork | Unsupported | Rejected with a stable error. | — |
-| Stopped clones, fork from snapshot | Unsupported | Needs identity rebinding and guest rekeying. | PR 4 (later) |
+| Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 |
+| Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 |
+| Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 |
+| Independent warm memory forks | Unsupported | VZ restores saved state only into an identical configuration (same machine identifier and MAC), and Apple documents concurrent VMs sharing an identifier as undefined. Repeated same-instance restore is the warm start. | — |
+| Stopped clones (fork from stopped) | Experimental | Rebinds machine identifier and MAC; rekeys SSH host keys on first start. Hostname and credentials not yet checked. | PR 4 |
+| Fork from stopped snapshot | Experimental | Graceful receipt and enabled agent required. Fresh machine identifier/MAC; SSH rekey on first cold boot. Stopped and Running targets supported. | PR 4 |
 | Update (PATCH), VM restart policy | Partial | Expiration/TTL and `restart_policy: always` or `never` are supported. Manual stop suppresses automatic restart. A shim that dies without removing its socket (for example, SIGKILL) reads as Stopped, so `always` restarts it. `on_failure` is rejected: no workload exit status is available. | PR 1 |
 | Exec health checks | Agent-gated | Supported at create and PATCH; `type: none` disables them. HTTP/TCP checks remain unsupported. | PR 3 |
 | Volumes, env and commands, credentials and auto-standby policies, passthrough, shaping | Unsupported | Rejected at create, start, or update. | — |
 | Ingress | Unsupported | Refused in macOS-only mode and for macOS targets. | — |
-| Snapshot schedules | Unsupported | Refused at admission. | — |
+| Snapshot schedules | Experimental | State-based stopped or memory capture, existing cadence/retention, host save/restore gate and live configuration probe. New path needs native QA. | PR 4 |
 
-Limits that apply everywhere: preserved machine identity (one active instance per
-identifier, no rekeying), no clean-base sanitation audit, and no external review verdict.
+Limits that apply everywhere: one active VM per machine identifier, no clean-base
+sanitation audit, and no external review verdict. Same-instance memory restore
+preserves identity; stopped clones get fresh machine identifiers and SSH host keys.

lib/forkvm/copy.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"strings"
 	"sync/atomic"
+
+	"github.com/kernel/hypeman/lib/paths"
 )
 
 var (
@@ -172,7 +174,7 @@ func copyRegularFile(state *copyState, srcPath, dstPath string, perms fs.FileMod
 }
 
 func shouldSkipDirectory(relPath string) bool {
-	return relPath == "logs"
+	return relPath == "logs" || relPath == paths.RestoreStagingDir
 }
 
 func shouldSkipRegularFile(relPath string) bool {

lib/hypervisor/snapshot_support.go

@@ -0,0 +1,8 @@
+package hypervisor
+
+import "context"
+
+// SnapshotSupportValidator checks configuration-specific support before pausing.
+type SnapshotSupportValidator interface {
+	ValidateSnapshotSupport(context.Context) error
+}

lib/hypervisor/snapshot_support_test.go

@@ -0,0 +1,34 @@
+package hypervisor
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type fakeSnapshotSupportValidator struct {
+	fakeHypervisor
+	err    error
+	called bool
+}
+
+func (h *fakeSnapshotSupportValidator) ValidateSnapshotSupport(context.Context) error {
+	h.called = true
+	return h.err
+}
+
+func TestTracingPreservesSnapshotSupportValidation(t *testing.T) {
+	probeErr := errors.New("configuration probe failed")
+	hv := &fakeSnapshotSupportValidator{err: probeErr}
+	wrapped := WrapHypervisor(TypeVZ, hv)
+	validator, ok := wrapped.(SnapshotSupportValidator)
+	require.True(t, ok)
+	require.ErrorIs(t, validator.ValidateSnapshotSupport(context.Background()), probeErr)
+	require.True(t, hv.called)
+	hv.err = nil
+	require.NoError(t, validator.ValidateSnapshotSupport(context.Background()))
+	unsupported := WrapHypervisor(TypeCloudHypervisor, fakeHypervisor{}).(SnapshotSupportValidator)
+	require.ErrorIs(t, unsupported.ValidateSnapshotSupport(context.Background()), ErrNotSupported)
+}

lib/hypervisor/tracing.go

@@ -219,6 +219,18 @@ func (h *tracingHypervisor) Resume(ctx context.Context) (err error) {
 	return h.next.Resume(ctx)
 }
 
+func (h *tracingHypervisor) ValidateSnapshotSupport(ctx context.Context) (err error) {
+	validator, ok := h.next.(SnapshotSupportValidator)
+	if !ok {
+		return ErrNotSupported
+	}
+	ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.validate_snapshot_support",
+		h.spanAttrs(attribute.String("operation", "validate_snapshot_support"))...,
+	)
+	defer func() { finishTraceSpan(span, err) }()
+	return validator.ValidateSnapshotSupport(ctx)
+}
+
 func (h *tracingHypervisor) Snapshot(ctx context.Context, destPath string) (err error) {
 	ctx, span := startTraceSpan(ctx, h.tracer, "hypervisor.snapshot",
 		h.spanAttrs(

lib/hypervisor/vz/client.go

@@ -262,7 +262,28 @@ func (c *Client) GetVMInfo(ctx context.Context) (*hypervisor.VMInfo, error) {
 }
 
 func (c *Client) Pause(ctx context.Context) error {
-	return c.doPut(ctx, "/api/v1/vm.pause", nil)
+	if err := c.doPut(ctx, "/api/v1/vm.pause", nil); err != nil {
+		return err
+	}
+	return c.waitForRawState(ctx, "Paused")
+}
+
+func (c *Client) ValidateSnapshotSupport(ctx context.Context) error {
+	body, err := c.doGet(ctx, "/api/v1/vm.snapshot-support")
+	if err != nil {
+		return err
+	}
+	var response struct {
+		Supported bool   `json:"supported"`
+		Reason    string `json:"reason"`
+	}
+	if err := json.Unmarshal(body, &response); err != nil {
+		return fmt.Errorf("decode snapshot support: %w", err)
+	}
+	if !response.Supported {
+		return fmt.Errorf("%w: VZ configuration cannot be saved/restored: %s", hypervisor.ErrNotSupported, response.Reason)
+	}
+	return nil
 }
 
 func (c *Client) Resume(ctx context.Context) error {
@@ -279,16 +300,16 @@ func (c *Client) Resume(ctx context.Context) error {
 }
 
 const (
-	resumeReadyTimeout      = 10 * time.Second
-	resumeReadyPollInterval = 25 * time.Millisecond
+	stateReadyTimeout      = 10 * time.Second
+	stateReadyPollInterval = 25 * time.Millisecond
 )
 
 // waitForRawState polls vm.info until the shim's raw state string equals want or
-// resumeReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState
+// stateReadyTimeout elapses. It checks the raw state (not the hypervisor.VMState
 // mapping, which collapses "Resuming" into Running) so the post-condition is the
 // guest actually being in the requested state.
 func (c *Client) waitForRawState(ctx context.Context, want string) error {
-	deadline := time.Now().Add(resumeReadyTimeout)
+	deadline := time.Now().Add(stateReadyTimeout)
 	for {
 		state, err := c.rawVMState(ctx)
 		if err == nil && state == want {
@@ -298,12 +319,12 @@ func (c *Client) waitForRawState(ctx context.Context, want string) error {
 			if err != nil {
 				return fmt.Errorf("wait for vm state %q: %w", want, err)
 			}
-			return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, resumeReadyTimeout, state)
+			return fmt.Errorf("vm did not reach state %q within %s (last state %q)", want, stateReadyTimeout, state)
 		}
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
-		case <-time.After(resumeReadyPollInterval):
+		case <-time.After(stateReadyPollInterval):
 		}
 	}
 }