feat(driver): a cold suspend produces the portable workspace checkpoint, and a resume can restore from it

cmd/runnerd/main.go

@@ -3,6 +3,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"log"
@@ -14,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/tokencanopy/rainier/checkpoint"
 	"github.com/tokencanopy/rainier/internal/driver"
 	"github.com/tokencanopy/rainier/internal/runnerd"
 )
@@ -68,6 +70,14 @@ func main() {
 		"cgroup v2 mount point; empty means /sys/fs/cgroup. It is where host-side metering reads each microVM's cpu.stat and memory.current from")
 	microvmSeccompOff := flag.Bool("microvm-seccomp-off", os.Getenv("RAINIER_MICROVM_SECCOMP_OFF") == "1",
 		"turn OFF the jailer's seccomp filter on the Firecracker process. Seccomp is on by default and this exists for diagnosing a filter rejection on a new kernel, not for production")
+	checkpointStoreDir := flag.String("checkpoint-store-dir", envDefault("RAINIER_CHECKPOINT_STORE_DIR", ""),
+		"host directory holding the portable workspace checkpoints a cold suspend writes (required when --driver=microvm). A cold suspend does not report success until a session's workspace is a committed, verified checkpoint in here (ADR-0003 §4.4), and a deep-dormant resume rebuilds the workspace image from it. A hosted cell puts these in regional object storage instead")
+	checkpointKeyFile := flag.String("checkpoint-key-file", envDefault("RAINIER_CHECKPOINT_KEY_FILE", ""),
+		"file holding the 32-byte key every workspace checkpoint is wrapped under, as 64 hex characters or 32 raw bytes, mode 0600 (required when --driver=microvm). Without it the checkpoints in --checkpoint-store-dir would be a tenant's files in the clear on this host; with it, losing this file means losing every checkpoint it wrapped")
+	checkpointRestoreUID := flag.Int("checkpoint-restore-uid", envIntDefault("RAINIER_CHECKPOINT_RESTORE_UID", 1000),
+		"uid a restored workspace's files are given to before they become a filesystem: the user the session image runs its agent as. A checkpoint records modes and not owners, so without this a restored workspace comes back owned by root and the agent cannot write it. 0 means leave every file owned by this runner")
+	checkpointRestoreGID := flag.Int("checkpoint-restore-gid", envIntDefault("RAINIER_CHECKPOINT_RESTORE_GID", 1000),
+		"gid a restored workspace's files are given to; see --checkpoint-restore-uid")
 	var microvmControlPlane capabilityFlag
 	flag.Var(&microvmControlPlane, "microvm-control-plane-cidr",
 		"a regional control-plane range a microVM guest must not be able to reach; repeatable, or set RAINIER_MICROVM_CONTROL_PLANE_CIDRS to a comma-separated list")
@@ -127,7 +137,19 @@ func main() {
 		case *microvmImageURL != "":
 			imageSource = driver.HTTPImageSource{Base: *microvmImageURL}
 		}
+		// Where a cold suspend's workspace checkpoint goes, and what wraps it.
+		// Both are REQUIRED here, and this is a Fatal for the same reason the
+		// kernel and the rootfs are: a runner that started without them would
+		// accept placements and then fail every cold suspend — with a tenant's
+		// work still inside a VM the durability barrier will not let it
+		// terminate.
+		ckpt, err := checkpointConfig(*checkpointStoreDir, *checkpointKeyFile,
+			*checkpointRestoreUID, *checkpointRestoreGID)
+		if err != nil {
+			log.Fatalf("--driver=microvm: %v", err)
+		}
 		mvm, err := driver.NewMicrovm(driver.MicrovmOpts{
+			Checkpoint:  ckpt,
 			KernelPath:  *kernelPath,
 			BaseRootfs:  *rootfsPath,
 			StateDir:    *microvmStateDir,
@@ -214,6 +236,43 @@ func main() {
 	}
 }
 
+// checkpointConfig builds the microVM driver's checkpoint configuration from
+// the two required flags, or says which one is missing and why it matters.
+//
+// It is a function of its arguments rather than of the flag variables so that
+// the refusals are testable: every one of them is a condition a real deployment
+// gets wrong at some point, and "the runner started anyway" is the outcome none
+// of them may have.
+func checkpointConfig(storeDir, keyFile string, uid, gid int) (*driver.CheckpointOpts, error) {
+	switch {
+	case storeDir == "":
+		return nil, errors.New("--checkpoint-store-dir is required: a cold suspend turns a session's workspace into a portable checkpoint before it terminates the VM (ADR-0003 §4.4), and there is deliberately no default — a temp-directory one would put a tenant's only durable copy somewhere the host reaps")
+	case keyFile == "":
+		return nil, errors.New("--checkpoint-key-file is required: a workspace checkpoint is encrypted under it, and an unencrypted one would be a tenant's files in the clear on this host")
+	case uid < 0 || gid < 0:
+		return nil, errors.New("--checkpoint-restore-uid and --checkpoint-restore-gid must not be negative")
+	}
+	store, err := driver.NewDirBlobStore(storeDir)
+	if err != nil {
+		return nil, err
+	}
+	key, err := driver.LoadCheckpointKey(keyFile)
+	if err != nil {
+		return nil, err
+	}
+	keys, err := checkpoint.NewStaticKeyWrapper(driver.SelfHostedCheckpointKeyRef, key)
+	if err != nil {
+		return nil, err
+	}
+	return &driver.CheckpointOpts{
+		Store:    store,
+		Keys:     keys,
+		KeyRef:   driver.SelfHostedCheckpointKeyRef,
+		OwnerUID: uid,
+		OwnerGID: gid,
+	}, nil
+}
+
 // egressProxyEndpoint resolves the one host-side destination a microVM
 // guest's firewall allows, from --microvm-egress-proxy or, failing that, from
 // --proxy-url.

cmd/runnerd/main_test.go

@@ -2,84 +2,108 @@
 package main
 
 import (
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 )
 
-// TestEgressProxyEndpoint covers the one flag that decides what a microVM
-// guest's firewall lets out.
-func TestEgressProxyEndpoint(t *testing.T) {
+// The two flags a microVM runner cannot start without, and what each refusal
+// has to say. They are Fatals in main for the reason the kernel and the rootfs
+// are: a runner that started without a checkpoint store would accept placements
+// and then fail every cold suspend, with a tenant's work still inside a VM the
+// durability barrier will not let it terminate.
+
+func keyFile(t *testing.T) string {
+	t.Helper()
+	p := filepath.Join(t.TempDir(), "checkpoint.key")
+	key := make([]byte, 32)
+	for i := range key {
+		key[i] = byte(i + 1)
+	}
+	if err := os.WriteFile(p, key, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	return p
+}
+
+func TestCheckpointConfigBuildsTheDriversPorts(t *testing.T) {
+	dir := filepath.Join(t.TempDir(), "checkpoints")
+	opts, err := checkpointConfig(dir, keyFile(t), 1000, 1000)
+	if err != nil {
+		t.Fatalf("checkpointConfig: %v", err)
+	}
+	if opts.Store == nil || opts.Keys == nil {
+		t.Fatalf("the configuration is %+v, want a store and a key wrapper", opts)
+	}
+	if opts.KeyRef == "" {
+		t.Error("the configuration names no key reference; a manifest has to record one")
+	}
+	if opts.OwnerUID != 1000 || opts.OwnerGID != 1000 {
+		t.Errorf("the restored workspace would be given to %d:%d, want 1000:1000", opts.OwnerUID, opts.OwnerGID)
+	}
+	// The store directory is created, because a runner that started without one
+	// would discover it at the worst possible moment.
+	info, err := os.Stat(dir)
+	if err != nil {
+		t.Fatalf("the store directory was not created: %v", err)
+	}
+	if info.Mode().Perm() != 0o700 {
+		t.Errorf("the store directory is mode %v, want 0700", info.Mode().Perm())
+	}
+}
+
+func TestCheckpointConfigRefusesWhatARunnerCannotStartWith(t *testing.T) {
+	good := keyFile(t)
+	loose := filepath.Join(t.TempDir(), "loose.key")
+	if err := os.WriteFile(loose, make([]byte, 32), 0o644); err != nil {
+		t.Fatal(err)
+	}
+
 	cases := []struct {
-		name     string
-		explicit string
-		proxyURL string
-		wantAddr string
-		wantPort int
-		wantErr  string
+		name  string
+		dir   string
+		key   string
+		uid   int
+		gid   int
+		wants string
 	}{
-		{
-			name:     "no proxy at all is a runner with no exception",
-			wantAddr: "",
-			wantPort: 0,
-		},
-		{
-			name:     "an explicit address wins",
-			explicit: "10.44.0.9:3128",
-			proxyURL: "http://egressd:3129",
-			wantAddr: "10.44.0.9",
-			wantPort: 3128,
-		},
-		{
-			name:     "a proxy URL that is already an address",
-			proxyURL: "http://10.44.0.9:3129",
-			wantAddr: "10.44.0.9",
-			wantPort: 3129,
-		},
-		{
-			name:     "a proxy URL with no port takes the scheme's",
-			proxyURL: "https://10.44.0.9",
-			wantAddr: "10.44.0.9",
-			wantPort: 443,
-		},
-		{
-			name:     "a proxy URL naming a host must be given as an address",
-			proxyURL: "http://egressd:3129",
-			wantErr:  "names a host and not an address",
-		},
-		{
-			name:     "an explicit endpoint that is not ip:port",
-			explicit: "10.44.0.9",
-			wantErr:  "not ip:port",
-		},
-		{
-			name:     "an explicit endpoint with a nonsense port",
-			explicit: "10.44.0.9:not-a-port",
-			wantErr:  "not a port",
-		},
-		{
-			name:     "an IPv6 proxy no guest could reach",
-			explicit: "[fd00::1]:3128",
-			wantErr:  "IPv6",
-		},
+		{"no store directory", "", good, 1000, 1000, "--checkpoint-store-dir is required"},
+		{"no key file", t.TempDir(), "", 1000, 1000, "--checkpoint-key-file is required"},
+		{"a key file that is not there", t.TempDir(), filepath.Join(t.TempDir(), "nope"), 1000, 1000, "checkpoint key file"},
+		{"a key anyone can read", t.TempDir(), loose, 1000, 1000, "readable by group or other"},
+		{"a negative uid", t.TempDir(), good, -1, 1000, "must not be negative"},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
-			addr, port, err := egressProxyEndpoint(tc.explicit, tc.proxyURL)
-			if tc.wantErr != "" {
-				if err == nil {
-					t.Fatalf("egressProxyEndpoint(%q, %q) = %q/%d, want an error naming %q", tc.explicit, tc.proxyURL, addr, port, tc.wantErr)
-				}
-				if !strings.Contains(err.Error(), tc.wantErr) {
-					t.Fatalf("error = %q, want it to name %q", err, tc.wantErr)
-				}
-				return
-			}
-			if err != nil {
-				t.Fatalf("egressProxyEndpoint(%q, %q): %v", tc.explicit, tc.proxyURL, err)
+			_, err := checkpointConfig(tc.dir, tc.key, tc.uid, tc.gid)
+			if err == nil {
+				t.Fatal("the configuration was accepted")
 			}
-			if addr != tc.wantAddr || port != tc.wantPort {
-				t.Fatalf("egressProxyEndpoint(%q, %q) = %q/%d, want %q/%d", tc.explicit, tc.proxyURL, addr, port, tc.wantAddr, tc.wantPort)
+			if !strings.Contains(err.Error(), tc.wants) {
+				t.Errorf("the refusal is %q, want it to name %q", err, tc.wants)
 			}
 		})
 	}
 }
+
+// TestCheckpointConfigSaysWhyRatherThanWhat. These lines are what an operator
+// reads at three in the morning, so each one names the consequence and not just
+// the missing argument.
+func TestCheckpointConfigSaysWhyRatherThanWhat(t *testing.T) {
+	_, err := checkpointConfig("", keyFile(t), 1000, 1000)
+	if err == nil {
+		t.Fatal("an empty store directory was accepted")
+	}
+	for _, want := range []string{"portable checkpoint", "no default"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the refusal %q does not explain %q", err, want)
+		}
+	}
+	if _, err = checkpointConfig(t.TempDir(), "", 1000, 1000); err == nil {
+		t.Fatal("an empty key file was accepted")
+	}
+	if !strings.Contains(err.Error(), "in the clear") {
+		t.Errorf("the refusal %q does not explain what an unencrypted checkpoint would be", err)
+	}
+}

cmd/sessiond/coldsuspend_test.go

@@ -13,13 +13,17 @@ import (
 // what is exercised is the frame's whole journey — OnControl's kind
 // dispatch, the `cold` flag, the handler, and the two answers — rather than
 // quiesceCold called directly.
-func coldWired(execs execKiller, boots *bootstrapper) (*rpcDispatcher, *recordingSender) {
+// stream is the workspace streamer the handler runs before it answers ready; a
+// nil one is a sandbox that does not stream, which is what these two tests
+// want — the secrets and the execs are their subject, and the stream has a file
+// of its own (workspacestream_test.go).
+func coldWired(execs execKiller, boots *bootstrapper, stream workspaceStreamer) (*rpcDispatcher, *recordingSender) {
 	d := newRPCDispatcher()
 	sender := &recordingSender{}
 	d.online(sender)
 	d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) {
 		if ev.Cold {
-			quiesceCold(execs, d, nil, boots, ev.ID)
+			quiesceCold(execs, d, nil, boots, ev.ID, stream)
 			return
 		}
 		quiesceExecs(execs, d, ev.ID)
@@ -50,7 +54,7 @@ func TestAColdSuspendForgetsTheDeliveredSecrets(t *testing.T) {
 	boots.remember([]string{"DEPLOY_KEY"})
 
 	execs := &recordingExecs{}
-	d, sender := coldWired(execs, boots)
+	d, sender := coldWired(execs, boots, nil)
 	d.OnControl(suspendingFrame(t, true, 91))
 
 	if _, set := os.LookupEnv("DEPLOY_KEY"); set {
@@ -85,7 +89,7 @@ func TestAWarmSuspendKeepsTheDeliveredSecrets(t *testing.T) {
 	boots.remember([]string{"DEPLOY_KEY"})
 
 	execs := &recordingExecs{}
-	d, sender := coldWired(execs, boots)
+	d, sender := coldWired(execs, boots, nil)
 	d.OnControl(suspendingFrame(t, false, 92))
 
 	if os.Getenv("DEPLOY_KEY") != "value_example" {

cmd/sessiond/flush_test.go

@@ -28,7 +28,7 @@ func flushWired(execs execKiller, boots *bootstrapper, agents agentFlusher) (*rp
 	})
 	d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) {
 		if ev.Cold {
-			quiesceCold(execs, d, nil, boots, ev.ID)
+			quiesceCold(execs, d, nil, boots, ev.ID, nil)
 			return
 		}
 		quiesceExecs(execs, d, ev.ID)